├── .eslintignore ├── .eslintrc.js ├── .github └── workflows │ └── ci.yml ├── .gitignore ├── README.md ├── bin └── macaca-macos.ts ├── build.sh ├── index.ts ├── package.json ├── playground ├── .eslintrc.js ├── README.md └── play-calculator.js ├── resource ├── applescript │ ├── readme.md │ ├── scpt │ │ └── mouseClick.scpt │ ├── scptd │ │ └── window.scptd │ │ │ └── Contents │ │ │ ├── Info.plist │ │ │ └── Resources │ │ │ ├── Scripts │ │ │ └── main.scpt │ │ │ └── description.rtfd │ │ │ └── TXT.rtf │ └── src │ │ ├── mouseClick.applescript │ │ └── window.applescript ├── javascript │ └── mouse.js └── swift │ ├── build.sh │ ├── mouse-drag-arm64 │ ├── mouse-drag-x64 │ ├── ocr-arm64 │ ├── ocr-x64 │ ├── readme.md │ └── src │ ├── LeftMouseDragged │ ├── .gitignore │ ├── Package.swift │ └── Sources │ │ └── main.swift │ └── MacosOcr │ ├── .gitignore │ ├── Package.swift │ └── Sources │ └── main.swift ├── src ├── core │ ├── enums.ts │ ├── helper.ts │ ├── jxa │ │ ├── exec-jxa.ts │ │ ├── jxaUtil.ts │ │ └── osaUtil.ts │ └── mixin.ts ├── driver │ ├── app.ts │ ├── clipboard.ts │ ├── keyboard.ts │ ├── mouse.ts │ ├── network.ts │ ├── screen.ts │ └── video.ts └── macaca-macos.ts ├── test ├── jxaUtil.test.ts ├── macaca-macos.test.ts ├── mocha.opts └── osaUtil.test.ts └── tsconfig.json /.eslintignore: -------------------------------------------------------------------------------- 1 | **/*.d.ts 2 | node_modules 3 | dist 4 | coverage 5 | .nyc_output 6 | resource 7 | -------------------------------------------------------------------------------- /.eslintrc.js: -------------------------------------------------------------------------------- 1 | 2 | module.exports = { 3 | root: true, 4 | extends: 'eslint-config-egg/typescript', 5 | rules: { 6 | 'valid-jsdoc': 0, 7 | 'no-script-url': 0, 8 | 'no-multi-spaces': 0, 9 | 'default-case': 0, 10 | 'no-case-declarations': 0, 11 | 'one-var-declaration-per-line': 0, 12 | 'no-restricted-syntax': 0, 13 | 'jsdoc/require-param': 0, 14 | 'jsdoc/check-param-names': 0, 15 | 'jsdoc/require-param-description': 0, 16 | 'arrow-parens': 0, 17 | 'prefer-promise-reject-errors': 0, 18 | 'no-control-regex': 0, 19 | 'no-use-before-define': 0, 20 | 'array-callback-return': 0, 21 | 'no-bitwise': 0, 22 | 'no-self-compare': 0, 23 | '@typescript-eslint/no-var-requires': 0, 24 | '@typescript-eslint/ban-ts-ignore': 0, 25 | '@typescript-eslint/no-use-before-define': 0, 26 | 'one-var': 0, 27 | 'no-sparse-arrays': 0, 28 | 'no-useless-concat': 0, 29 | '@typescript-eslint/ban-types': 0, 30 | }, 31 | }; 32 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | workflow_dispatch: 5 | 6 | push: 7 | branches: 8 | - '**' 9 | paths-ignore: 10 | - '**.md' 11 | 12 | jobs: 13 | Runner: 14 | timeout-minutes: 10 15 | runs-on: macOS-latest 16 | steps: 17 | - name: Checkout Git Source 18 | uses: actions/checkout@v3 19 | 20 | - name: Setup Node.js 21 | uses: actions/setup-node@v3 22 | with: 23 | node-version: ${{ matrix.node-version }} 24 | 25 | - name: Install dependencies 26 | run: | 27 | npm i npm@6 -g 28 | npm i 29 | 30 | - name: Continuous integration 31 | run: npm run test 32 | 33 | - name: Code coverage 34 | uses: codecov/codecov-action@v3.0.0 35 | with: 36 | token: ${{ secrets.CODECOV_TOKEN }} -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | .nyc_output 3 | coverage 4 | .idea 5 | .run 6 | dist 7 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # macaca-macos 2 | 3 | --- 4 | 5 | [![NPM version][npm-image]][npm-url] 6 | [![CI][ci-image]][ci-url] 7 | [![Test coverage][codecov-image]][codecov-url] 8 | [![node version][node-image]][node-url] 9 | 10 | [npm-image]: https://img.shields.io/npm/v/macaca-macos.svg?logo=npm 11 | [npm-url]: https://npmjs.org/package/macaca-macos 12 | [ci-image]: https://github.com/macacajs/macaca-macos/actions/workflows/ci.yml/badge.svg 13 | [ci-url]: https://github.com/macacajs/macaca-macos/actions/workflows/ci.yml 14 | [codecov-image]: https://img.shields.io/codecov/c/github/macacajs/macaca-macos.svg?logo=codecov 15 | [codecov-url]: https://codecov.io/gh/macacajs/macaca-macos 16 | [node-image]: https://img.shields.io/badge/node.js-%3E=_16-green.svg?logo=node.js 17 | [node-url]: http://nodejs.org/download/ 18 | 19 | > Integrate [robotjs](https://github.com/octalmage/robotjs), [osascript](https://developer.apple.com/documentation/uniformtypeidentifiers/uttype/3551541-osascript) and shell of the operating system to automatically control the MacOS, such as controlling the mouse, keyboard, window operation, etc. 20 | 21 | ## Installment 22 | 23 | ```bash 24 | $ npm i macaca-macos --save-dev 25 | ``` 26 | 27 | ## Usage 28 | ```ts 29 | import MacacaMacOS from 'macaca-macos'; 30 | 31 | const driver = new MacacaMacOS(); 32 | await driver.startApp('/System/Applications/Notes.app'); 33 | ``` 34 | 35 | 36 | 37 | ## Contributors 38 | 39 | |[
xudafeng](https://github.com/xudafeng)
|[
yihuineng](https://github.com/yihuineng)
|[
snapre](https://github.com/snapre)
| 40 | | :---: | :---: | :---: | 41 | 42 | 43 | This project follows the git-contributor [spec](https://github.com/xudafeng/git-contributor), auto updated at `Mon Aug 29 2022 00:09:09 GMT+0800`. 44 | 45 | 46 | 47 | ## Star History 48 | 49 | [![Star History Chart](https://api.star-history.com/svg?repos=macacajs/macaca-macos&type=Date)](https://star-history.com/#macacajs/macaca-macos&Date) 50 | -------------------------------------------------------------------------------- /bin/macaca-macos.ts: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ts-node 2 | 3 | /** 4 | * @description macaca-macos cli工具 5 | */ 6 | 7 | import MacacaMacOS from '..'; 8 | import { Helper } from '../src/core/helper'; 9 | import path from 'path'; 10 | 11 | process.setMaxListeners(0); 12 | process.on('uncaughtException', function(err) { 13 | console.log('Caught exception: ' + err); 14 | process.exit(1); 15 | }); 16 | 17 | const { program } = require('commander'); 18 | const os = require('os'); 19 | const chalk = require('chalk'); 20 | 21 | const getLogoStr = () => { 22 | const lines = [ 23 | ' __ __ __ __ ___ ____ ', 24 | '| \\/ | __ _ ___ __ _ ___ __ _ | \\/ | __ _ ___ / _ \\/ ___| ', 25 | '| |\\/| |/ _` |/ __/ _` |/ __/ _` | | |\\/| |/ _` |/ __| | | \\___ \\', 26 | '| | | | (_| | (_| (_| | (_| (_| | | | | | (_| | (__| |_| |___) |', 27 | '|_| |_|\\__,_|\\___\\__,_|\\___\\__,_| |_| |_|\\__,_|\\___|\\___/|____/ ', 28 | ]; 29 | return chalk.green(lines.join(os.EOL)); 30 | }; 31 | 32 | program 33 | .addHelpText('before', getLogoStr()) 34 | .version(Helper.getPkgVersion()); 35 | 36 | program 37 | .command('relative_mouse_pos [appName]') 38 | .alias('rmp') 39 | .description('获取鼠标在APP上的相对位置(app界面左上角坐标: 0, 0)') 40 | .option('-c, --color', '顺带获取点位颜色hex值', false) 41 | .action(async (appName, opts) => { 42 | const { color } = opts; 43 | const driver = new MacacaMacOS(); 44 | const realPos = driver.mouseGetPos(); 45 | const appPos = await driver.getAppSizePosition(appName); 46 | let relativePos = `${realPos.x - appPos.topLeftX},${realPos.y - appPos.topLeftY}`; 47 | if (color) { 48 | const colorHex = driver.getPixelColor(realPos.x, realPos.y); 49 | relativePos = `${relativePos} ${colorHex}`; 50 | } 51 | await driver.setClipText(relativePos); 52 | console.log(`${appName}窗口相对坐标: ${relativePos} 已复制到剪贴板`); 53 | }); 54 | 55 | program 56 | .command('resize [appName]') 57 | .description('设置窗口大小和位置') 58 | .option('-w, --width ', '宽', parseInt) 59 | .option('-h, --height ', '高', parseInt) 60 | .option('-p, --position ', '设置app左上角起点 x,y', '0,100') 61 | .action(async (appName, opts) => { 62 | const { position, width, height } = opts; 63 | const driver = new MacacaMacOS(); 64 | await driver.resizePosition({ 65 | name: appName, 66 | topLeftX: Number.parseInt(position.split(',')[0]), 67 | topLeftY: Number.parseInt(position.split(',')[1]), 68 | width, height, 69 | }); 70 | console.log('success'); 71 | }); 72 | 73 | program 74 | .command('ocr [target]') 75 | .description('app或图片ocr') 76 | .action(async (target) => { 77 | const driver = new MacacaMacOS(); 78 | let res; 79 | if (target.endsWith('.png') || target.endsWith('.jpg')) { 80 | target = target.startsWith('/') ? target : path.resolve(process.cwd(), target); 81 | res = await driver.screenOcr({ 82 | picFile: target, 83 | }); 84 | } else { 85 | const appName = target; 86 | await driver.focusApp(appName); 87 | const rect = await driver.getAppSizePosition(appName); 88 | res = await driver.screenOcr({ 89 | rectangle: `${rect.topLeftX},${rect.topLeftY},${rect.width},${rect.height}`, 90 | }); 91 | } 92 | console.log(JSON.stringify(res, null, 2)); 93 | }); 94 | 95 | program.parse(process.argv); 96 | -------------------------------------------------------------------------------- /build.sh: -------------------------------------------------------------------------------- 1 | # 1. clean dist 2 | rm -rf ./dist 3 | 4 | # 2. tsc compile 5 | `npm bin`/tsc 6 | 7 | # 3. move resource 8 | cp -r ./resource ./dist/resource 9 | 10 | # 4. delete unused resource files 11 | rm -rf ./dist/resource/swift/src 12 | rm -rf ./dist/resource/swift/build.sh 13 | 14 | # 4. replace ts-node to node 15 | grep -rl 'ts-node' ./dist/bin | xargs sed -i '' 's/ts-node/node/g' 16 | -------------------------------------------------------------------------------- /index.ts: -------------------------------------------------------------------------------- 1 | import MacacaMacOS from './src/macaca-macos'; 2 | 3 | export default MacacaMacOS; 4 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "macaca-macos", 3 | "version": "0.3.0", 4 | "description": "Macaca MacOS driver", 5 | "keywords": [ 6 | "macos", 7 | "macaca" 8 | ], 9 | "files": [ 10 | "dist" 11 | ], 12 | "main": "dist/index", 13 | "typings": "dist/index.d.ts", 14 | "repository": { 15 | "type": "git", 16 | "url": "https://github.com/macacajs/macaca-macos" 17 | }, 18 | "dependencies": { 19 | "applescript": "^1.0.0", 20 | "chalk": "^4.1.0", 21 | "commander": "^7.2.0", 22 | "npm-update": "3", 23 | "shelljs": "^0.8.5" 24 | }, 25 | "optionalDependencies": { 26 | "robotjs": "^0.6.0" 27 | }, 28 | "devDependencies": { 29 | "@jxa/global-type": "^1.3.5", 30 | "@jxa/types": "^1.3.5", 31 | "@types/mocha": "9", 32 | "@types/node": "^16.11.12", 33 | "eslint": "8", 34 | "eslint-config-egg": "12", 35 | "git-contributor": "1", 36 | "husky": "^1.3.1", 37 | "macaca-ecosystem": "1", 38 | "mocha": "8", 39 | "nyc": "^13.1.0", 40 | "ts-node": "^10.9.1", 41 | "typescript": "4" 42 | }, 43 | "bin": { 44 | "macaca-macos": "./dist/bin/macaca-macos.js" 45 | }, 46 | "scripts": { 47 | "test": "nyc --reporter=lcov --reporter=text mocha 'test/**/*.test.ts' --require ts-node/register --recursive --timeout 600000", 48 | "lint": "eslint . --ext .ts,js", 49 | "build": "sh ./build.sh", 50 | "prepublishOnly": "npm run build", 51 | "contributor": "git-contributor" 52 | }, 53 | "husky": { 54 | "hooks": { 55 | "pre-commit": "npm run lint" 56 | } 57 | }, 58 | "license": "MIT" 59 | } 60 | -------------------------------------------------------------------------------- /playground/.eslintrc.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | module.exports = { 4 | root: true, 5 | extends: 'eslint-config-egg', 6 | rules: { 7 | 'valid-jsdoc': 0, 8 | 'no-script-url': 0, 9 | 'no-multi-spaces': 0, 10 | 'default-case': 0, 11 | 'no-case-declarations': 0, 12 | 'one-var-declaration-per-line': 0, 13 | 'no-restricted-syntax': 0, 14 | 'jsdoc/require-param': 0, 15 | 'jsdoc/check-param-names': 0, 16 | 'jsdoc/require-param-description': 0, 17 | 'arrow-parens': 0, 18 | 'prefer-promise-reject-errors': 0, 19 | 'no-control-regex': 0, 20 | 'no-use-before-define': 0, 21 | 'array-callback-return': 0, 22 | 'no-bitwise': 0, 23 | 'no-self-compare': 0, 24 | 'one-var': 0, 25 | 'no-trailing-spaces': [ 'warn', { skipBlankLines: true }], 26 | 'no-return-await': 0, 27 | }, 28 | }; 29 | -------------------------------------------------------------------------------- /playground/README.md: -------------------------------------------------------------------------------- 1 | # Macaca MacOS Playground 2 | 3 | --- 4 | 5 | ## Calculator 6 | 7 | ```bash 8 | $ node ./play-calculator.js 9 | ``` 10 | -------------------------------------------------------------------------------- /playground/play-calculator.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const { default: MacacaMacOS } = require('../dist'); 4 | 5 | const sleep = ms => new Promise(resolve => setTimeout(resolve, ms)); 6 | 7 | async function main() { 8 | const driver = new MacacaMacOS(); 9 | await driver.startApp('/System/Applications/Calculator.app'); 10 | const isRunning = await driver.isAppRunning('Calculator'); 11 | console.log('app is running: %s', isRunning); 12 | await sleep(3E3); 13 | await driver.keyboardTap('numpad_1'); 14 | await sleep(3E3); 15 | await driver.keyboardTap('numpad_1'); 16 | await sleep(3E3); 17 | await driver.keyboardTap('numpad_1'); 18 | await sleep(3E3); 19 | await driver.keyboardTap('numpad_1'); 20 | } 21 | 22 | main().then().catch(console.log); 23 | -------------------------------------------------------------------------------- /resource/applescript/readme.md: -------------------------------------------------------------------------------- 1 | # applescript 2 | 3 | - src 目录 4 | - 脚本源码，(部分脚本无法直接执行需要通过 系统脚本编辑器软件 编译后执行) 5 | - 部分可通过 osaUtil 直接执行 6 | - scpt 目录 7 | - 使用 Mac系统脚本编辑器软件保存的脚本(已经过编译，可直接通过命令行运行) 8 | - shell方式 osascript ${file} ...args 9 | - scptd 目录 10 | - 使用 Mac系统脚本编辑器软件 导出的脚本包(经过编译，可通过jxa Library引用) 11 | - jxa方式 const window = Library('window'); 12 | 13 | --- 14 | 15 | - 如需通过Library 使用 请将scptd目录下的文件拷贝至 ~/Library/Script Libraries 目录 16 | -------------------------------------------------------------------------------- /resource/applescript/scpt/mouseClick.scpt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/macacajs/macaca-macos/21a9783ca2b54f05d748ecb0e31bb2ee7701091c/resource/applescript/scpt/mouseClick.scpt -------------------------------------------------------------------------------- /resource/applescript/scptd/window.scptd/Contents/Info.plist: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | CFBundleIdentifier 6 | com.apple.ScriptEditor.id.window 7 | CFBundleName 8 | window 9 | CFBundleShortVersionString 10 | 1.0 11 | WindowState 12 | 13 | bundleDividerCollapsed 14 | 15 | bundlePositionOfDivider 16 | 0.0 17 | dividerCollapsed 18 | 19 | eventLogLevel 20 | 2 21 | name 22 | ScriptWindowState 23 | positionOfDivider 24 | 443 25 | savedFrame 26 | 20 395 700 678 0 0 1728 1079 27 | selectedTab 28 | result 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /resource/applescript/scptd/window.scptd/Contents/Resources/Scripts/main.scpt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/macacajs/macaca-macos/21a9783ca2b54f05d748ecb0e31bb2ee7701091c/resource/applescript/scptd/window.scptd/Contents/Resources/Scripts/main.scpt -------------------------------------------------------------------------------- /resource/applescript/scptd/window.scptd/Contents/Resources/description.rtfd/TXT.rtf: -------------------------------------------------------------------------------- 1 | {\rtf1\ansi\ansicpg936\cocoartf2639 2 | \cocoatextscaling0\cocoaplatform0{\fonttbl} 3 | {\colortbl;\red255\green255\blue255;} 4 | {\*\expandedcolortbl;;} 5 | } -------------------------------------------------------------------------------- /resource/applescript/src/mouseClick.applescript: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env osascript 2 | 3 | on run argv 4 | set argList to {} 5 | repeat with arg in argv 6 | set end of argList to quoted form of arg 7 | end repeat 8 | set {TID, text item delimiters} to {text item delimiters, space} 9 | set argList to argList as text 10 | set text item delimiters to TID 11 | 12 | tell application "System Events" 13 | click at {item 1 of argv, item 2 of argv} 14 | end tell 15 | end run 16 | -------------------------------------------------------------------------------- /resource/applescript/src/window.applescript: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env osascript 2 | 3 | -- 之所以有两种方法，是因为这两种方法并非总是生效的，具体软件具体情况不同 4 | -- 移动窗口 5 | on moveBounds(name, topLeftX, topLeftY, bottomRightX, bottomRightY) 6 | tell application name 7 | set bounds of front window to {topLeftX, topLeftY, bottomRightX, bottomRightY} 8 | end tell 9 | end moveBounds 10 | 11 | -- 设置窗口位置和大小 通过重定位 12 | on sizePosition(name, topLeftX, topLeftY, width, height) 13 | tell application "System Events" to tell application process name 14 | tell window 1 15 | set {position, size} to {{topLeftX, topLeftY}, {width, height}} 16 | end tell 17 | end tell 18 | end sizePosition 19 | 20 | -- 设置窗口位置 21 | on setPosition(name, topLeftX, topLeftY) 22 | tell application "System Events" to tell application process name 23 | tell window 1 24 | set {position} to {{topLeftX, topLeftY}} 25 | end tell 26 | end tell 27 | end setPosition 28 | -------------------------------------------------------------------------------- /resource/javascript/mouse.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env osascript -l JavaScript 2 | 3 | // 控制鼠标的点击（左、右），鼠标拖拽、移动事件 4 | // 使用示例： 5 | // click() 6 | // drag(1068 + 122, 38,1068, 38) 7 | // move(652, 568) 8 | 9 | ObjC.import('Cocoa') 10 | ObjC.import('stdlib') 11 | ObjC.import('CoreGraphics') 12 | 13 | 14 | // 获取鼠标坐标========================= 15 | 16 | function location(screenH = 1050) { 17 | const mouseLoc = $.NSEvent.mouseLocation //获取 鼠标当前的的坐标（浮点数） 18 | mouseLoc.mx = parseInt(mouseLoc.x) 19 | mouseLoc.my = screenH - Math.trunc(mouseLoc.y) //坐标需要屏幕高度减获取的坐标 20 | return mouseLoc 21 | } 22 | 23 | // 鼠标的基本操作========================= 24 | 25 | const { mx, my } = location() 26 | const left_mouse_down = $.kCGEventLeftMouseDown //鼠标左键按下事件 27 | const right_mouse_down = $.kCGEventRightMouseDown //鼠标左键按下事件 28 | const left_mouse_up = $.kCGEventLeftMouseUp 29 | const right_mouse_up = $.kCGEventRightMouseUp 30 | const left_mouse_drag = $.kCGEventLeftMouseDragged 31 | const mouse_move = $.kCGEventMouseMoved 32 | const mouse_scroll = $.KCGEventScrollWheel 33 | 34 | // 用于注册鼠标事件 35 | function mouse_event(event_type, coords) { 36 | const nil = $() 37 | // const nil = 10 38 | // usleep(200000) 39 | const event = $.CGEventCreateMouseEvent( 40 | nil, 41 | event_type, 42 | coords, 43 | $.kCGMouseButtonLeft 44 | ) 45 | $.CGEventPost($.kCGHIDEventTap, event) 46 | delay(0.01)//添加一点延迟，保证稳定 47 | // $.CFRelease(event) 48 | } 49 | 50 | function down(x = mx, y = my, r = false) { 51 | const coords = { x: x, y: y } //坐标对象 52 | const mouse_down = r ? right_mouse_down : left_mouse_down 53 | mouse_event(mouse_down, coords) 54 | } 55 | 56 | function up(x = mx, y = my, r = false) { 57 | const coords = { x: x, y: y } //坐标对象 58 | const mouse_up = r ? right_mouse_up : left_mouse_up 59 | mouse_event(mouse_up, coords) 60 | } 61 | 62 | function click(opts = {}) { 63 | const { x = mx, y = my, r = false } = opts; 64 | down(x, y, r) 65 | up(x, y, r) 66 | } 67 | 68 | // drag从特定位置按下，拖拽到指定位置 69 | function drag(tx, ty, cx = location().mx, cy = location().my) { 70 | const t_coords = { x: tx, y: ty } // 拖拽末尾坐标 71 | down(cx, cy) 72 | mouse_event(left_mouse_drag, t_coords) 73 | delay(0.5) 74 | up(tx, ty) 75 | } 76 | 77 | // move是在鼠标没有点击的状态下进行移动 78 | function move(x, y) { 79 | const coords = { x: x, y: y } 80 | mouse_event(mouse_move, coords) 81 | } 82 | 83 | exports.location = location; 84 | exports.click = click; 85 | exports.move = move; 86 | exports.drag = drag; 87 | -------------------------------------------------------------------------------- /resource/swift/build.sh: -------------------------------------------------------------------------------- 1 | # 1. build MacosOcr project 2 | echo "☕️ start build ocr ..." 3 | cd ./src/MacosOcr 4 | swift build -c release 5 | cd ../../ 6 | cp ./src/MacosOcr/.build/release/MacosOcr ./ocr 7 | 8 | # 2. build LeftMouseDragged project 9 | echo "☕️ start build mouse-drag ..." 10 | cd ./src/LeftMouseDragged 11 | swift build -c release 12 | cd ../../ 13 | cp ./src/LeftMouseDragged/.build/release/LeftMouseDragged ./mouse-drag -------------------------------------------------------------------------------- /resource/swift/mouse-drag-arm64: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/macacajs/macaca-macos/21a9783ca2b54f05d748ecb0e31bb2ee7701091c/resource/swift/mouse-drag-arm64 -------------------------------------------------------------------------------- /resource/swift/mouse-drag-x64: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/macacajs/macaca-macos/21a9783ca2b54f05d748ecb0e31bb2ee7701091c/resource/swift/mouse-drag-x64 -------------------------------------------------------------------------------- /resource/swift/ocr-arm64: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/macacajs/macaca-macos/21a9783ca2b54f05d748ecb0e31bb2ee7701091c/resource/swift/ocr-arm64 -------------------------------------------------------------------------------- /resource/swift/ocr-x64: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/macacajs/macaca-macos/21a9783ca2b54f05d748ecb0e31bb2ee7701091c/resource/swift/ocr-x64 -------------------------------------------------------------------------------- /resource/swift/readme.md: -------------------------------------------------------------------------------- 1 | ## Build swift project 2 | ```bash 3 | bash ./build.sh 4 | ``` 5 | 6 | ## Executable binary create by swift 7 | - OCR 8 | - ocr ability powered by macOS. 9 | - usage： ocr /path/to/img 10 | 11 | - mouse-drag 12 | - mouse drag to (+x,+y) relate to current point 13 | - mouse-drag delay x y 14 | -------------------------------------------------------------------------------- /resource/swift/src/LeftMouseDragged/.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | /.build 3 | /Packages 4 | xcuserdata/ 5 | DerivedData/ 6 | .swiftpm/configuration/registries.json 7 | .swiftpm/xcode/package.xcworkspace/contents.xcworkspacedata 8 | .netrc 9 | -------------------------------------------------------------------------------- /resource/swift/src/LeftMouseDragged/Package.swift: -------------------------------------------------------------------------------- 1 | // swift-tools-version: 5.9 2 | // The swift-tools-version declares the minimum version of Swift required to build this package. 3 | 4 | import PackageDescription 5 | 6 | let package = Package( 7 | name: "LeftMouseDragged", 8 | targets: [ 9 | // Targets are the basic building blocks of a package, defining a module or a test suite. 10 | // Targets can depend on other targets in this package and products from dependencies. 11 | .executableTarget( 12 | name: "LeftMouseDragged"), 13 | ] 14 | ) 15 | -------------------------------------------------------------------------------- /resource/swift/src/LeftMouseDragged/Sources/main.swift: -------------------------------------------------------------------------------- 1 | // The Swift Programming Language 2 | // https://docs.swift.org/swift-book 3 | 4 | import AppKit 5 | import Cocoa 6 | 7 | func taskDelay(_ millSeconds: Int, _ task: @escaping () -> Void) { 8 | DispatchQueue.main.asyncAfter(deadline: .now() + Double(millSeconds) / 1000.0, execute: task) 9 | } 10 | 11 | func postMouseEvent(button:CGMouseButton, type:CGEventType, point: CGPoint,clickCount:Int64 = 1) 12 | { 13 | let event = createMouseEvent(button: button, type: type, point: point,clickCount:clickCount) 14 | event.post(tap: CGEventTapLocation.cghidEventTap) 15 | } 16 | 17 | func createMouseEvent(button:CGMouseButton, type:CGEventType, point: CGPoint,clickCount:Int64 = 1) -> CGEvent 18 | { 19 | let event : CGEvent = CGEvent(mouseEventSource: CGEventSource.init(stateID: CGEventSourceStateID.privateState), mouseType: type, mouseCursorPosition: point, mouseButton: button)! 20 | event.setIntegerValueField(CGEventField.mouseEventClickState, value: clickCount) 21 | return event 22 | } 23 | 24 | func mouseDragged(point:CGPoint,toPoint:CGPoint,button:CGMouseButton,postDelay:Int){ 25 | let toMaxX:Bool = toPoint.x - point.x > 0 26 | let toMaxY:Bool = toPoint.y - point.y > 0 27 | 28 | var tempPointY = point.y 29 | var tempPointX = point.x 30 | 31 | postMouseEvent(button: button, type: button == .left ? .leftMouseDown : .rightMouseDown, point: point,clickCount: 1); 32 | 33 | let blockOperation = BlockOperation() 34 | 35 | blockOperation.addExecutionBlock { 36 | while toMaxY ? (toPoint.y > tempPointY) : (toPoint.y < tempPointY){ 37 | Thread.sleep(forTimeInterval: 0.0001 * Double(postDelay)) 38 | toMaxY ? (tempPointY += 1) : (tempPointY -= 1) 39 | postMouseEvent(button: button, type: button == .left ? .leftMouseDragged : .rightMouseDragged, point: CGPoint(x: tempPointX, y: tempPointY),clickCount: 1); 40 | } 41 | } 42 | blockOperation.addExecutionBlock { 43 | while toMaxX ? (toPoint.x > tempPointX) : (toPoint.x < tempPointX) { 44 | Thread.sleep(forTimeInterval: 0.0001 * Double(postDelay)) 45 | toMaxX ? (tempPointX += 1) : (tempPointX -= 1) 46 | postMouseEvent(button: button, type: button == .left ? .leftMouseDragged : .rightMouseDragged, point: CGPoint(x: tempPointX, y: tempPointY),clickCount: 1); 47 | } 48 | 49 | } 50 | blockOperation.completionBlock = { 51 | postMouseEvent(button: button, type: button == .left ? .leftMouseUp : .rightMouseUp, point: toPoint,clickCount: 1); 52 | } 53 | blockOperation.start() 54 | } 55 | 56 | func printJson(_ data: Any) { 57 | let jsonData = try! JSONSerialization.data(withJSONObject: data) 58 | print(String(data: jsonData, encoding: String.Encoding.utf8) ?? "") 59 | } 60 | 61 | func main(args: [String]) -> Int32 { 62 | guard CommandLine.arguments.count == 6 || CommandLine.arguments.count == 4 else { 63 | fputs(String(format: "usage1: %1$@ postDelay x1 y1 x2 y2, dragged from (x1, y1) to (x1, y2), when move a px, sleep(postDelay)\n", CommandLine.arguments[0]), stderr) 64 | fputs(String(format: "usage2: %1$@ postDelay x y, dragged from mouseLocation to (mouseLocation.x + x, mouseLocation.y + y), when move a px, sleep(postDelay)\n", CommandLine.arguments[0]), stderr) 65 | return 1 66 | } 67 | let postDelay = Int(args[1]) ?? 0 68 | var mouseLoc = NSEvent.mouseLocation 69 | 70 | if (CommandLine.arguments.count == 4) { 71 | mouseLoc.y = NSHeight(NSScreen.screens[0].frame) - mouseLoc.y; 72 | let newLoc = CGPoint(x: mouseLoc.x+CGFloat(Int(args[2]) ?? 0), y: mouseLoc.y+CGFloat(Int(args[3]) ?? 0)) 73 | mouseDragged( 74 | point: CGPoint(x: mouseLoc.x, y: mouseLoc.y), 75 | toPoint: newLoc, 76 | button: .left, 77 | postDelay: postDelay 78 | ) 79 | printJson([ 80 | "success": true, 81 | "from": [ "x1": Int(mouseLoc.x), "y1": Int(mouseLoc.y) ], 82 | "to": [ "x2": Int(newLoc.x), "y2": Int(newLoc.y) ], 83 | ]) 84 | return 0 85 | } 86 | 87 | let x1 = Int(args[2]) ?? Int(mouseLoc.x) 88 | let y1 = Int(args[3]) ?? Int(mouseLoc.y) 89 | let x2 = Int(args[4]) ?? Int(mouseLoc.x) 90 | let y2 = Int(args[5]) ?? Int(mouseLoc.y) 91 | 92 | if (x1 != x2 || y1 != y2) { 93 | mouseDragged( 94 | point: CGPoint(x: x1, y: y1), 95 | toPoint: CGPoint(x: x2, y: y2), 96 | button: .left, 97 | postDelay: postDelay 98 | ) 99 | } 100 | printJson([ 101 | "success": true, 102 | "from": [ "x1": x1, "y1": y1 ], 103 | "to": [ "x2": x2, "y2": y2 ], 104 | ]) 105 | return 0 106 | } 107 | 108 | exit(main(args: CommandLine.arguments)) 109 | -------------------------------------------------------------------------------- /resource/swift/src/MacosOcr/.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | /.build 3 | /Packages 4 | xcuserdata/ 5 | DerivedData/ 6 | .swiftpm/configuration/registries.json 7 | .swiftpm/xcode/package.xcworkspace/contents.xcworkspacedata 8 | .netrc 9 | -------------------------------------------------------------------------------- /resource/swift/src/MacosOcr/Package.swift: -------------------------------------------------------------------------------- 1 | // swift-tools-version: 5.9 2 | // The swift-tools-version declares the minimum version of Swift required to build this package. 3 | 4 | import PackageDescription 5 | 6 | let package = Package( 7 | name: "MacosOcr", 8 | platforms: [ 9 | .macOS(.v10_15) 10 | ], 11 | targets: [ 12 | // Targets are the basic building blocks of a package, defining a module or a test suite. 13 | // Targets can depend on other targets in this package and products from dependencies. 14 | .executableTarget( 15 | name: "MacosOcr"), 16 | ] 17 | ) 18 | -------------------------------------------------------------------------------- /resource/swift/src/MacosOcr/Sources/main.swift: -------------------------------------------------------------------------------- 1 | // The Swift Programming Language 2 | // https://docs.swift.org/swift-book 3 | 4 | import Cocoa 5 | import Vision 6 | // import ArgumentParser 7 | 8 | // https://developer.apple.com/documentation/vision/vnrecognizetextrequest 9 | 10 | let MODE: VNRequestTextRecognitionLevel = VNRequestTextRecognitionLevel.accurate // or .fast 11 | let USE_LANG_CORRECTION = false 12 | var REVISION:Int 13 | if #available(macOS 13, *) { 14 | REVISION = VNRecognizeTextRequestRevision3 15 | } else if #available(macOS 11, *) { 16 | REVISION = VNRecognizeTextRequestRevision2 17 | } else { 18 | REVISION = VNRecognizeAnimalsRequestRevision1 19 | } 20 | 21 | func main(args: [String]) -> Int32 { 22 | guard CommandLine.arguments.count == 2 else { 23 | fputs(String(format: "usage: %1$@ image\n", CommandLine.arguments[0]), stderr) 24 | return 1 25 | } 26 | 27 | let src = args[1] 28 | 29 | guard let img = NSImage(byReferencingFile: src) else { 30 | fputs("Error: failed to load image '\(src)'\n", stderr) 31 | return 1 32 | } 33 | 34 | guard let imgRef = img.cgImage(forProposedRect: &img.alignmentRect, context: nil, hints: nil) else { 35 | fputs("Error: failed to convert NSImage to CGImage for '\(src)'\n", stderr) 36 | return 1 37 | } 38 | 39 | let request: VNRecognizeTextRequest = VNRecognizeTextRequest { (request, error) in 40 | let observations: [VNRecognizedTextObservation] = request.results as? [VNRecognizedTextObservation] ?? [] 41 | var results: [Any] = [] 42 | for observation: VNRecognizedTextObservation in observations { 43 | let candidate: VNRecognizedText = observation.topCandidates(1)[0] 44 | let value = [ 45 | "word": candidate.string, 46 | "rect": [ 47 | "left": Int(Float(observation.topLeft.x) * Float(imgRef.width)), 48 | "top": Int(Float(1 - observation.topLeft.y) * Float(imgRef.height)), 49 | "width": Int(Float(observation.topRight.x - observation.topLeft.x) * Float(imgRef.width)), 50 | "height": Int(Float(observation.topLeft.y - observation.bottomLeft.y) * Float(imgRef.height)), 51 | ], 52 | "confidence": candidate.confidence, 53 | ] 54 | results.append(value) 55 | } 56 | let data = try! JSONSerialization.data(withJSONObject: results) 57 | print(String(data: data, encoding: String.Encoding.utf8) ?? "") 58 | } 59 | request.recognitionLevel = MODE 60 | request.usesLanguageCorrection = USE_LANG_CORRECTION 61 | request.revision = REVISION 62 | request.recognitionLanguages = ["zh-Hans", "en-US"] 63 | //request.minimumTextHeight = 0 64 | //request.customWords = [String] 65 | 66 | try? VNImageRequestHandler(cgImage: imgRef, options: [:]).perform([request]) 67 | return 0 68 | } 69 | 70 | exit(main(args: CommandLine.arguments)) 71 | 72 | // struct Banner: ParsableCommand { 73 | // static let configuration = CommandConfiguration( 74 | // abstract: "A Swift command-line tool to manage blog post banners", 75 | // subcommands: [Generate.self]) 76 | 77 | // init() { } 78 | // } 79 | 80 | // // Banner.main() 81 | 82 | // struct Generate: ParsableCommand { 83 | 84 | // public static let configuration = CommandConfiguration(abstract: "Generate a blog post banner from the given input") 85 | 86 | // @Argument(help: "The title of the blog post") 87 | // private var title: String 88 | 89 | // @Option(name: .shortAndLong, default: nil, help: "The week of the blog post as used in the file name") 90 | // private var week: Int? 91 | 92 | // @Flag(name: .long, help: "Show extra logging for debugging purposes") 93 | // private var verbose: Bool 94 | 95 | // func run() throws { 96 | // if verbose { 97 | // let weekDescription = week.map { "and week \($0)" } 98 | // print("Creating a banner for title \"\(title)\" \(weekDescription ?? "")") 99 | // } 100 | // } 101 | // } 102 | -------------------------------------------------------------------------------- /src/core/enums.ts: -------------------------------------------------------------------------------- 1 | export enum EDriver { 2 | Swift = 'swift', 3 | RobotJs = 'robotjs', 4 | JXA = 'jxa', 5 | AppleScript = 'applescript', 6 | } 7 | -------------------------------------------------------------------------------- /src/core/helper.ts: -------------------------------------------------------------------------------- 1 | import os from 'os'; 2 | import fs from 'fs'; 3 | import npmUpdate from 'npm-update'; 4 | 5 | const path = require('path'); 6 | const shell = require('shelljs'); 7 | 8 | export class Helper { 9 | 10 | /** 11 | * 检测4k屏 12 | */ 13 | static isHdpiDisplay(): boolean { 14 | const res = shell.exec('system_profiler SPDisplaysDataType | grep Resolution', { silent: true }).stdout.trim(); 15 | if (res) { 16 | // 多个显示器只关注主显示器 17 | const main = res.split('\n')[0].trim(); 18 | if (main.includes('Retina')) { 19 | return true; 20 | } 21 | // 一般情况下 4k 高大于等于 2160 像素, 2k达不到这个高度，除非这是个造型很变态的2k显示器... 22 | // 不用宽度做判断(存在宽屏显示器) 23 | const mainY = Number.parseInt(main.split(' x ')[1].split(' ')[0]); 24 | return mainY >= 2160; 25 | } 26 | console.log('无显示器信息'); 27 | } 28 | 29 | static async isDeprecated() { 30 | await npmUpdate({ 31 | pkg: this.getPkg(), 32 | }); 33 | } 34 | 35 | static getPkg() { 36 | let pkg: any = {}; 37 | if (__dirname.includes('/dist/src/core')) { 38 | pkg = require('../../../package.json'); 39 | } else { 40 | pkg = require('../../package.json'); 41 | } 42 | return pkg; 43 | } 44 | 45 | /** 46 | * 获取版本 47 | */ 48 | static getPkgVersion(): string { 49 | const pkg = this.getPkg(); 50 | return pkg.version; 51 | } 52 | 53 | /** 54 | * 调试日志 55 | * @param items 56 | */ 57 | static debug(...items) { 58 | if (process.env.MACACA_MACOS_DEBUG) { 59 | console.log(...items); 60 | } 61 | } 62 | 63 | static getResourcePath(): string { 64 | return path.resolve(__dirname, '../../resource'); 65 | } 66 | 67 | /** 68 | * 支持每秒打印时间 69 | * @param ms 70 | * @param tick 71 | */ 72 | static async sleep(ms: number, tick = false) { 73 | if (ms > 10000 && tick) { 74 | for (let i = 0; (i * 1000) < ms; i++) { 75 | await this.sleep(1000); 76 | console.log(`${i}s`); 77 | } 78 | } else { 79 | return new Promise(resolve => setTimeout(resolve, ms)); 80 | } 81 | } 82 | 83 | /** 84 | * 判断是否为异步函数 85 | * @param func 86 | */ 87 | static isAsyncFunc(func: Function) { 88 | return func && typeof func === 'function' && func[Symbol.toStringTag] === 'AsyncFunction'; 89 | } 90 | 91 | static hasUnicode(str: string) { 92 | for (let i = 0; i < str.length; i++) { 93 | if (str.charCodeAt(i) > 255) { 94 | return true; 95 | } 96 | } 97 | return false; 98 | } 99 | 100 | static tmpdir() { 101 | // mac 环境需要转换 102 | if (os.platform() === 'darwin') { 103 | return fs.realpathSync(os.tmpdir()); 104 | } 105 | return os.tmpdir(); 106 | } 107 | 108 | /** 109 | * 等待直到条件方法返回true 110 | * @param func 111 | * @param waitMs 112 | * @param intervalMs 113 | */ 114 | static async waitUntil(func: Function, waitMs = 5E3, intervalMs = 1E3) { 115 | const start = Date.now(); 116 | const end = start + waitMs; 117 | const isAsyncFunc = this.isAsyncFunc(func); 118 | const fn = () => { 119 | return new Promise(async (resolve, reject) => { 120 | const continuation = (res, rej) => { 121 | const now = Date.now(); 122 | if (now < end) { 123 | res(this.sleep(intervalMs).then(fn)); 124 | } else { 125 | const funcStr = func.toString().replace(/ +/g, ' ').replace(/\n/g, ''); 126 | rej(`Wait For Condition: ${funcStr} timeout ${waitMs} ms`); 127 | } 128 | }; 129 | let isOk; 130 | if (isAsyncFunc) { 131 | isOk = await func().catch(() => { 132 | continuation(resolve, reject); 133 | }); 134 | } else { 135 | isOk = func(); 136 | } 137 | if (isOk) { 138 | resolve(true); 139 | } else { 140 | continuation(resolve, reject); 141 | } 142 | }); 143 | }; 144 | return fn(); 145 | } 146 | } 147 | -------------------------------------------------------------------------------- /src/core/jxa/exec-jxa.ts: -------------------------------------------------------------------------------- 1 | import { execFile } from 'child_process'; 2 | import fs from 'fs'; 3 | import os from 'os'; 4 | import { Helper } from '../helper'; 5 | 6 | const shell = require('shelljs'); 7 | 8 | const executeInOsa = async (code: string, args: any[]): Promise => { 9 | // 检查并更新lib 10 | if (code.includes('Library(')) { 11 | const scptdDir = `${Helper.getResourcePath()}/applescript/scptd`; 12 | const userLibDir = `${os.homedir()}/Library/Script\ Libraries`; 13 | const versionFile = `${userLibDir}/macaca-macos.txt`; 14 | const version = Helper.getPkgVersion(); 15 | if (!fs.existsSync(userLibDir)) { 16 | shell.mkdir('-p', userLibDir); 17 | } 18 | if (fs.existsSync(versionFile)) { 19 | const userVersion = fs.readFileSync(versionFile, 'utf-8').toString().trim(); 20 | if (userVersion !== version) { 21 | shell.cp('-R', `${scptdDir}/*`, userLibDir); 22 | shell.rm('-rf', versionFile); 23 | fs.writeFileSync(versionFile, version); 24 | } 25 | } else { 26 | shell.cp('-R', `${scptdDir}/*`, userLibDir); 27 | fs.writeFileSync(versionFile, version); 28 | } 29 | } 30 | const envs: any = { 31 | OSA_ARGS: JSON.stringify(args), 32 | }; 33 | return new Promise((resolve, reject) => { 34 | const child = execFile( 35 | '/usr/bin/osascript', 36 | [ '-l', 'JavaScript' ], 37 | { 38 | env: envs, 39 | maxBuffer: 1E8, 40 | }, 41 | (err: Error, stdout: any, stderr: any) => { 42 | if (err) { 43 | return reject(err); 44 | } 45 | if (stderr) { 46 | console.error(stderr); 47 | } 48 | 49 | if (!stdout) { 50 | resolve(undefined); 51 | } 52 | 53 | try { 54 | const result = JSON.parse(stdout.toString().trim()).result; 55 | resolve(result); 56 | } catch (errorOutput) { 57 | resolve(stdout.toString().trim()); 58 | } 59 | }, 60 | ); 61 | child.stdin.write(code); 62 | child.stdin.end(); 63 | }); 64 | }; 65 | 66 | const requireFunc = ` 67 | ObjC.import('Foundation'); 68 | var fm = $.NSFileManager.defaultManager; 69 | var requireHack = function (path) { 70 | var contents = fm.contentsAtPath(path.toString()); // NSData 71 | contents = $.NSString.alloc.initWithDataEncoding(contents, $.NSUTF8StringEncoding); 72 | 73 | var module = {exports: {}}; 74 | var exports = module.exports; 75 | eval(ObjC.unwrap(contents)); 76 | 77 | return module.exports; 78 | }; 79 | `; 80 | 81 | export const execJxa = async (jxaCodeFunction: (...args: any[]) => void, args: any[] = []) => { 82 | const code = ` 83 | ObjC.import('stdlib'); 84 | ${requireFunc} 85 | var args = JSON.parse($.getenv('OSA_ARGS')); 86 | var fn = (${jxaCodeFunction.toString()}); 87 | var out = fn.apply(null, args); 88 | JSON.stringify({ result: out }); 89 | `; 90 | return executeInOsa(code, args).catch(e => { 91 | console.warn(e.message); 92 | }); 93 | }; 94 | -------------------------------------------------------------------------------- /src/core/jxa/jxaUtil.ts: -------------------------------------------------------------------------------- 1 | // eslint-disable-next-line @typescript-eslint/ban-ts-comment 2 | // @ts-nocheck 3 | 4 | // open when coding 5 | // import { Application } from '@jxa/types'; 6 | // import '@jxa/global-type'; 7 | 8 | import { execJxa } from './exec-jxa'; 9 | import { Helper } from '../helper'; 10 | 11 | declare global { 12 | function Application(name: string): object; 13 | 14 | function Library(name: string): object; 15 | 16 | function Path(name: string): object; 17 | 18 | function requireHack(name: string): object; 19 | 20 | function delay(delay?: number): void; 21 | 22 | const ObjC: any; 23 | const $: any; 24 | } 25 | 26 | export const jxaUtil = { 27 | /** 28 | * 设置剪贴板内容 29 | * @param str 30 | */ 31 | async setClipText(str: string) { 32 | await execJxa((str) => { 33 | const app = Application.currentApplication(); 34 | app.includeStandardAdditions = true; 35 | return app.setTheClipboardTo(str || '[NONE]'); 36 | }, [ str ]); 37 | }, 38 | 39 | async getClipText() { 40 | const res: any = await execJxa(() => { 41 | const app = Application.currentApplication(); 42 | app.includeStandardAdditions = true; 43 | return app.theClipboard(); 44 | }, []); 45 | Helper.debug('clipText:', res); 46 | return res; 47 | }, 48 | 49 | /** 50 | * 键盘按键和组合键 51 | * https://eastmanreference.com/complete-list-of-applescript-key-codes 52 | * @param key 53 | * @param modified command | option | control | shift 54 | */ 55 | async keyTap(key: string | number, modified: string[] = []) { 56 | // keyCode 57 | if (typeof key === 'number') { 58 | await execJxa((key, modified) => { 59 | const sys = Application('System Events'); 60 | sys.keyCode(key, { 61 | using: modified.map(it => { 62 | return `${it} down`; 63 | }), 64 | }); 65 | }, [ key, modified ]); 66 | } else { 67 | await execJxa((key, modified) => { 68 | const sys = Application('System Events'); 69 | sys.keystroke(key, { 70 | using: modified.map(it => { 71 | return `${it} down`; 72 | }), 73 | }); 74 | }, [ key, modified ]); 75 | } 76 | }, 77 | 78 | /** 79 | * 字符串输入 80 | * - 不支持Unicode字符 81 | * @param str 82 | * @param delay 83 | */ 84 | async typeString(str, delay = false) { 85 | if (delay) { 86 | await execJxa((str) => { 87 | const sys = Application('System Events'); 88 | for (let i = 0; i < str.length; i++) { 89 | sys.keystroke(str[i]); 90 | delay(0.1); 91 | } 92 | }, [ str ]); 93 | } else { 94 | await execJxa((str) => { 95 | const sys = Application('System Events'); 96 | sys.keystroke(str); 97 | }, [ str ]); 98 | } 99 | }, 100 | 101 | /** 102 | * 关闭app 103 | */ 104 | async safeQuitApp(appName: string) { 105 | await execJxa((appName) => { 106 | const app = Application(appName); 107 | if (app.running()) { 108 | app.quit(); 109 | } 110 | }, [ appName ]); 111 | }, 112 | 113 | /** 114 | * 窗口重定位(默认左上角) 115 | * @param opts 116 | */ 117 | async resizePosition(opts: { 118 | name: string; 119 | topLeftX?: number; 120 | topLeftY?: number; 121 | width?: number; 122 | height?: number; 123 | }) { 124 | opts.topLeftX = opts.topLeftX || 0; 125 | opts.topLeftY = opts.topLeftY || 0; 126 | if (opts.width && opts.height) { 127 | await execJxa((opts) => { 128 | const window = Library('window'); 129 | const app = Application(opts.name); 130 | if (app.running()) { 131 | app.activate(); 132 | window.sizePosition(opts.name, opts.topLeftX, opts.topLeftY, opts.width, opts.height); 133 | } 134 | }, [ opts ]); 135 | } else { 136 | // 仅移动位置 137 | await execJxa((opts) => { 138 | const window = Library('window'); 139 | const app = Application(opts.name); 140 | if (app.running()) { 141 | app.activate(); 142 | window.setPosition(opts.name, opts.topLeftX, opts.topLeftY); 143 | } 144 | }, [ opts ]); 145 | } 146 | }, 147 | 148 | /** 149 | * 获取当前系统用户信息 150 | */ 151 | async getUserName() { 152 | const res: any = await execJxa(() => { 153 | const sys = Application('System Events'); 154 | return sys.currentUser().name(); 155 | }, []); 156 | return res; 157 | }, 158 | 159 | /** 160 | * app运行状态 161 | * @param appName 162 | */ 163 | async isAppRunning(appName: string) { 164 | const res: boolean = await execJxa((appName) => { 165 | return Application(appName).running(); 166 | }, [ appName ]); 167 | return res; 168 | }, 169 | 170 | /** 171 | * 对话框, 接受用户输入并返回 172 | */ 173 | async prompt(message: string): Promise { 174 | return execJxa((msg) => { 175 | const app = Application.currentApplication(); 176 | app.includeStandardAdditions = true; 177 | try { 178 | return app.displayDialog(msg, { defaultAnswer: '' }).textReturned; 179 | } catch (e) { 180 | return null; 181 | } 182 | }, [ message ]); 183 | }, 184 | 185 | /** 186 | * 告警 187 | * @param title 188 | * @param msg 189 | * @param type 190 | */ 191 | async alert(title: string, msg: string, type = 'info'): Promise { 192 | let icon; 193 | if (type === 'warn') { 194 | icon = '⚠️'; 195 | } else if (type === 'error') { 196 | icon = '❌'; 197 | } else { 198 | icon = '🤖'; 199 | } 200 | const message = `${icon} ${type}: ${msg}`; 201 | return execJxa((title, msg) => { 202 | const app = Application.currentApplication(); 203 | app.includeStandardAdditions = true; 204 | app.displayAlert(title, { message: msg }); 205 | }, [ title, message ]); 206 | }, 207 | 208 | async confirm(msg: string): Promise { 209 | return execJxa((msg) => { 210 | const app = Application.currentApplication(); 211 | app.includeStandardAdditions = true; 212 | try { 213 | app.displayDialog(msg); 214 | return true; 215 | } catch (e) { 216 | return false; 217 | } 218 | }, [ msg ]); 219 | }, 220 | 221 | /** 222 | * 浏览器可以 223 | * ⚠️ electron app 似乎获取不到窗口ID 224 | * @param appName 'Google Chrome' | 'Safari' | 'Firefox' 225 | */ 226 | async getWindowIdByAppName(appName: string) { 227 | const res: any = await execJxa((appName) => { 228 | const app = Application(appName); 229 | app.includeStandardAdditions = true; 230 | if (app.running()) { 231 | app.activate(); 232 | if (app.windows[0]) { 233 | const window = app.windows[0]; 234 | return window.id(); 235 | } 236 | return app.id(); 237 | } 238 | }, [ appName ]); 239 | Helper.debug(res); 240 | return res; 241 | }, 242 | /** 243 | * 激活聚焦 244 | * @param appName 245 | */ 246 | async focusApp(appName: string) { 247 | await execJxa((appName) => { 248 | const app = Application(appName); 249 | app.includeStandardAdditions = true; 250 | if (app.running()) { 251 | app.activate(); 252 | } 253 | }, [ appName ]); 254 | }, 255 | 256 | /** 257 | * 鼠标操作(默认当前位置左键) 258 | * - 不稳定，会失效 259 | */ 260 | async click(opts: { 261 | x: number; 262 | y: number; 263 | r?: boolean; 264 | } = {}) { 265 | const mouseLib = `${Helper.getResourcePath()}/javascript/mouse.js`; 266 | await execJxa((lib, opts) => { 267 | const mouse = requireHack(lib); 268 | mouse.click(opts); 269 | }, [ mouseLib, opts ]); 270 | }, 271 | 272 | // FIXME not work 273 | async drag(x: number, y: number) { 274 | const mouseLib = `${Helper.getResourcePath()}/javascript/mouse.js`; 275 | await execJxa((lib, x, y) => { 276 | const mouse = requireHack(lib); 277 | mouse.drag(x, y); 278 | }, [ mouseLib, x, y ]); 279 | }, 280 | 281 | async asSafeActivate(appName: string) { 282 | await execJxa((appName) => { 283 | const win = Library('window'); 284 | return win.safeActivate(appName); 285 | }, [ appName ]); 286 | }, 287 | }; 288 | -------------------------------------------------------------------------------- /src/core/jxa/osaUtil.ts: -------------------------------------------------------------------------------- 1 | import { Helper } from '../helper'; 2 | 3 | const applescript = require('applescript'); 4 | const shell = require('shelljs'); 5 | const scptDir = `${Helper.getResourcePath()}/applescript/scpt`; 6 | 7 | // 直接执行 AppleScript 脚本/脚本包 8 | export const osaUtil = { 9 | 10 | async execAppleScriptStr(funcStr: string): Promise { 11 | return new Promise((resolve, reject) => { 12 | applescript.execString(funcStr, (err, rtn) => { 13 | if (err) { 14 | console.error(err); 15 | reject('Something went wrong!'); 16 | } 17 | resolve(rtn); 18 | }); 19 | }); 20 | }, 21 | 22 | /** 23 | * 文件要求以 applescript 结尾的源码 24 | * @param file 25 | * @param args 26 | */ 27 | async execAppleScriptFile(file: string, args = []): Promise { 28 | if (!file.endsWith('.applescript')) { 29 | console.error('只支持applescript格式文件'); 30 | return; 31 | } 32 | return new Promise((resolve, reject) => { 33 | applescript.execFile(file, args, (err, rtn) => { 34 | if (err) { 35 | console.error(err); 36 | reject('Something went wrong!'); 37 | } 38 | resolve(rtn); 39 | }); 40 | }); 41 | }, 42 | 43 | /** 44 | * 执行scpt文件 45 | */ 46 | execScptFile(file: string, args = []) { 47 | if (!file.endsWith('.scpt')) { 48 | console.error('只支持scpt格式文件'); 49 | return; 50 | } 51 | shell.chmod('+x', file); 52 | const cmd = `osascript '${file}' ${args.join(' ')} -ss`; 53 | Helper.debug(cmd); 54 | const res = shell.exec(cmd, { silent: true }).stdout; 55 | Helper.debug(res); 56 | return res; 57 | }, 58 | 59 | /** 60 | * applescript 实现 61 | * jxa运行applescript lib会有问题 只能通过shell执行 62 | * - 适用原生MacOS的应用（实际点击的是坐标所在的UI元素）,不适用Electron 63 | * - FIXME 似乎只能点击到到执行脚本的应用窗口，否则卡死 64 | * @param opts 65 | */ 66 | click(opts: { 67 | x: number; 68 | y: number; 69 | }) { 70 | Helper.debug(opts); 71 | this.execScptFile(`${scptDir}/mouseClick.scpt`, [ opts.x, opts.y ]); 72 | }, 73 | 74 | /** 75 | * 隐藏所有APP(显示桌面) 76 | */ 77 | async hideAllApp() { 78 | await this.execAppleScriptStr(` 79 | tell application "Finder" 80 | set visible of every process whose visible is true and name is not "Finder" to false 81 | set the collapsed of windows to true 82 | end tell 83 | `); 84 | }, 85 | 86 | /** 87 | * 启动app 88 | * @param name 89 | */ 90 | async safeLaunchApp(name: string) { 91 | await this.execAppleScriptStr(` 92 | set appName to "${name}" 93 | set startIt to false 94 | tell application "System Events" 95 | if not (exists process appName) then 96 | set startIt to true 97 | else if frontmost of process appName then 98 | set visible of process appName to false 99 | else 100 | set frontmost of process appName to true 101 | end if 102 | end tell 103 | if startIt then 104 | tell application appName to activate 105 | end if 106 | `); 107 | }, 108 | 109 | async focusApp(name: string) { 110 | await this.execAppleScriptStr(` 111 | tell application "${name}" 112 | reopen 113 | activate 114 | end tell 115 | `); 116 | }, 117 | 118 | /** 119 | * 获取所有app的位置和长宽 120 | * - 不可见的窗口无法查到 121 | * - 一个app可能会有多个窗口 122 | */ 123 | async getAllAppSizePosition() { 124 | const res = await this.execAppleScriptStr(` 125 | tell application "System Events" 126 | set _P to a reference to (processes whose background only = false) 127 | set _W to a reference to windows of _P 128 | set res to [_P's name, _W's size, _W's position] 129 | return res 130 | end tell 131 | `); 132 | const result = []; 133 | const names = res[0]; 134 | const sizes = res[1]; 135 | const positions = res[2]; 136 | for (let i = 0; i < names.length; i++) { 137 | result.push({ 138 | name: names[i], 139 | size: sizes[i], 140 | position: positions[i], 141 | }); 142 | } 143 | return result; 144 | }, 145 | }; 146 | -------------------------------------------------------------------------------- /src/core/mixin.ts: -------------------------------------------------------------------------------- 1 | export default function applyMixins(derivedCtor: any, constructors: any[]) { 2 | constructors.forEach((baseCtor) => { 3 | Object.getOwnPropertyNames(baseCtor.prototype).forEach((name) => { 4 | Object.defineProperty( 5 | derivedCtor.prototype, 6 | name, 7 | Object.getOwnPropertyDescriptor(baseCtor.prototype, name) || Object.create(null), 8 | ); 9 | }); 10 | }); 11 | } 12 | -------------------------------------------------------------------------------- /src/driver/app.ts: -------------------------------------------------------------------------------- 1 | import shell from 'shelljs'; 2 | import { Helper } from '../core/helper'; 3 | import { EDriver } from '../core/enums'; 4 | import { osaUtil } from '../core/jxa/osaUtil'; 5 | import { jxaUtil } from '../core/jxa/jxaUtil'; 6 | 7 | export default class AppDriver { 8 | async hideAllApp() { 9 | await osaUtil.hideAllApp(); 10 | } 11 | 12 | async startApp(appNameOrFile) { 13 | if (appNameOrFile.startsWith('/')) { 14 | return shell.exec(`open ${appNameOrFile}`, { silent: true }); 15 | } 16 | return await osaUtil.safeLaunchApp(appNameOrFile); 17 | } 18 | 19 | async focusApp(name: string, opts: { 20 | driver?: EDriver; 21 | } = {}) { 22 | const { 23 | driver = EDriver.AppleScript, 24 | } = opts; 25 | if (driver === EDriver.AppleScript) { 26 | return osaUtil.focusApp(name); 27 | } else if (driver === EDriver.JXA) { 28 | return jxaUtil.focusApp(name); 29 | } 30 | } 31 | 32 | async isAppRunning(name: string) { 33 | return jxaUtil.isAppRunning(name); 34 | } 35 | 36 | /** 37 | * 关闭app 38 | */ 39 | async safeQuitAppByName(name: string) { 40 | if (await this.isAppRunning(name)) { 41 | await jxaUtil.safeQuitApp(name); 42 | // 等待退出完毕 43 | await Helper.waitUntil(async () => { 44 | return !(await this.isAppRunning(name)); 45 | }); 46 | } 47 | } 48 | 49 | /** 50 | * 仅返回第一个窗口 51 | * @param name 52 | */ 53 | async getAppSizePosition(name: string) { 54 | const ress = await osaUtil.getAllAppSizePosition(); 55 | const res = ress.find(it => { 56 | return it.name === name; 57 | }); 58 | if (res && res.position.length > 0) { 59 | return { 60 | topLeftX: res.position[0][0], 61 | topLeftY: res.position[0][1], 62 | width: res.size[0][0], 63 | height: res.size[0][1], 64 | }; 65 | } 66 | } 67 | 68 | /** 69 | * 设置app窗口位置 70 | * @param opts 71 | */ 72 | async resizePosition(opts: { 73 | name: string; 74 | topLeftX?: number; 75 | topLeftY?: number; 76 | width?: number; 77 | height?: number; 78 | }) { 79 | // app 窗口存在 80 | await Helper.waitUntil(async () => { 81 | return this.getAppSizePosition(opts.name); 82 | }, 10E3); 83 | await jxaUtil.resizePosition(opts); 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /src/driver/clipboard.ts: -------------------------------------------------------------------------------- 1 | import { jxaUtil } from '../core/jxa/jxaUtil'; 2 | 3 | export default class ClipboardDriver { 4 | /** 5 | * 获取剪切板的文本 6 | */ 7 | async getClipText(): Promise { 8 | return jxaUtil.getClipText(); 9 | } 10 | 11 | /** 12 | * 设置文本到剪切板 13 | * @param str string 14 | */ 15 | async setClipText(str: string) { 16 | return jxaUtil.setClipText(str); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/driver/keyboard.ts: -------------------------------------------------------------------------------- 1 | import robot from 'robotjs'; 2 | import { jxaUtil } from '../core/jxa/jxaUtil'; 3 | import { Helper } from '../core/helper'; 4 | 5 | export default class KeyboardDriver { 6 | /** 7 | * 字符串输入 8 | * @param str 9 | * @param delay 10 | * @param robotJs 11 | */ 12 | async keyboardTypeString(str: string, delay = false, robotJs = false) { 13 | const hasUnicode = Helper.hasUnicode(str); 14 | if (robotJs) { 15 | // 已知问题 16 | // ⚠️ robotJs 在 keyTap 执行之后会不稳定 17 | if (!hasUnicode && delay) { 18 | // delay 不支持中文delay输入 19 | await robot.typeStringDelayed(str, 5E3); 20 | } else { 21 | // 支持中文 22 | robot.typeString(str); 23 | } 24 | } else { 25 | if (hasUnicode) { 26 | const temp = await jxaUtil.getClipText(); 27 | await jxaUtil.setClipText(str); 28 | await this.keyboardTap('v', [ 'command' ]); 29 | await this.keyboardTap('right'); 30 | await jxaUtil.setClipText(temp); // 恢复 31 | } else { 32 | await jxaUtil.typeString(str, delay); 33 | } 34 | } 35 | } 36 | 37 | /** 38 | * @param key 39 | * @param modified 40 | */ 41 | async keyboardTap(key: string, modified: string[] = []) { 42 | robot.keyTap(key, modified); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/driver/mouse.ts: -------------------------------------------------------------------------------- 1 | import robot from 'robotjs'; 2 | import shell from 'shelljs'; 3 | import { Helper } from '../core/helper'; 4 | import { EDriver } from '../core/enums'; 5 | import { osaUtil } from '../core/jxa/osaUtil'; 6 | import { jxaUtil } from '../core/jxa/jxaUtil'; 7 | import ScreenDriver from './screen'; 8 | import os from 'os'; 9 | import assert from 'assert'; 10 | import fs from 'fs'; 11 | 12 | export default class MouseDriver { 13 | mouseMoveTo(x: number, y: number) { 14 | robot.moveMouseSmooth(x, y); 15 | } 16 | 17 | /** 18 | * 鼠标点击 当前鼠标所在位置 19 | * @param opts 20 | */ 21 | mouseClick(opts: { 22 | button?: string; // left | middle | right 23 | doubleClick?: boolean; // for robotJs only 24 | driver?: EDriver; 25 | } = {}) { 26 | const { 27 | driver = EDriver.RobotJs, 28 | button = 'left', 29 | doubleClick = false, 30 | } = opts; 31 | if (driver === EDriver.AppleScript) { 32 | const pos = this.mouseGetPos(); 33 | Helper.debug('click', pos); 34 | return osaUtil.click(pos); 35 | } 36 | robot.mouseClick(button, doubleClick); 37 | } 38 | 39 | /** 40 | * 点击屏幕/目标区域的文案 41 | * 依赖 ocr 42 | * @param opts 43 | */ 44 | async mouseClickText(opts: { 45 | text: string; // 目标文案 46 | index?: number; // 重复项指针 47 | rectangle?: string; // 截图目标区域 通过矩形框 x,y,width,height 默认全屏 48 | clickOpts?: any; 49 | shiftX?: number; // 偏移量 50 | shiftY?: number; 51 | }) { 52 | const { 53 | text, index, rectangle, 54 | clickOpts = {}, 55 | shiftX = 0, 56 | shiftY = 0, 57 | } = opts; 58 | const res = await new ScreenDriver().getTextsPosition({ 59 | texts: [ text ], 60 | index, 61 | rectangle, 62 | }); 63 | if (res.length) { 64 | let { x, y } = res[0]; 65 | // 绝对位置 66 | if (rectangle) { 67 | x = Number.parseInt(rectangle.split(',')[0]) + x; 68 | y = Number.parseInt(rectangle.split(',')[1]) + y; 69 | } 70 | // 偏移 71 | x = x + shiftX; 72 | y = y + shiftY; 73 | this.mouseMoveTo(x, y); 74 | this.mouseClick(clickOpts); 75 | return true; 76 | } 77 | } 78 | 79 | /** 80 | * 从当前位置拖拽到目标位置 81 | * @param x 82 | * @param y 83 | * @param opts 84 | */ 85 | async mouseDrag(x: number, y: number, opts: { 86 | driver?: EDriver; 87 | } = {}) { 88 | if (opts.driver === EDriver.JXA) { 89 | await jxaUtil.drag(x, y); 90 | } else if (opts.driver === EDriver.RobotJs) { 91 | // Robotjs 存在无法拖动app窗口的问题 92 | robot.mouseToggle('down'); 93 | robot.dragMouse(x, y); 94 | robot.mouseToggle('up'); 95 | } 96 | // default swift 97 | const curr_pos = this.mouseGetPos(); 98 | const cmdFile = `${Helper.getResourcePath()}/swift/mouse-drag-${os.arch()}`; 99 | assert(fs.existsSync(cmdFile), `不支持的架构: ${os.arch()}`); 100 | shell.exec(`${cmdFile} 10 ${curr_pos.x} ${curr_pos.y} ${x} ${y}`, { silent: true }); 101 | } 102 | 103 | mouseGetPos() { 104 | return robot.getMousePos(); 105 | } 106 | 107 | mouseScroll(x: number, y: number) { 108 | robot.scrollMouse(x, y); 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /src/driver/network.ts: -------------------------------------------------------------------------------- 1 | import shell from 'shelljs'; 2 | import { Helper } from '../core/helper'; 3 | 4 | export default class NetworkDriver { 5 | wifiDeviceName; 6 | /** 7 | * 获取wifi设备名 8 | * macos 一般为 en0 或 en1 9 | */ 10 | getWifiDeviceName(): string { 11 | if (this.wifiDeviceName) { 12 | return this.wifiDeviceName; 13 | } 14 | const name = shell.exec('networksetup -listnetworkserviceorder | sed -n \'/Wi-Fi/s|.*Device: \\(.*\\)).*|\\1|p\'', { silent: true }).stdout; 15 | Helper.debug(name); 16 | this.wifiDeviceName = name.trim(); 17 | return this.wifiDeviceName; 18 | } 19 | 20 | wifiTurnOff() { 21 | const cmd = `networksetup -setairportpower ${this.getWifiDeviceName()} off`; 22 | shell.exec(cmd, { silent: true }); 23 | } 24 | 25 | wifiTurnOn() { 26 | const cmd = `networksetup -setairportpower ${this.getWifiDeviceName()} on`; 27 | shell.exec(cmd, { silent: true }); 28 | } 29 | 30 | /** 31 | * wifi 状态获取 32 | */ 33 | isWifiOn(): boolean { 34 | // Wi-Fi Power (en0): On 35 | const cmd = `networksetup -getairportpower ${this.getWifiDeviceName()}`; 36 | const status = shell.exec(cmd, { silent: true }).stdout; 37 | Helper.debug(`wifi status: ${status.trim()}`); 38 | return !!status.includes('On'); 39 | } 40 | 41 | /** 42 | * 转换开关状态 43 | */ 44 | wifiToggle() { 45 | const name = this.getWifiDeviceName(); 46 | const cmd = `networksetup -getairportpower ${name} | grep "On" && networksetup -setairportpower ${name} off || networksetup -setairportpower ${name} on`; 47 | shell.exec(cmd, { silent: true }); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/driver/screen.ts: -------------------------------------------------------------------------------- 1 | import shell from 'shelljs'; 2 | import robot from 'robotjs'; 3 | import fs from 'fs'; 4 | import { Helper } from '../core/helper'; 5 | import os from 'os'; 6 | import assert from 'assert'; 7 | 8 | export default class ScreenDriver { 9 | 10 | hidpi = Helper.isHdpiDisplay(); 11 | 12 | /** 13 | * 暴露ocr方法，支持通过重写使用三方能力替代 14 | */ 15 | async fileOcr(imgFile: string): Promise<{ 16 | rect: { 17 | left: number; 18 | top: number; 19 | height: number; 20 | width: number; 21 | }; 22 | word: string; 23 | }[]> { 24 | const cmdFile = `${Helper.getResourcePath()}/swift/ocr-${os.arch()}`; 25 | assert(fs.existsSync(cmdFile), `不支持的架构: ${os.arch()}`); 26 | const resStr = shell.exec(`${cmdFile} ${imgFile}`, { silent: true }).stdout; 27 | return JSON.parse(resStr); 28 | } 29 | 30 | /** 31 | * 使用系统ocr能力 32 | */ 33 | async screenOcr(opts: { 34 | picFile?: string; 35 | rectangle?: string; // 通过矩形框 x,y,width,height 36 | count?: number; // 支持多次结果合并返回,增强识别率 37 | } = {}) { 38 | const { picFile, rectangle, count = 1 } = opts; 39 | const saveFile = picFile || `${Helper.tmpdir()}/${Date.now()}.png`; 40 | if (!fs.existsSync(saveFile)) { 41 | this.screenShot(saveFile, { rectangle }); 42 | } 43 | const ocrRes = []; 44 | for (let i = 0; i < count; i++) { 45 | const res = await this.fileOcr(saveFile); 46 | ocrRes.push(...res); 47 | } 48 | return { 49 | imgFile: saveFile, 50 | ocrRes, 51 | }; 52 | } 53 | 54 | /** 55 | * 检查文案存在 56 | */ 57 | async checkTextExist(opts: { 58 | text: string; 59 | picFile?: string; // 可直接指定图片 60 | rectangle?: string; // 截图目标区域 通过矩形框 x,y,width,height 默认全屏 61 | }): Promise { 62 | const { text, picFile, rectangle } = opts; 63 | const res = await this.getTextsPosition({ 64 | texts: [ text ], 65 | picFile, rectangle, 66 | }); 67 | return !!res.length; 68 | } 69 | 70 | /** 71 | * 获取文案在截图区域的位置 72 | * @param opts 73 | */ 74 | async getTextsPosition(opts: { 75 | texts: string[]; // 目标文案 76 | index?: number; // 重复项指针 77 | contains?: boolean; // 包含即可 78 | picFile?: string; // 可直接指定图片 79 | rectangle?: string; // 截图目标区域 通过矩形框 x,y,width,height 默认全屏 80 | }) { 81 | const { 82 | texts, rectangle, picFile, 83 | index = 0, 84 | contains = true, 85 | } = opts; 86 | // 获取文案位置 87 | const { ocrRes } = await this.screenOcr({ picFile, rectangle }); 88 | const resultList = []; 89 | // 找多个目标 90 | for (const text of texts) { 91 | const hitItems = ocrRes.filter(it => { 92 | return contains ? it.word.includes(text) : it.word.trim() === text; 93 | }); 94 | // 支持倒数 95 | const idx = index < 0 ? hitItems.length + index : index; 96 | const hitItem = hitItems[idx]; 97 | // 相对位置 98 | if (hitItem) { 99 | const { left, top, height, width } = hitItem.rect; 100 | let xx = left + width / 2; 101 | let yy = top + height / 2; 102 | if (this.hidpi) { 103 | xx = xx / 2; 104 | yy = yy / 2; 105 | } 106 | resultList.push({ x: Math.floor(xx), y: Math.floor(yy), word: hitItem.word }); 107 | } 108 | } 109 | return resultList; 110 | } 111 | 112 | /** 113 | * 含头部的系统状态栏 114 | */ 115 | screenGetSize(): { 116 | width: number; 117 | height: number; 118 | } { 119 | return robot.getScreenSize(); 120 | } 121 | 122 | screenCaptureBuffer(area: { 123 | topLeftX: number; 124 | topLeftY: number; 125 | width: number; 126 | height: number; 127 | }) { 128 | const img = robot.screen.capture(area.topLeftX, area.topLeftY, area.width, area.height); 129 | return img.image; 130 | } 131 | 132 | /** 133 | * 截图 134 | * @param picFile 135 | * @param opts 136 | */ 137 | screenShot(picFile: string, opts: { 138 | rectangle?: string; // 通过矩形框 x,y,width,height 139 | } = {}) { 140 | const { rectangle } = opts; 141 | let cmd = 'screencapture -x -r'; 142 | if (rectangle) { 143 | cmd += ` -R ${rectangle}`; 144 | } 145 | cmd += ` ${picFile}`; 146 | shell.exec(cmd); 147 | return picFile; 148 | } 149 | 150 | getPixelColor(x, y) { 151 | return robot.getPixelColor(x, y); 152 | } 153 | } 154 | -------------------------------------------------------------------------------- /src/driver/video.ts: -------------------------------------------------------------------------------- 1 | import fs from 'fs'; 2 | import shell from 'shelljs'; 3 | import { Helper } from '../core/helper'; 4 | import mixin from '../core/mixin'; 5 | import KeyboardDriver from './keyboard'; 6 | 7 | class VideoDriver extends KeyboardDriver { 8 | recordingVideoFile: string; 9 | /** 10 | * 开始录像，返回mov文件路径 11 | * @param opts 12 | */ 13 | startVideo(opts: { 14 | movFile?: string; // 指定保存的mov文件 15 | rectangle?: string; // 通过矩形框 x,y,width,height 16 | seconds?: number; // 指定时长 17 | } = {}) { 18 | if (this.recordingVideoFile) { 19 | console.error('存在录制中的录像'); 20 | return; 21 | } 22 | const { rectangle, seconds, movFile } = opts; 23 | // 静音 录像 显示点击 24 | let args = [ 25 | '-x', 26 | '-r', 27 | '-v', 28 | '-k', 29 | ]; 30 | if (rectangle) { 31 | args = args.concat([ 32 | '-R', 33 | rectangle, 34 | ]); 35 | } 36 | if (seconds) { 37 | args = args.concat([ 38 | '-V', 39 | `${seconds}`, 40 | ]); 41 | } 42 | const saveFile = movFile || `${Helper.tmpdir()}/${Date.now()}.mov`; 43 | args.push(saveFile); 44 | const cmd = `screencapture ${args.join(' ')}`; 45 | shell.exec(cmd, { silent: true, async: true }); 46 | this.recordingVideoFile = saveFile; 47 | } 48 | 49 | /** 50 | * 结束当前录像 51 | * @param destFile 52 | */ 53 | async saveVideo(destFile?: string): Promise { 54 | if ( 55 | destFile 56 | && !destFile.endsWith('.mov') 57 | && !destFile.endsWith('.mp4') 58 | ) { 59 | console.error('仅支持mov和mp4格式'); 60 | return; 61 | } 62 | // 结束录制 63 | const movFile = this.recordingVideoFile; 64 | this.recordingVideoFile = null; 65 | if (!movFile) { 66 | console.error('未开始录像'); 67 | return; 68 | } 69 | // 触发 screencapture 结束录像并保存录像文件 70 | await this.keyboardTap('escape', [ 'command', 'control' ]); 71 | await Helper.waitUntil(() => { 72 | return fs.existsSync(movFile); 73 | }); 74 | // 完成 screencapture 程序退出 75 | await this.keyboardTap('escape'); 76 | if (destFile) { 77 | // ffmpeg 转换 78 | if (destFile.endsWith('.mp4')) { 79 | const cmd = `ffmpeg -i ${movFile} -vcodec h264 -an -crf 20 -preset ultrafast -strict -2 -y ${destFile}`; 80 | shell.exec(cmd, { silent: true }); 81 | } else { 82 | shell.cp(movFile, destFile); 83 | } 84 | return destFile; 85 | } 86 | return movFile; 87 | } 88 | } 89 | 90 | mixin(VideoDriver, [ 91 | KeyboardDriver, 92 | ]); 93 | 94 | export default VideoDriver; 95 | -------------------------------------------------------------------------------- /src/macaca-macos.ts: -------------------------------------------------------------------------------- 1 | import mixin from './core/mixin'; 2 | import AppDriver from './driver/app'; 3 | import MouseDriver from './driver/mouse'; 4 | import KeyboardDriver from './driver/keyboard'; 5 | import ClipboardDriver from './driver/clipboard'; 6 | import VideoDriver from './driver/video'; 7 | import ScreenDriver from './driver/screen'; 8 | import { jxaUtil } from './core/jxa/jxaUtil'; 9 | import { osaUtil } from './core/jxa/osaUtil'; 10 | import NetworkDriver from './driver/network'; 11 | 12 | class MacacaMacOS { 13 | static jxaUtil = jxaUtil; 14 | static osaUtil = osaUtil; 15 | } 16 | 17 | interface MacacaMacOS extends AppDriver, MouseDriver, KeyboardDriver, ClipboardDriver, VideoDriver, ScreenDriver, NetworkDriver {} 18 | 19 | mixin(MacacaMacOS, [ 20 | AppDriver, 21 | MouseDriver, 22 | KeyboardDriver, 23 | ClipboardDriver, 24 | VideoDriver, 25 | ScreenDriver, 26 | NetworkDriver, 27 | ]); 28 | 29 | export default MacacaMacOS; 30 | -------------------------------------------------------------------------------- /test/jxaUtil.test.ts: -------------------------------------------------------------------------------- 1 | import { jxaUtil } from '../src/core/jxa/jxaUtil'; 2 | import MacacaMacOS from '../src/macaca-macos'; 3 | import assert from 'assert'; 4 | 5 | describe.skip('jxaUtil unit testing', function() { 6 | 7 | it('focusApp should be ok', async () => { 8 | await jxaUtil.focusApp('Notes'); 9 | const win = await new MacacaMacOS().getAppSizePosition('Notes'); 10 | assert(win); 11 | }); 12 | 13 | }); 14 | -------------------------------------------------------------------------------- /test/macaca-macos.test.ts: -------------------------------------------------------------------------------- 1 | import MacacaMacOS from '../src/macaca-macos'; 2 | import { Helper } from '../src/core/helper'; 3 | import { EDriver } from '../src/core/enums'; 4 | import assert from 'assert'; 5 | 6 | describe('macaca-macos unit testing', function() { 7 | this.timeout(0); 8 | process.env.MACACA_MACOS_DEBUG = 'true'; 9 | const driver = new MacacaMacOS(); 10 | let res: any; 11 | 12 | it.skip('isAppRunning should be ok', async () => { 13 | this.timeout(0); 14 | assert(driver); 15 | await driver.startApp('/System/Applications/Notes.app'); 16 | res = await driver.isAppRunning('Notes'); 17 | assert(res); 18 | }); 19 | 20 | it('mouseGetPos should be ok', async () => { 21 | this.timeout(0); 22 | const res = driver.mouseGetPos(); 23 | assert(res); 24 | }); 25 | 26 | // 测试 27 | it.skip('mouseClickText should be ok', async () => { 28 | this.timeout(0); 29 | const res = driver.mouseClickText({ 30 | text: '测试', 31 | shiftX: 100, 32 | }); 33 | assert(res); 34 | }); 35 | 36 | it.skip('checkTextExist should be ok', async () => { 37 | this.timeout(0); 38 | const res = driver.checkTextExist({ 39 | text: '文案存在', 40 | }); 41 | assert(res); 42 | }); 43 | 44 | it.skip('overwrite should be ok', async () => { 45 | this.timeout(0); 46 | driver.fileOcr = async (imgFile) => { 47 | console.log(imgFile); 48 | return [ 49 | { 50 | word: '哈哈', 51 | rect: { 52 | left: 0, 53 | top: 1, 54 | height: 100, 55 | width: 100, 56 | }, 57 | }, 58 | ]; 59 | }; 60 | const res = driver.checkTextExist({ 61 | text: '哈哈', 62 | }); 63 | assert(res); 64 | }); 65 | 66 | it.skip('mouse drag should be ok', async function() { 67 | this.timeout(0); 68 | await Helper.sleep(3E3); 69 | await driver.mouseDrag(100, 100); 70 | const res = driver.mouseGetPos(); 71 | console.log(res); 72 | assert(res.x === 100 && res.y === 100); 73 | }); 74 | 75 | it.skip('screen ocr should be ok', async function() { 76 | this.timeout(0); 77 | const res = await driver.screenOcr(); 78 | console.log(JSON.stringify(res, null, 2)); 79 | assert(res.ocrRes); 80 | }); 81 | 82 | it.skip('AppleScript mouseClick should be ok', async () => { 83 | this.timeout(0); 84 | driver.mouseClick({ 85 | driver: EDriver.AppleScript, 86 | }); 87 | }); 88 | 89 | it('Clipboard actions should be ok', async () => { 90 | this.timeout(0); 91 | const str = 'Hello world.'; 92 | await driver.setClipText(str); 93 | const res = await driver.getClipText(); 94 | assert.equal(res, str, '剪贴板内容不符合预期'); 95 | }); 96 | 97 | it('video should be ok', async function() { 98 | this.timeout(0); 99 | driver.startVideo({ rectangle: '0,0,400,600' }); 100 | await Helper.sleep(10E3); 101 | const mov = await driver.saveVideo(); 102 | console.log(mov); 103 | assert(mov); 104 | }); 105 | 106 | it('focusApp should be ok', async function() { 107 | this.timeout(0); 108 | await driver.focusApp('Notes'); 109 | console.log('end'); 110 | }); 111 | 112 | describe.skip('Network driver test', function() { 113 | this.timeout(0); 114 | it('wifi device name should work', async function() { 115 | this.timeout(0); 116 | const name = driver.getWifiDeviceName(); 117 | assert(name, 'wifi设备查询异常'); 118 | }); 119 | 120 | it('wifi device turn on should work', async function() { 121 | this.timeout(0); 122 | driver.wifiTurnOn(); 123 | const isWifiOn = driver.isWifiOn(); 124 | assert(isWifiOn, 'wifi状态检查异常'); 125 | }); 126 | 127 | }); 128 | }); 129 | -------------------------------------------------------------------------------- /test/mocha.opts: -------------------------------------------------------------------------------- 1 | --reporter spec 2 | -------------------------------------------------------------------------------- /test/osaUtil.test.ts: -------------------------------------------------------------------------------- 1 | import { osaUtil } from '../src/core/jxa/osaUtil'; 2 | import MacacaMacOS from '../src/macaca-macos'; 3 | import assert from 'assert'; 4 | 5 | describe('osaUtil unit testing', function() { 6 | 7 | it('getAllAppSizePosition should be ok', async () => { 8 | const res = await osaUtil.getAllAppSizePosition(); 9 | console.log(res); 10 | assert(res); 11 | }); 12 | 13 | it('focusApp should be ok', async () => { 14 | await osaUtil.focusApp('Notes'); 15 | const win = await new MacacaMacOS().getAppSizePosition('Notes'); 16 | console.log(win); 17 | assert(win); 18 | }); 19 | 20 | }); 21 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES2018", 4 | "module": "commonjs", 5 | "moduleResolution": "node", 6 | "experimentalDecorators": true, 7 | "emitDecoratorMetadata": true, 8 | "resolveJsonModule": true, 9 | "inlineSourceMap":true, 10 | "noImplicitThis": true, 11 | "esModuleInterop": true, 12 | "noUnusedLocals": true, 13 | "stripInternal": true, 14 | "pretty": true, 15 | "declaration": true, 16 | "removeComments": false, 17 | "types": [ "mocha", "node" ], 18 | "outDir": "./dist" 19 | }, 20 | "include": [ 21 | "src", 22 | "bin", 23 | "index.ts" 24 | ] 25 | } 26 | --------------------------------------------------------------------------------