├── .github └── workflows │ ├── cargo_check.yml │ ├── changelog.yml │ ├── ci.yml │ ├── clippy.yml │ └── rustfmt.yml ├── .gitignore ├── CHANGELOG.md ├── Cargo.toml ├── Keyboard_commands.md ├── LICENSE ├── README.md ├── build.rs ├── examples ├── display_mouse_coords.rs ├── find_image_move_mouse_and_input.rs ├── open_github_page.rs └── opencl_example.rs ├── src ├── core │ ├── keyboard │ │ ├── linux │ │ │ └── mod.rs │ │ ├── macos │ │ │ └── mod.rs │ │ ├── mod.rs │ │ └── windows │ │ │ └── mod.rs │ ├── mod.rs │ ├── mouse │ │ ├── linux │ │ │ └── mod.rs │ │ ├── macos │ │ │ └── mod.rs │ │ ├── mod.rs │ │ ├── mouse_position.rs │ │ └── windows │ │ │ └── mod.rs │ ├── screen │ │ ├── linux │ │ │ └── mod.rs │ │ ├── macos │ │ │ └── mod.rs │ │ ├── mod.rs │ │ └── windows │ │ │ └── mod.rs │ └── template_match │ │ ├── fft_ncc.rs │ │ ├── mod.rs │ │ ├── open_cl.rs │ │ ├── opencl_kernel.rs │ │ ├── opencl_v2.rs │ │ ├── segmented_ncc.rs │ │ └── slow_ncc.rs ├── data │ ├── mod.rs │ └── opencl.rs ├── errors.rs ├── imgtools │ └── mod.rs ├── lib.rs └── rustautogui_impl │ ├── keyboard_impl.rs │ ├── mod.rs │ ├── mouse_impl.rs │ └── template_match_impl │ ├── find_img_impl.rs │ ├── load_img_impl.rs │ └── mod.rs ├── tests ├── mouse_tests.rs ├── multi_test.rs ├── testing_images │ ├── algorithm_tests │ │ ├── Darts_main.png │ │ ├── Darts_template1.png │ │ ├── Darts_template2.png │ │ ├── Darts_template3.png │ │ ├── Socket_main.png │ │ ├── Socket_template1.png │ │ ├── Socket_template2.png │ │ ├── Socket_template3.png │ │ ├── Split_main.png │ │ ├── Split_template1.png │ │ ├── Split_template2.png │ │ ├── Split_template3.png │ │ ├── Split_template4.png │ │ └── Split_template5.png │ └── gui_tests │ │ ├── step_1_l.png │ │ ├── step_1_m.png │ │ ├── step_1_w.png │ │ ├── step_2_l.png │ │ ├── step_2_m.png │ │ ├── step_2_w.png │ │ ├── step_3_l.png │ │ ├── step_3_m.png │ │ └── step_3_w.png └── tmpl_match_tests.rs └── testspeed.gif /.github/workflows/cargo_check.yml: -------------------------------------------------------------------------------- 1 | on: 2 | push: 3 | branches: [main] 4 | pull_request: 5 | merge_group: 6 | 7 | name: Cargo.toml validation 8 | 9 | jobs: 10 | check-cargo-toml: 11 | name: Check Cargo.toml consistency 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/checkout@v4 15 | 16 | - name: Validate Cargo.toml 17 | run: | 18 | # Check [features] default = ["full"] 19 | DEFAULT_FEATURES=$(grep -A 1 "^\[features\]" Cargo.toml | grep "default" || true) 20 | echo "Found default features: $DEFAULT_FEATURES" 21 | if [[ "$DEFAULT_FEATURES" != *'default = ["full"]'* ]]; then 22 | echo "::error ::[features] default must be [\"full\"]" 23 | exit 1 24 | fi 25 | echo "Cargo.toml checks passed successfully!" -------------------------------------------------------------------------------- /.github/workflows/changelog.yml: -------------------------------------------------------------------------------- 1 | on: 2 | pull_request: 3 | # Run on labeled/unlabeled in addition to defaults to detect 4 | # adding/removing skip-changelog labels. 5 | types: [opened, reopened, labeled, unlabeled, synchronize] 6 | merge_group: 7 | 8 | name: Changelog check 9 | 10 | jobs: 11 | changelog: 12 | name: Changelog check 13 | runs-on: ubuntu-latest 14 | steps: 15 | - name: Checkout sources 16 | uses: actions/checkout@v4 17 | 18 | - name: Check changelog update 19 | uses: dangoslen/changelog-enforcer@v3 20 | with: 21 | skipLabels: skip-changelog 22 | missingUpdateErrorMessage: "Please add a changelog entry to the appropriate section of the CHANGELOG.md file." 23 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | on: 2 | push: 3 | branches: [main] 4 | pull_request: 5 | merge_group: 6 | 7 | name: Continuous integration 8 | 9 | jobs: 10 | ci: 11 | name: CI 12 | runs-on: ubuntu-latest 13 | needs: [check] 14 | if: always() 15 | steps: 16 | - name: Done 17 | run: jq --exit-status 'all(.result == "success")' <<< '${{ toJson(needs) }}' 18 | 19 | check: 20 | strategy: 21 | matrix: 22 | include: 23 | # Linux 24 | - { target: x86_64-unknown-linux-gnu, os: ubuntu-latest, feature: lite } 25 | - { target: x86_64-unknown-linux-gnu, os: ubuntu-latest, feature: full } 26 | - { target: x86_64-unknown-linux-gnu, os: ubuntu-latest, feature: opencl } 27 | - { target: x86_64-unknown-linux-gnu, os: ubuntu-latest, feature: dev } 28 | 29 | # macOS 30 | - { target: x86_64-apple-darwin, os: macos-latest, feature: lite } 31 | - { target: x86_64-apple-darwin, os: macos-latest, feature: full } 32 | - { target: x86_64-apple-darwin, os: macos-latest, feature: opencl } 33 | - { target: x86_64-apple-darwin, os: macos-latest, feature: dev } 34 | 35 | - { target: aarch64-apple-darwin, os: macos-latest, feature: lite } 36 | - { target: aarch64-apple-darwin, os: macos-latest, feature: full } 37 | - { target: aarch64-apple-darwin, os: macos-latest, feature: opencl } 38 | - { target: aarch64-apple-darwin, os: macos-latest, feature: dev } 39 | 40 | # Windows (only lite and full) 41 | - { target: x86_64-pc-windows-msvc, os: windows-latest, feature: lite } 42 | - { target: x86_64-pc-windows-msvc, os: windows-latest, feature: full } 43 | runs-on: ${{ matrix.os }} 44 | steps: 45 | - uses: actions/checkout@v4 46 | 47 | - name: Display build info 48 | run: | 49 | echo "========================================" 50 | echo "Building for OS: ${{ matrix.os }}" 51 | echo "Target: ${{ matrix.target }}" 52 | echo "Feature: ${{ matrix.feature }}" 53 | echo "========================================" 54 | shell: bash 55 | 56 | 57 | - name: Install x11 dev packages (linux) 58 | run: | 59 | sudo apt update 60 | sudo apt install -y libx11-dev libxtst-dev ocl-icd-opencl-dev 61 | # Only install on Ubuntu 62 | if: matrix.os == 'ubuntu-latest' 63 | 64 | - name: Install Rust toolchain 65 | run: rustup target add ${{ matrix.target }} 66 | 67 | - name: Cache Dependencies 68 | uses: Swatinem/rust-cache@v2 69 | with: 70 | key: v2.1-${{ matrix.target }}-${{ matrix.feature }} 71 | 72 | - uses: actions-rs/cargo@v1 73 | with: 74 | command: build 75 | args: --target ${{ matrix.target }} --release --examples --no-default-features --features ${{ matrix.feature }} -------------------------------------------------------------------------------- /.github/workflows/clippy.yml: -------------------------------------------------------------------------------- 1 | on: 2 | push: 3 | branches: [main] 4 | pull_request: 5 | merge_group: 6 | 7 | name: Clippy 8 | 9 | jobs: 10 | ci: 11 | name: CI 12 | runs-on: ubuntu-latest 13 | needs: [clippy-check] 14 | if: always() 15 | steps: 16 | - name: Done 17 | run: jq --exit-status 'all(.result == "success")' <<< '${{ toJson(needs) }}' 18 | 19 | clippy-check: 20 | strategy: 21 | matrix: 22 | include: 23 | - { target: x86_64-unknown-linux-gnu, os: ubuntu-latest } 24 | - { target: x86_64-apple-darwin, os: macos-latest } 25 | - { target: aarch64-apple-darwin, os: macos-latest } 26 | - { target: x86_64-pc-windows-msvc, os: windows-latest } 27 | runs-on: ${{ matrix.os }} 28 | steps: 29 | - uses: actions/checkout@v4 30 | 31 | - name: Install Rust toolchain 32 | run: rustup target add ${{ matrix.target }} 33 | 34 | - name: Cache Dependencies 35 | uses: Swatinem/rust-cache@v2 36 | with: 37 | key: v2.1-${{ matrix.target }} 38 | 39 | - uses: actions-rs/cargo@v1 40 | with: 41 | command: clippy 42 | args: --target ${{ matrix.target }} 43 | -------------------------------------------------------------------------------- /.github/workflows/rustfmt.yml: -------------------------------------------------------------------------------- 1 | on: 2 | push: 3 | branches: [main] 4 | pull_request: 5 | merge_group: 6 | 7 | name: Code formatting check 8 | 9 | jobs: 10 | fmt: 11 | name: Rustfmt 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/checkout@v4 15 | - name: Use the latest stable rustc 16 | run: rustup update stable && rustup default stable 17 | 18 | - run: cargo fmt --all -- --check 19 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /Cargo.lock 3 | /*.png 4 | /src/main.rs 5 | /debug 6 | # /testing_images* 7 | *ipynb* 8 | /testing_points/ 9 | *.zip -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | All notable changes to this project will be documented in this file. 3 | 4 | ## [2.5.0] - 2025-04-27 5 | ### Added 6 | - Added **OpenCL** implementation of the algorithm. Now you can run the template matching process of GPU to achieve better performance. Two variants of algorithm included 7 | - Added additional fields to MatchMode enum: SegmentedOcl and SegmentedOclV2 - only available with opencl feature 8 | - Added optimization to segmented template match with automatic threshold detection 9 | - Added *_custom template preparation functions which allow user to input threshold themselves if they dont want automatic detection 10 | - Code cleanup / restructure 11 | - Added feature "lite". This includes much lighter version of library with no template matching possibility, just keyboard and mouse included 12 | - Added feature "dev" which opens core functions and structs as pub 13 | 14 | ## [2.4.0] - 2025-04-05 15 | ### Added / Changed / Fixed 16 | - Changed: function drag_mouse to drag_mouse_to_pos. Reason is the new drag_mouse function that moves mouse in relation to current position 17 | - made MouseClick enum public 18 | - Added functions: 19 | - - move_mouse() - Moves mouse relative to its current position. -x left, +x right, -y up, +y down 20 | - - move_mouse_to() -- Accepts Option, Option. None value keeps the same x or y. Usefull for horizontal and vertical movement 21 | - - drag_mouse() - performs mouse drag action relative to current position. Same rules as in move_mouse function 22 | - - drag_mouse_to() - same as in mouse_move_to, accepts Options. 23 | - - drag_mouse_to_pos() - same as in move_mouse_to_pos(). This is the old drag_mouse() func 24 | - - get_mouse_position() - returns Result<(i32, i32)> current mouse position 25 | - - click() - Mouse click - choose button with MouseClick enum 26 | - - click_down() - accepts MouseClick enum (does not work on macOS) 27 | - - click_up() - accepts MouseClick enum (does not work on macOS) 28 | - - key_down() - executes key press down only 29 | - - key_up() - executes key press up only 30 | - move_mouse_to_pos() remains the same, while drag_mouse_to_pos() is new name for the old version of drag_mouse() function 31 | 32 | ## [2.3.0] - 2025-03-30 33 | ### Fixed / Removed 34 | - Rework of Segmented NCC template match. Completely removed argument of Max segments and made it always work robustly, never sacrificing precision for speed. Additionally fixed part of formulas which will additionally reduce false positives, regardless of max segments. The fix also improves algorithms speed when compared to previous version, if max_segments is not taken into consideration. The speed gain is due to much less checks and verifications in the algorithm. 35 | - Fixed returned values from find image on screen, where its correctly returning positions adjusted for screen region and template size, where previously that worked only on find image and move mouse 36 | - Removed completely the change prepared template function. 37 | 38 | 39 | ## [2.2.2] - 2025-03-27 40 | ### Fixed 41 | - use `&str` && `&[]` more wide 42 | 43 | ## [2.2.1] - 2025-03-26 44 | ### Fixed 45 | - macOS alias check turned off till fixed 46 | 47 | 48 | ## [2.2.0] - 2025-03-25 49 | ### Added / Fixed 50 | - Added ability to store multiple images (stored in Hashmap in struct) and give them alias. Can be stored from path, Imagebuffer or encoded u8 vec 51 | - Added corresponsing find_stored_image_on_screen() and find_stored_image_on_screen_and_move_mouse() which additionaly take alias parameter 52 | - Added prepare_template_from_imagebuffer() which accepts Imagebuffers RGB, RGBa and Luma(black and white) 53 | - Added prepare_template_from_raw_encoded() which can load from encoded u8 vec 54 | - Added search for image with implemented loop 55 | - Added Super/Win key commands for Linux 56 | - Added F1-F20 keys for Linux 57 | - Added another example 58 | - Added custom error type 59 | - Made template search for macOS retina displays more robust. Now 2 variants of template are stored, the provided one and resized one. The search is done for both of them. The reason for this is, it cannot be known if user is providing template image which he got as a snip which needs to be resized, or from a (for instance) downloaded image which does not require resize 60 | - updated to latest versions of dependencies 61 | - Fix: find_image_and_move_mouse now returns vec of all found locations instead of just top location 62 | - Fix: README code examples fixed 63 | - Fix: check for out of bounds on windows mouse move fixed 64 | - imgtools::convert_image_to_bw was renamed to convert_rgba_to_bw 65 | - cleaned up and moved lots of things to private. Only available modules to public now are RustAutoGui, MatchMode, imgtools, normalized_x_corr and function print_mouse_position() 66 | - Win keys currently seem to not work on windows. They will be left in the code and accessible to call, since they issue is most likely not related to code and could be resolved. On keyboard_commands.md they are now labeled as not implemented 67 | 68 | 69 | 70 | 71 | 72 | 73 | ## [2.1.1] - 2025-03.16 74 | ### Fixed 75 | - added some missing keys for keyboard 76 | - a more detailed list of available key commands is now available in Keyboard_commands.md 77 | - old keyboard_commands.txt file has been removed 78 | - negative movement_times for mouse will not produce errors anymore. Instead, they are treated as 0.0 value 79 | 80 | ## [2.1.0] - 2025-03.15 81 | ### Added/Fixed 82 | - added scroll left / right functionality 83 | - added drag mouse functionalty (click down -> move to location -> click up) 84 | - added get_screen_size() method 85 | - Fix for keyboard. Works on US layout only at the moment. Shifted argument from keyboard_input() method removed 86 | - fixed double click on MacOS 87 | - fix: added another check for region boundaries that should prevent the code to run an assertion, rather returning an error instead. 88 | - fix: find_image_and_move_mouse now returns correct position 89 | - changed move_mouse to accept u32 instead of i32 as x, y parameters 90 | - included warnings 91 | 92 | 93 | ## [2.0.1] - 2025-03.14 94 | ### Fixed 95 | - Fixed readme code examples 96 | - fixed Segmented normalized cross correlation doing false matches. 97 | 98 | ## [2.0.0] - 2025-03.10 99 | ### Added/Fixed 100 | - complete rework of the code which will not be compatible with old versions. 101 | - introduced graceful exits, except for some situations like not having x11 activated on linux 102 | - most of methods return Result<> now. 103 | 104 | ## [1.0.1] - 2025-03.07 105 | ### Fixed 106 | - fixed wrong creation of debug folder even when not in debug mode 107 | 108 | ## [1.0.0] - 2025-02.25 109 | ### Added 110 | - Segmented correlation template matching mode included 111 | 112 | ## [0.3.2] - 2024-09.03 113 | ### Fixed 114 | - fix MACOS capture screen on retina 115 | 116 | 117 | ## [0.3.1] - 2024-08.01 118 | ### Added 119 | -small optimization to template prepare 120 | 121 | ## [0.3.0] - 2024-07.27 122 | ### Removed 123 | -removed egui and eframe dependencies. Unnecessary and used just to create one window to show mouse position. Simply printing it now. 124 | 125 | ## [0.2.2] - 2024-07.27 126 | ### Added 127 | -scroll up and scroll down functions 128 | 129 | ## [0.2.1] - 2024-07.26 130 | ### Added 131 | -multi key press function 132 | 133 | ## [0.2.0] - 2024-07-26 134 | ### Added 135 | - macOS support 136 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "rustautogui" 3 | authors = ["Davor Marusic"] 4 | version = "2.5.0" 5 | edition = "2021" 6 | description = "Highly optimized GUI automation library for controlling the mouse and keyboard, with template matching support." 7 | license = "MIT" 8 | readme = "README.md" 9 | homepage = "https://github.com/DavorMar/rustautogui" 10 | keywords = ["automation", "autogui", "gui", "mouse", "keyboard"] 11 | categories = ["gui", "development-tools"] 12 | exclude = ["testspeed.gif", "tests/testing_images/"] 13 | repository = "https://github.com/DavorMar/rustautogui" 14 | 15 | 16 | [dependencies] 17 | rustfft = { version = "6.2.0", optional = true } 18 | rayon = { version = "1.10.0", optional = true } 19 | num-complex = { version = "0.4.6", optional = true } 20 | ocl = { version = "0.19.7", optional = true } 21 | image = { version = "0.25", optional = true} 22 | rand = "0.9" 23 | 24 | [target.'cfg(target_os = "linux")'.dependencies] 25 | x11 = "2.21.0" 26 | 27 | [target.'cfg(target_os = "macos")'.dependencies] 28 | core-graphics = {version = "0.23.2", features = ["highsierra"]} 29 | 30 | 31 | [target.'cfg(windows)'.dependencies] 32 | winapi = { version = "0.3", features = ["winuser", "windef"] } 33 | 34 | 35 | [features] 36 | default = ["full"] 37 | full = ["rustfft", "num-complex", "rayon", "image"] 38 | lite = [] 39 | opencl = ["ocl", "full"] 40 | dev = ["opencl"] 41 | 42 | 43 | [[example]] 44 | name = "opencl_example" 45 | path = "examples/opencl_example.rs" 46 | required-features = ["opencl"] 47 | 48 | 49 | [[example]] 50 | name = "find_image_move_mouse_and_input" 51 | path = "examples/find_image_move_mouse_and_input.rs" 52 | required-features = ["full"] 53 | 54 | 55 | [[example]] 56 | name = "open_github_page" 57 | path = "examples/open_github_page.rs" 58 | required-features = ["full"] -------------------------------------------------------------------------------- /Keyboard_commands.md: -------------------------------------------------------------------------------- 1 | | Key | Windows | Linux | MacOS | 2 | |------------------|---------|-------|-------| 3 | | space / " " | X | X | X | 4 | | A-Z | X | X | X | 5 | | a-z | X | X | X | 6 | | 0-9 | X | X | X | 7 | | ! | X | X | X | 8 | | @ | X | X | X | 9 | | # | X | X | X | 10 | | $ | X | X | X | 11 | | % | X | X | X | 12 | | ^ | X | X | X | 13 | | & | X | X | X | 14 | | * | X | X | X | 15 | | ( | X | X | X | 16 | | ) | X | X | X | 17 | | - | X | X | X | 18 | | _ | X | X | X | 19 | | = | X | X | X | 20 | | + | X | X | X | 21 | | [ | X | X | X | 22 | | ] | X | X | X | 23 | | { | X | X | X | 24 | | } | X | X | X | 25 | | ; | X | X | X | 26 | | : | X | X | X | 27 | | ' | X | X | X | 28 | | " | X | X | X | 29 | | , | X | X | X | 30 | | < | X | X | X | 31 | | . | X | X | X | 32 | | > | X | X | X | 33 | | / | X | X | X | 34 | | ? | X | X | X | 35 | | \ | X | X | X | 36 | | \| | X | X | X | 37 | | ` | X | X | X | 38 | | ~ | X | X | X | 39 | | return / enter | X | X | X | 40 | | backspace | X | X | X | 41 | | tab | X | X | X | 42 | | escape / esc | X | X | X | 43 | | up(_arrow) | X | X | X | 44 | | down(_arrow) | X | X | X | 45 | | left(_arrow) | X | X | X | 46 | | right(_arrow) | X | X | X | 47 | | home | X | X | X | 48 | | end | X | X | X | 49 | | page_up / pgup | X | X | X | 50 | | page_down / pgdn | X | X | X | 51 | | insert | X | X | X | 52 | | delete / del | X | X | X | 53 | | f1 - f12 | X | X | X | 54 | | f13 - f20 | X | X | X | 55 | | f21 - f24 | X | - | - | 56 | | control_l | X | X | X | 57 | | control_r | X | X | X | 58 | | ctrl / control | X | X | X | 59 | | alt | X | X | X | 60 | | alt_l | X | X | X | 61 | | alt_r | X | X | X | 62 | | shift | X | X | X | 63 | | shift_l | X | X | X | 64 | | shift_r | X | X | X | 65 | | caps_lock | X | X | X | 66 | | printscreen | X | - | X | 67 | | printscrn | X | - | X | 68 | | prtsc/prtscr | X | - | X | 69 | | scroll_lock | X | - | X | 70 | | numlock | X | - | X | 71 | | pause | X | - | X | 72 | | clear | X | - | - | 73 | | win | - | X | - |# ATM win keys do not work on windows 74 | | winleft | - | X | - | 75 | | win_l | - | X | - | 76 | | winright | - | X | - | 77 | | win_r | - | X | - | 78 | | super | - | X | - | 79 | | super_r | - | X | - | 80 | | super | - | X | - | 81 | | super_l | - | X | - | 82 | | command | - | - | X | # (⌘) 83 | | command_l | - | - | X | # (⌘) 84 | | command_r | - | - | X | # (⌘) 85 | | option | - | - | X | #(⌥) 86 | | option_l | - | - | X | #(⌥) 87 | | option_r | - | - | X | #(⌥) 88 | | function | - | - | X | #(⌥) 89 | | num0 - num9 | X | - | - | 90 | | browserback | X | - | - | 91 | | browserforward | X | - | - | 92 | | browserrefresh | X | - | - | 93 | | browserstop | X | - | - | 94 | | browsersearch | X | - | - | 95 | | browserfavorites | X | - | - | 96 | | browserhome | X | - | - | 97 | | volumemute | X | - | X | 98 | | volumedown | X | - | X | 99 | | volumeup | X | - | X | 100 | | nexttrack | X | - | - | 101 | | prevtrack | X | - | - | 102 | | stop | X | - | - | 103 | | playpause | X | - | - | 104 | | launchmail | X | - | - | 105 | | launchmediaselect| X | - | - | 106 | | launchapp1 | X | - | - | 107 | | launchapp2 | X | - | - | 108 | | separator | X | - | - | 109 | | kana | X | - | - | 110 | | hanguel | X | - | - | 111 | | hangul | X | - | - | 112 | | junja | X | - | - | 113 | | final | X | - | - | 114 | | hanja | X | - | - | 115 | | kanji | X | - | - | 116 | | convert | X | - | - | 117 | | nonconvert | X | - | - | 118 | | accept | X | - | - | 119 | | modechange | X | - | - | 120 | | kanji | X | - | - | 121 | | execute | X | - | - | 122 | | help | X | - | X | 123 | | apps | X | - | - | 124 | | sleep | X | - | - | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Davor Marušić 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /build.rs: -------------------------------------------------------------------------------- 1 | // build.rs 2 | use std::env; 3 | 4 | fn main() { 5 | //required for X11 binding connection 6 | if env::var("CARGO_CFG_TARGET_OS").unwrap() == "linux" { 7 | println!("cargo:rustc-link-lib=X11"); 8 | println!("cargo:rustc-link-lib=Xtst"); 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /examples/display_mouse_coords.rs: -------------------------------------------------------------------------------- 1 | use rustautogui::print_mouse_position; 2 | 3 | #[allow(unused_must_use)] 4 | fn main() { 5 | print_mouse_position(); 6 | } 7 | -------------------------------------------------------------------------------- /examples/find_image_move_mouse_and_input.rs: -------------------------------------------------------------------------------- 1 | fn main() { 2 | #[cfg(not(feature = "lite"))] 3 | { 4 | use rustautogui; 5 | use rustautogui::imgtools; 6 | // initialize autogui 7 | let mut gui = rustautogui::RustAutoGui::new(false).unwrap(); 8 | 9 | // displays (x, y) size of screen 10 | // especially useful on linux where screen from all monitors is grabbed 11 | gui.get_screen_size(); 12 | 13 | { 14 | // load the image searching for. Region is Option<(startx, starty, width, height)> of search. Matchmode FFT or Segmented (not implemented before 1.0 version), max segments, only important for Segmented match mode 15 | gui.prepare_template_from_file( 16 | "test.png", 17 | Some((0, 0, 500, 300)), 18 | rustautogui::MatchMode::FFT, 19 | ) 20 | .unwrap(); 21 | } 22 | // or another way to prepare template 23 | { 24 | let img = imgtools::load_image_rgba("test.png").unwrap(); // just loading this way for example 25 | gui.prepare_template_from_imagebuffer( 26 | img, 27 | Some((0, 0, 700, 500)), 28 | rustautogui::MatchMode::Segmented, 29 | ) 30 | .unwrap(); 31 | } 32 | 33 | // or segmented variant with no region 34 | gui.prepare_template_from_file("test.png", None, rustautogui::MatchMode::FFT) 35 | .unwrap(); 36 | 37 | // change prepare template settings, like region, matchmode or max segments 38 | 39 | // automatically move mouse to found template position, execute movement for 1 second 40 | gui.find_image_on_screen_and_move_mouse(0.9, 1.0).unwrap(); 41 | 42 | //move mouse to position (in this case to bottom left of stanard 1920x1080 monitor), move the mouse for 1 second 43 | gui.move_mouse_to_pos(1920, 1080, 1.0).unwrap(); 44 | 45 | // execute left click, move the mouse to x (500), y (500) position for 1 second, release mouse click 46 | // used for actions like moving icons or files 47 | // suggestion: dont use very small moving time values, especially on mac 48 | gui.drag_mouse(500, 500, 1.0).unwrap(); 49 | 50 | // execute mouse left click 51 | gui.left_click().unwrap(); 52 | 53 | // execute mouse right click 54 | gui.right_click().unwrap(); 55 | 56 | // execute mouse middle click 57 | gui.middle_click().unwrap(); 58 | 59 | // execute a double (left) mouse click 60 | gui.double_click().unwrap(); 61 | 62 | // mouse scrolls 63 | gui.scroll_down(1).unwrap(); 64 | gui.scroll_up(5).unwrap(); 65 | gui.scroll_right(8).unwrap(); 66 | gui.scroll_left(10).unwrap(); 67 | 68 | // input keyboard string 69 | gui.keyboard_input("test.com").unwrap(); 70 | 71 | // press a keyboard command, in this case enter(return) 72 | gui.keyboard_command("return").unwrap(); 73 | 74 | // two key press 75 | gui.keyboard_multi_key("alt", "tab", None).unwrap(); 76 | 77 | // three key press 78 | gui.keyboard_multi_key("shift", "control", Some("e")) 79 | .unwrap(); 80 | 81 | // maybe you would want to loop search until image is found and break the loop then 82 | loop { 83 | let pos = gui.find_image_on_screen_and_move_mouse(0.9, 1.0).unwrap(); 84 | match pos { 85 | Some(_) => break, 86 | None => (), 87 | } 88 | } 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /examples/open_github_page.rs: -------------------------------------------------------------------------------- 1 | // code that opens chrome, new tab, goes to github, mouse overs star (click commented out), scrolls to the bottom and clicks on terms 2 | // no image files included for examples 3 | 4 | fn main() { 5 | #[cfg(not(feature = "lite"))] 6 | { 7 | use rustautogui::RustAutoGui; 8 | use std::thread; 9 | use std::time; 10 | let mut rustautogui = RustAutoGui::new(false).unwrap(); // initialize 11 | let (screen_w, screen_h) = rustautogui.get_screen_size(); 12 | 13 | // load, process and store image of star(can be image of project name text) 14 | // regions are written in this way to cover any screen size, not hardcoding them 15 | rustautogui 16 | .store_template_from_file( 17 | "/home/davor/Pictures/stars.png", 18 | Some(( 19 | (0.2 * screen_w as f32) as u32, // start x 20 | 0, // start y 21 | (0.5 * screen_w as f32) as u32, // width 22 | (0.4 * screen_h as f32) as u32, // height 23 | )), 24 | rustautogui::MatchMode::Segmented, 25 | "stars", 26 | ) 27 | .unwrap(); 28 | 29 | // just for example doing single prepare, but you would want to store it also 30 | // load, process and store image of terms 31 | rustautogui 32 | .prepare_template_from_file( 33 | "/home/davor/Pictures/terms.png", 34 | Some(( 35 | (0.1 * screen_w as f32) as u32, // start x 36 | (0.7 * screen_h as f32) as u32, // start y 37 | (0.5 * screen_w as f32) as u32, // width 38 | (0.3 * screen_h as f32) as u32, // height 39 | )), 40 | rustautogui::MatchMode::Segmented, 41 | ) 42 | .unwrap(); 43 | 44 | // press windows/super key 45 | rustautogui.keyboard_command("win").unwrap(); 46 | 47 | thread::sleep(time::Duration::from_millis(500)); 48 | 49 | // write in chrome 50 | rustautogui.keyboard_input("chrome").unwrap(); 51 | 52 | thread::sleep(time::Duration::from_millis(500)); 53 | 54 | // run chrome with clicking return/enter 55 | rustautogui.keyboard_command("return").unwrap(); 56 | 57 | thread::sleep(time::Duration::from_millis(500)); 58 | 59 | // open new tab with ctrl+t 60 | rustautogui.keyboard_multi_key("ctrl", "t", None).unwrap(); 61 | 62 | thread::sleep(time::Duration::from_millis(500)); 63 | 64 | // input github in url (url input area selected automatically by new tab opening) 65 | rustautogui 66 | .keyboard_input("https://github.com/DavorMar/rustautogui") 67 | .unwrap(); 68 | 69 | thread::sleep(time::Duration::from_millis(500)); 70 | 71 | // return/enter to open github 72 | rustautogui.keyboard_command("return").unwrap(); 73 | 74 | thread::sleep(time::Duration::from_millis(500)); 75 | 76 | // loop till image of star is found or timeout of 15 seconds is hit 77 | rustautogui 78 | .loop_find_stored_image_on_screen_and_move_mouse(0.95, 1.0, 15, "stars") 79 | .unwrap(); 80 | 81 | thread::sleep(time::Duration::from_millis(500)); 82 | // rustautogui.left_click().unwrap(); 83 | thread::sleep(time::Duration::from_millis(500)); 84 | 85 | // scroll down 80 times to the bottom of the page where small terms button is 86 | rustautogui.scroll_down(80).unwrap(); 87 | 88 | // loop till image found with timeout of 15 seconds 89 | rustautogui 90 | .loop_find_image_on_screen_and_move_mouse(0.95, 1.0, 15) 91 | .unwrap(); 92 | 93 | thread::sleep(time::Duration::from_millis(500)); 94 | rustautogui.left_click().unwrap(); 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /examples/opencl_example.rs: -------------------------------------------------------------------------------- 1 | #![cfg(feature = "opencl")] 2 | 3 | use rustautogui; 4 | 5 | fn main() { 6 | //initiate gui 7 | let mut gui = rustautogui::RustAutoGui::new(false).unwrap(); 8 | // prepare and store template with Opencl matchmode VERSION 2 and custom threshold of 0.8 9 | gui.store_template_from_file_custom( 10 | "test.png", 11 | Some((500, 100, 1000, 700)), 12 | rustautogui::MatchMode::SegmentedOclV2, 13 | "test_img", 14 | 0.8, 15 | ) 16 | .unwrap(); 17 | // store another image with Opencl VERSION 1 with automatic determination of threshold 18 | gui.store_template_from_file( 19 | "test2.png", 20 | None, 21 | rustautogui::MatchMode::SegmentedOcl, 22 | "test_img2", 23 | ) 24 | .unwrap(); 25 | 26 | // store third template with no opencl, which will run on cpu 27 | gui.store_template_from_file( 28 | "test3.png", 29 | None, 30 | rustautogui::MatchMode::Segmented, 31 | "test_img3", 32 | ) 33 | .unwrap(); 34 | 35 | // execute image searches in completely same pattern. Using opencl is determined in preparation phase 36 | gui.find_stored_image_on_screen(0.9, "test_img").unwrap(); 37 | gui.find_stored_image_on_screen(0.9, "test_img2").unwrap(); 38 | gui.find_stored_image_on_screen(0.9, "test_img3").unwrap(); 39 | } 40 | -------------------------------------------------------------------------------- /src/core/keyboard/mod.rs: -------------------------------------------------------------------------------- 1 | use std::collections::HashMap; 2 | 3 | use crate::errors::AutoGuiError; 4 | 5 | #[cfg(target_os = "windows")] 6 | pub mod windows; 7 | #[cfg(target_os = "windows")] 8 | pub use windows::Keyboard; 9 | 10 | #[cfg(target_os = "linux")] 11 | pub mod linux; 12 | #[cfg(target_os = "linux")] 13 | pub use linux::Keyboard; 14 | 15 | #[cfg(target_os = "macos")] 16 | pub mod macos; 17 | #[cfg(target_os = "macos")] 18 | pub use macos::Keyboard; 19 | 20 | #[cfg(any(target_os = "macos", target_os = "windows"))] 21 | fn get_keymap_key<'a>(target: &'a Keyboard, key: &str) -> Result<&'a (u16, bool), AutoGuiError> { 22 | let values = target 23 | .keymap 24 | .get(key) 25 | .ok_or(AutoGuiError::UnSupportedKey(format!( 26 | "{} key/command is not supported", 27 | key 28 | )))?; 29 | Ok(values) 30 | } 31 | 32 | #[cfg(target_os = "linux")] 33 | fn get_keymap_key<'a>(target: &'a Keyboard, key: &str) -> Result<&'a (String, bool), AutoGuiError> { 34 | let values = target 35 | .keymap 36 | .get(key) 37 | .ok_or(AutoGuiError::UnSupportedKey(format!( 38 | "{} key/command is not supported", 39 | key 40 | )))?; 41 | Ok(values) 42 | } 43 | -------------------------------------------------------------------------------- /src/core/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod keyboard; 2 | pub mod mouse; 3 | pub mod screen; 4 | #[cfg(not(feature = "lite"))] 5 | pub mod template_match; 6 | -------------------------------------------------------------------------------- /src/core/mouse/linux/mod.rs: -------------------------------------------------------------------------------- 1 | use crate::errors::AutoGuiError; 2 | 3 | use super::{MouseClick, MouseScroll}; 4 | use std::time::Instant; 5 | use std::{ptr, thread, time::Duration}; 6 | use x11::xlib::{ 7 | CurrentTime, RevertToParent, Window, XDefaultRootWindow, XFlush, XQueryPointer, XSetInputFocus, 8 | XTranslateCoordinates, XWarpPointer, _XDisplay, 9 | }; 10 | use x11::xtest::{XTestFakeButtonEvent, XTestQueryExtension}; 11 | 12 | #[derive(Debug)] 13 | pub struct Mouse { 14 | screen: *mut _XDisplay, 15 | root_window: u64, 16 | } 17 | 18 | impl Mouse { 19 | pub fn new(screen: *mut _XDisplay, root_window: u64) -> Self { 20 | Self { 21 | screen, 22 | root_window, 23 | } 24 | } 25 | 26 | /// moves mouse to x, y pixel coordinate on screen 27 | pub fn move_mouse_to_pos(&self, x: i32, y: i32, moving_time: f32) -> Result<(), AutoGuiError> { 28 | // if no moving time, then instant move is executed 29 | unsafe { 30 | if moving_time <= 0.0 { 31 | XWarpPointer(self.screen, 0, self.root_window, 0, 0, 0, 0, x, y); 32 | XFlush(self.screen); 33 | return Ok(()); 34 | } 35 | } 36 | 37 | // if moving time is included, loop is executed that moves step by step 38 | let start = Instant::now(); 39 | let start_location = self.get_mouse_position()?; 40 | let distance_x = x - start_location.0; 41 | let distance_y = y - start_location.1; 42 | loop { 43 | let duration = start.elapsed().as_secs_f32(); 44 | 45 | let time_passed_percentage = duration / moving_time; 46 | if time_passed_percentage > 10.0 { 47 | continue; 48 | } 49 | let new_x = start_location.0 as f32 + (time_passed_percentage * distance_x as f32); 50 | let new_y = start_location.1 as f32 + (time_passed_percentage * distance_y as f32); 51 | unsafe { 52 | if time_passed_percentage >= 1.0 { 53 | XWarpPointer(self.screen, 0, self.root_window, 0, 0, 0, 0, x, y); 54 | XFlush(self.screen); 55 | break; 56 | } else { 57 | XWarpPointer( 58 | self.screen, 59 | 0, 60 | self.root_window, 61 | 0, 62 | 0, 63 | 0, 64 | 0, 65 | new_x as i32, 66 | new_y as i32, 67 | ); 68 | XFlush(self.screen); 69 | } 70 | } 71 | } 72 | Ok(()) 73 | } 74 | 75 | pub fn drag_mouse(&self, x: i32, y: i32, moving_time: f32) -> Result<(), AutoGuiError> { 76 | let mut event_base = 0; 77 | let mut error_base = 0; 78 | unsafe { 79 | if XTestQueryExtension( 80 | self.screen, 81 | &mut event_base, 82 | &mut error_base, 83 | &mut event_base, 84 | &mut error_base, 85 | ) == 0 86 | { 87 | return Err(AutoGuiError::OSFailure( 88 | "Xtest extension is not available".to_string(), 89 | )); 90 | } 91 | if let Some(window) = self.get_window_under_cursor()? { 92 | self.set_focus_to_window(window); 93 | } 94 | // Press the mouse button 95 | XTestFakeButtonEvent(self.screen, 1, 1, CurrentTime); 96 | XFlush(self.screen); 97 | } 98 | thread::sleep(Duration::from_millis(50)); 99 | self.move_mouse_to_pos(x, y, moving_time)?; 100 | unsafe { 101 | // Release the mouse button 102 | XTestFakeButtonEvent(self.screen, 1, 0, CurrentTime); 103 | XFlush(self.screen); 104 | } 105 | Ok(()) 106 | } 107 | 108 | /// returns x, y pixel coordinate of mouse position 109 | pub fn get_mouse_position(&self) -> Result<(i32, i32), AutoGuiError> { 110 | unsafe { 111 | let mut root_return = 0; 112 | let mut child_return = 0; 113 | let mut root_x = 0; 114 | let mut root_y = 0; 115 | let mut win_x = 0; 116 | let mut win_y = 0; 117 | let mut mask_return = 0; 118 | 119 | let status = XQueryPointer( 120 | self.screen, 121 | self.root_window, 122 | &mut root_return, 123 | &mut child_return, 124 | &mut root_x, 125 | &mut root_y, 126 | &mut win_x, 127 | &mut win_y, 128 | &mut mask_return, 129 | ); 130 | 131 | if status == 0 { 132 | return Err(AutoGuiError::OSFailure( 133 | "Unable to query pointer position".to_string(), 134 | )); 135 | } 136 | 137 | Ok((root_x, root_y)) 138 | } 139 | } 140 | 141 | /// click mouse, either left, right or middle 142 | pub fn mouse_click(&self, button: MouseClick) -> Result<(), AutoGuiError> { 143 | let button = match button { 144 | MouseClick::LEFT => 1, 145 | MouseClick::MIDDLE => 2, 146 | MouseClick::RIGHT => 3, 147 | }; 148 | 149 | let mut event_base = 0; 150 | let mut error_base = 0; 151 | unsafe { 152 | if XTestQueryExtension( 153 | self.screen, 154 | &mut event_base, 155 | &mut error_base, 156 | &mut event_base, 157 | &mut error_base, 158 | ) == 0 159 | { 160 | return Err(AutoGuiError::OSFailure( 161 | "Xtest extension is not available".to_string(), 162 | )); 163 | } 164 | if let Some(window) = self.get_window_under_cursor()? { 165 | self.set_focus_to_window(window); 166 | } 167 | // Press the mouse button 168 | XTestFakeButtonEvent(self.screen, button, 1, CurrentTime); 169 | XFlush(self.screen); 170 | 171 | // Release the mouse button 172 | XTestFakeButtonEvent(self.screen, button, 0, CurrentTime); 173 | XFlush(self.screen); 174 | } 175 | Ok(()) 176 | } 177 | 178 | pub fn mouse_down(&self, button: MouseClick) -> Result<(), AutoGuiError> { 179 | let button = match button { 180 | MouseClick::LEFT => 1, 181 | MouseClick::MIDDLE => 2, 182 | MouseClick::RIGHT => 3, 183 | }; 184 | let mut event_base = 0; 185 | let mut error_base = 0; 186 | unsafe { 187 | if XTestQueryExtension( 188 | self.screen, 189 | &mut event_base, 190 | &mut error_base, 191 | &mut event_base, 192 | &mut error_base, 193 | ) == 0 194 | { 195 | return Err(AutoGuiError::OSFailure( 196 | "Xtest extension is not available".to_string(), 197 | )); 198 | } 199 | if let Some(window) = self.get_window_under_cursor()? { 200 | self.set_focus_to_window(window); 201 | } 202 | // Press the mouse button 203 | XTestFakeButtonEvent(self.screen, button, 1, CurrentTime); 204 | XFlush(self.screen); 205 | } 206 | Ok(()) 207 | } 208 | 209 | pub fn mouse_up(&self, button: MouseClick) -> Result<(), AutoGuiError> { 210 | let button = match button { 211 | MouseClick::LEFT => 1, 212 | MouseClick::MIDDLE => 2, 213 | MouseClick::RIGHT => 3, 214 | }; 215 | let mut event_base = 0; 216 | let mut error_base = 0; 217 | unsafe { 218 | if XTestQueryExtension( 219 | self.screen, 220 | &mut event_base, 221 | &mut error_base, 222 | &mut event_base, 223 | &mut error_base, 224 | ) == 0 225 | { 226 | return Err(AutoGuiError::OSFailure( 227 | "Xtest extension is not available".to_string(), 228 | )); 229 | } 230 | if let Some(window) = self.get_window_under_cursor()? { 231 | self.set_focus_to_window(window); 232 | } 233 | // Press the mouse button 234 | XTestFakeButtonEvent(self.screen, button, 0, CurrentTime); 235 | XFlush(self.screen); 236 | } 237 | Ok(()) 238 | } 239 | 240 | pub fn scroll(&self, direction: MouseScroll, intensity: u32) { 241 | let button = match direction { 242 | MouseScroll::UP => 4, 243 | MouseScroll::DOWN => 5, 244 | MouseScroll::LEFT => 6, 245 | MouseScroll::RIGHT => 7, 246 | }; 247 | let mut event_base = 0; 248 | let mut error_base = 0; 249 | unsafe { 250 | if XTestQueryExtension( 251 | self.screen, 252 | &mut event_base, 253 | &mut error_base, 254 | &mut event_base, 255 | &mut error_base, 256 | ) == 0 257 | { 258 | eprintln!("XTest extension not available"); 259 | return; 260 | } 261 | // if let Some(window) = self.get_window_under_cursor() { 262 | // self.set_focus_to_window(window); 263 | // } 264 | // Press the mouse button 265 | for _ in 0..intensity { 266 | XTestFakeButtonEvent(self.screen, button, 1, CurrentTime); 267 | XFlush(self.screen); 268 | 269 | // Release the mouse button 270 | XTestFakeButtonEvent(self.screen, button, 0, CurrentTime); 271 | XFlush(self.screen); 272 | } 273 | } 274 | } 275 | 276 | /// return window that is at cursor position. Used when executing left click to also 277 | /// change focused window 278 | fn get_window_under_cursor(&self) -> Result, AutoGuiError> { 279 | let mut child: Window = 0; 280 | let mut win_x: i32 = 0; 281 | let mut win_y: i32 = 0; 282 | 283 | unsafe { 284 | let (pos_x, pos_y) = self.get_mouse_position()?; 285 | if XTranslateCoordinates( 286 | self.screen, 287 | XDefaultRootWindow(self.screen), 288 | XDefaultRootWindow(self.screen), 289 | pos_x, 290 | pos_y, 291 | &mut win_x, 292 | &mut win_y, 293 | &mut child, 294 | ) != 0 295 | { 296 | if child != 0 { 297 | return Ok(Some(child)); 298 | } 299 | } 300 | Ok(None) 301 | } 302 | } 303 | 304 | /// change focused window. Used when clicking a window 305 | fn set_focus_to_window(&self, window: Window) { 306 | unsafe { 307 | XSetInputFocus(self.screen, window, RevertToParent, CurrentTime); 308 | XFlush(self.screen); 309 | thread::sleep(Duration::from_millis(50)); 310 | } 311 | } 312 | } 313 | -------------------------------------------------------------------------------- /src/core/mouse/macos/mod.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | thread::sleep, 3 | time::{Duration, Instant}, 4 | }; 5 | 6 | use crate::core::mouse::{MouseClick, MouseScroll}; 7 | use crate::errors::AutoGuiError; 8 | use core_graphics::{ 9 | event::{CGEvent, CGEventTapLocation, CGEventType, CGMouseButton, ScrollEventUnit}, 10 | event_source::{CGEventSource, CGEventSourceStateID}, 11 | geometry::CGPoint, 12 | }; 13 | 14 | pub struct Mouse {} 15 | 16 | impl Mouse { 17 | pub fn new() -> Self { 18 | Self {} 19 | } 20 | /// moves mouse to x, y pixel coordinate on screen 21 | 22 | pub fn move_mouse_to_pos(x: i32, y: i32, moving_time: f32) -> Result<(), AutoGuiError> { 23 | if moving_time <= 0.0 { 24 | Mouse::move_mouse(x, y) 25 | } else { 26 | let start_location = Mouse::get_mouse_position()?; 27 | let distance_x = x - start_location.0; 28 | let distance_y = y - start_location.1; 29 | let start = Instant::now(); 30 | loop { 31 | let duration = start.elapsed().as_secs_f32(); 32 | 33 | let time_passed_percentage = duration / moving_time; 34 | if time_passed_percentage > 10.0 { 35 | continue; 36 | } 37 | let new_x = start_location.0 as f32 + (time_passed_percentage * distance_x as f32); 38 | let new_y = start_location.1 as f32 + (time_passed_percentage * distance_y as f32); 39 | if time_passed_percentage >= 1.0 { 40 | Mouse::move_mouse(x, y)?; 41 | break; 42 | } else { 43 | Mouse::move_mouse(new_x as i32, new_y as i32)?; 44 | } 45 | } 46 | Ok(()) 47 | } 48 | } 49 | 50 | pub fn drag_mouse(x: i32, y: i32, moving_time: f32) -> Result<(), AutoGuiError> { 51 | let (cg_button, down, up) = ( 52 | CGMouseButton::Left, 53 | CGEventType::LeftMouseDown, 54 | CGEventType::LeftMouseUp, 55 | ); 56 | let drag = CGEventType::LeftMouseDragged; 57 | // needed as input for where to click 58 | let mouse_pos = Mouse::get_mouse_position()?; 59 | // click down 60 | let cg_event_source = 61 | CGEventSource::new(CGEventSourceStateID::HIDSystemState).map_err(|_| { 62 | AutoGuiError::OSFailure( 63 | "Error creating CGEventSource on mouse movement".to_string(), 64 | ) 65 | })?; 66 | 67 | let click_down = CGEvent::new_mouse_event( 68 | cg_event_source.clone(), 69 | down, 70 | CGPoint::new(mouse_pos.0 as f64, mouse_pos.1 as f64), 71 | cg_button, 72 | ) 73 | .map_err(|_| AutoGuiError::OSFailure("Failed the mouse click down CGevent".to_string()))?; 74 | click_down.post(CGEventTapLocation::HID); 75 | 76 | sleep(Duration::from_millis(100)); 77 | 78 | // Move mouse with dragging event 79 | let distance = (((x - mouse_pos.0).pow(2) + (y - mouse_pos.1).pow(2)) as f32).sqrt(); 80 | let steps = distance / 20.0; // Adjust for smoothness 81 | 82 | let dx = (x - mouse_pos.0) as f64 / steps as f64; 83 | let dy = (y - mouse_pos.1) as f64 / steps as f64; 84 | 85 | for i in 1..=steps as i32 { 86 | let new_x = mouse_pos.0 as f64 + dx * i as f64; 87 | let new_y = mouse_pos.1 as f64 + dy * i as f64; 88 | 89 | let drag_event = CGEvent::new_mouse_event( 90 | cg_event_source.clone(), 91 | drag, // Use LeftMouseDragged instead of just moving 92 | CGPoint::new(new_x, new_y), 93 | cg_button, 94 | ) 95 | .map_err(|_| AutoGuiError::OSFailure("Failed to create drag CGEvent".to_string()))?; 96 | drag_event.post(CGEventTapLocation::HID); 97 | 98 | sleep(Duration::from_millis( 99 | (moving_time * 1000.0 / steps as f32) as u64, 100 | )); 101 | } 102 | 103 | //click up 104 | let mouse_pos = Mouse::get_mouse_position()?; 105 | let cg_event_source = 106 | CGEventSource::new(CGEventSourceStateID::HIDSystemState).map_err(|_| { 107 | AutoGuiError::OSFailure( 108 | "Error creating CGEventSource on mouse movement".to_string(), 109 | ) 110 | })?; 111 | 112 | let click_up = CGEvent::new_mouse_event( 113 | cg_event_source, 114 | up, 115 | CGPoint::new(mouse_pos.0 as f64, mouse_pos.1 as f64), 116 | cg_button, 117 | ) 118 | .map_err(|_| AutoGuiError::OSFailure("Failed the mouse click up CGevent".to_string()))?; 119 | click_up.post(CGEventTapLocation::HID); 120 | 121 | sleep(Duration::from_millis(20)); 122 | 123 | Ok(()) 124 | } 125 | 126 | // separate private function called by move to pos 127 | fn move_mouse(x: i32, y: i32) -> Result<(), AutoGuiError> { 128 | let gc_event_source = 129 | CGEventSource::new(CGEventSourceStateID::HIDSystemState).map_err(|_| { 130 | AutoGuiError::OSFailure( 131 | "Error creating CGEventSource on mouse movement".to_string(), 132 | ) 133 | })?; 134 | 135 | let event = CGEvent::new_mouse_event( 136 | gc_event_source, 137 | CGEventType::MouseMoved, 138 | CGPoint::new(x as f64, y as f64), 139 | CGMouseButton::Left, 140 | ) 141 | .map_err(|_| AutoGuiError::OSFailure("Failed creating CGEvent".to_string()))?; 142 | event.post(CGEventTapLocation::HID); 143 | 144 | sleep(Duration::from_millis(20)); 145 | Ok(()) 146 | } 147 | 148 | /// Gets the current mouse position. 149 | pub fn get_mouse_position() -> Result<(i32, i32), AutoGuiError> { 150 | let gc_event_source = 151 | CGEventSource::new(CGEventSourceStateID::HIDSystemState).map_err(|_| { 152 | AutoGuiError::OSFailure( 153 | "Error creating CGEventSource on mouse movement".to_string(), 154 | ) 155 | })?; 156 | let event = CGEvent::new(gc_event_source) 157 | .map_err(|_| AutoGuiError::OSFailure("Failed creating CGevent".to_string()))?; 158 | let point = event.location(); 159 | Ok((point.x as i32, point.y as i32)) 160 | } 161 | 162 | /// execute left, right or middle mouse click 163 | pub fn mouse_click(button: MouseClick) -> Result<(), AutoGuiError> { 164 | let (cg_button, down, up) = match button { 165 | MouseClick::LEFT => ( 166 | CGMouseButton::Left, 167 | CGEventType::LeftMouseDown, 168 | CGEventType::LeftMouseUp, 169 | ), 170 | MouseClick::RIGHT => ( 171 | CGMouseButton::Right, 172 | CGEventType::RightMouseDown, 173 | CGEventType::RightMouseUp, 174 | ), 175 | MouseClick::MIDDLE => ( 176 | CGMouseButton::Center, 177 | CGEventType::OtherMouseDown, 178 | CGEventType::OtherMouseUp, 179 | ), 180 | }; 181 | 182 | // needed as input for where to click 183 | let mouse_pos = Mouse::get_mouse_position()?; 184 | 185 | let cg_event_source = 186 | CGEventSource::new(CGEventSourceStateID::HIDSystemState).map_err(|_| { 187 | AutoGuiError::OSFailure( 188 | "Error creating CGEventSource on mouse movement".to_string(), 189 | ) 190 | })?; 191 | let click_down = CGEvent::new_mouse_event( 192 | cg_event_source, 193 | down, 194 | CGPoint::new(mouse_pos.0 as f64, mouse_pos.1 as f64), 195 | cg_button, 196 | ) 197 | .map_err(|_| AutoGuiError::OSFailure("Failed the mouse click down CGevent".to_string()))?; 198 | click_down.post(CGEventTapLocation::HID); 199 | 200 | sleep(Duration::from_millis(20)); 201 | 202 | let cg_event_source = 203 | CGEventSource::new(CGEventSourceStateID::HIDSystemState).map_err(|_| { 204 | AutoGuiError::OSFailure( 205 | "Error creating CGEventSource on mouse movement".to_string(), 206 | ) 207 | })?; 208 | 209 | let click_up = CGEvent::new_mouse_event( 210 | cg_event_source, 211 | up, 212 | CGPoint::new(mouse_pos.0 as f64, mouse_pos.1 as f64), 213 | cg_button, 214 | ) 215 | .map_err(|_| AutoGuiError::OSFailure("Failed the mouse click up CGevent".to_string()))?; 216 | 217 | click_up.post(CGEventTapLocation::HID); 218 | 219 | sleep(Duration::from_millis(20)); 220 | Ok(()) 221 | } 222 | 223 | pub fn mouse_down(button: MouseClick) -> Result<(), AutoGuiError> { 224 | let (cg_button, down) = match button { 225 | MouseClick::LEFT => (CGMouseButton::Left, CGEventType::LeftMouseDown), 226 | MouseClick::RIGHT => (CGMouseButton::Right, CGEventType::RightMouseDown), 227 | MouseClick::MIDDLE => (CGMouseButton::Center, CGEventType::OtherMouseDown), 228 | }; 229 | 230 | // needed as input for where to click 231 | let mouse_pos = Mouse::get_mouse_position()?; 232 | 233 | let cg_event_source = 234 | CGEventSource::new(CGEventSourceStateID::HIDSystemState).map_err(|_| { 235 | AutoGuiError::OSFailure( 236 | "Error creating CGEventSource on mouse movement".to_string(), 237 | ) 238 | })?; 239 | let click_down = CGEvent::new_mouse_event( 240 | cg_event_source, 241 | down, 242 | CGPoint::new(mouse_pos.0 as f64, mouse_pos.1 as f64), 243 | cg_button, 244 | ) 245 | .map_err(|_| AutoGuiError::OSFailure("Failed the mouse click down CGevent".to_string()))?; 246 | click_down.post(CGEventTapLocation::HID); 247 | sleep(Duration::from_millis(20)); 248 | Ok(()) 249 | } 250 | 251 | pub fn mouse_up(button: MouseClick) -> Result<(), AutoGuiError> { 252 | let (cg_button, up) = match button { 253 | MouseClick::LEFT => (CGMouseButton::Left, CGEventType::LeftMouseUp), 254 | MouseClick::RIGHT => (CGMouseButton::Right, CGEventType::RightMouseUp), 255 | MouseClick::MIDDLE => (CGMouseButton::Center, CGEventType::OtherMouseUp), 256 | }; 257 | 258 | // needed as input for where to click 259 | let mouse_pos = Mouse::get_mouse_position()?; 260 | 261 | let cg_event_source = 262 | CGEventSource::new(CGEventSourceStateID::HIDSystemState).map_err(|_| { 263 | AutoGuiError::OSFailure( 264 | "Error creating CGEventSource on mouse movement".to_string(), 265 | ) 266 | })?; 267 | 268 | let click_up = CGEvent::new_mouse_event( 269 | cg_event_source, 270 | up, 271 | CGPoint::new(mouse_pos.0 as f64, mouse_pos.1 as f64), 272 | cg_button, 273 | ) 274 | .map_err(|_| AutoGuiError::OSFailure("Failed the mouse click up CGevent".to_string()))?; 275 | 276 | click_up.post(CGEventTapLocation::HID); 277 | 278 | sleep(Duration::from_millis(20)); 279 | Ok(()) 280 | } 281 | 282 | pub fn scroll(direction: MouseScroll, intensity: u32) -> Result<(), AutoGuiError> { 283 | let delta = match direction { 284 | MouseScroll::UP => (10 * intensity as i32, 0), 285 | MouseScroll::DOWN => (-10 * intensity as i32, 0), 286 | MouseScroll::LEFT => (0, 10 * intensity as i32), 287 | MouseScroll::RIGHT => (0, -10 * intensity as i32), 288 | }; 289 | let cg_event_source = 290 | CGEventSource::new(CGEventSourceStateID::HIDSystemState).map_err(|_| { 291 | AutoGuiError::OSFailure( 292 | "Error creating CGEventSource on mouse movement".to_string(), 293 | ) 294 | })?; 295 | 296 | let scroll = CGEvent::new_scroll_event( 297 | cg_event_source, 298 | ScrollEventUnit::PIXEL, 299 | 2, 300 | delta.0, 301 | delta.1, 302 | 0, 303 | ) 304 | .map_err(|_| AutoGuiError::OSFailure("Failed creating mouse scroll CGevent".to_string()))?; 305 | scroll.post(CGEventTapLocation::HID); 306 | sleep(Duration::from_millis(20)); 307 | Ok(()) 308 | } 309 | 310 | pub fn double_click() -> Result<(), AutoGuiError> { 311 | let source = CGEventSource::new(CGEventSourceStateID::HIDSystemState).map_err(|_| { 312 | AutoGuiError::OSFailure( 313 | "Failed creating CGEventSource on mouse double click".to_string(), 314 | ) 315 | })?; 316 | let pos = Mouse::get_mouse_position()?; 317 | 318 | // needed first to get focus of the window 319 | Self::mouse_click(MouseClick::LEFT)?; 320 | sleep(Duration::from_millis(50)); 321 | 322 | // MacOs does double click wierldy. 323 | // good explanation at https://stackoverflow.com/questions/1483657/performing-a-double-click-using-cgeventcreatemouseevent 324 | // basically, the x.set_integer_value_field defines event as double click. Sending 2 times a left click does not work 325 | let mouse_down = CGEvent::new_mouse_event( 326 | source.clone(), 327 | CGEventType::LeftMouseDown, 328 | CGPoint::new(pos.0 as f64, pos.1 as f64), 329 | CGMouseButton::Left, 330 | ) 331 | .map_err(|_| { 332 | AutoGuiError::OSFailure( 333 | "Failed creating CGevent for mouse click down action".to_string(), 334 | ) 335 | })?; 336 | mouse_down.set_integer_value_field(1, 2); 337 | 338 | let mouse_up = CGEvent::new_mouse_event( 339 | source.clone(), 340 | CGEventType::LeftMouseUp, 341 | CGPoint::new(pos.0 as f64, pos.1 as f64), 342 | CGMouseButton::Left, 343 | ) 344 | .map_err(|_| { 345 | AutoGuiError::OSFailure("Failed creating CGevent for mouse up click".to_string()) 346 | })?; 347 | mouse_up.set_integer_value_field(1, 2); 348 | 349 | mouse_down.post(CGEventTapLocation::HID); 350 | sleep(Duration::from_millis(10)); 351 | mouse_up.post(CGEventTapLocation::HID); 352 | sleep(Duration::from_millis(50)); 353 | 354 | Ok(()) 355 | } 356 | } 357 | -------------------------------------------------------------------------------- /src/core/mouse/mod.rs: -------------------------------------------------------------------------------- 1 | #[cfg(target_os = "windows")] 2 | pub mod windows; 3 | 4 | #[cfg(target_os = "linux")] 5 | pub mod linux; 6 | 7 | #[cfg(target_os = "macos")] 8 | pub mod macos; 9 | 10 | #[cfg(target_os = "windows")] 11 | pub use windows::Mouse; 12 | 13 | #[cfg(target_os = "macos")] 14 | pub use macos::Mouse; 15 | 16 | #[cfg(target_os = "linux")] 17 | pub use linux::Mouse; 18 | 19 | pub enum MouseClick { 20 | LEFT, 21 | RIGHT, 22 | MIDDLE, 23 | } 24 | 25 | pub enum MouseScroll { 26 | UP, 27 | DOWN, 28 | LEFT, 29 | RIGHT, 30 | } 31 | 32 | pub mod mouse_position; 33 | -------------------------------------------------------------------------------- /src/core/mouse/mouse_position.rs: -------------------------------------------------------------------------------- 1 | #[cfg(target_os = "linux")] 2 | use super::Mouse; 3 | use crate::errors::AutoGuiError; 4 | #[cfg(target_os = "linux")] 5 | use std::ptr; 6 | #[cfg(target_os = "linux")] 7 | use x11::xlib::*; 8 | 9 | #[cfg(any(target_os = "windows", target_os = "macos"))] 10 | use crate::core::mouse::Mouse; 11 | 12 | use std::thread::sleep; 13 | use std::time::Duration; 14 | 15 | /* 16 | 17 | small helper function to open a window that shows mouse position 18 | 19 | 20 | 21 | example : 22 | fn main() { 23 | mouse::mouse_position::show_mouse_position_window(); 24 | } 25 | thats all 26 | */ 27 | #[cfg(target_os = "linux")] 28 | struct DisplayWrapper { 29 | display: *mut x11::xlib::Display, 30 | } 31 | //created so display gets dropped when code finishes 32 | #[cfg(target_os = "linux")] 33 | impl DisplayWrapper { 34 | fn new() -> Self { 35 | unsafe { 36 | let display = XOpenDisplay(ptr::null()); 37 | if display.is_null() { 38 | panic!("Unable to open X display"); 39 | } 40 | DisplayWrapper { display } 41 | } 42 | } 43 | } 44 | #[cfg(target_os = "linux")] 45 | impl Drop for DisplayWrapper { 46 | fn drop(&mut self) { 47 | unsafe { 48 | XCloseDisplay(self.display); 49 | } 50 | } 51 | } 52 | 53 | pub fn print_mouse_position() -> Result<(), AutoGuiError> { 54 | #[cfg(target_os = "linux")] 55 | { 56 | let display_wrapper = DisplayWrapper::new(); 57 | 58 | unsafe { 59 | let screen = XDefaultScreen(display_wrapper.display); 60 | let root = XRootWindow(display_wrapper.display, screen); 61 | let mouse = Mouse::new(display_wrapper.display, root); 62 | loop { 63 | let (x, y) = mouse.get_mouse_position()?; 64 | println!("{x}, {y}"); 65 | sleep(Duration::from_millis(20)); 66 | } 67 | } 68 | } 69 | #[cfg(target_os = "windows")] 70 | { 71 | loop { 72 | let (x, y) = Mouse::get_mouse_position(); 73 | println!("{x}, {y}"); 74 | sleep(Duration::from_millis(20)); 75 | } 76 | } 77 | #[cfg(target_os = "macos")] 78 | { 79 | loop { 80 | let (x, y) = Mouse::get_mouse_position()?; 81 | println!("{x}, {y}"); 82 | sleep(Duration::from_millis(20)); 83 | } 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /src/core/mouse/windows/mod.rs: -------------------------------------------------------------------------------- 1 | use crate::core::mouse::{MouseClick, MouseScroll}; 2 | use std::mem::{size_of, zeroed}; 3 | use std::{thread, time, time::Instant}; 4 | use winapi::shared::windef::POINT; 5 | use winapi::um::winuser::{ 6 | SendInput, SetCursorPos, INPUT, INPUT_MOUSE, MOUSEEVENTF_HWHEEL, MOUSEEVENTF_LEFTDOWN, 7 | MOUSEEVENTF_LEFTUP, MOUSEEVENTF_MIDDLEDOWN, MOUSEEVENTF_MIDDLEUP, MOUSEEVENTF_RIGHTDOWN, 8 | MOUSEEVENTF_RIGHTUP, MOUSEEVENTF_WHEEL, 9 | }; 10 | #[derive(Debug)] 11 | pub struct Mouse {} 12 | impl Mouse { 13 | #[allow(unused_variables)] 14 | pub fn new() -> Mouse { 15 | Mouse {} 16 | } 17 | 18 | /// moves mouse to x, y pixel coordinate on screen 19 | pub fn move_mouse_to_pos(x: i32, y: i32, moving_time: f32) { 20 | // if no moving time, then instant move is executed 21 | unsafe { 22 | if moving_time <= 0.0 { 23 | SetCursorPos(x, y); 24 | return; 25 | } 26 | }; 27 | // if moving time is included, loop is executed that moves step by step 28 | let start = Instant::now(); 29 | let start_location = Mouse::get_mouse_position(); 30 | let distance_x = x - start_location.0; 31 | let distance_y = y - start_location.1; 32 | 33 | loop { 34 | let duration = start.elapsed().as_secs_f32(); 35 | 36 | let time_passed_percentage = duration / moving_time; 37 | // on first iterations, time passed percentage gets values greater than 10, probably because duration is a 38 | // very small number. Probably could do if duration < 0.05 or similar 39 | if time_passed_percentage > 10.0 { 40 | continue; 41 | } 42 | let new_x = start_location.0 as f32 + (time_passed_percentage * distance_x as f32); 43 | let new_y = start_location.1 as f32 + (time_passed_percentage * distance_y as f32); 44 | 45 | unsafe { 46 | if time_passed_percentage >= 1.0 { 47 | SetCursorPos(x, y); 48 | break; 49 | } else { 50 | SetCursorPos(new_x as i32, new_y as i32); 51 | } 52 | } 53 | } 54 | } 55 | 56 | pub fn drag_mouse(x: i32, y: i32, moving_time: f32) { 57 | let (down, up) = (MOUSEEVENTF_LEFTDOWN, MOUSEEVENTF_LEFTUP); 58 | unsafe { 59 | // set up the first input event (mouse down) 60 | let mut input_down: INPUT = zeroed(); 61 | input_down.type_ = INPUT_MOUSE; 62 | input_down.u.mi_mut().dwFlags = down; 63 | SendInput(1, &mut input_down, size_of::() as i32); 64 | // wait a bit after click down, before moving 65 | thread::sleep(time::Duration::from_millis(80)); 66 | Mouse::move_mouse_to_pos(x, y, moving_time); 67 | thread::sleep(time::Duration::from_millis(50)); 68 | // set up the second input event (mouse up) 69 | let mut input_up: INPUT = zeroed(); 70 | input_up.type_ = INPUT_MOUSE; 71 | input_up.u.mi_mut().dwFlags = up; 72 | // send the input events 73 | SendInput(2, &mut input_up, size_of::() as i32); 74 | } 75 | } 76 | 77 | /// returns x, y pixel coordinate of mouse position 78 | pub fn get_mouse_position() -> (i32, i32) { 79 | unsafe { 80 | let mut point = POINT { x: 0, y: 0 }; 81 | winapi::um::winuser::GetCursorPos(&mut point); 82 | (point.x, point.y) 83 | } 84 | } 85 | 86 | /// click mouse, either left, right or middle "MouseClick::LEFT/RIGHT/MIDDLE enumerator" 87 | pub fn mouse_click(button: MouseClick) { 88 | // create event type depending on click type 89 | let (down, up) = match button { 90 | MouseClick::LEFT => (MOUSEEVENTF_LEFTDOWN, MOUSEEVENTF_LEFTUP), 91 | MouseClick::RIGHT => (MOUSEEVENTF_RIGHTDOWN, MOUSEEVENTF_RIGHTUP), 92 | MouseClick::MIDDLE => (MOUSEEVENTF_MIDDLEDOWN, MOUSEEVENTF_MIDDLEUP), 93 | }; 94 | unsafe { 95 | // create an array of INPUT structures 96 | let mut inputs: [INPUT; 2] = [zeroed(), zeroed()]; 97 | // set up the first input event (mouse down) 98 | inputs[0].type_ = INPUT_MOUSE; 99 | inputs[0].u.mi_mut().dwFlags = down; 100 | // set up the second input event (mouse up) 101 | inputs[1].type_ = INPUT_MOUSE; 102 | inputs[1].u.mi_mut().dwFlags = up; 103 | // send the input events 104 | SendInput(2, inputs.as_mut_ptr(), size_of::() as i32); 105 | } 106 | } 107 | 108 | pub fn mouse_down(button: MouseClick) { 109 | // create event type depending on click type 110 | let down = match button { 111 | MouseClick::LEFT => MOUSEEVENTF_LEFTDOWN, 112 | MouseClick::RIGHT => MOUSEEVENTF_RIGHTDOWN, 113 | MouseClick::MIDDLE => MOUSEEVENTF_MIDDLEDOWN, 114 | }; 115 | unsafe { 116 | // create an array of INPUT structures 117 | let mut input: INPUT = zeroed(); 118 | 119 | input.type_ = INPUT_MOUSE; 120 | input.u.mi_mut().dwFlags = down; 121 | 122 | // send the input events 123 | SendInput(1, &mut input, size_of::() as i32); 124 | } 125 | } 126 | 127 | pub fn mouse_up(button: MouseClick) { 128 | // create event type depending on click type 129 | let up = match button { 130 | MouseClick::LEFT => MOUSEEVENTF_LEFTUP, 131 | MouseClick::RIGHT => MOUSEEVENTF_RIGHTUP, 132 | MouseClick::MIDDLE => MOUSEEVENTF_MIDDLEUP, 133 | }; 134 | unsafe { 135 | // create an array of INPUT structures 136 | let mut input: INPUT = zeroed(); 137 | // set up thefirstut vent (mous; 138 | input.type_ = INPUT_MOUSE; 139 | input.u.mi_mut().dwFlags = up; 140 | 141 | // send the input events 142 | SendInput(1, &mut input, size_of::() as i32); 143 | } 144 | } 145 | 146 | pub fn scroll(direction: MouseScroll, intensity: u32) { 147 | // direction , H or W wheel, depending on axis scrolled 148 | let (amount, wheel_direction) = match direction { 149 | MouseScroll::UP => (120, MOUSEEVENTF_WHEEL), 150 | MouseScroll::DOWN => (-120, MOUSEEVENTF_WHEEL), 151 | MouseScroll::LEFT => (-120, MOUSEEVENTF_HWHEEL), 152 | MouseScroll::RIGHT => (120, MOUSEEVENTF_HWHEEL), 153 | }; 154 | let amount = amount * intensity as i32; 155 | unsafe { 156 | let mut scroll_input: INPUT = zeroed(); 157 | 158 | scroll_input.type_ = INPUT_MOUSE; 159 | scroll_input.u.mi_mut().dwFlags = wheel_direction; 160 | scroll_input.u.mi_mut().mouseData = amount as u32; 161 | SendInput(1, &mut scroll_input, size_of::() as i32); 162 | } 163 | } 164 | } 165 | -------------------------------------------------------------------------------- /src/core/screen/linux/mod.rs: -------------------------------------------------------------------------------- 1 | #[cfg(not(feature = "lite"))] 2 | extern crate image; 3 | extern crate x11; 4 | #[cfg(not(feature = "lite"))] 5 | use crate::errors::ImageProcessingError; 6 | use crate::{errors::AutoGuiError, imgtools}; 7 | use core::error; 8 | #[cfg(not(feature = "lite"))] 9 | use image::{GrayImage, ImageBuffer, Luma, Rgba}; 10 | #[cfg(not(feature = "lite"))] 11 | use rayon::prelude::*; 12 | use std::ptr; 13 | use x11::xlib::{ 14 | XCloseDisplay, XDefaultScreen, XDestroyImage, XDisplayHeight, XDisplayWidth, XGetImage, 15 | XOpenDisplay, XRootWindow, ZPixmap, _XDisplay, 16 | }; 17 | 18 | #[cfg(not(feature = "lite"))] 19 | const ALLPLANES: u64 = 0xFFFFFFFFFFFFFFFF; 20 | 21 | #[derive(Debug, Clone)] 22 | pub struct Screen { 23 | pub screen_width: i32, 24 | pub screen_height: i32, 25 | pub display: *mut _XDisplay, 26 | pub root_window: u64, 27 | #[cfg(not(feature = "lite"))] 28 | pub screen_data: ScreenImgData, 29 | } 30 | #[derive(Debug, Clone)] 31 | #[cfg(not(feature = "lite"))] 32 | pub struct ScreenImgData { 33 | pub pixel_data: Vec, 34 | pub screen_region_width: u32, 35 | pub screen_region_height: u32, 36 | } 37 | 38 | impl Screen { 39 | pub fn new() -> Self { 40 | unsafe { 41 | // open the display (usually ":0"). This display pointer will be passed 42 | // to mouse and keyboard structs aswell 43 | let display: *mut _XDisplay = XOpenDisplay(ptr::null()); 44 | if display.is_null() { 45 | panic!("Error grabbing display. Unable to open X display. Possible x11 issue, check if it is activated and that you're not running wayland"); 46 | } 47 | 48 | // get root window 49 | let screen = XDefaultScreen(display); 50 | let root = XRootWindow(display, screen); 51 | 52 | let screen_width = XDisplayWidth(display, screen); 53 | let screen_height = XDisplayHeight(display, screen); 54 | #[cfg(not(feature = "lite"))] 55 | let img_data = ScreenImgData { 56 | pixel_data: vec![0u8; (screen_width * screen_height * 4) as usize], 57 | screen_region_width: 0, 58 | screen_region_height: 0, 59 | }; 60 | Screen { 61 | screen_width: screen_width, 62 | screen_height: screen_height, 63 | display: display, 64 | root_window: root, 65 | #[cfg(not(feature = "lite"))] 66 | screen_data: img_data, 67 | } 68 | } 69 | } 70 | 71 | /// returns screen dimensions. All monitors included 72 | pub fn dimension(&self) -> (i32, i32) { 73 | let dimensions = (self.screen_width, self.screen_height); 74 | dimensions 75 | } 76 | 77 | #[cfg(not(feature = "lite"))] 78 | #[allow(dead_code)] 79 | /// return region dimension which is set up when template is precalculated 80 | pub fn region_dimension(&self) -> (u32, u32) { 81 | let dimensions = ( 82 | self.screen_data.screen_region_width, 83 | self.screen_data.screen_region_height, 84 | ); 85 | dimensions 86 | } 87 | 88 | pub fn destroy(&self) { 89 | unsafe { 90 | XCloseDisplay(self.display); 91 | } 92 | } 93 | 94 | #[allow(dead_code)] 95 | /// executes convert_bitmap_to_rgba, meaning it converts Vector of values to RGBA and crops the image 96 | /// as inputted region area. Not used anywhere at the moment 97 | #[cfg(not(feature = "lite"))] 98 | pub fn grab_screen_image( 99 | &mut self, 100 | region: (u32, u32, u32, u32), 101 | ) -> Result, Vec>, AutoGuiError> { 102 | let (x, y, width, height) = region; 103 | self.screen_data.screen_region_width = width; 104 | self.screen_data.screen_region_height = height; 105 | self.capture_screen()?; 106 | let image = self.convert_bitmap_to_rgba()?; 107 | let cropped_image: ImageBuffer, Vec> = 108 | imgtools::cut_screen_region(x, y, width, height, &image); 109 | Ok(cropped_image) 110 | } 111 | 112 | /// executes convert_bitmap_to_grayscale, meaning it converts Vector of values to grayscale and crops the image 113 | /// as inputted region area 114 | #[cfg(not(feature = "lite"))] 115 | pub fn grab_screen_image_grayscale( 116 | &mut self, 117 | region: &(u32, u32, u32, u32), 118 | ) -> Result, Vec>, AutoGuiError> { 119 | let (x, y, width, height) = region; 120 | self.screen_data.screen_region_width = *width; 121 | self.screen_data.screen_region_height = *height; 122 | self.capture_screen()?; 123 | let image: ImageBuffer, Vec> = self.convert_bitmap_to_grayscale()?; 124 | let cropped_image: ImageBuffer, Vec> = 125 | imgtools::cut_screen_region(*x, *y, *width, *height, &image); 126 | Ok(cropped_image) 127 | } 128 | #[cfg(not(feature = "lite"))] 129 | /// captures and saves screenshot of monitors 130 | pub fn grab_screenshot(&mut self, image_path: &str) -> Result<(), AutoGuiError> { 131 | self.capture_screen()?; 132 | let image = self.convert_bitmap_to_rgba()?; 133 | Ok(image.save(image_path)?) 134 | } 135 | #[cfg(not(feature = "lite"))] 136 | /// first order capture screen function. it captures screen image and stores it as vector in self.pixel_data 137 | fn capture_screen(&mut self) -> Result<(), AutoGuiError> { 138 | unsafe { 139 | let ximage = XGetImage( 140 | self.display, 141 | self.root_window, 142 | 0, 143 | 0, 144 | self.screen_width as u32, 145 | self.screen_height as u32, 146 | ALLPLANES, 147 | ZPixmap, 148 | ); 149 | if ximage.is_null() { 150 | return Err(AutoGuiError::OSFailure("Error grabbing display image. Unable to get X image. Possible x11 error, check if you're running on x11 and not wayland".to_string())); 151 | } 152 | 153 | // get the image data 154 | let data = (*ximage).data as *mut u8; 155 | let data_len = 156 | ((*ximage).width * (*ximage).height * ((*ximage).bits_per_pixel / 8)) as usize; 157 | let slice = std::slice::from_raw_parts(data, data_len); 158 | // create an image buffer from the captured data 159 | let mut img = ImageBuffer::, Vec>::new( 160 | (*ximage).width as u32, 161 | (*ximage).height as u32, 162 | ); 163 | let (image_width, image_height) = img.dimensions(); 164 | 165 | let mut pixel_data: Vec = 166 | Vec::with_capacity((image_width * image_height * 4) as usize); 167 | for (x, y, _pixel) in img.enumerate_pixels_mut() { 168 | let index = ((y * image_width + x) * 4) as usize; 169 | pixel_data.push(slice[index + 2]); // R 170 | pixel_data.push(slice[index + 1]); // G 171 | pixel_data.push(slice[index]); // B 172 | pixel_data.push(255); // A 173 | } 174 | self.screen_data.pixel_data = pixel_data; 175 | XDestroyImage(ximage); 176 | } 177 | return Ok(()); 178 | } 179 | #[cfg(not(feature = "lite"))] 180 | /// convert vector to Luma Imagebuffer 181 | fn convert_bitmap_to_grayscale(&self) -> Result, Vec>, AutoGuiError> { 182 | let mut grayscale_data = 183 | Vec::with_capacity((self.screen_width * self.screen_height) as usize); 184 | for chunk in self.screen_data.pixel_data.chunks_exact(4) { 185 | let r = chunk[2] as u32; 186 | let g = chunk[1] as u32; 187 | let b = chunk[0] as u32; 188 | // calculate the grayscale value using the luminance formula 189 | let gray_value = ((r * 30 + g * 59 + b * 11) / 100) as u8; 190 | grayscale_data.push(gray_value); 191 | } 192 | GrayImage::from_raw( 193 | self.screen_width as u32, 194 | self.screen_height as u32, 195 | grayscale_data, 196 | ) 197 | .ok_or(ImageProcessingError::new("Failed conversion to grayscale").into()) 198 | } 199 | #[cfg(not(feature = "lite"))] 200 | /// convert vector to RGBA ImageBuffer 201 | fn convert_bitmap_to_rgba(&self) -> Result, Vec>, AutoGuiError> { 202 | ImageBuffer::from_raw( 203 | self.screen_width as u32, 204 | self.screen_height as u32, 205 | self.screen_data.pixel_data.clone(), 206 | ) 207 | .ok_or(ImageProcessingError::new("Failed conversion to RGBa").into()) 208 | } 209 | } 210 | -------------------------------------------------------------------------------- /src/core/screen/macos/mod.rs: -------------------------------------------------------------------------------- 1 | use core_graphics::display; 2 | use core_graphics::display::CGDisplay; 3 | 4 | use crate::{errors::AutoGuiError, imgtools}; 5 | 6 | #[cfg(not(feature = "lite"))] 7 | use image::{ 8 | imageops::{resize, FilterType::Nearest}, 9 | GrayImage, ImageBuffer, Luma, Rgba, 10 | }; 11 | 12 | #[derive(Debug, Clone)] 13 | pub struct Screen { 14 | pub screen_width: i32, 15 | pub screen_height: i32, 16 | #[cfg(not(feature = "lite"))] 17 | pub screen_data: ScreenImgData, 18 | } 19 | #[cfg(not(feature = "lite"))] 20 | #[derive(Debug, Clone)] 21 | pub struct ScreenImgData { 22 | pub display: CGDisplay, 23 | pub pixel_data: Vec, 24 | pub scaling_factor_x: f32, // difference between logical and phyisical resolution 25 | pub scaling_factor_y: f32, 26 | pub screen_region_width: u32, 27 | pub screen_region_height: u32, 28 | } 29 | 30 | impl Screen { 31 | pub fn new() -> Result { 32 | unsafe { 33 | let main_display_id = display::CGMainDisplayID(); 34 | let main_display = CGDisplay::new(main_display_id); 35 | // because of retina display, and scaling factors, image captured can be double the size 36 | // for that detection of retina is needed to divide all the pixel positions 37 | // by the factor. As far as i understood it should actually always be 2 but leaving it like this 38 | // shouldnt produce errors and covers any different case 39 | #[allow(unused_variables)] 40 | let image = main_display.image().ok_or(AutoGuiError::OSFailure( 41 | "Failed to create CGImage from display".to_string(), 42 | ))?; 43 | 44 | let screen_width = main_display.pixels_wide() as i32; 45 | let screen_height = main_display.pixels_high() as i32; 46 | 47 | #[cfg(not(feature = "lite"))] 48 | let screen_data = ScreenImgData { 49 | display: main_display, 50 | pixel_data: vec![0u8; (screen_width * screen_height * 4) as usize], 51 | scaling_factor_x: image.width() as f32 / screen_width as f32, 52 | scaling_factor_y: image.height() as f32 / screen_height as f32, 53 | screen_region_width: 0, 54 | screen_region_height: 0, 55 | }; 56 | Ok(Self { 57 | screen_height, 58 | screen_width, 59 | 60 | #[cfg(not(feature = "lite"))] 61 | screen_data, 62 | }) 63 | } 64 | } 65 | 66 | /// returns screen dimensions. All monitors included 67 | pub fn dimension(&self) -> (i32, i32) { 68 | let dimensions = (self.screen_width, self.screen_height); 69 | dimensions 70 | } 71 | #[cfg(not(feature = "lite"))] 72 | #[allow(dead_code)] 73 | /// return region dimension which is set up when template is precalculated 74 | pub fn region_dimension(&self) -> (u32, u32) { 75 | let dimensions = ( 76 | self.screen_data.screen_region_width, 77 | self.screen_data.screen_region_height, 78 | ); 79 | dimensions 80 | } 81 | #[cfg(not(feature = "lite"))] 82 | #[allow(dead_code)] 83 | /// executes convert_bitmap_to_rgba, meaning it converts Vector of values to RGBA and crops the image 84 | /// as inputted region area. Not used anywhere at the moment 85 | pub fn grab_screen_image( 86 | &mut self, 87 | region: (u32, u32, u32, u32), 88 | ) -> Result, Vec>, AutoGuiError> { 89 | let (x, y, width, height) = region; 90 | self.screen_data.screen_region_width = width; 91 | self.screen_data.screen_region_height = height; 92 | self.capture_screen()?; 93 | let image = self.convert_bitmap_to_rgba()?; 94 | let cropped_image: ImageBuffer, Vec> = 95 | imgtools::cut_screen_region(x, y, width, height, &image); 96 | Ok(cropped_image) 97 | } 98 | #[cfg(not(feature = "lite"))] 99 | /// executes convert_bitmap_to_grayscale, meaning it converts Vector of values to grayscale and crops the image 100 | /// as inputted region area 101 | pub fn grab_screen_image_grayscale( 102 | &mut self, 103 | region: &(u32, u32, u32, u32), 104 | ) -> Result, Vec>, AutoGuiError> { 105 | let (x, y, width, height) = region; 106 | self.screen_data.screen_region_width = *width; 107 | self.screen_data.screen_region_height = *height; 108 | self.capture_screen()?; 109 | let image: ImageBuffer, Vec> = self.convert_bitmap_to_grayscale()?; 110 | let cropped_image: ImageBuffer, Vec> = 111 | imgtools::cut_screen_region(*x, *y, *width, *height, &image); 112 | Ok(cropped_image) 113 | } 114 | #[cfg(not(feature = "lite"))] 115 | /// captures and saves screenshot of monitors 116 | pub fn grab_screenshot(&mut self, image_path: &str) -> Result<(), AutoGuiError> { 117 | self.capture_screen()?; 118 | let image = self.convert_bitmap_to_rgba()?; 119 | Ok(image.save(image_path)?) 120 | } 121 | #[cfg(not(feature = "lite"))] 122 | /// first order capture screen function. it captures screen image and stores it as vector in self.pixel_data 123 | fn capture_screen(&mut self) -> Result<(), AutoGuiError> { 124 | let image = self 125 | .screen_data 126 | .display 127 | .image() 128 | .ok_or(AutoGuiError::OSFailure( 129 | "Failed to capture screen image".to_string(), 130 | ))?; 131 | 132 | let pixel_data: Vec = image 133 | .data() 134 | .bytes() 135 | .chunks(4) 136 | .flat_map(|chunk| { 137 | // reorder color components 138 | if let &[b, g, r, a] = chunk { 139 | vec![r, g, b, a] 140 | } else { 141 | unreachable!() 142 | } 143 | }) 144 | .collect(); 145 | self.screen_data.pixel_data = pixel_data; 146 | Ok(()) 147 | } 148 | #[cfg(not(feature = "lite"))] 149 | /// convert vector to Luma Imagebuffer 150 | fn convert_bitmap_to_grayscale(&self) -> Result, Vec>, AutoGuiError> { 151 | let mut grayscale_data = 152 | Vec::with_capacity((self.screen_width * self.screen_height) as usize); 153 | for chunk in self.screen_data.pixel_data.chunks_exact(4) { 154 | let r = chunk[2] as u32; 155 | let g = chunk[1] as u32; 156 | let b = chunk[0] as u32; 157 | // calculate the grayscale value using the luminance formula 158 | let gray_value = ((r * 30 + g * 59 + b * 11) / 100) as u8; 159 | grayscale_data.push(gray_value); 160 | } 161 | let mut image = GrayImage::from_raw( 162 | (self.screen_data.scaling_factor_x * self.screen_width as f32) as u32, 163 | (self.screen_data.scaling_factor_y * self.screen_height as f32) as u32, 164 | grayscale_data, 165 | ) 166 | .ok_or(AutoGuiError::ImgError( 167 | "Could not convert image to grayscale".to_string(), 168 | ))?; 169 | let image = resize( 170 | &mut image, 171 | self.screen_width as u32, 172 | self.screen_height as u32, 173 | Nearest, 174 | ); 175 | Ok(image) 176 | } 177 | #[cfg(not(feature = "lite"))] 178 | /// convert vector to RGBA ImageBuffer 179 | fn convert_bitmap_to_rgba(&self) -> Result, Vec>, AutoGuiError> { 180 | ImageBuffer::from_raw( 181 | (self.screen_data.scaling_factor_x * self.screen_width as f32) as u32, 182 | (self.screen_data.scaling_factor_y * self.screen_height as f32) as u32, 183 | self.screen_data.pixel_data.clone(), 184 | ) 185 | .ok_or(AutoGuiError::ImgError( 186 | "Could not convert image to rgba".to_string(), 187 | )) 188 | } 189 | } 190 | -------------------------------------------------------------------------------- /src/core/screen/mod.rs: -------------------------------------------------------------------------------- 1 | #[cfg(target_os = "windows")] 2 | pub mod windows; 3 | #[cfg(target_os = "windows")] 4 | pub use windows::Screen; 5 | 6 | #[cfg(target_os = "linux")] 7 | pub mod linux; 8 | #[cfg(target_os = "linux")] 9 | pub use linux::Screen; 10 | 11 | #[cfg(target_os = "macos")] 12 | pub mod macos; 13 | #[cfg(target_os = "macos")] 14 | pub use macos::Screen; 15 | -------------------------------------------------------------------------------- /src/core/screen/windows/mod.rs: -------------------------------------------------------------------------------- 1 | #[cfg(not(feature = "lite"))] 2 | extern crate rayon; 3 | extern crate winapi; 4 | 5 | #[cfg(not(feature = "lite"))] 6 | use image::{GrayImage, ImageBuffer, ImageError, Luma, Rgba}; 7 | use std::mem::size_of; 8 | use std::ptr::null_mut; 9 | use winapi::shared::minwindef::{DWORD, HGLOBAL, LPVOID, UINT}; 10 | use winapi::um::wingdi::DIB_RGB_COLORS; 11 | use winapi::um::wingdi::{ 12 | BitBlt, CreateCompatibleBitmap, CreateCompatibleDC, DeleteDC, DeleteObject, GetDIBits, 13 | SelectObject, BITMAPINFO, BITMAPINFOHEADER, BI_RGB, RGBQUAD, SRCCOPY, 14 | }; 15 | use winapi::um::winuser::{GetDC, ReleaseDC}; 16 | 17 | use crate::{imgtools, AutoGuiError}; 18 | 19 | #[derive(Debug, Clone)] 20 | pub struct Screen { 21 | pub screen_width: i32, 22 | pub screen_height: i32, 23 | #[cfg(not(feature = "lite"))] 24 | pub screen_data: ScreenImgData, 25 | } 26 | #[derive(Debug, Clone)] 27 | #[cfg(not(feature = "lite"))] 28 | pub struct ScreenImgData { 29 | pub screen_region_width: u32, 30 | pub screen_region_height: u32, 31 | pub pixel_data: Vec, 32 | h_screen_dc: *mut winapi::shared::windef::HDC__, 33 | h_memory_dc: *mut winapi::shared::windef::HDC__, 34 | h_bitmap: *mut winapi::shared::windef::HBITMAP__, 35 | } 36 | 37 | impl Screen { 38 | ///Creates struct that holds information about screen 39 | pub fn new() -> Result { 40 | unsafe { 41 | let screen_width: i32 = winapi::um::winuser::GetSystemMetrics(0); 42 | let screen_height = winapi::um::winuser::GetSystemMetrics(1); 43 | 44 | #[cfg(not(feature = "lite"))] 45 | let screen_data = ScreenImgData { 46 | screen_region_height: screen_height as u32, 47 | screen_region_width: screen_width as u32, 48 | pixel_data: vec![0u8; (screen_width * screen_height * 4) as usize], 49 | // capture Device Context is a windows struct type that hold information that is written to the screen or printer 50 | h_screen_dc: GetDC(null_mut()), 51 | // here we create a compatible device context in memory, which will have same properties, and we will tell windows to write a screen to it 52 | h_memory_dc: CreateCompatibleDC(GetDC(null_mut())), 53 | h_bitmap: CreateCompatibleBitmap(GetDC(null_mut()), screen_width, screen_height), 54 | }; 55 | Ok(Screen { 56 | screen_height, 57 | screen_width, 58 | #[cfg(not(feature = "lite"))] 59 | screen_data, 60 | }) 61 | } 62 | } 63 | pub fn dimension(&self) -> (i32, i32) { 64 | (self.screen_width, self.screen_height) 65 | } 66 | #[cfg(not(feature = "lite"))] 67 | #[allow(dead_code)] 68 | pub fn region_dimension(&self) -> (u32, u32) { 69 | ( 70 | self.screen_data.screen_region_width, 71 | self.screen_data.screen_region_height, 72 | ) 73 | } 74 | #[cfg(not(feature = "lite"))] 75 | /// clear memory and delete screen 76 | pub fn destroy(&self) { 77 | unsafe { 78 | DeleteObject(self.screen_data.h_bitmap as HGLOBAL); 79 | DeleteDC(self.screen_data.h_memory_dc); 80 | ReleaseDC(null_mut(), self.screen_data.h_screen_dc); 81 | } 82 | } 83 | #[cfg(not(feature = "lite"))] 84 | #[allow(dead_code)] 85 | /// captures screen and returns Imagebuffer in RGBA cropped for the selected region 86 | pub fn grab_screen_image( 87 | &mut self, 88 | region: (u32, u32, u32, u32), 89 | ) -> Result, Vec>, AutoGuiError> { 90 | let (x, y, width, height) = region; 91 | self.screen_data.screen_region_width = width; 92 | self.screen_data.screen_region_height = height; 93 | self.capture_screen(); 94 | let image = self.convert_bitmap_to_rgba()?; 95 | 96 | let cropped_image: ImageBuffer, Vec> = 97 | imgtools::cut_screen_region(x, y, width, height, &image); 98 | Ok(cropped_image) 99 | } 100 | #[cfg(not(feature = "lite"))] 101 | /// captures screen, and returns grayscale Imagebuffer cropped for the selected region 102 | pub fn grab_screen_image_grayscale( 103 | &mut self, 104 | region: &(u32, u32, u32, u32), 105 | ) -> Result, Vec>, AutoGuiError> { 106 | let (x, y, width, height) = region; 107 | self.screen_data.screen_region_width = *width; 108 | self.screen_data.screen_region_height = *height; 109 | self.capture_screen(); 110 | let image = self.convert_bitmap_to_grayscale()?; 111 | 112 | let cropped_image: ImageBuffer, Vec> = 113 | imgtools::cut_screen_region(*x, *y, *width, *height, &image); 114 | Ok(cropped_image) 115 | } 116 | #[cfg(not(feature = "lite"))] 117 | /// grabs screen image and saves file at provided 118 | pub fn grab_screenshot(&mut self, image_path: &str) -> Result<(), AutoGuiError> { 119 | self.capture_screen(); 120 | let image = self.convert_bitmap_to_rgba()?; 121 | Ok(image.save(image_path)?) 122 | } 123 | #[cfg(not(feature = "lite"))] 124 | fn capture_screen(&mut self) { 125 | unsafe { 126 | // here we select the memory device context and the bitmap as main ones 127 | SelectObject( 128 | self.screen_data.h_memory_dc, 129 | self.screen_data.h_bitmap as HGLOBAL, 130 | ); 131 | // this function writes data to memory device context 132 | BitBlt( 133 | self.screen_data.h_memory_dc, 134 | 0, 135 | 0, 136 | self.screen_width, 137 | self.screen_height, 138 | self.screen_data.h_screen_dc, 139 | 0, 140 | 0, 141 | SRCCOPY, 142 | ); 143 | let mut bitmap_info = BITMAPINFO { 144 | bmiHeader: BITMAPINFOHEADER { 145 | biSize: size_of::() as DWORD, 146 | biWidth: self.screen_width, 147 | biHeight: -self.screen_height, // Negative to indicate top-down DIB 148 | biPlanes: 1, 149 | biBitCount: 32, 150 | biCompression: BI_RGB, 151 | biSizeImage: 0, 152 | biXPelsPerMeter: 0, 153 | biYPelsPerMeter: 0, 154 | biClrUsed: 0, 155 | biClrImportant: 0, 156 | }, 157 | bmiColors: [RGBQUAD { 158 | rgbBlue: 0, 159 | rgbGreen: 0, 160 | rgbRed: 0, 161 | rgbReserved: 0, 162 | }; 1], 163 | }; 164 | 165 | // Allocate buffer for the bitmap data 166 | let mut bitmap_data: Vec = 167 | vec![0u8; (self.screen_width * self.screen_height * 4) as usize]; 168 | 169 | // Get the bitmap data 170 | GetDIBits( 171 | self.screen_data.h_memory_dc, 172 | self.screen_data.h_bitmap, 173 | 0, 174 | self.screen_height as UINT, 175 | bitmap_data.as_mut_ptr() as LPVOID, 176 | &mut bitmap_info, 177 | DIB_RGB_COLORS, 178 | ); 179 | 180 | self.screen_data.pixel_data = bitmap_data 181 | } 182 | } 183 | #[cfg(not(feature = "lite"))] 184 | fn convert_bitmap_to_grayscale(&self) -> Result, Vec>, AutoGuiError> { 185 | let mut grayscale_data = 186 | Vec::with_capacity((self.screen_width * self.screen_height) as usize); 187 | for chunk in self.screen_data.pixel_data.chunks_exact(4) { 188 | let r = chunk[2] as u32; 189 | let g = chunk[1] as u32; 190 | let b = chunk[0] as u32; 191 | // calculate the grayscale value using the luminance formula 192 | let gray_value = ((r * 30 + g * 59 + b * 11) / 100) as u8; 193 | grayscale_data.push(gray_value); 194 | } 195 | 196 | GrayImage::from_raw( 197 | self.screen_width as u32, 198 | self.screen_height as u32, 199 | grayscale_data, 200 | ) 201 | .ok_or(AutoGuiError::ImgError( 202 | "could not convert image to grayscale".to_string(), 203 | )) 204 | } 205 | #[cfg(not(feature = "lite"))] 206 | fn convert_bitmap_to_rgba(&self) -> Result, Vec>, AutoGuiError> { 207 | ImageBuffer::from_raw( 208 | self.screen_width as u32, 209 | self.screen_height as u32, 210 | self.screen_data.pixel_data.clone(), 211 | ) 212 | .ok_or(AutoGuiError::ImgError( 213 | "failed to convert to RGBA".to_string(), 214 | )) 215 | } 216 | } 217 | -------------------------------------------------------------------------------- /src/core/template_match/fft_ncc.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Fast Normalized Cross correlation algorithm 3 | * Author of the algorithm: J.P.Lewis 4 | * http://scribblethink.org/Work/nvisionInterface/vi95_lewis.pdf 5 | */ 6 | 7 | use crate::{data::FFTData, imgtools}; 8 | use core::cmp::max; 9 | use image::{ImageBuffer, Luma}; 10 | use rayon::prelude::*; 11 | use rustfft::{num_complex::Complex, Fft, FftPlanner}; 12 | 13 | use super::{compute_integral_images, sum_region}; 14 | 15 | pub fn fft_ncc( 16 | image: &ImageBuffer, Vec>, 17 | precision: f32, 18 | prepared_data: &FFTData, 19 | ) -> Vec<(u32, u32, f64)> { 20 | // retreive all precalculated template data, most importantly template with already fft and conjugation calculated 21 | // sum squared deviations will be needed for denominator 22 | 23 | let mut planner = FftPlanner::::new(); 24 | let fft: std::sync::Arc> = 25 | planner.plan_fft_forward((prepared_data.padded_size * prepared_data.padded_size) as usize); 26 | let (image_width, image_height) = image.dimensions(); 27 | let image_vec: Vec> = imgtools::imagebuffer_to_vec(image); 28 | 29 | if (image_width < prepared_data.template_width) 30 | || (image_height < prepared_data.template_height) 31 | { 32 | return Vec::new(); 33 | } 34 | 35 | // compute needed integral images for denominator calculation 36 | let (image_integral, squared_image_integral) = compute_integral_images(&image_vec); 37 | 38 | //// calculating zero mean image 39 | let sum_image: u64 = sum_region(&image_integral, 0, 0, image_width, image_height); 40 | 41 | // calculating zero mean image , meaning image pixel values - image zero value 42 | let image_average_total = sum_image as f32 / (image_height * image_width) as f32; //@audit check image_height*image_width != 0 43 | let mut zero_mean_image: Vec> = 44 | vec![vec![0.0; image_width as usize]; image_height as usize]; 45 | for y in 0..image_height { 46 | for x in 0..image_width { 47 | let image_pixel_value = image.get_pixel(x, y)[0] as f32; 48 | zero_mean_image[y as usize][x as usize] = image_pixel_value - image_average_total; 49 | } 50 | } 51 | 52 | // padding to least squares and placing image in top left corner, same as template 53 | let mut image_padded: Vec> = vec![ 54 | Complex::new(0.0, 0.0); 55 | (prepared_data.padded_size * prepared_data.padded_size) 56 | as usize 57 | ]; 58 | for dy in 0..image_height { 59 | for dx in 0..image_width { 60 | let image_pixel_value = zero_mean_image[dy as usize][dx as usize]; 61 | image_padded[dy as usize * prepared_data.padded_size as usize + dx as usize] = 62 | Complex::new(image_pixel_value, 0.0); 63 | } 64 | } 65 | 66 | // conver image into frequency domain 67 | let ifft: std::sync::Arc> = 68 | planner.plan_fft_inverse((prepared_data.padded_size * prepared_data.padded_size) as usize); 69 | fft.process(&mut image_padded); 70 | 71 | // calculate F(image) * F(template).conjugate 72 | let product_freq: Vec> = image_padded 73 | .iter() 74 | .zip(prepared_data.template_conj_freq.iter()) 75 | .map(|(&img_val, &tmpl_val)| img_val * tmpl_val) 76 | .collect(); 77 | // do inverse fft 78 | let mut fft_result: Vec> = product_freq.clone(); 79 | ifft.process(&mut fft_result); 80 | 81 | // flatten for multithreading 82 | let coords: Vec<(u32, u32)> = (0..=(image_height - prepared_data.template_height)) //@audit could underflow if image_height = 0 83 | .flat_map(|y| (0..=(image_width - prepared_data.template_width)).map(move |x| (x, y))) //@audit could underflow if image_width = 0 84 | .collect(); 85 | // multithreading pixel by pixel template sliding, where correlations are filtered by precision 86 | // sending all needed data to calculate nominator and denominator at each of pixel positions 87 | let mut found_points: Vec<(u32, u32, f64)> = coords 88 | .par_iter() 89 | .map(|&(x, y)| { 90 | let corr = fft_correlation_calculation( 91 | &image_integral, 92 | &squared_image_integral, 93 | prepared_data.template_width, 94 | prepared_data.template_height, 95 | prepared_data.template_sum_squared_deviations, 96 | x, 97 | y, 98 | prepared_data.padded_size, 99 | &fft_result, 100 | ); 101 | 102 | (x, y, corr) 103 | }) 104 | .filter(|&(_, _, corr)| corr > precision as f64) 105 | .collect(); 106 | found_points.sort_by(|a, b| b.2.partial_cmp(&a.2).unwrap()); 107 | 108 | found_points 109 | } 110 | 111 | #[allow(dead_code)] 112 | fn fft_correlation_calculation( 113 | image_integral: &[Vec], 114 | squared_image_integral: &[Vec], 115 | template_width: u32, 116 | template_height: u32, 117 | template_sum_squared_deviations: f32, 118 | x: u32, // big image x value 119 | y: u32, // big image y value, 120 | padded_size: u32, 121 | fft_result: &[Complex], 122 | ) -> f64 { 123 | /// Function for calculation of correlation at each pixel position 124 | ////////// denominator calculation 125 | let sum_image: u64 = sum_region(image_integral, x, y, template_width, template_height); 126 | 127 | let sum_squared_image: u64 = sum_region( 128 | squared_image_integral, 129 | x, 130 | y, 131 | template_width, 132 | template_height, 133 | ); 134 | let image_sum_squared_deviations = sum_squared_image as f64 135 | - (sum_image as f64).powi(2) / (template_height * template_width) as f64; //@audit check template_height*template_width!=0 136 | let denominator = 137 | (image_sum_squared_deviations * template_sum_squared_deviations as f64).sqrt(); 138 | 139 | /////////////// NOMINATOR CALCULATION 140 | 141 | // fft result is calculated invert of whole image and template that were padded and zero valued 142 | // each pixel position shows value for that template position 143 | let numerator_value = 144 | fft_result[(y * padded_size) as usize + x as usize].re / (padded_size * padded_size) as f32; //@audit guess the padded_size is always non zero but could be checked 145 | let mut corr = numerator_value as f64 / denominator; 146 | 147 | if corr > 2.0 { 148 | corr = -100.0; 149 | } 150 | corr 151 | } 152 | 153 | pub fn prepare_template_picture( 154 | template: &ImageBuffer, Vec>, 155 | image_width: u32, 156 | image_height: u32, 157 | ) -> FFTData { 158 | /// precalculate all the neccessary data so its not slowing down main process 159 | /// returning template in frequency domain, with calculated conjugate 160 | let (template_width, template_height) = template.dimensions(); 161 | let padded_width = image_width.next_power_of_two(); 162 | let padded_height = image_height.next_power_of_two(); 163 | let padded_size = max(padded_width, padded_height); 164 | 165 | let mut sum_template = 0.0; 166 | // calculate needed sums 167 | for y in 0..template_height { 168 | for x in 0..template_width { 169 | let template_value = template.get_pixel(x, y)[0] as f32; 170 | sum_template += template_value; 171 | } 172 | } 173 | let mean_template_value = sum_template / (template_height * template_width) as f32; 174 | // create zero mean template 175 | let mut zero_mean_template: Vec> = 176 | vec![vec![0.0; template_width as usize]; template_height as usize]; 177 | let mut template_sum_squared_deviations: f32 = 0.0; 178 | for y in 0..template_height { 179 | for x in 0..template_width { 180 | let template_value = template.get_pixel(x, y)[0] as f32; 181 | let squared_deviation = (template_value - mean_template_value).powf(2.0); 182 | template_sum_squared_deviations += squared_deviation; 183 | 184 | // set zero mean value on new template 185 | zero_mean_template[y as usize][x as usize] = template_value - mean_template_value; 186 | } 187 | } 188 | // pad the zero mean template 189 | let mut template_padded: Vec> = 190 | vec![Complex::new(0.0, 0.0); (padded_size * padded_size) as usize]; 191 | for dy in 0..template_height { 192 | for dx in 0..template_width { 193 | let template_pixel_value = zero_mean_template[dy as usize][dx as usize]; 194 | template_padded[dy as usize * padded_size as usize + dx as usize] = 195 | Complex::new(template_pixel_value, 0.0); 196 | } 197 | } 198 | // convert template to frequency domain 199 | let mut planner = FftPlanner::::new(); 200 | let fft: std::sync::Arc> = 201 | planner.plan_fft_forward((padded_size * padded_size) as usize); 202 | fft.process(&mut template_padded); 203 | // calculate template conjugate 204 | let template_conj_freq: Vec> = 205 | template_padded.iter().map(|&val| val.conj()).collect(); 206 | 207 | FFTData { 208 | template_conj_freq, 209 | template_sum_squared_deviations, 210 | template_width, 211 | template_height, 212 | padded_size, 213 | } 214 | } 215 | -------------------------------------------------------------------------------- /src/core/template_match/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod fft_ncc; 2 | #[cfg(feature = "opencl")] 3 | pub mod open_cl; 4 | pub mod opencl_kernel; 5 | #[cfg(feature = "opencl")] 6 | pub mod opencl_v2; 7 | pub mod segmented_ncc; 8 | pub mod slow_ncc; 9 | 10 | #[allow(dead_code)] 11 | fn compute_integral_image(image: &[Vec]) -> Vec> { 12 | /* 13 | Function that takes an image as input and computes an integral table (sum table). 14 | Table is calculated in a way : f(x,y) = sum where f(x1<=x, y1<=y), meaning it always 15 | sums all the pixels that are above and left of the value, including the value. Meaning if we take middle of 16 | the picture as point, the top left corner will be summed and be represented in that value. 17 | This is done a bit faster with an algorithm that says: 18 | s(x,y) = f(x,y) + s(x-1,y) + s(x, y-1) - s(x-1, y-1), meaning it takes pixel value and adds 19 | already calculated sum value left from it, above from it and subtracts the pixel value that is 20 | above and left from it. 21 | for x=0 or y=0 we just omit part of the formula, and for x=0 and y=0 we insert just the pixel value. 22 | This table is made in order to more easily compute sums of images for further calculations in the algorithm. 23 | example: 24 | [1 1 1 1] 25 | [1 1 1 1] 26 | [1 1 1 1] 27 | [1 1 1 1] 28 | becomes 29 | [ 1 2 3 4 ] 30 | [ 2 4 6 8 ] 31 | [ 3 6 9 12 ] 32 | [ 4 8 12 16 ] 33 | */ 34 | 35 | let height = image.len() as u32; 36 | let width = if height > 0 { image[0].len() as u32 } else { 0 }; 37 | 38 | let mut integral_image = vec![vec![0u64; width as usize]; height as usize]; 39 | 40 | for y in 0..height { 41 | for x in 0..width { 42 | let pixel_value = image[y as usize][x as usize] as u64; 43 | let integral_value = if x == 0 && y == 0 { 44 | pixel_value 45 | } else if x == 0 { 46 | pixel_value + integral_image[(y - 1) as usize][x as usize] 47 | } else if y == 0 { 48 | pixel_value + integral_image[y as usize][(x - 1) as usize] 49 | } else { 50 | pixel_value 51 | + integral_image[(y - 1) as usize][x as usize] 52 | + integral_image[y as usize][(x - 1) as usize] 53 | - integral_image[(y - 1) as usize][(x - 1) as usize] 54 | }; 55 | integral_image[y as usize][x as usize] = integral_value; 56 | } 57 | } 58 | 59 | integral_image 60 | } 61 | 62 | #[allow(dead_code)] 63 | fn compute_squared_integral_image(image: &[Vec]) -> Vec> { 64 | /* 65 | Same as compute_integral_image, except we always take squared value of pixel f(x,y). 66 | */ 67 | 68 | let height = image.len() as u32; 69 | let width = if height > 0 { image[0].len() as u32 } else { 0 }; 70 | 71 | let mut integral_image = vec![vec![0u64; width as usize]; height as usize]; 72 | 73 | for y in 0..height { 74 | for x in 0..width { 75 | let pixel_value = (image[y as usize][x as usize] as u64).pow(2); 76 | let integral_value = if x == 0 && y == 0 { 77 | pixel_value 78 | } else if x == 0 { 79 | pixel_value + integral_image[(y - 1) as usize][x as usize] 80 | } else if y == 0 { 81 | pixel_value + integral_image[y as usize][(x - 1) as usize] 82 | } else { 83 | pixel_value 84 | + integral_image[(y - 1) as usize][x as usize] 85 | + integral_image[y as usize][(x - 1) as usize] 86 | - integral_image[(y - 1) as usize][(x - 1) as usize] 87 | }; 88 | integral_image[y as usize][x as usize] = integral_value; 89 | } 90 | } 91 | 92 | integral_image 93 | } 94 | 95 | /// Compute both normal and squared integral image 96 | fn compute_integral_images(image: &[Vec]) -> (Vec>, Vec>) { 97 | let height = image.len() as u32; 98 | let width = if height > 0 { image[0].len() as u32 } else { 0 }; 99 | let mut integral_image = vec![vec![0u64; width as usize]; height as usize]; 100 | let mut squared_integral_image = vec![vec![0u64; width as usize]; height as usize]; 101 | for y in 0..height { 102 | for x in 0..width { 103 | let pixel_value = image[y as usize][x as usize] as u64; 104 | let pixel_value_squared = (image[y as usize][x as usize] as u64).pow(2); 105 | let (integral_value, squared_integral_value) = if x == 0 && y == 0 { 106 | (pixel_value, pixel_value_squared) 107 | } else if x == 0 { 108 | ( 109 | pixel_value + integral_image[(y - 1) as usize][x as usize], 110 | pixel_value_squared + squared_integral_image[(y - 1) as usize][x as usize], 111 | ) 112 | } else if y == 0 { 113 | ( 114 | pixel_value + integral_image[y as usize][(x - 1) as usize], 115 | pixel_value_squared + squared_integral_image[y as usize][(x - 1) as usize], 116 | ) 117 | } else { 118 | ( 119 | pixel_value 120 | + integral_image[(y - 1) as usize][x as usize] 121 | + integral_image[y as usize][(x - 1) as usize] 122 | - integral_image[(y - 1) as usize][(x - 1) as usize], 123 | pixel_value_squared 124 | + squared_integral_image[(y - 1) as usize][x as usize] 125 | + squared_integral_image[y as usize][(x - 1) as usize] 126 | - squared_integral_image[(y - 1) as usize][(x - 1) as usize], 127 | ) 128 | }; 129 | integral_image[y as usize][x as usize] = integral_value; 130 | squared_integral_image[y as usize][x as usize] = squared_integral_value; 131 | } 132 | } 133 | 134 | (integral_image, squared_integral_image) 135 | } 136 | 137 | fn sum_region(integral_image: &[Vec], x: u32, y: u32, width: u32, height: u32) -> u64 { 138 | /* 139 | Used to calculate sum region of an integral image. Bottom right pixel will have summed up value of everything above and left. 140 | In order to get exact sum value of subregion of the image, we take that sum from bottom right, 141 | subtract from it the value that is on the top right , in the first row above the image, subtract the value that is on 142 | the bottom left, in the first column left of the image, and add up the value that is top left in the 143 | first row and colum above and left of the image. 144 | [ 1 2 3 4 ] 145 | [ 2 4/ 6 8/] 146 | [ 3 6 9 12 ] 147 | [ 4 8/ 12 16/] 148 | - ive marked most important numbers with / that are important to calculate subimage that is like 149 | [9 12] (bottom right of the image) 150 | [12 16] 151 | In order to calculate subimage sum, we take 16 - 8 - 8 + 4, which is 4 152 | */ 153 | let mut sum = integral_image[(y + height - 1) as usize][(x + width - 1) as usize]; 154 | if x == 0 && y == 0 { 155 | // do nothing 156 | } else if y == 0 { 157 | sum -= integral_image[(y + height - 1) as usize][(x - 1) as usize]; 158 | } else if x == 0 { 159 | sum -= integral_image[(y - 1) as usize][(x + width - 1) as usize] 160 | } else { 161 | sum += integral_image[(y - 1) as usize][(x - 1) as usize]; 162 | sum -= integral_image[(y + height - 1) as usize][(x - 1) as usize]; 163 | sum -= integral_image[(y - 1) as usize][(x + width - 1) as usize]; 164 | } 165 | sum 166 | } 167 | -------------------------------------------------------------------------------- /src/core/template_match/open_cl.rs: -------------------------------------------------------------------------------- 1 | use super::opencl_v2; 2 | use crate::core::template_match::{compute_integral_images, sum_region}; 3 | use crate::data::SegmentedData; 4 | use crate::data::{GpuMemoryPointers, KernelStorage}; 5 | use image::{ImageBuffer, Luma}; 6 | use ocl; 7 | use ocl::{Buffer, Context, Kernel, Program, Queue}; 8 | 9 | pub enum OclVersion { 10 | V1, 11 | V2, 12 | } 13 | 14 | pub fn gui_opencl_ncc_template_match( 15 | queue: &Queue, 16 | program: &Program, 17 | max_workgroup_size: u32, 18 | kernel_storage: &KernelStorage, 19 | gpu_memory_pointers: &GpuMemoryPointers, 20 | precision: f32, 21 | image: &ImageBuffer, Vec>, 22 | template_data: &SegmentedData, 23 | ocl_version: OclVersion, 24 | ) -> ocl::Result> { 25 | let (image_width, image_height) = image.dimensions(); 26 | 27 | let (image_integral, squared_image_integral) = compute_integral_images_ocl(&image); 28 | 29 | let slow_expected_corr = precision * (template_data.expected_corr_slow - 0.001); 30 | match ocl_version { 31 | OclVersion::V1 => { 32 | let kernel = &kernel_storage.v1_kernel; 33 | let mut gpu_results = gui_opencl_ncc( 34 | kernel, 35 | &image_integral, 36 | &squared_image_integral, 37 | image_width, 38 | image_height, 39 | template_data.template_width, 40 | template_data.template_height, 41 | gpu_memory_pointers, 42 | precision, 43 | )?; 44 | gpu_results.retain(|&(_, _, value)| value >= slow_expected_corr); 45 | gpu_results.sort_by(|a, b| b.2.partial_cmp(&a.2).unwrap_or(std::cmp::Ordering::Equal)); 46 | return Ok(gpu_results); 47 | } 48 | OclVersion::V2 => { 49 | let slow_segment_count = template_data.template_segments_slow.len() as i32; 50 | let kernel = &kernel_storage.v2_kernel_fast; 51 | let segments_processed_by_thread_slow = 52 | (slow_segment_count / max_workgroup_size as i32).max(1); 53 | let remainder_segments_slow = if slow_segment_count > max_workgroup_size as i32 { 54 | slow_segment_count % max_workgroup_size as i32 55 | } else { 56 | 0 57 | }; 58 | return opencl_v2::gui_opencl_ncc_v2( 59 | kernel, 60 | &image_integral, 61 | &squared_image_integral, 62 | image_width, 63 | image_height, 64 | template_data.template_width, 65 | template_data.template_height, 66 | template_data.segment_sum_squared_deviations_slow, 67 | template_data.segments_mean_slow, 68 | slow_expected_corr, 69 | queue, 70 | program, 71 | gpu_memory_pointers, 72 | slow_segment_count, 73 | remainder_segments_slow, 74 | segments_processed_by_thread_slow, 75 | max_workgroup_size as i32, 76 | precision, 77 | ); 78 | } 79 | } 80 | } 81 | 82 | pub fn gui_opencl_ncc( 83 | kernel: &Kernel, 84 | image_integral: &[u64], 85 | squared_image_integral: &[u64], 86 | image_width: u32, 87 | image_height: u32, 88 | template_width: u32, 89 | template_height: u32, 90 | gpu_memory_pointers: &GpuMemoryPointers, 91 | precision: f32, 92 | ) -> ocl::Result> { 93 | let result_width = (image_width - template_width + 1) as usize; 94 | let result_height = (image_height - template_height + 1) as usize; 95 | let output_size = result_width * result_height; 96 | gpu_memory_pointers 97 | .buffer_image_integral 98 | .write(image_integral) 99 | .enq()?; 100 | gpu_memory_pointers 101 | .buffer_image_integral_squared 102 | .write(squared_image_integral) 103 | .enq()?; 104 | 105 | gpu_memory_pointers 106 | .buffer_precision 107 | .write(&vec![precision - 0.01]) 108 | .enq()?; 109 | 110 | unsafe { 111 | kernel.enq()?; 112 | } 113 | let mut results = vec![0.0f32; output_size]; 114 | gpu_memory_pointers 115 | .results_buffer 116 | .read(&mut results) 117 | .enq()?; 118 | 119 | let final_results: Vec<(u32, u32, f32)> = results 120 | .into_iter() 121 | .enumerate() 122 | .map(|(idx, corr)| { 123 | let x = (idx % result_width) as u32; 124 | let y = (idx / result_width) as u32; 125 | (x, y, corr) 126 | }) 127 | .collect(); 128 | 129 | Ok(final_results) 130 | } 131 | 132 | pub fn compute_integral_images_ocl(image: &ImageBuffer, Vec>) -> (Vec, Vec) { 133 | let (width, height) = image.dimensions(); 134 | let image = image.as_raw(); 135 | let mut integral_image = vec![0u64; (width * height) as usize]; 136 | let mut squared_integral_image = vec![0u64; (width * height) as usize]; 137 | for y in 0..height { 138 | for x in 0..width { 139 | let pixel_value = image[(y * width + x) as usize] as u64; 140 | let pixel_value_squared = (image[(y * width + x) as usize] as u64).pow(2); 141 | let (integral_value, squared_integral_value) = if x == 0 && y == 0 { 142 | (pixel_value, pixel_value_squared) 143 | } else if x == 0 { 144 | ( 145 | pixel_value + integral_image[((y - 1) * width + x) as usize], 146 | pixel_value_squared + squared_integral_image[((y - 1) * width + x) as usize], 147 | ) 148 | } else if y == 0 { 149 | ( 150 | pixel_value + integral_image[(y * width + (x - 1)) as usize], 151 | pixel_value_squared + squared_integral_image[(y * width + (x - 1)) as usize], 152 | ) 153 | } else { 154 | ( 155 | pixel_value 156 | + integral_image[((y - 1) * width + x) as usize] 157 | + integral_image[(y * width + (x - 1)) as usize] 158 | - integral_image[((y - 1) * width + (x - 1)) as usize], 159 | pixel_value_squared 160 | + squared_integral_image[((y - 1) * width + x) as usize] 161 | + squared_integral_image[(y * width + (x - 1)) as usize] 162 | - squared_integral_image[((y - 1) * width + (x - 1)) as usize], 163 | ) 164 | }; 165 | integral_image[(y * width + x) as usize] = integral_value; 166 | squared_integral_image[(y * width + x) as usize] = squared_integral_value; 167 | } 168 | } 169 | 170 | (integral_image, squared_integral_image) 171 | } 172 | -------------------------------------------------------------------------------- /src/core/template_match/opencl_kernel.rs: -------------------------------------------------------------------------------- 1 | #[cfg(feature = "opencl")] 2 | pub const OCL_KERNEL: &str = r#" 3 | inline ulong sum_region( 4 | __global const ulong* integral, 5 | int x, 6 | int y, 7 | int width, 8 | int height, 9 | int image_width 10 | ) { 11 | int x2 = x + width - 1; 12 | int y2 = y + height - 1; 13 | 14 | ulong br = integral[y2 * image_width + x2]; 15 | ulong bl = (x == 0) ? 0 : integral[y2 * image_width + (x - 1)]; 16 | ulong tr = (y == 0) ? 0 : integral[(y - 1) * image_width + x2]; 17 | ulong tl = (x == 0 || y == 0) ? 0 : integral[(y - 1) * image_width + (x - 1)]; 18 | long sum = (long)br + (long)tl - (long)bl - (long)tr; 19 | 20 | 21 | return (ulong)sum; 22 | } 23 | 24 | 25 | inline ulong sum_region_squared( 26 | __global const ulong* integral_sq, 27 | int x, 28 | int y, 29 | int width, 30 | int height, 31 | int image_width 32 | ) { 33 | int x2 = x + width - 1; 34 | int y2 = y + height - 1; 35 | 36 | ulong br = integral_sq[y2 * image_width + x2]; 37 | ulong bl = (x == 0) ? 0 : integral_sq[y2 * image_width + (x - 1)]; 38 | ulong tr = (y == 0) ? 0 : integral_sq[(y - 1) * image_width + x2]; 39 | ulong tl = (x == 0 || y == 0) ? 0 : integral_sq[(y - 1) * image_width + (x - 1)]; 40 | long sum = (long)br + (long)tl - (long)bl - (long)tr; 41 | return (ulong)sum; 42 | } 43 | 44 | 45 | __kernel void segmented_match_integral( 46 | __global const ulong* integral, 47 | __global const ulong* integral_sq, 48 | __global const int4* segments, 49 | __global const int4* segments_slow, 50 | __global const float* segment_values, 51 | __global const float* segment_values_slow, 52 | const int num_segments, 53 | const int num_segments_slow, 54 | const float template_mean, 55 | const float template_mean_slow, 56 | const float template_sq_dev, 57 | const float template_sq_dev_slow, 58 | __global float* results, 59 | const int image_width, 60 | const int image_height, 61 | const int template_width, 62 | const int template_height, 63 | const float min_expected_corr, 64 | __global float* precision_buff 65 | ) { 66 | float precision = precision_buff[0]; 67 | int idx = get_global_id(0); 68 | int result_width = image_width - template_width + 1; 69 | int result_height = image_height - template_height + 1; 70 | 71 | if (idx >= result_width * result_height) return; 72 | // results[idx] = 0.0; 73 | int x = idx % result_width; 74 | int y = idx / result_width; 75 | 76 | 77 | ulong patch_sum = sum_region(integral, x, y, template_width, template_height, image_width); 78 | ulong patch_sq_sum = sum_region_squared(integral_sq, x, y, template_width, template_height, image_width); 79 | 80 | 81 | 82 | float area = (float)(template_width * template_height); 83 | float mean_img = (float)(patch_sum) / area; 84 | float var_img = (float)(patch_sq_sum) - ((float)(patch_sum) * (float)(patch_sum)) / area; 85 | 86 | float nominator = 0.0f; 87 | for (int i = 0; i < num_segments; i++) { 88 | int4 seg = segments[i]; 89 | float seg_val = segment_values[i]; 90 | int seg_area = seg.z * seg.w; 91 | 92 | ulong region_sum = sum_region(integral, x + seg.x, y + seg.y, seg.z, seg.w, image_width); 93 | 94 | nominator += ((float)(region_sum) - mean_img * seg_area) * (seg_val - template_mean); 95 | } 96 | 97 | float denominator = sqrt(var_img * template_sq_dev); 98 | 99 | float corr = (denominator != 0.0f) ? (nominator / denominator) : -1.0f; 100 | 101 | 102 | 103 | if (corr < (min_expected_corr - 0.001)* precision) { 104 | results[idx] = corr; 105 | return; 106 | } else { 107 | float denominator_slow = sqrt(var_img * template_sq_dev_slow); 108 | float nominator_slow = 0.0f; 109 | for (int i = 0; i < num_segments_slow; i++) { 110 | int4 seg_slow = segments_slow[i]; 111 | float seg_val_slow = segment_values_slow[i]; 112 | int seg_area = seg_slow.z * seg_slow.w; 113 | 114 | ulong region_sum = sum_region(integral, x + seg_slow.x, y + seg_slow.y, seg_slow.z, seg_slow.w, image_width); 115 | 116 | nominator_slow += ((float)(region_sum) - mean_img * seg_area) * (seg_val_slow - template_mean); 117 | } 118 | float corr_slow = (denominator_slow != 0.0f) ? (nominator_slow / denominator_slow) : -1.0f; 119 | results[idx] = corr_slow; 120 | } 121 | } 122 | 123 | 124 | __kernel void v2_segmented_match_integral_fast_pass( 125 | __global const ulong* integral, 126 | __global const ulong* integral_sq, 127 | __global const int4* segments, 128 | __global const float* segment_values, 129 | const int num_segments, 130 | const float template_mean, 131 | const float template_sq_dev, 132 | __global int2* results, 133 | const int image_width, 134 | const int image_height, 135 | const int template_width, 136 | const int template_height, 137 | const float min_expected_corr, 138 | const int remainder_segments_fast, 139 | const int segments_per_thread_fast, 140 | const int pixels_per_workgroup, 141 | const int workgroup_size, 142 | __local ulong* sum_template_region_buff, 143 | __local ulong* sum_sq_template_region_buff, 144 | __local float* thread_segment_sum_buff, 145 | __global int* valid_corr_count, 146 | __global float* precision_buff 147 | ) { 148 | int global_id = get_global_id(0); 149 | int local_id = get_local_id(0); 150 | int workgroup_id = get_group_id(0); 151 | int result_w = image_width - template_width; 152 | if (local_id == 3 && global_id == 2) { 153 | valid_corr_count[0] == 0; 154 | } 155 | 156 | 157 | // num_segments is also count of threads per pixel for fast img 158 | if (local_id * segments_per_thread_fast + remainder_segments_fast >= num_segments * pixels_per_workgroup) return ; // this solves more segments per thread 159 | 160 | int pixel_pos = (workgroup_id * pixels_per_workgroup) + (local_id / num_segments) ; 161 | int image_x = pixel_pos % result_w; 162 | int image_y = pixel_pos / result_w; 163 | 164 | // first sum the region of template area for numerator calculations 165 | // we do it with first threads for each x,y position which workgroup processes 166 | // if there are 5 pixels processed, local_id 0-4 should process sum regions for each position, 5-9 for squared 167 | ulong patch_sum = 0; 168 | if (local_id < pixels_per_workgroup) { 169 | patch_sum = sum_region(integral, image_x, image_y, template_width, template_height, image_width); 170 | sum_template_region_buff[local_id] = patch_sum; 171 | 172 | } 173 | 174 | // there will never be less than 2 segments 175 | // meaning pixels per workgroup is never greater than workgroup_size / 2 176 | if (local_id >= pixels_per_workgroup && local_id < pixels_per_workgroup * 2) { 177 | ulong patch_sq_sum = sum_region_squared(integral_sq, image_x, image_y, template_width, template_height, image_width); 178 | sum_sq_template_region_buff[local_id % pixels_per_workgroup] = patch_sq_sum; 179 | } 180 | 181 | int result_width = image_width - template_width + 1; 182 | int result_height = image_height - template_height + 1; 183 | float area = (float)(template_width * template_height); 184 | 185 | // wait for threads to complete writing sum_area 186 | barrier(CLK_LOCAL_MEM_FENCE); 187 | 188 | 189 | float mean_img = (float)(sum_template_region_buff[local_id / num_segments]) / area; 190 | 191 | 192 | // this is to cover if we have more than 1 segment per thread. This method 193 | // with remainder allows us to keep all threads working 194 | int remainder_offset = 0; 195 | int remainder_addition = 0; 196 | if (remainder_segments_fast > 0) { 197 | if (local_id >= remainder_segments_fast) { 198 | remainder_offset = remainder_segments_fast; 199 | } else { 200 | remainder_offset = local_id; 201 | remainder_addition = 1; 202 | } 203 | 204 | } 205 | 206 | 207 | 208 | // AUDIT - DOUBLE CHECK THIS LOGIC 209 | int thread_segment_start = (local_id * segments_per_thread_fast + remainder_offset ) % num_segments; 210 | int thread_segment_end = thread_segment_start + segments_per_thread_fast + remainder_addition; 211 | 212 | float nominator = 0.0f; 213 | for (int i = thread_segment_start; i< thread_segment_end; i++) { 214 | 215 | int4 seg = segments[i]; 216 | float seg_val = segment_values[i]; 217 | int seg_area = seg.z* seg.w; 218 | ulong region_sum = sum_region(integral, image_x + seg.x, image_y + seg.y, seg.z, seg.w, image_width); 219 | 220 | 221 | nominator += ((float)(region_sum) - mean_img * seg_area) * (seg_val - template_mean); 222 | 223 | } 224 | 225 | thread_segment_sum_buff[local_id] = nominator; 226 | 227 | barrier(CLK_LOCAL_MEM_FENCE); 228 | 229 | 230 | 231 | if (local_id < pixels_per_workgroup) { 232 | float nominator_sum = 0.0f; 233 | int sum_start = local_id * num_segments; 234 | int sum_end = sum_start + (num_segments / segments_per_thread_fast ) - (remainder_segments_fast/segments_per_thread_fast); 235 | for (int i = sum_start; i< sum_end; i++) { 236 | nominator_sum = nominator_sum + thread_segment_sum_buff[i] ; 237 | } 238 | 239 | int pixel_pos_final = (workgroup_id * pixels_per_workgroup) + (local_id) ; 240 | int image_x = pixel_pos_final % result_w; 241 | int image_y = pixel_pos_final / result_w; 242 | 243 | float precision = precision_buff[0]; 244 | ulong patch_sq_sum_extracted = sum_sq_template_region_buff[local_id]; 245 | float var_img = (float)patch_sq_sum_extracted - ((float)patch_sum * (float)patch_sum)/ (float)area; 246 | float denominator = sqrt(var_img * (float)template_sq_dev); 247 | float corr = (denominator != 0.0f) ? (nominator_sum / denominator) : -1.0f; 248 | 249 | if (corr >= (min_expected_corr - 0.01) * precision && corr < 2) { 250 | 251 | int index = atomic_add(valid_corr_count, 1); 252 | results[index] = (int2)(image_x, image_y); 253 | 254 | } 255 | } 256 | } 257 | 258 | 259 | 260 | __kernel void v2_segmented_match_integral_slow_pass ( 261 | __global const ulong* integral, 262 | __global const ulong* integral_sq, 263 | __global const int4* segments, 264 | __global const float* segment_values, 265 | const int num_segments, 266 | const float template_mean, 267 | const float template_sq_dev, 268 | __global int2* position_results, 269 | __global float* corr_results, 270 | const int image_width, 271 | const int image_height, 272 | const int template_width, 273 | const int template_height, 274 | const float min_expected_corr, 275 | const int remainder_segments_slow, 276 | const int segments_per_thread_slow, 277 | const int workgroup_size, 278 | __local ulong* sum_template_region_buff, 279 | __local ulong* sum_sq_template_region_buff, 280 | __local float* thread_segment_sum_buff, 281 | __global int* valid_corr_count_slow, 282 | __global int* valid_corr_count_fast, 283 | __global int2* fast_pass_results, 284 | __global float* precision_buff 285 | ) { 286 | 287 | int global_id = get_global_id(0); 288 | int local_id = get_local_id(0); 289 | int workgroup_id = get_group_id(0); 290 | 291 | 292 | 293 | 294 | 295 | int image_x = fast_pass_results[workgroup_id].x; 296 | int image_y = fast_pass_results[workgroup_id].y; 297 | 298 | int result_w = image_width - template_width; 299 | // num_segments is also count of threads per pixel for fast img 300 | if (local_id * segments_per_thread_slow + remainder_segments_slow >= num_segments) return ; // this solves more segments per thread 301 | 302 | 303 | // first sum the region of template area for numerator calculations 304 | // we do it with first threads for each x,y position which workgroup processes 305 | // if there are 5 pixels processed, local_id 0-4 should process sum regions for each position, 5-9 for squared 306 | ulong patch_sum = 0; 307 | if (local_id == 0) { 308 | patch_sum = sum_region(integral, image_x, image_y, template_width, template_height, image_width); 309 | sum_template_region_buff[0] = patch_sum; 310 | 311 | } 312 | 313 | // there will never be less than 2 segments 314 | // meaning pixels per workgroup is never greater than workgroup_size / 2 315 | if (local_id == 1) { 316 | ulong patch_sq_sum = sum_region_squared(integral_sq, image_x, image_y, template_width, template_height, image_width); 317 | sum_sq_template_region_buff[0] = patch_sq_sum; 318 | } 319 | int result_width = image_width - template_width + 1; 320 | int result_height = image_height - template_height + 1; 321 | float area = (float)(template_width * template_height); 322 | // wait for threads to complete writing sum_area 323 | barrier(CLK_LOCAL_MEM_FENCE); 324 | float mean_img = (float)(sum_template_region_buff[0]) / area; 325 | // this is to cover if we have more than 1 segment per thread. This method 326 | 327 | 328 | // with remainder allows us to keep all threads working 329 | int remainder_offset = 0; 330 | int remainder_addition = 0; 331 | if (remainder_segments_slow > 0) { 332 | if (local_id >= remainder_segments_slow) { 333 | remainder_offset = remainder_segments_slow; 334 | } else { 335 | remainder_offset = local_id; 336 | remainder_addition = 1; 337 | } 338 | 339 | } 340 | 341 | int thread_segment_start = (local_id * segments_per_thread_slow + remainder_offset ) % num_segments; 342 | int thread_segment_end = thread_segment_start + segments_per_thread_slow + remainder_addition; 343 | 344 | 345 | float nominator = 0.0f; 346 | for (int i = thread_segment_start; i< thread_segment_end; i++) { 347 | 348 | int4 seg = segments[i]; 349 | float seg_val = segment_values[i]; 350 | int seg_area = seg.z* seg.w; 351 | ulong region_sum = sum_region(integral, image_x + seg.x, image_y + seg.y, seg.z, seg.w, image_width); 352 | 353 | 354 | nominator += ((float)(region_sum) - mean_img * seg_area) * (seg_val - template_mean); 355 | 356 | } 357 | 358 | thread_segment_sum_buff[local_id] = nominator; 359 | barrier(CLK_LOCAL_MEM_FENCE); 360 | if (local_id == 0) { 361 | float nominator_sum = 0.0f; 362 | int sum_start = 0; 363 | int sum_end = sum_start + (num_segments / segments_per_thread_slow ) - (remainder_segments_slow/segments_per_thread_slow); 364 | for (int i = sum_start; i< sum_end; i++) { 365 | nominator_sum = nominator_sum + thread_segment_sum_buff[i] ; 366 | } 367 | 368 | 369 | 370 | 371 | ulong patch_sq_sum_extracted = sum_sq_template_region_buff[0]; 372 | float var_img = (float)patch_sq_sum_extracted - ((float)patch_sum * (float)patch_sum)/ (float)area; 373 | float denominator = sqrt(var_img * (float)template_sq_dev); 374 | float corr = (denominator != 0.0f) ? (nominator_sum / denominator) : -1.0f; 375 | float precision = precision_buff[0]; 376 | 377 | if (corr >= (min_expected_corr - 0.001) * precision && corr < 2) { 378 | int index = atomic_add(valid_corr_count_slow, 1); 379 | position_results[index] = (int2)(image_x, image_y); 380 | corr_results[index] = corr; 381 | } 382 | } 383 | } 384 | 385 | 386 | "#; 387 | -------------------------------------------------------------------------------- /src/core/template_match/opencl_v2.rs: -------------------------------------------------------------------------------- 1 | use crate::data::GpuMemoryPointers; 2 | use ocl; 3 | use ocl::core::Int2; 4 | use ocl::{Buffer, Context, Device, Kernel, Program, Queue}; 5 | 6 | /* 7 | Version 2 splits work by doing 2 kernel runs, one for slow 2nd for fast. It utilizes workgroups to maximum, unless number of segments is between (workgroup_size / 2 ) and (workgroup_size ) 8 | Each workgroup processes 1 or more pixel positions. 9 | Each thread processes 1 or more segments. 10 | 3 cases with workgroup size of 256: 11 | case 1: 12 | number of segments: 512 // greater than workgroup size 13 | each workgroup processes 1 pixel position 14 | each thread processes 2 segments for that position 15 | 16 | case 2: 17 | number of segments 200 // between workgroup_size / 2 and workgroup_size 18 | each workgroup processes 1 pixel pos 19 | each thread processes 1 segment 20 | 56 threads are doing nothing - waste of computing power 21 | 22 | case 3: 23 | num of seg: 4 24 | each workgroup processes 256/4 = 64 pixel positions // smaller than workgroup_size /2 25 | each thread processes 1 segment 26 | all threads utilized 27 | 28 | additionally: 29 | num of segments = 312 // greater than 256 and unusual num for workgroup sizes 30 | we calculate remainder 312 - 256 = 56 31 | 56 threads calculate 2 segments. The rest (256) calculate 1 segment. 32 | 33 | Issues: 34 | Very large images can lead to freezing everything , especially on linux, due to large thread spawn 35 | Very succeptible to how fast image is segmented. It can lead to giving too many false positives for slow pass 36 | which slows down the process. This is why its best used with tweaked threshold of prepared image , to skip 37 | false positives, but too high threshold can slow down fast process itself 38 | */ 39 | 40 | pub fn gui_opencl_ncc_v2( 41 | v2_kernel_fast_pass: &Kernel, 42 | image_integral: &[u64], 43 | squared_image_integral: &[u64], 44 | image_width: u32, 45 | image_height: u32, 46 | template_width: u32, 47 | template_height: u32, 48 | segments_sum_squared_deviation_slow: f32, 49 | segments_mean_slow: f32, 50 | slow_expected_corr: f32, 51 | queue: &Queue, 52 | program: &Program, 53 | gpu_memory_pointers: &GpuMemoryPointers, 54 | slow_segment_count: i32, 55 | remainder_segments_slow: i32, 56 | segments_processed_by_thread_slow: i32, 57 | workgroup_size: i32, 58 | precision: f32, 59 | ) -> ocl::Result> { 60 | gpu_memory_pointers 61 | .buffer_image_integral 62 | .write(image_integral) 63 | .enq()?; 64 | gpu_memory_pointers 65 | .buffer_image_integral_squared 66 | .write(squared_image_integral) 67 | .enq()?; 68 | gpu_memory_pointers 69 | .buffer_precision 70 | .write(&vec![precision]) 71 | .enq()?; 72 | 73 | unsafe { 74 | v2_kernel_fast_pass.enq()?; 75 | } 76 | // get how many points have been found with fast pass 77 | let mut valid_corr_count_host = vec![0i32; 1]; 78 | gpu_memory_pointers 79 | .buffer_valid_corr_count_fast 80 | .read(&mut valid_corr_count_host) 81 | .enq()?; 82 | let valid_corr_count = valid_corr_count_host[0] as usize; 83 | // gather those points 84 | if valid_corr_count == 0 { 85 | let final_results: Vec<(u32, u32, f32)> = Vec::new(); 86 | 87 | return Ok(final_results); 88 | } 89 | 90 | let new_global_work_size = valid_corr_count * workgroup_size as usize; 91 | // Some temporary value determined to limit count of threads - almost i32::max 92 | // if new_global_work_size >= 2_000_000_000 { 93 | // return Err(ocl::Error::from("Too high global work size on slow pass. Try tuning your segmentation threshold higher up or use smaller template")); 94 | // } 95 | 96 | let v2_kernel_slow_pass = Kernel::builder() 97 | .program(&program) 98 | .name("v2_segmented_match_integral_slow_pass") 99 | .queue(queue.clone()) 100 | .global_work_size(new_global_work_size) 101 | .arg(&gpu_memory_pointers.buffer_image_integral) 102 | .arg(&gpu_memory_pointers.buffer_image_integral_squared) 103 | .arg(&gpu_memory_pointers.segments_slow_buffer) 104 | .arg(&gpu_memory_pointers.segment_slow_values_buffer) 105 | .arg(&slow_segment_count) 106 | .arg(&(segments_mean_slow as f32)) 107 | .arg(&(segments_sum_squared_deviation_slow as f32)) 108 | .arg(&gpu_memory_pointers.buffer_results_slow_positions_v2) 109 | .arg(&gpu_memory_pointers.buffer_results_slow_corrs_v2) 110 | .arg(&(image_width as i32)) 111 | .arg(&(image_height as i32)) 112 | .arg(&(template_width as i32)) 113 | .arg(&(template_height as i32)) 114 | .arg(&(slow_expected_corr as f32)) 115 | .arg(&remainder_segments_slow) 116 | .arg(&segments_processed_by_thread_slow) 117 | .arg(&workgroup_size) 118 | .arg_local::(1) // sum_template_region_buff 119 | .arg_local::(1) // sum_sq_template_region_buff 120 | .arg_local::(workgroup_size as usize) // thread_segment_sum_buff 121 | .arg(&gpu_memory_pointers.buffer_valid_corr_count_slow) 122 | .arg(&gpu_memory_pointers.buffer_valid_corr_count_fast) // <-- atomic int 123 | .arg(&gpu_memory_pointers.buffer_results_fast_v2) 124 | .arg(&gpu_memory_pointers.buffer_precision) 125 | .build()?; 126 | unsafe { 127 | v2_kernel_slow_pass.enq()?; 128 | } 129 | 130 | let mut valid_corr_count_host_slow = vec![0i32; 1]; 131 | gpu_memory_pointers 132 | .buffer_valid_corr_count_slow 133 | .read(&mut valid_corr_count_host_slow) 134 | .enq()?; 135 | let valid_corr_count_slow = valid_corr_count_host_slow[0] as usize; 136 | if valid_corr_count_slow > 0 { 137 | let mut slow_pass_corrs = vec![0.0; valid_corr_count_slow]; 138 | let mut slow_pass_positions = vec![ocl::core::Int2::zero(); valid_corr_count_slow]; 139 | gpu_memory_pointers 140 | .buffer_results_slow_positions_v2 141 | .read(&mut slow_pass_positions) 142 | .enq()?; 143 | 144 | gpu_memory_pointers 145 | .buffer_results_slow_corrs_v2 146 | .read(&mut slow_pass_corrs) 147 | .enq()?; 148 | gpu_memory_pointers 149 | .buffer_results_slow_corrs_v2 150 | .write(&vec![0.0f32; valid_corr_count_slow]) 151 | .enq()?; 152 | 153 | gpu_memory_pointers 154 | .buffer_results_slow_positions_v2 155 | .write(&vec![Int2::zero(); valid_corr_count_slow]) 156 | .enq()?; 157 | 158 | let mut result_vec: Vec<(u32, u32, f32)> = slow_pass_positions 159 | .iter() 160 | .zip(slow_pass_corrs.iter()) 161 | .map(|(pos, &corr)| (pos[0] as u32, pos[1] as u32, corr)) 162 | .collect(); 163 | result_vec.retain(|&(_, _, value)| value >= (slow_expected_corr - 0.01) * precision); 164 | 165 | result_vec 166 | .sort_unstable_by(|a, b| b.2.partial_cmp(&a.2).unwrap_or(std::cmp::Ordering::Equal)); 167 | return Ok(result_vec); 168 | } else { 169 | let final_results: Vec<(u32, u32, f32)> = Vec::new(); 170 | return Ok(final_results); 171 | } 172 | } 173 | -------------------------------------------------------------------------------- /src/core/template_match/slow_ncc.rs: -------------------------------------------------------------------------------- 1 | use crate::core::template_match::{ 2 | compute_integral_image, compute_squared_integral_image, sum_region, 3 | }; 4 | 5 | use crate::imgtools; 6 | use rayon::prelude::*; 7 | 8 | use image::{ImageBuffer, Luma}; 9 | 10 | #[allow(dead_code)] 11 | pub fn slow_ncc_template_match( 12 | image: &ImageBuffer, Vec>, 13 | template: &ImageBuffer, Vec>, 14 | ) -> (u32, u32, f64) { 15 | let (image_width, image_height) = image.dimensions(); 16 | let (template_width, template_height) = template.dimensions(); 17 | let mut best_match_x = 0; 18 | let mut best_match_y = 0; 19 | let mut min_corr = -100.0; 20 | 21 | // Compute integral images 22 | let image_vec: Vec> = imgtools::imagebuffer_to_vec(image); 23 | 24 | let image_integral = compute_integral_image(&image_vec); 25 | let squared_image_integral = compute_squared_integral_image(&image_vec); 26 | 27 | let mut sum_template = 0; 28 | let mut sum_squared_template = 0; 29 | 30 | for y in 0..template_height { 31 | for x in 0..template_width { 32 | let template_value = template.get_pixel(x, y)[0] as f64; 33 | sum_template += template_value as u64; 34 | sum_squared_template += (template_value as u64).pow(2); 35 | } 36 | } 37 | 38 | // let mean_template_value = 39 | // sum_template as f64 / (template_height as f64 * template_width as f64); 40 | 41 | // let mut template_sum_squared_deviations = 0.0; 42 | // for y in 0..template_height { 43 | // for x in 0..template_width { 44 | // let template_value = template.get_pixel(x, y)[0] as f64; 45 | // let squared_deviation = (template_value - mean_template_value).powf(2.0); 46 | // template_sum_squared_deviations += squared_deviation; 47 | // } 48 | // } 49 | 50 | let template_sum_squared_deviations = sum_squared_template as f64 51 | - (sum_template as f64).powi(2) / (template_height as f64 * template_width as f64); //@audit unlikely but check if template_height*template_width != 0, also powi() result varies on platform, but think it is CPU so it will be deterministic after all if not used on Raspberry Pi for example 52 | 53 | // Slide the template over the image 54 | let coords: Vec<(u32, u32)> = (0..=(image_height - template_height)) //@audit if image_height is 0 this will underflow 55 | .flat_map(|y| (0..=(image_width - template_width)).map(move |x| (x, y))) //@audit if image_width is 0 this will underflow 56 | .collect(); 57 | 58 | let results: Vec<(u32, u32, f64)> = coords 59 | .par_iter() 60 | .map(|&(x, y)| { 61 | let corr = calculate_corr_value( 62 | image, 63 | &image_integral, 64 | &squared_image_integral, 65 | template, 66 | template_width, 67 | template_height, 68 | sum_template, 69 | template_sum_squared_deviations, 70 | x, 71 | y, 72 | ); 73 | (x, y, corr) 74 | }) 75 | .collect(); 76 | for (x, y, corr) in results { 77 | if corr > min_corr { 78 | min_corr = corr; 79 | best_match_x = x; 80 | best_match_y = y; 81 | } 82 | } 83 | (best_match_x, best_match_y, min_corr) 84 | } 85 | 86 | fn calculate_corr_value( 87 | image: &ImageBuffer, Vec>, 88 | image_integral: &[Vec], 89 | squared_image_integral: &[Vec], 90 | template: &ImageBuffer, Vec>, 91 | template_width: u32, 92 | template_height: u32, 93 | sum_template: u64, 94 | template_sum_squared_deviations: f64, 95 | x: u32, // big image x value 96 | y: u32, // big image y value 97 | ) -> f64 { 98 | let sum_image: u64 = sum_region(image_integral, x, y, template_width, template_height); 99 | let sum_squared_image: u64 = sum_region( 100 | squared_image_integral, 101 | x, 102 | y, 103 | template_width, 104 | template_height, 105 | ); 106 | let mean_image = sum_image as f64 / (template_height * template_width) as f64; 107 | let mean_template = sum_template as f64 / (template_height * template_width) as f64; 108 | let mut numerator = 0.0; 109 | for y1 in 0..template_height { 110 | for x1 in 0..template_width { 111 | let image_pixel = image.get_pixel(x + x1, y + y1)[0]; 112 | let template_pixel = template.get_pixel(x1, y1)[0]; 113 | 114 | numerator += 115 | (image_pixel as f64 - mean_image) * (template_pixel as f64 - mean_template); 116 | } 117 | } 118 | 119 | let image_sum_squared_deviations = sum_squared_image as f64 120 | - (sum_image as f64).powi(2) / (template_height * template_width) as f64; //@audit same as above 121 | let denominator = (image_sum_squared_deviations * template_sum_squared_deviations).sqrt(); 122 | 123 | let corr = numerator / denominator; 124 | 125 | corr.abs() 126 | } 127 | -------------------------------------------------------------------------------- /src/data/mod.rs: -------------------------------------------------------------------------------- 1 | #[cfg(feature = "opencl")] 2 | pub mod opencl; 3 | #[cfg(not(feature = "lite"))] 4 | use crate::MatchMode; 5 | use crate::RustAutoGui; 6 | #[cfg(feature = "opencl")] 7 | pub use opencl::*; 8 | #[cfg(not(feature = "lite"))] 9 | use rustfft::{num_complex::Complex, num_traits::ToPrimitive}; 10 | 11 | #[cfg(not(feature = "lite"))] 12 | use image::{ImageBuffer, Luma}; 13 | #[cfg(not(feature = "lite"))] 14 | use std::collections::HashMap; 15 | 16 | #[cfg(not(feature = "lite"))] 17 | pub struct TemplateMatchingData { 18 | pub template: Option, Vec>>, 19 | pub prepared_data: PreparedData, // used direct load and search 20 | pub prepared_data_stored: HashMap, //prepared data, region, matchmode 21 | pub match_mode: Option, 22 | pub region: (u32, u32, u32, u32), 23 | pub alias_used: String, 24 | } 25 | 26 | #[cfg(not(feature = "lite"))] 27 | pub struct BackupData { 28 | pub starting_data: PreparedData, 29 | pub starting_region: (u32, u32, u32, u32), 30 | pub starting_match_mode: Option, 31 | pub starting_template_height: u32, 32 | pub starting_template_width: u32, 33 | pub starting_alias_used: String, 34 | } 35 | #[cfg(not(feature = "lite"))] 36 | impl BackupData { 37 | pub fn update_rustautogui(self, target: &mut RustAutoGui) { 38 | target.template_data.prepared_data = self.starting_data.clone(); 39 | target.template_data.region = self.starting_region; 40 | target.template_data.match_mode = self.starting_match_mode; 41 | target.screen.screen_data.screen_region_width = self.starting_region.2; 42 | target.screen.screen_data.screen_region_height = self.starting_region.3; 43 | target.template_width = self.starting_template_width; 44 | target.template_height = self.starting_template_height; 45 | target.template_data.alias_used = self.starting_alias_used; 46 | } 47 | } 48 | 49 | #[cfg(not(feature = "lite"))] 50 | pub enum PreparedData { 51 | Segmented(SegmentedData), 52 | FFT(FFTData), 53 | None, 54 | } 55 | #[cfg(not(feature = "lite"))] 56 | impl Clone for PreparedData { 57 | fn clone(&self) -> Self { 58 | match self { 59 | PreparedData::Segmented(data) => PreparedData::Segmented(data.clone()), 60 | PreparedData::FFT(data) => PreparedData::FFT(data.clone()), 61 | PreparedData::None => PreparedData::None, 62 | } 63 | } 64 | } 65 | 66 | #[cfg(not(feature = "lite"))] 67 | pub struct SegmentedData { 68 | pub template_segments_fast: Vec<(u32, u32, u32, u32, f32)>, 69 | pub template_segments_slow: Vec<(u32, u32, u32, u32, f32)>, 70 | pub template_width: u32, 71 | pub template_height: u32, 72 | pub segment_sum_squared_deviations_fast: f32, 73 | pub segment_sum_squared_deviations_slow: f32, 74 | pub expected_corr_fast: f32, 75 | pub expected_corr_slow: f32, 76 | pub segments_mean_fast: f32, 77 | pub segments_mean_slow: f32, 78 | } 79 | #[cfg(not(feature = "lite"))] 80 | impl Clone for SegmentedData { 81 | fn clone(&self) -> Self { 82 | Self { 83 | template_segments_fast: self.template_segments_fast.clone(), 84 | template_segments_slow: self.template_segments_slow.clone(), 85 | template_width: self.template_width, 86 | template_height: self.template_height, 87 | segment_sum_squared_deviations_fast: self.segment_sum_squared_deviations_fast, 88 | segment_sum_squared_deviations_slow: self.segment_sum_squared_deviations_slow, 89 | expected_corr_fast: self.expected_corr_fast, 90 | expected_corr_slow: self.expected_corr_slow, 91 | segments_mean_fast: self.segments_mean_fast, 92 | segments_mean_slow: self.segments_mean_slow, 93 | } 94 | } 95 | } 96 | #[cfg(not(feature = "lite"))] 97 | pub struct FFTData { 98 | pub template_conj_freq: Vec>, 99 | pub template_sum_squared_deviations: f32, 100 | pub template_width: u32, 101 | pub template_height: u32, 102 | pub padded_size: u32, 103 | } 104 | #[cfg(not(feature = "lite"))] 105 | impl Clone for FFTData { 106 | fn clone(&self) -> Self { 107 | Self { 108 | template_conj_freq: self.template_conj_freq.clone(), 109 | template_sum_squared_deviations: self.template_sum_squared_deviations, 110 | template_width: self.template_width, 111 | template_height: self.template_height, 112 | padded_size: self.padded_size, 113 | } 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /src/data/opencl.rs: -------------------------------------------------------------------------------- 1 | use crate::AutoGuiError; 2 | use ocl::{Buffer, Context, Program, Queue}; 3 | use std::collections::HashMap; 4 | 5 | pub struct OpenClData { 6 | pub device_list: Vec, 7 | pub ocl_program: Program, 8 | #[allow(dead_code)] 9 | pub ocl_context: Context, 10 | pub ocl_queue: Queue, 11 | pub ocl_buffer_storage: HashMap, 12 | pub ocl_kernel_storage: HashMap, 13 | pub ocl_workgroup_size: u32, 14 | } 15 | 16 | #[derive(Debug)] 17 | pub struct KernelStorage { 18 | pub v1_kernel: ocl::Kernel, 19 | pub v2_kernel_fast: ocl::Kernel, 20 | } 21 | 22 | impl KernelStorage { 23 | pub fn new( 24 | gpu_memory_pointers: &GpuMemoryPointers, 25 | program: &ocl::Program, 26 | queue: &ocl::Queue, 27 | image_width: u32, 28 | image_height: u32, 29 | template_width: u32, 30 | template_height: u32, 31 | fast_segment_count: u32, 32 | slow_segment_count: u32, 33 | segments_mean_fast: f32, 34 | segments_mean_slow: f32, 35 | segment_sum_squared_deviation_fast: f32, 36 | segment_sum_squared_deviation_slow: f32, 37 | fast_expected_corr: f32, 38 | max_workgroup_size: usize, 39 | ) -> Result { 40 | let result_width = (image_width - template_width + 1) as usize; 41 | let result_height = (image_height - template_height + 1) as usize; 42 | let output_size = result_width * result_height; 43 | let kernel_v1 = ocl::Kernel::builder() 44 | .program(&program) 45 | .name("segmented_match_integral") 46 | .queue(queue.clone()) 47 | .global_work_size(output_size) 48 | .arg(&gpu_memory_pointers.buffer_image_integral) 49 | .arg(&gpu_memory_pointers.buffer_image_integral_squared) 50 | .arg(&gpu_memory_pointers.segments_fast_buffer) 51 | .arg(&gpu_memory_pointers.segments_slow_buffer) 52 | .arg(&gpu_memory_pointers.segment_fast_values_buffer) 53 | .arg(&gpu_memory_pointers.segment_slow_values_buffer) 54 | .arg(&(fast_segment_count as i32)) 55 | .arg(&(slow_segment_count as i32)) 56 | .arg(&(segments_mean_fast as f32)) 57 | .arg(&(segments_mean_slow as f32)) 58 | .arg(&(segment_sum_squared_deviation_fast as f32)) 59 | .arg(&(segment_sum_squared_deviation_slow as f32)) 60 | .arg(&gpu_memory_pointers.results_buffer) 61 | .arg(&(image_width as i32)) 62 | .arg(&(image_height as i32)) 63 | .arg(&(template_width as i32)) 64 | .arg(&(template_height as i32)) 65 | .arg(&(fast_expected_corr as f32 - 0.01)) 66 | .arg(&gpu_memory_pointers.buffer_precision) 67 | .build()?; 68 | 69 | let mut remainder_segments_fast = 0; 70 | 71 | let mut segments_processed_by_thread_fast = 1; 72 | 73 | let mut pixels_processed_by_workgroup = 1; 74 | let max_workgroup_size = max_workgroup_size; 75 | 76 | // if we have more segments than workgroup size, then that workgroup only processes 77 | // that single pixel. Each thread inside workgroup processes certain amount of equally distributed segments 78 | if fast_segment_count as usize > max_workgroup_size { 79 | segments_processed_by_thread_fast = fast_segment_count as usize / max_workgroup_size; 80 | remainder_segments_fast = (fast_segment_count as usize % max_workgroup_size) as i32; 81 | // else, if we have low thread count then 1 workgroup can process multiple pixels. IE workgroup with 256 threads 82 | // can process 64 pixels with 4 segments 83 | } else { 84 | pixels_processed_by_workgroup = max_workgroup_size / fast_segment_count as usize; 85 | // threads per pixel = fast_segmented_count 86 | } 87 | let global_workgroup_count = 88 | (output_size + pixels_processed_by_workgroup - 1) / pixels_processed_by_workgroup; 89 | // total amount of threads that need to be spawned 90 | let global_work_size = global_workgroup_count as usize * max_workgroup_size; 91 | 92 | // if the workgroup finds a succesfull correlation with fast pass, it will have to calculate it 93 | // with the slow pass aswell for that same x,y pos. But if we had low fast segment count 94 | // that workgroup will not be utilized nicely. Will have to rework this part 95 | 96 | let v2_kernel_fast_pass = ocl::Kernel::builder() 97 | .program(&program) 98 | .name("v2_segmented_match_integral_fast_pass") 99 | .queue(queue.clone()) 100 | .global_work_size(global_work_size) 101 | .arg(&gpu_memory_pointers.buffer_image_integral) 102 | .arg(&gpu_memory_pointers.buffer_image_integral_squared) 103 | .arg(&gpu_memory_pointers.segments_fast_buffer) 104 | .arg(&gpu_memory_pointers.segment_fast_values_buffer) 105 | .arg(&(fast_segment_count as i32)) 106 | .arg(&(segments_mean_fast as f32)) 107 | .arg(&(segment_sum_squared_deviation_fast as f32)) 108 | .arg(&gpu_memory_pointers.buffer_results_fast_v2) ///////////////////////CHANGE THIS TO ONE FROM GPUMEMPOINTERS STRUCT 109 | .arg(&(image_width as i32)) 110 | .arg(&(image_height as i32)) 111 | .arg(&(template_width as i32)) 112 | .arg(&(template_height as i32)) 113 | .arg(&(fast_expected_corr as f32) - 0.01) 114 | .arg(&remainder_segments_fast) 115 | .arg(&(segments_processed_by_thread_fast as i32)) 116 | .arg(&(pixels_processed_by_workgroup as i32)) 117 | .arg(&(max_workgroup_size as i32)) 118 | .arg_local::(pixels_processed_by_workgroup) // sum_template_region_buff 119 | .arg_local::(pixels_processed_by_workgroup) // sum_sq_template_region_buff 120 | .arg_local::(max_workgroup_size) // thread_segment_sum_buff 121 | .arg(&gpu_memory_pointers.buffer_valid_corr_count_fast) // <-- atomic int 122 | .arg(&gpu_memory_pointers.buffer_precision) 123 | .build()?; 124 | 125 | Ok(Self { 126 | v1_kernel: kernel_v1, 127 | v2_kernel_fast: v2_kernel_fast_pass, 128 | }) 129 | } 130 | } 131 | 132 | #[derive(Debug)] 133 | pub struct GpuMemoryPointers { 134 | pub segments_fast_buffer: Buffer, 135 | pub segments_slow_buffer: Buffer, 136 | pub segment_fast_values_buffer: Buffer, 137 | pub segment_slow_values_buffer: Buffer, 138 | pub results_buffer: Buffer, 139 | pub buffer_image_integral: Buffer, 140 | pub buffer_image_integral_squared: Buffer, 141 | pub buffer_results_fast_v2: Buffer, 142 | pub buffer_results_slow_positions_v2: Buffer, 143 | pub buffer_results_slow_corrs_v2: Buffer, 144 | pub buffer_valid_corr_count_fast: Buffer, 145 | pub buffer_valid_corr_count_slow: Buffer, 146 | pub buffer_precision: Buffer, 147 | } 148 | impl GpuMemoryPointers { 149 | pub fn new( 150 | image_width: u32, 151 | image_height: u32, 152 | template_width: u32, 153 | template_height: u32, 154 | queue: &Queue, 155 | template_segments_slow: &[(u32, u32, u32, u32, f32)], 156 | template_segments_fast: &[(u32, u32, u32, u32, f32)], 157 | ) -> Result { 158 | let result_width = (image_width - template_width + 1) as usize; 159 | let result_height = (image_height - template_height + 1) as usize; 160 | let output_size = result_width * result_height; 161 | let segment_fast_int4: Vec = template_segments_fast 162 | .iter() 163 | .map(|&(x, y, w, h, _)| ocl::prm::Int4::new(x as i32, y as i32, w as i32, h as i32)) 164 | .collect(); 165 | 166 | let segment_slow_int4: Vec = template_segments_slow 167 | .iter() 168 | .map(|&(x, y, w, h, _)| ocl::prm::Int4::new(x as i32, y as i32, w as i32, h as i32)) 169 | .collect(); 170 | 171 | let segment_values_fast: Vec = template_segments_fast 172 | .iter() 173 | .map(|&(_, _, _, _, v)| v) 174 | .collect(); 175 | let segment_values_slow: Vec = template_segments_slow 176 | .iter() 177 | .map(|&(_, _, _, _, v)| v) 178 | .collect(); 179 | 180 | let buffer_segments_fast: Buffer = Buffer::::builder() 181 | .queue(queue.clone()) 182 | .len(segment_fast_int4.len()) 183 | .copy_host_slice(&segment_fast_int4) 184 | .build()?; 185 | 186 | let buffer_segments_slow: Buffer = Buffer::::builder() 187 | .queue(queue.clone()) 188 | .len(segment_slow_int4.len()) 189 | .copy_host_slice(&segment_slow_int4) 190 | .build()?; 191 | 192 | let buffer_segment_values_fast: Buffer = Buffer::::builder() 193 | .queue(queue.clone()) 194 | .len(segment_values_fast.len()) 195 | .copy_host_slice(&segment_values_fast) 196 | .build()?; 197 | 198 | let buffer_segment_values_slow: Buffer = Buffer::::builder() 199 | .queue(queue.clone()) 200 | .len(segment_values_slow.len()) 201 | .copy_host_slice(&segment_values_slow) 202 | .build()?; 203 | 204 | let buffer_results = Buffer::::builder() 205 | .queue(queue.clone()) 206 | .len(output_size) 207 | .build()?; 208 | 209 | let buffer_image_integral = Buffer::::builder() 210 | .queue(queue.clone()) 211 | .len(image_width * image_height) 212 | .build()?; 213 | 214 | let buffer_image_integral_squared = Buffer::::builder() 215 | .queue(queue.clone()) 216 | .len(image_width * image_height) 217 | .build()?; 218 | 219 | // BUFFERS FOR v2 ALGORITHM ADDITIONALLY 220 | let buffer_results_fast = Buffer::::builder() 221 | .queue(queue.clone()) 222 | .len(output_size) 223 | .build()?; 224 | 225 | let buffer_results_slow_positions = Buffer::::builder() 226 | .queue(queue.clone()) 227 | .len(output_size) 228 | .build()?; 229 | 230 | let buffer_results_slow_corrs = Buffer::::builder() 231 | .queue(queue.clone()) 232 | .len(output_size) 233 | .build()?; 234 | 235 | let valid_corr_count_buf_fast: Buffer = Buffer::builder() 236 | .queue(queue.clone()) 237 | .flags(ocl::flags::MEM_READ_WRITE) 238 | .len(1) 239 | .fill_val(0i32) // Init to 0 240 | .build()?; 241 | 242 | let valid_corr_count_buf_slow: Buffer = Buffer::builder() 243 | .queue(queue.clone()) 244 | .flags(ocl::flags::MEM_READ_WRITE) 245 | .len(1) 246 | .fill_val(0i32) // Init to 0 247 | .build()?; 248 | 249 | let precision_buff: Buffer = Buffer::builder() 250 | .queue(queue.clone()) 251 | .flags(ocl::flags::MEM_READ_WRITE) 252 | .len(1) 253 | .fill_val(0.99) // Init to 0 254 | .build()?; 255 | 256 | Ok(Self { 257 | segments_fast_buffer: buffer_segments_fast, 258 | segments_slow_buffer: buffer_segments_slow, 259 | segment_fast_values_buffer: buffer_segment_values_fast, 260 | segment_slow_values_buffer: buffer_segment_values_slow, 261 | results_buffer: buffer_results, 262 | buffer_image_integral, 263 | buffer_image_integral_squared, 264 | buffer_results_fast_v2: buffer_results_fast, 265 | buffer_results_slow_positions_v2: buffer_results_slow_positions, 266 | buffer_results_slow_corrs_v2: buffer_results_slow_corrs, 267 | buffer_valid_corr_count_fast: valid_corr_count_buf_fast, 268 | buffer_valid_corr_count_slow: valid_corr_count_buf_slow, 269 | buffer_precision: precision_buff, 270 | }) 271 | } 272 | } 273 | 274 | #[derive(Debug)] 275 | pub struct DevicesInfo { 276 | #[allow(dead_code)] 277 | device: ocl::Device, 278 | pub index: u32, 279 | pub global_mem_size: u32, 280 | pub clock_frequency: u32, 281 | pub compute_units: u32, 282 | pub brand: String, 283 | pub name: String, 284 | pub score: u32, 285 | } 286 | #[cfg(feature = "opencl")] 287 | impl DevicesInfo { 288 | pub fn new( 289 | device: ocl::Device, 290 | index: u32, 291 | global_mem_size: u32, 292 | clock_frequency: u32, 293 | compute_units: u32, 294 | brand: String, 295 | name: String, 296 | score: u32, 297 | ) -> Self { 298 | Self { 299 | device, 300 | index, 301 | global_mem_size, 302 | clock_frequency, 303 | compute_units, 304 | brand, 305 | name, 306 | score, 307 | } 308 | } 309 | pub fn print_device(&self) -> String { 310 | format!( 311 | "Device brand: {}, name: {}\nMemory: {} MB, Compute units: {}, Clock Freq :{} mhz, index: {}, score: {}", 312 | self.brand, self.name, self.global_mem_size, self.compute_units, self.clock_frequency, self.index, self.score 313 | ) 314 | } 315 | } 316 | -------------------------------------------------------------------------------- /src/errors.rs: -------------------------------------------------------------------------------- 1 | use std::{ 2 | ffi::{CString, NulError}, 3 | fmt, 4 | }; 5 | 6 | #[cfg(feature = "opencl")] 7 | use ocl; 8 | 9 | #[derive(Debug)] 10 | pub enum AutoGuiError { 11 | OSFailure(String), 12 | UnSupportedKey(String), 13 | IoError(std::io::Error), 14 | AliasError(String), 15 | OutOfBoundsError(String), 16 | #[cfg(not(feature = "lite"))] 17 | ImageError(ImageProcessingError), 18 | ImgError(String), 19 | NulError(NulError), 20 | #[cfg(feature = "opencl")] 21 | OclError(ocl::Error), 22 | } 23 | 24 | impl fmt::Display for AutoGuiError { 25 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 26 | match self { 27 | AutoGuiError::OSFailure(err) => write!(f, "OS Failure: {}", err), 28 | AutoGuiError::UnSupportedKey(err) => write!(f, "Key not supported: {}", err), 29 | AutoGuiError::IoError(err) => write!(f, "IO Error: {}", err), 30 | AutoGuiError::AliasError(err) => write!(f, "Alias Error: {}", err), 31 | AutoGuiError::OutOfBoundsError(err) => write!(f, "Out of bounds error: {}", err), 32 | #[cfg(not(feature = "lite"))] 33 | AutoGuiError::ImageError(err) => write!(f, "Image Error: {}", err), 34 | AutoGuiError::ImgError(err) => write!(f, "Image Error: {}", err), 35 | AutoGuiError::NulError(err) => write!(f, "Convert to C String nulerror: {}", err), 36 | #[cfg(feature = "opencl")] 37 | AutoGuiError::OclError(err) => write!(f, "OpenCL Error: {}", err), 38 | } 39 | } 40 | } 41 | 42 | impl From for AutoGuiError { 43 | fn from(err: NulError) -> Self { 44 | AutoGuiError::NulError(err) 45 | } 46 | } 47 | #[cfg(not(feature = "lite"))] 48 | impl From for AutoGuiError { 49 | fn from(err: image::ImageError) -> Self { 50 | AutoGuiError::ImageError(ImageProcessingError::External(err)) 51 | } 52 | } 53 | #[cfg(not(feature = "lite"))] 54 | impl From for AutoGuiError { 55 | fn from(err: ImageProcessingError) -> Self { 56 | AutoGuiError::ImageError(err) 57 | } 58 | } 59 | 60 | impl From for AutoGuiError { 61 | fn from(err: std::io::Error) -> Self { 62 | AutoGuiError::IoError(err) 63 | } 64 | } 65 | #[cfg(feature = "opencl")] 66 | impl From for AutoGuiError { 67 | fn from(err: ocl::Error) -> Self { 68 | AutoGuiError::OclError(err) 69 | } 70 | } 71 | 72 | #[derive(Debug)] 73 | #[cfg(not(feature = "lite"))] 74 | pub enum ImageProcessingError { 75 | External(image::ImageError), 76 | Custom(String), 77 | } 78 | #[cfg(not(feature = "lite"))] 79 | impl ImageProcessingError { 80 | pub fn new(msg: &str) -> Self { 81 | Self::Custom(msg.to_string()) 82 | } 83 | } 84 | #[cfg(not(feature = "lite"))] 85 | impl fmt::Display for ImageProcessingError { 86 | fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { 87 | match self { 88 | ImageProcessingError::External(err) => write!(f, "{}", err), 89 | ImageProcessingError::Custom(err) => write!(f, "{}", err), 90 | } 91 | } 92 | } 93 | 94 | impl std::error::Error for AutoGuiError {} 95 | #[cfg(not(feature = "lite"))] 96 | impl std::error::Error for ImageProcessingError {} 97 | -------------------------------------------------------------------------------- /src/imgtools/mod.rs: -------------------------------------------------------------------------------- 1 | /* 2 | Functions used throughout the code that have more of a general purpose, like 3 | loading images from disk, converting image to black-white or RGB, cutting image 4 | and converting image to vector. 5 | */ 6 | use crate::errors::AutoGuiError; 7 | #[cfg(not(feature = "lite"))] 8 | use image::{ 9 | error::LimitError, DynamicImage, GrayImage, ImageBuffer, Luma, Pixel, Primitive, Rgb, Rgba, 10 | }; 11 | #[cfg(not(feature = "lite"))] 12 | use rustfft::{num_complex::Complex, num_traits::ToPrimitive}; 13 | 14 | #[cfg(not(feature = "lite"))] 15 | /// Loads image from the provided path and converts to black-white format 16 | /// Returns image in image::ImageBuffer format 17 | pub fn load_image_bw(location: &str) -> Result, Vec>, AutoGuiError> { 18 | let img = image::ImageReader::open(location)?; 19 | 20 | let img = img.decode()?; 21 | 22 | let gray_image: ImageBuffer, Vec> = img.to_luma8(); 23 | Ok(gray_image) 24 | } 25 | #[cfg(not(feature = "lite"))] 26 | /// Loads image from the provided path and converts to RGBA format 27 | /// Returns image in image::ImageBuffer format 28 | pub fn load_image_rgba(location: &str) -> Result, Vec>, AutoGuiError> { 29 | let img = image::ImageReader::open(location)?; 30 | let img = img.decode()?; 31 | Ok(img.to_rgba8()) // return rgba image 32 | } 33 | #[cfg(not(feature = "lite"))] 34 | pub fn check_imagebuffer_color_scheme( 35 | image: &ImageBuffer>, 36 | ) -> Result 37 | where 38 | P: Pixel + 'static, 39 | T: Primitive + ToPrimitive + 'static, 40 | { 41 | let buff_len = image.as_raw().len() as u32; 42 | let (img_w, img_h) = image.dimensions(); 43 | if (img_w * img_h) == 0 { 44 | let err = "Error: The buffer provided is empty and has no size".to_string(); 45 | return Err(AutoGuiError::ImgError(err)); 46 | } 47 | Ok(buff_len / (img_w * img_h)) 48 | } 49 | #[cfg(not(feature = "lite"))] 50 | pub fn convert_t_imgbuffer_to_luma( 51 | image: &ImageBuffer>, 52 | color_scheme: u32, 53 | ) -> Result, Vec>, AutoGuiError> 54 | where 55 | P: Pixel + 'static, 56 | T: Primitive + ToPrimitive + 'static, 57 | { 58 | let (img_w, img_h) = image.dimensions(); 59 | match color_scheme { 60 | 1 => { 61 | // Black and white image (Luma) 62 | // convert from Vec to Vec 63 | let raw_img: Result, AutoGuiError> = image 64 | .as_raw() 65 | .iter() 66 | .map(|x| { 67 | x.to_u8().ok_or(AutoGuiError::ImgError( 68 | "Pixel conversion to raw failed".to_string(), 69 | )) 70 | }) 71 | .collect(); 72 | 73 | ImageBuffer::, Vec>::from_raw(img_w, img_h, raw_img?).ok_or( 74 | AutoGuiError::ImgError("failed to convert to Luma".to_string()), 75 | ) 76 | } 77 | 3 => { 78 | // Rgb 79 | let raw_img: Result, AutoGuiError> = image 80 | .as_raw() 81 | .iter() 82 | .map(|x| { 83 | x.to_u8().ok_or(AutoGuiError::ImgError( 84 | "Pixel conversion to raw failed".to_string(), 85 | )) 86 | }) 87 | .collect(); 88 | let rgb_img = ImageBuffer::, Vec>::from_raw(img_w, img_h, raw_img?).ok_or( 89 | AutoGuiError::ImgError("Failed conversion to RGB".to_string()), 90 | )?; 91 | Ok(DynamicImage::ImageRgb8(rgb_img).to_luma8()) 92 | } 93 | 4 => { 94 | // Rgba 95 | let raw_img: Result, AutoGuiError> = image 96 | .as_raw() 97 | .iter() 98 | .map(|x| { 99 | x.to_u8().ok_or(AutoGuiError::ImgError( 100 | "Pixel conversion to raw failed".to_string(), 101 | )) 102 | }) 103 | .collect(); 104 | let rgba_img = ImageBuffer::, Vec>::from_raw(img_w, img_h, raw_img?) 105 | .ok_or(AutoGuiError::ImgError( 106 | "Failed conversion to RGBA".to_string(), 107 | ))?; 108 | Ok(DynamicImage::ImageRgba8(rgba_img).to_luma8()) 109 | } 110 | _ => Err(AutoGuiError::ImgError( 111 | "Unknown image format. Load works only for Rgb/Rgba/Luma(BW) formats".to_string(), 112 | )), 113 | } 114 | } 115 | #[cfg(not(feature = "lite"))] 116 | /// Does conversion from ImageBuffer RGBA to ImageBuffer Black and White(Luma) 117 | pub fn convert_rgba_to_bw( 118 | image: ImageBuffer, Vec>, 119 | ) -> Result, Vec>, AutoGuiError> { 120 | let (img_w, img_h) = image.dimensions(); 121 | let raw_img: Result, AutoGuiError> = image 122 | .as_raw() 123 | .iter() 124 | .map(|x| { 125 | x.to_u8().ok_or(AutoGuiError::ImgError( 126 | "Pixel conversion to raw failed".to_string(), 127 | )) 128 | }) 129 | .collect(); 130 | let rgba_img = ImageBuffer::, Vec>::from_raw(img_w, img_h, raw_img?).ok_or( 131 | AutoGuiError::ImgError("Failed to convert to RGBA".to_string()), 132 | )?; 133 | Ok(DynamicImage::ImageRgba8(rgba_img).to_luma8()) 134 | } 135 | #[cfg(not(feature = "lite"))] 136 | /// Does conversion from ImageBuffer RGBA to ImageBuffer Black and White(Luma) 137 | pub fn convert_rgba_to_bw_old( 138 | image: ImageBuffer, Vec>, 139 | ) -> Result, Vec>, AutoGuiError> { 140 | let mut grayscale_data: Vec = Vec::with_capacity(image.len()); 141 | let image_width = image.width(); 142 | let image_height = image.height(); 143 | for chunk in image.chunks_exact(4) { 144 | let r = chunk[2] as u32; 145 | let g = chunk[1] as u32; 146 | let b = chunk[0] as u32; 147 | // Calculate the grayscale value using the luminance formula 148 | let gray_value = ((r * 30 + g * 59 + b * 11) / 100) as u8; 149 | grayscale_data.push(gray_value); 150 | } 151 | GrayImage::from_raw(image_width, image_height, grayscale_data).ok_or(AutoGuiError::ImgError( 152 | "Failed to convert to grayscale".to_string(), 153 | )) 154 | } 155 | 156 | #[cfg(not(feature = "lite"))] 157 | /// Cuts Region of image. Inputs are top left x , y pixel coordinates on image, 158 | /// width and height of region and the image being cut. 159 | /// Returns image os same datatype 160 | pub fn cut_screen_region( 161 | x: u32, 162 | y: u32, 163 | width: u32, 164 | height: u32, 165 | screen_image: &ImageBuffer>, 166 | ) -> ImageBuffer> 167 | where 168 | T: Pixel + 'static, 169 | { 170 | assert!(x + width <= screen_image.width()); 171 | assert!(y + height <= screen_image.height()); 172 | 173 | let mut sub_image: ImageBuffer> = ImageBuffer::new(width, height); 174 | 175 | // copy pixels from the original image buffer to the sub-image buffer 176 | for y_sub in 0..height { 177 | for x_sub in 0..width { 178 | let pixel = screen_image.get_pixel(x + x_sub, y + y_sub); 179 | sub_image.put_pixel(x_sub, y_sub, *pixel); 180 | } 181 | } 182 | sub_image 183 | } 184 | 185 | #[cfg(not(feature = "lite"))] 186 | ///Converts Imagebuffer to Vector format 187 | pub fn imagebuffer_to_vec( 188 | image: &ImageBuffer, Vec>, 189 | ) -> Vec> { 190 | let (width, height) = image.dimensions(); 191 | let zero_pixel = image.get_pixel(0, 0)[0]; 192 | let mut vec: Vec> = vec![vec![zero_pixel; width as usize]; height as usize]; 193 | 194 | for y in 0..height { 195 | for x in 0..width { 196 | vec[y as usize][x as usize] = image.get_pixel(x, y)[0]; 197 | } 198 | } 199 | vec 200 | } 201 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | #![allow(unused_doc_comments, unused_imports)] 2 | #![doc = include_str!("../README.md")] 3 | 4 | #[cfg(all(feature = "lite", feature = "opencl"))] 5 | compile_error!("Features `lite` and `opencl` cannot be enabled at the same time."); 6 | 7 | // Regular private modules 8 | #[cfg(not(any(test, feature = "dev")))] 9 | mod core; 10 | #[cfg(not(any(test, feature = "dev")))] 11 | mod data; 12 | 13 | // Public modules during testing 14 | #[cfg(any(test, feature = "dev"))] 15 | pub mod core; 16 | 17 | #[cfg(any(test, feature = "dev"))] 18 | pub mod data; 19 | 20 | pub mod errors; 21 | pub mod imgtools; 22 | mod rustautogui_impl; 23 | 24 | #[cfg(not(feature = "lite"))] 25 | use data::*; 26 | 27 | use crate::errors::*; 28 | use std::{collections::HashMap, env}; 29 | 30 | #[cfg(not(feature = "lite"))] 31 | use core::template_match; 32 | use core::{ 33 | keyboard::Keyboard, 34 | mouse::{mouse_position, Mouse, MouseScroll}, 35 | screen::Screen, 36 | }; 37 | 38 | // opencl stuff 39 | #[cfg(feature = "opencl")] 40 | use crate::data::{DevicesInfo, OpenClData}; 41 | #[cfg(feature = "opencl")] 42 | use ocl::{enums, Buffer, Context, Kernel, Program, Queue}; 43 | 44 | pub use core::mouse::mouse_position::print_mouse_position; 45 | pub use core::mouse::MouseClick; 46 | 47 | #[cfg(not(feature = "lite"))] 48 | const DEFAULT_ALIAS: &str = "default_rsgui_!#123#!"; 49 | #[cfg(not(feature = "lite"))] 50 | const DEFAULT_BCKP_ALIAS: &str = "bckp_tmpl_.#!123!#."; 51 | 52 | /// Matchmode Segmented correlation and Fourier transform correlation 53 | #[derive(PartialEq, Debug)] 54 | #[cfg(not(feature = "lite"))] 55 | pub enum MatchMode { 56 | Segmented, 57 | FFT, 58 | #[cfg(feature = "opencl")] 59 | SegmentedOcl, 60 | #[cfg(feature = "opencl")] 61 | SegmentedOclV2, 62 | } 63 | #[cfg(not(feature = "lite"))] 64 | impl Clone for MatchMode { 65 | fn clone(&self) -> Self { 66 | match self { 67 | MatchMode::Segmented => MatchMode::Segmented, 68 | MatchMode::FFT => MatchMode::FFT, 69 | #[cfg(feature = "opencl")] 70 | MatchMode::SegmentedOcl => MatchMode::SegmentedOcl, 71 | #[cfg(feature = "opencl")] 72 | MatchMode::SegmentedOclV2 => MatchMode::SegmentedOclV2, 73 | } 74 | } 75 | } 76 | 77 | /// Main struct for Rustautogui 78 | /// Struct gets assigned keyboard, mouse and struct to it implemented functions execute commands from each of assigned substructs 79 | /// executes also correlation algorithms when doing find_image_on_screen 80 | #[allow(dead_code)] 81 | pub struct RustAutoGui { 82 | #[cfg(not(feature = "lite"))] 83 | template_data: TemplateMatchingData, 84 | debug: bool, 85 | template_height: u32, 86 | template_width: u32, 87 | keyboard: Keyboard, 88 | mouse: Mouse, 89 | screen: Screen, 90 | 91 | suppress_warnings: bool, 92 | 93 | #[cfg(feature = "opencl")] 94 | opencl_data: OpenClData, 95 | } 96 | impl RustAutoGui { 97 | /// initiation of screen, keyboard and mouse that are assigned to new rustautogui struct. 98 | /// all the other struct fields are initiated as 0 or None 99 | pub fn new(debug: bool) -> Result { 100 | // initiation of screen, keyboard and mouse 101 | // on windows there is no need to share display pointer accross other structs 102 | #[cfg(any(target_os = "windows", target_os = "macos"))] 103 | let screen = Screen::new()?; 104 | #[cfg(any(target_os = "windows", target_os = "macos"))] 105 | let keyboard = Keyboard::new(); 106 | #[cfg(any(target_os = "windows", target_os = "macos"))] 107 | let mouse_struct: Mouse = Mouse::new(); 108 | 109 | #[cfg(target_os = "linux")] 110 | let screen = Screen::new(); 111 | #[cfg(target_os = "linux")] 112 | let keyboard = Keyboard::new(screen.display); 113 | #[cfg(target_os = "linux")] 114 | let mouse_struct: Mouse = Mouse::new(screen.display, screen.root_window); 115 | 116 | // check for env variable to suppress warnings, otherwise set default false value 117 | let suppress_warnings = env::var("RUSTAUTOGUI_SUPPRESS_WARNINGS") 118 | .map(|val| val == "1" || val.eq_ignore_ascii_case("true")) 119 | .unwrap_or(false); // Default: warnings are NOT suppressed 120 | 121 | // OCL INITIALIZATION 122 | #[cfg(feature = "opencl")] 123 | let opencl_data = Self::setup_opencl(None)?; 124 | 125 | #[cfg(not(feature = "lite"))] 126 | let template_match_data = TemplateMatchingData { 127 | template: None, 128 | prepared_data: PreparedData::None, 129 | prepared_data_stored: HashMap::new(), 130 | match_mode: None, 131 | region: (0, 0, 0, 0), 132 | alias_used: DEFAULT_ALIAS.to_string(), 133 | }; 134 | 135 | Ok(Self { 136 | #[cfg(not(feature = "lite"))] 137 | template_data: template_match_data, 138 | debug, 139 | template_width: 0, 140 | template_height: 0, 141 | keyboard, 142 | mouse: mouse_struct, 143 | screen, 144 | suppress_warnings, 145 | 146 | #[cfg(feature = "opencl")] 147 | opencl_data: opencl_data, 148 | }) 149 | } 150 | 151 | #[cfg(feature = "dev")] 152 | pub fn dev_setup_opencl(device_id: Option) -> Result { 153 | Self::setup_opencl(device_id) 154 | } 155 | 156 | #[cfg(feature = "opencl")] 157 | fn setup_opencl(device_id: Option) -> Result { 158 | let context = Context::builder().build()?; 159 | let available_devices = context.devices(); 160 | let device_count = available_devices.len(); 161 | let mut device_list: Vec = Vec::new(); 162 | let mut highest_score = 0; 163 | let mut best_device_index = 0; 164 | let mut max_workgroup_size = 0; 165 | for (i, device) in available_devices.into_iter().enumerate() { 166 | let device_type = device.info(enums::DeviceInfo::Type)?.to_string(); 167 | let workgroup_size: u32 = device 168 | .info(enums::DeviceInfo::MaxWorkGroupSize)? 169 | .to_string() 170 | .parse() 171 | .map_err(|_| AutoGuiError::OSFailure("Failed to read GPU data".to_string()))?; 172 | let global_mem: u64 = device 173 | .info(enums::DeviceInfo::GlobalMemSize)? 174 | .to_string() 175 | .parse() 176 | .map_err(|_| AutoGuiError::OSFailure("Failed to read GPU data".to_string()))?; 177 | let compute_units: u32 = device 178 | .info(enums::DeviceInfo::MaxComputeUnits)? 179 | .to_string() 180 | .parse() 181 | .map_err(|_| AutoGuiError::OSFailure("Failed to read GPU data".to_string()))?; 182 | 183 | let clock_frequency = device 184 | .info(enums::DeviceInfo::MaxClockFrequency)? 185 | .to_string() 186 | .parse() 187 | .map_err(|_| AutoGuiError::OSFailure("Failed to read GPU data".to_string()))?; 188 | let device_vendor = device.info(enums::DeviceInfo::Vendor)?.to_string(); 189 | let device_name = device.info(enums::DeviceInfo::Name)?.to_string(); 190 | let global_mem_gb = global_mem / 1_048_576; 191 | let score = global_mem_gb as u32 * 2 + compute_units * 10 + clock_frequency; 192 | let gui_device = DevicesInfo::new( 193 | device, 194 | i as u32, 195 | global_mem_gb as u32, 196 | clock_frequency, 197 | compute_units, 198 | device_vendor, 199 | device_name, 200 | score, 201 | ); 202 | 203 | device_list.push(gui_device); 204 | match device_id { 205 | Some(x) => { 206 | if x as usize > device_count { 207 | return Err(ocl::Error::from("No device found for the given index"))?; 208 | } 209 | if i == x as usize { 210 | highest_score = score; 211 | best_device_index = i; 212 | max_workgroup_size = workgroup_size; 213 | } 214 | } 215 | None => { 216 | if score >= highest_score && device_type.contains("GPU") { 217 | highest_score = score; 218 | best_device_index = i; 219 | max_workgroup_size = workgroup_size; 220 | } 221 | } 222 | } 223 | } 224 | let used_device = context.devices()[best_device_index as usize]; 225 | let queue = Queue::new(&context, used_device, None)?; 226 | let program_source = template_match::opencl_kernel::OCL_KERNEL; 227 | let program = Program::builder().src(program_source).build(&context)?; 228 | 229 | let opencl_data = OpenClData { 230 | device_list: device_list, 231 | ocl_program: program, 232 | ocl_context: context, 233 | ocl_queue: queue, 234 | ocl_buffer_storage: HashMap::new(), 235 | ocl_kernel_storage: HashMap::new(), 236 | ocl_workgroup_size: max_workgroup_size, 237 | }; 238 | Ok(opencl_data) 239 | } 240 | 241 | /// set true to turn off warnings. 242 | pub fn set_suppress_warnings(&mut self, suppress: bool) { 243 | self.suppress_warnings = suppress; 244 | } 245 | 246 | /// changes debug mode. True activates debug 247 | pub fn change_debug_state(&mut self, state: bool) { 248 | self.debug = state; 249 | } 250 | 251 | /// returns screen width and height 252 | pub fn get_screen_size(&mut self) -> (i32, i32) { 253 | self.screen.dimension() 254 | } 255 | #[cfg(not(feature = "lite"))] 256 | /// saves screenshot and saves it at provided path 257 | pub fn save_screenshot(&mut self, path: &str) -> Result<(), AutoGuiError> { 258 | self.screen.grab_screenshot(path)?; 259 | Ok(()) 260 | } 261 | #[cfg(feature = "opencl")] 262 | pub fn list_devices(&self) { 263 | for (i, item) in (&self.opencl_data.device_list).iter().enumerate() { 264 | println!("Device {i}:"); 265 | println!("{}", item.print_device()); 266 | println!("\n"); 267 | } 268 | } 269 | #[cfg(feature = "opencl")] 270 | pub fn change_ocl_device(&mut self, device_index: u32) -> Result<(), AutoGuiError> { 271 | let new_opencl_data = Self::setup_opencl(Some(device_index))?; 272 | self.opencl_data = new_opencl_data; 273 | 274 | self.template_data.template = None; 275 | self.template_data.prepared_data = PreparedData::None; 276 | self.template_data.prepared_data_stored = HashMap::new(); 277 | self.template_width = 0; 278 | self.template_height = 0; 279 | self.template_data.alias_used = DEFAULT_ALIAS.to_string(); 280 | self.template_data.region = (0, 0, 0, 0); 281 | self.template_data.match_mode = None; 282 | 283 | Ok(()) 284 | } 285 | #[cfg(not(feature = "lite"))] 286 | /// checks if region selected out of screen bounds, if template size > screen size (redundant) 287 | /// and if template size > region size 288 | fn check_if_region_out_of_bound( 289 | &mut self, 290 | template_width: u32, 291 | template_height: u32, 292 | region_x: u32, 293 | region_y: u32, 294 | region_width: u32, 295 | region_height: u32, 296 | ) -> Result<(), AutoGuiError> { 297 | if (region_x + region_width > self.screen.screen_width as u32) 298 | | (region_y + region_height > self.screen.screen_height as u32) 299 | { 300 | return Err(AutoGuiError::OutOfBoundsError( 301 | "Region size larger than screen size".to_string(), 302 | )); 303 | } 304 | 305 | // this is a redundant check since this case should be covered by the 306 | // next region check, but leaving it 307 | if (template_width > (self.screen.screen_width as u32)) 308 | | (template_height > (self.screen.screen_height as u32)) 309 | { 310 | return Err(AutoGuiError::OutOfBoundsError( 311 | "Template size larger than screen size".to_string(), 312 | )); 313 | } 314 | #[cfg(not(feature = "lite"))] 315 | if (template_width > region_width) | (template_height > region_height) { 316 | return Err(AutoGuiError::OutOfBoundsError( 317 | "Template size larger than region size".to_string(), 318 | )); 319 | } 320 | #[cfg(not(feature = "lite"))] 321 | if template_height * template_width == 0 { 322 | Err(ImageProcessingError::Custom( 323 | "Template size = 0. Please check loaded template if its correct".to_string(), 324 | ))?; 325 | } 326 | Ok(()) 327 | } 328 | } 329 | 330 | #[cfg(target_os = "linux")] 331 | impl Drop for RustAutoGui { 332 | fn drop(&mut self) { 333 | self.screen.destroy(); 334 | } 335 | } 336 | 337 | #[cfg(not(feature = "lite"))] 338 | #[cfg(target_os = "windows")] 339 | impl Drop for RustAutoGui { 340 | fn drop(&mut self) { 341 | self.screen.destroy(); 342 | } 343 | } 344 | -------------------------------------------------------------------------------- /src/rustautogui_impl/keyboard_impl.rs: -------------------------------------------------------------------------------- 1 | use crate::AutoGuiError; 2 | 3 | impl crate::RustAutoGui { 4 | /// accepts string and mimics keyboard key presses for each character in string 5 | pub fn keyboard_input(&self, input: &str) -> Result<(), AutoGuiError> { 6 | let input_string = String::from(input); 7 | for letter in input_string.chars() { 8 | self.keyboard.send_char(&letter)?; 9 | } 10 | Ok(()) 11 | } 12 | 13 | /// executes keyboard command like "return" or "escape" 14 | pub fn keyboard_command(&self, input: &str) -> Result<(), AutoGuiError> { 15 | let input_string = String::from(input); 16 | // return automatically the result of send_command function 17 | self.keyboard.send_command(&input_string) 18 | } 19 | 20 | pub fn keyboard_multi_key( 21 | &self, 22 | input1: &str, 23 | input2: &str, 24 | input3: Option<&str>, 25 | ) -> Result<(), AutoGuiError> { 26 | let input3 = input3.map(String::from); 27 | 28 | self.keyboard.send_multi_key(input1, input2, input3) 29 | } 30 | 31 | pub fn key_down(&self, key: &str) -> Result<(), AutoGuiError> { 32 | self.keyboard.key_down(key) 33 | } 34 | 35 | pub fn key_up(&self, key: &str) -> Result<(), AutoGuiError> { 36 | self.keyboard.key_up(key) 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/rustautogui_impl/mod.rs: -------------------------------------------------------------------------------- 1 | use crate::RustAutoGui; 2 | 3 | pub mod keyboard_impl; 4 | pub mod mouse_impl; 5 | pub mod template_match_impl; 6 | -------------------------------------------------------------------------------- /src/rustautogui_impl/mouse_impl.rs: -------------------------------------------------------------------------------- 1 | #![allow(clippy::unit_arg)] 2 | 3 | use crate::core::mouse::{mouse_position, Mouse, MouseClick, MouseScroll}; 4 | use crate::AutoGuiError; 5 | 6 | impl crate::RustAutoGui { 7 | pub fn get_mouse_position(&self) -> Result<(i32, i32), AutoGuiError> { 8 | #[cfg(target_os = "linux")] 9 | return self.mouse.get_mouse_position(); 10 | #[cfg(target_os = "windows")] 11 | return Ok(Mouse::get_mouse_position()); 12 | #[cfg(target_os = "macos")] 13 | return Mouse::get_mouse_position(); 14 | } 15 | 16 | /// Move mouse to x,y pixel coordinate 17 | pub fn move_mouse_to_pos(&self, x: u32, y: u32, moving_time: f32) -> Result<(), AutoGuiError> { 18 | if (x as i32 > self.screen.screen_width) | (y as i32 > self.screen.screen_height) { 19 | return Err(AutoGuiError::OutOfBoundsError(format!( 20 | "Out of bounds at positions x,y :{}, {}", 21 | x, y 22 | ))); 23 | } 24 | 25 | #[cfg(target_os = "windows")] 26 | { 27 | Mouse::move_mouse_to_pos(x as i32, y as i32, moving_time); 28 | Ok(()) 29 | } 30 | #[cfg(target_os = "linux")] 31 | return self 32 | .mouse 33 | .move_mouse_to_pos(x as i32, y as i32, moving_time); 34 | #[cfg(target_os = "macos")] 35 | return Mouse::move_mouse_to_pos(x as i32, y as i32, moving_time); 36 | } 37 | 38 | /// Very similar to move mouse to pos, but takes Option and Option, where None value just keeps the current mouse x or y value 39 | /// So in case you want to more easily move mouse horizontally or vertically 40 | pub fn move_mouse_to( 41 | &self, 42 | x: Option, 43 | y: Option, 44 | moving_time: f32, 45 | ) -> Result<(), AutoGuiError> { 46 | let (pos_x, pos_y) = self.get_mouse_position()?; 47 | 48 | let x = if let Some(x) = x { x as i32 } else { pos_x }; 49 | 50 | let y = if let Some(y) = y { y as i32 } else { pos_y }; 51 | 52 | if (x > self.screen.screen_width) | (y > self.screen.screen_height) { 53 | return Err(AutoGuiError::OutOfBoundsError(format!( 54 | "Out of bounds at positions x,y :{}, {}", 55 | x, y 56 | ))); 57 | } 58 | 59 | #[cfg(target_os = "windows")] 60 | { 61 | Mouse::move_mouse_to_pos(x, y, moving_time); 62 | Ok(()) 63 | } 64 | #[cfg(target_os = "linux")] 65 | return self.mouse.move_mouse_to_pos(x, y, moving_time); 66 | #[cfg(target_os = "macos")] 67 | return Mouse::move_mouse_to_pos(x, y, moving_time); 68 | } 69 | 70 | /// Move mouse in relative position. Accepts both positive and negative values, where negative X moves left, positive moves right 71 | /// and negative Y moves up, positive down 72 | pub fn move_mouse(&self, x: i32, y: i32, moving_time: f32) -> Result<(), AutoGuiError> { 73 | let (pos_x, pos_y) = self.get_mouse_position()?; 74 | 75 | let x = x + pos_x; 76 | let y = y + pos_y; 77 | 78 | if (x > self.screen.screen_width) | (y > self.screen.screen_height) | (x < 0) | (y < 0) { 79 | return Err(AutoGuiError::OutOfBoundsError( 80 | format!("Out of bounds at positions x,y :{}, {}", x, y), // "Mouse movement out of screen boundaries".to_string(), 81 | )); 82 | } 83 | 84 | #[cfg(target_os = "windows")] 85 | { 86 | Mouse::move_mouse_to_pos(x, y, moving_time); 87 | Ok(()) 88 | } 89 | #[cfg(target_os = "linux")] 90 | return self.mouse.move_mouse_to_pos(x, y, moving_time); 91 | #[cfg(target_os = "macos")] 92 | return Mouse::move_mouse_to_pos(x, y, moving_time); 93 | } 94 | 95 | /// executes left click down, move to position relative to current position, left click up 96 | pub fn drag_mouse(&self, x: i32, y: i32, moving_time: f32) -> Result<(), AutoGuiError> { 97 | let (pos_x, pos_y) = self.get_mouse_position()?; 98 | 99 | let x = x + pos_x; 100 | let y = y + pos_y; 101 | if (x > self.screen.screen_width) | (y > self.screen.screen_height) | (x < 0) | (y < 0) { 102 | return Err(AutoGuiError::OutOfBoundsError( 103 | format!("Out of bounds at positions x,y :{}, {}", x, y), // "Mouse movement out of screen boundaries".to_string(), 104 | )); 105 | }; 106 | #[cfg(target_os = "windows")] 107 | { 108 | Mouse::drag_mouse(x, y, moving_time); 109 | 110 | Ok(()) 111 | } 112 | #[cfg(target_os = "macos")] 113 | { 114 | if moving_time < 0.5 && !self.suppress_warnings { 115 | eprintln!("WARNING:Small moving time values may cause issues on mouse drag"); 116 | } 117 | return Mouse::drag_mouse(x as i32, y as i32, moving_time); 118 | } 119 | #[cfg(target_os = "linux")] 120 | { 121 | if moving_time < 0.5 { 122 | if !self.suppress_warnings { 123 | eprintln!("WARNING:Small moving time values may cause issues on mouse drag"); 124 | } 125 | } 126 | return self.mouse.drag_mouse(x as i32, y as i32, moving_time); 127 | } 128 | } 129 | 130 | /// Moves to position x,y. None values maintain current position. Useful for vertical and horizontal movement 131 | pub fn drag_mouse_to( 132 | &self, 133 | x: Option, 134 | y: Option, 135 | moving_time: f32, 136 | ) -> Result<(), AutoGuiError> { 137 | let (pos_x, pos_y) = self.get_mouse_position()?; 138 | 139 | let x = if let Some(x) = x { x as i32 } else { pos_x }; 140 | 141 | let y = if let Some(y) = y { y as i32 } else { pos_y }; 142 | 143 | if (x > self.screen.screen_width) | (y > self.screen.screen_height) { 144 | return Err(AutoGuiError::OutOfBoundsError(format!( 145 | "Out of bounds at positions x,y :{}, {}", 146 | x, y 147 | ))); 148 | } 149 | #[cfg(target_os = "windows")] 150 | { 151 | Mouse::drag_mouse(x, y, moving_time); 152 | 153 | Ok(()) 154 | } 155 | #[cfg(target_os = "macos")] 156 | { 157 | if moving_time < 0.5 && !self.suppress_warnings { 158 | eprintln!("WARNING:Small moving time values may cause issues on mouse drag"); 159 | } 160 | return Mouse::drag_mouse(x as i32, y as i32, moving_time); 161 | } 162 | #[cfg(target_os = "linux")] 163 | { 164 | if moving_time < 0.5 { 165 | if !self.suppress_warnings { 166 | eprintln!("WARNING:Small moving time values may cause issues on mouse drag"); 167 | } 168 | } 169 | return self.mouse.drag_mouse(x as i32, y as i32, moving_time); 170 | } 171 | } 172 | 173 | /// moves mouse to x, y pixel coordinate 174 | pub fn drag_mouse_to_pos(&self, x: u32, y: u32, moving_time: f32) -> Result<(), AutoGuiError> { 175 | if (x as i32 > self.screen.screen_width) | (y as i32 > self.screen.screen_height) { 176 | return Err(AutoGuiError::OutOfBoundsError( 177 | "Drag Mouse out of screen boundaries".to_string(), 178 | )); 179 | } 180 | 181 | #[cfg(target_os = "windows")] 182 | { 183 | Mouse::drag_mouse(x as i32, y as i32, moving_time); 184 | 185 | Ok(()) 186 | } 187 | #[cfg(target_os = "macos")] 188 | { 189 | if moving_time < 0.5 && !self.suppress_warnings { 190 | eprintln!("WARNING:Small moving time values may cause issues on mouse drag"); 191 | } 192 | return Mouse::drag_mouse(x as i32, y as i32, moving_time); 193 | } 194 | #[cfg(target_os = "linux")] 195 | { 196 | if moving_time < 0.5 { 197 | if !self.suppress_warnings { 198 | eprintln!("WARNING:Small moving time values may cause issues on mouse drag"); 199 | } 200 | } 201 | return self.mouse.drag_mouse(x as i32, y as i32, moving_time); 202 | } 203 | } 204 | 205 | /// Mouse click. Choose button Mouseclick::{LEFT,RIGHT,MIDDLE} 206 | pub fn click(&self, button: MouseClick) -> Result<(), AutoGuiError> { 207 | #[cfg(target_os = "linux")] 208 | return self.mouse.mouse_click(button); 209 | #[cfg(target_os = "windows")] 210 | return Ok(Mouse::mouse_click(button)); 211 | #[cfg(target_os = "macos")] 212 | return Mouse::mouse_click(button); 213 | } 214 | 215 | /// executes left mouse click 216 | pub fn left_click(&self) -> Result<(), AutoGuiError> { 217 | #[cfg(target_os = "linux")] 218 | return self.mouse.mouse_click(MouseClick::LEFT); 219 | #[cfg(target_os = "windows")] 220 | return Ok(Mouse::mouse_click(MouseClick::LEFT)); 221 | #[cfg(target_os = "macos")] 222 | return Mouse::mouse_click(MouseClick::LEFT); 223 | } 224 | 225 | /// executes right mouse click 226 | pub fn right_click(&self) -> Result<(), AutoGuiError> { 227 | #[cfg(target_os = "linux")] 228 | return self.mouse.mouse_click(MouseClick::RIGHT); 229 | #[cfg(target_os = "macos")] 230 | return Mouse::mouse_click(MouseClick::RIGHT); 231 | #[cfg(target_os = "windows")] 232 | return Ok(Mouse::mouse_click(MouseClick::RIGHT)); 233 | } 234 | 235 | /// executes middle mouse click 236 | pub fn middle_click(&self) -> Result<(), AutoGuiError> { 237 | #[cfg(target_os = "linux")] 238 | return self.mouse.mouse_click(MouseClick::MIDDLE); 239 | #[cfg(target_os = "windows")] 240 | return Ok(Mouse::mouse_click(MouseClick::MIDDLE)); 241 | #[cfg(target_os = "macos")] 242 | return Mouse::mouse_click(MouseClick::MIDDLE); 243 | } 244 | 245 | /// executes double left mouse click 246 | pub fn double_click(&self) -> Result<(), AutoGuiError> { 247 | #[cfg(target_os = "linux")] 248 | { 249 | self.mouse.mouse_click(MouseClick::LEFT)?; 250 | self.mouse.mouse_click(MouseClick::LEFT) 251 | } 252 | #[cfg(target_os = "windows")] 253 | { 254 | Mouse::mouse_click(MouseClick::LEFT); 255 | Mouse::mouse_click(MouseClick::LEFT); 256 | Ok(()) 257 | } 258 | #[cfg(target_os = "macos")] 259 | Mouse::double_click() 260 | } 261 | 262 | pub fn click_down(&self, button: MouseClick) -> Result<(), AutoGuiError> { 263 | #[cfg(target_os = "linux")] 264 | return self.mouse.mouse_down(button); 265 | #[cfg(target_os = "macos")] 266 | return Mouse::mouse_down(button); 267 | #[cfg(target_os = "windows")] 268 | return Ok(Mouse::mouse_down(button)); 269 | } 270 | pub fn click_up(&self, button: MouseClick) -> Result<(), AutoGuiError> { 271 | #[cfg(target_os = "linux")] 272 | return self.mouse.mouse_up(button); 273 | #[cfg(target_os = "macos")] 274 | return Mouse::mouse_up(button); 275 | #[cfg(target_os = "windows")] 276 | return Ok(Mouse::mouse_up(button)); 277 | } 278 | 279 | pub fn scroll_up(&self, intensity: u32) -> Result<(), AutoGuiError> { 280 | #[cfg(target_os = "linux")] 281 | return Ok(self.mouse.scroll(MouseScroll::UP, intensity)); 282 | #[cfg(target_os = "windows")] 283 | return Ok(Mouse::scroll(MouseScroll::UP, intensity)); 284 | #[cfg(target_os = "macos")] 285 | return Mouse::scroll(MouseScroll::UP, intensity); 286 | } 287 | 288 | pub fn scroll_down(&self, intensity: u32) -> Result<(), AutoGuiError> { 289 | #[cfg(target_os = "linux")] 290 | return Ok(self.mouse.scroll(MouseScroll::DOWN, intensity)); 291 | #[cfg(target_os = "windows")] 292 | return Ok(Mouse::scroll(MouseScroll::DOWN, intensity)); 293 | #[cfg(target_os = "macos")] 294 | return Mouse::scroll(MouseScroll::DOWN, intensity); 295 | } 296 | 297 | pub fn scroll_left(&self, intensity: u32) -> Result<(), AutoGuiError> { 298 | #[cfg(target_os = "linux")] 299 | return Ok(self.mouse.scroll(MouseScroll::LEFT, intensity)); 300 | #[cfg(target_os = "windows")] 301 | return Ok(Mouse::scroll(MouseScroll::LEFT, intensity)); 302 | #[cfg(target_os = "macos")] 303 | return Mouse::scroll(MouseScroll::LEFT, intensity); 304 | } 305 | 306 | pub fn scroll_right(&self, intensity: u32) -> Result<(), AutoGuiError> { 307 | #[cfg(target_os = "linux")] 308 | return Ok(self.mouse.scroll(MouseScroll::RIGHT, intensity)); 309 | #[cfg(target_os = "windows")] 310 | return Ok(Mouse::scroll(MouseScroll::RIGHT, intensity)); 311 | #[cfg(target_os = "macos")] 312 | return Mouse::scroll(MouseScroll::RIGHT, intensity); 313 | } 314 | } 315 | -------------------------------------------------------------------------------- /src/rustautogui_impl/template_match_impl/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod find_img_impl; 2 | pub mod load_img_impl; 3 | -------------------------------------------------------------------------------- /tests/mouse_tests.rs: -------------------------------------------------------------------------------- 1 | // run with cargo test --tests --release -- --nocapture 2 | 3 | #[cfg(feature = "dev")] 4 | pub mod mouse_tests { 5 | use rustautogui; 6 | 7 | #[test] 8 | fn execute_tests() { 9 | let mut gui = rustautogui::RustAutoGui::new(false).unwrap(); 10 | let (s_w, s_h) = gui.get_screen_size(); 11 | 12 | let center_x = (s_w / 2) as u32; 13 | let center_y = (s_h / 2) as u32; 14 | 15 | // makes a square movement around the center 16 | gui.move_mouse_to_pos(center_x, center_y, 0.5).unwrap(); 17 | gui.move_mouse(-(center_x as i32 / 2), 0, 0.5).unwrap(); 18 | gui.move_mouse(0, -(center_y as i32 / 2), 0.5).unwrap(); 19 | gui.move_mouse_to(Some(center_x + center_x / 2), None, 0.5) 20 | .unwrap(); 21 | gui.move_mouse_to_pos(center_x + center_x / 2, center_y + center_y / 2, 0.5) 22 | .unwrap(); 23 | gui.move_mouse(-3 * s_w / 4, 0, 1.5).unwrap(); 24 | gui.move_mouse_to(None, Some(s_h as u32 / 2), 0.5).unwrap(); 25 | gui.move_mouse(s_w - 1, 0, 1.5).unwrap(); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /tests/multi_test.rs: -------------------------------------------------------------------------------- 1 | // run with cargo test --tests --release -- --nocapture 2 | 3 | #[cfg(feature = "dev")] 4 | mod multi_test { 5 | use rustautogui::{self, imgtools, RustAutoGui}; 6 | 7 | #[test] 8 | fn main_test() { 9 | let mut gui = rustautogui::RustAutoGui::new(true).unwrap(); 10 | gui.list_devices(); 11 | load_imgs(&mut gui); 12 | gui.loop_find_stored_image_on_screen_and_move_mouse(0.9, 1.0, 10, "step_0") 13 | .unwrap(); 14 | gui.left_click().unwrap(); 15 | gui.loop_find_stored_image_on_screen_and_move_mouse(0.9, 0.5, 10, "step_1") 16 | .unwrap(); 17 | gui.left_click().unwrap(); 18 | gui.loop_find_stored_image_on_screen_and_move_mouse(0.9, 1.5, 10, "step_2") 19 | .unwrap(); 20 | gui.left_click().unwrap(); 21 | gui.keyboard_input("test!@#45<>/\\|{}[]&*()_+").unwrap(); 22 | } 23 | 24 | fn load_imgs(gui: &mut RustAutoGui) { 25 | #[cfg(target_os = "windows")] 26 | let insert = 'w'; 27 | #[cfg(target_os = "macos")] 28 | let insert = "m"; 29 | #[cfg(target_os = "linux")] 30 | let insert = 'l'; 31 | let img: image::ImageBuffer, Vec> = imgtools::load_image_bw( 32 | format!("tests/testing_images/gui_tests/step_1_{}.png", insert).as_str(), 33 | ) 34 | .unwrap(); 35 | gui.store_template_from_imagebuffer_custom( 36 | img, 37 | None, 38 | rustautogui::MatchMode::SegmentedOcl, 39 | "step_0", 40 | 0.9, 41 | ) 42 | .unwrap(); 43 | 44 | gui.store_template_from_file( 45 | format!("tests/testing_images/gui_tests/step_2_{}.png", insert).as_str(), 46 | None, 47 | rustautogui::MatchMode::SegmentedOclV2, 48 | "step_1", 49 | ) 50 | .unwrap(); 51 | 52 | gui.store_template_from_file_custom( 53 | format!("tests/testing_images/gui_tests/step_3_{}.png", insert).as_str(), 54 | None, 55 | rustautogui::MatchMode::Segmented, 56 | "step_2", 57 | 0.8, 58 | ) 59 | .unwrap(); 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /tests/testing_images/algorithm_tests/Darts_main.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DavorMar/rustautogui/863b47af9751af9285d4c17dfe87699feb2bcc1c/tests/testing_images/algorithm_tests/Darts_main.png -------------------------------------------------------------------------------- /tests/testing_images/algorithm_tests/Darts_template1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DavorMar/rustautogui/863b47af9751af9285d4c17dfe87699feb2bcc1c/tests/testing_images/algorithm_tests/Darts_template1.png -------------------------------------------------------------------------------- /tests/testing_images/algorithm_tests/Darts_template2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DavorMar/rustautogui/863b47af9751af9285d4c17dfe87699feb2bcc1c/tests/testing_images/algorithm_tests/Darts_template2.png -------------------------------------------------------------------------------- /tests/testing_images/algorithm_tests/Darts_template3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DavorMar/rustautogui/863b47af9751af9285d4c17dfe87699feb2bcc1c/tests/testing_images/algorithm_tests/Darts_template3.png -------------------------------------------------------------------------------- /tests/testing_images/algorithm_tests/Socket_main.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DavorMar/rustautogui/863b47af9751af9285d4c17dfe87699feb2bcc1c/tests/testing_images/algorithm_tests/Socket_main.png -------------------------------------------------------------------------------- /tests/testing_images/algorithm_tests/Socket_template1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DavorMar/rustautogui/863b47af9751af9285d4c17dfe87699feb2bcc1c/tests/testing_images/algorithm_tests/Socket_template1.png -------------------------------------------------------------------------------- /tests/testing_images/algorithm_tests/Socket_template2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DavorMar/rustautogui/863b47af9751af9285d4c17dfe87699feb2bcc1c/tests/testing_images/algorithm_tests/Socket_template2.png -------------------------------------------------------------------------------- /tests/testing_images/algorithm_tests/Socket_template3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DavorMar/rustautogui/863b47af9751af9285d4c17dfe87699feb2bcc1c/tests/testing_images/algorithm_tests/Socket_template3.png -------------------------------------------------------------------------------- /tests/testing_images/algorithm_tests/Split_main.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DavorMar/rustautogui/863b47af9751af9285d4c17dfe87699feb2bcc1c/tests/testing_images/algorithm_tests/Split_main.png -------------------------------------------------------------------------------- /tests/testing_images/algorithm_tests/Split_template1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DavorMar/rustautogui/863b47af9751af9285d4c17dfe87699feb2bcc1c/tests/testing_images/algorithm_tests/Split_template1.png -------------------------------------------------------------------------------- /tests/testing_images/algorithm_tests/Split_template2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DavorMar/rustautogui/863b47af9751af9285d4c17dfe87699feb2bcc1c/tests/testing_images/algorithm_tests/Split_template2.png -------------------------------------------------------------------------------- /tests/testing_images/algorithm_tests/Split_template3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DavorMar/rustautogui/863b47af9751af9285d4c17dfe87699feb2bcc1c/tests/testing_images/algorithm_tests/Split_template3.png -------------------------------------------------------------------------------- /tests/testing_images/algorithm_tests/Split_template4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DavorMar/rustautogui/863b47af9751af9285d4c17dfe87699feb2bcc1c/tests/testing_images/algorithm_tests/Split_template4.png -------------------------------------------------------------------------------- /tests/testing_images/algorithm_tests/Split_template5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DavorMar/rustautogui/863b47af9751af9285d4c17dfe87699feb2bcc1c/tests/testing_images/algorithm_tests/Split_template5.png -------------------------------------------------------------------------------- /tests/testing_images/gui_tests/step_1_l.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DavorMar/rustautogui/863b47af9751af9285d4c17dfe87699feb2bcc1c/tests/testing_images/gui_tests/step_1_l.png -------------------------------------------------------------------------------- /tests/testing_images/gui_tests/step_1_m.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DavorMar/rustautogui/863b47af9751af9285d4c17dfe87699feb2bcc1c/tests/testing_images/gui_tests/step_1_m.png -------------------------------------------------------------------------------- /tests/testing_images/gui_tests/step_1_w.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DavorMar/rustautogui/863b47af9751af9285d4c17dfe87699feb2bcc1c/tests/testing_images/gui_tests/step_1_w.png -------------------------------------------------------------------------------- /tests/testing_images/gui_tests/step_2_l.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DavorMar/rustautogui/863b47af9751af9285d4c17dfe87699feb2bcc1c/tests/testing_images/gui_tests/step_2_l.png -------------------------------------------------------------------------------- /tests/testing_images/gui_tests/step_2_m.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DavorMar/rustautogui/863b47af9751af9285d4c17dfe87699feb2bcc1c/tests/testing_images/gui_tests/step_2_m.png -------------------------------------------------------------------------------- /tests/testing_images/gui_tests/step_2_w.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DavorMar/rustautogui/863b47af9751af9285d4c17dfe87699feb2bcc1c/tests/testing_images/gui_tests/step_2_w.png -------------------------------------------------------------------------------- /tests/testing_images/gui_tests/step_3_l.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DavorMar/rustautogui/863b47af9751af9285d4c17dfe87699feb2bcc1c/tests/testing_images/gui_tests/step_3_l.png -------------------------------------------------------------------------------- /tests/testing_images/gui_tests/step_3_m.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DavorMar/rustautogui/863b47af9751af9285d4c17dfe87699feb2bcc1c/tests/testing_images/gui_tests/step_3_m.png -------------------------------------------------------------------------------- /tests/testing_images/gui_tests/step_3_w.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DavorMar/rustautogui/863b47af9751af9285d4c17dfe87699feb2bcc1c/tests/testing_images/gui_tests/step_3_w.png -------------------------------------------------------------------------------- /tests/tmpl_match_tests.rs: -------------------------------------------------------------------------------- 1 | // run with cargo test --tests --release -- --nocapture 2 | 3 | #[cfg(feature = "dev")] 4 | pub mod tmpl_match_tests { 5 | use rustautogui::core::template_match::open_cl::OclVersion; 6 | use rustautogui::core::template_match::*; 7 | use rustautogui::data::{opencl::KernelStorage, opencl::*, PreparedData}; 8 | use rustautogui::imgtools; 9 | 10 | #[test] 11 | fn testing_speeds() { 12 | let image_paths = vec![ 13 | "tests/testing_images/algorithm_tests/Darts_main.png", 14 | "tests/testing_images/algorithm_tests/Darts_main.png", 15 | "tests/testing_images/algorithm_tests/Darts_main.png", 16 | "tests/testing_images/algorithm_tests/Socket_main.png", 17 | "tests/testing_images/algorithm_tests/Socket_main.png", 18 | "tests/testing_images/algorithm_tests/Socket_main.png", 19 | "tests/testing_images/algorithm_tests/Split_main.png", 20 | "tests/testing_images/algorithm_tests/Split_main.png", 21 | // "tests/testing_images/algorithm_tests/Split_main.png", 22 | "tests/testing_images/algorithm_tests/Split_main.png", 23 | "tests/testing_images/algorithm_tests/Split_main.png", 24 | ]; 25 | let template_paths = vec![ 26 | "tests/testing_images/algorithm_tests/Darts_template1.png", 27 | "tests/testing_images/algorithm_tests/Darts_template2.png", 28 | "tests/testing_images/algorithm_tests/Darts_template3.png", 29 | "tests/testing_images/algorithm_tests/Socket_template1.png", 30 | "tests/testing_images/algorithm_tests/Socket_template2.png", 31 | "tests/testing_images/algorithm_tests/Socket_template3.png", 32 | "tests/testing_images/algorithm_tests/Split_template1.png", 33 | "tests/testing_images/algorithm_tests/Split_template2.png", 34 | // "tests/testing_images/algorithm_tests/Split_template3.png", 35 | "tests/testing_images/algorithm_tests/Split_template4.png", 36 | "tests/testing_images/algorithm_tests/Split_template5.png", 37 | ]; 38 | let target_positions: Vec<(i32, i32)> = vec![ 39 | (206, 1), 40 | (60, 270), 41 | (454, 31), 42 | (197, 345), 43 | (81, 825), 44 | (359, 666), 45 | (969, 688), 46 | (713, 1389), 47 | (1273, 1667), 48 | (41, 53), 49 | ]; 50 | #[cfg(not(feature = "lite"))] 51 | for ((img_val, tmpl_val), target_position) in 52 | image_paths.iter().zip(template_paths).zip(target_positions) 53 | { 54 | testing_run(*img_val, tmpl_val, target_position); 55 | } 56 | } 57 | 58 | fn segmented_run( 59 | template: &image::ImageBuffer, Vec>, 60 | main_image: &image::ImageBuffer, Vec>, 61 | target_positions: (i32, i32), 62 | template_path: &str, 63 | custom: bool, 64 | ) { 65 | let mut threshold = None; 66 | let mut insert_str = String::new(); 67 | if custom { 68 | threshold = Some(0.5); 69 | insert_str.push_str("custom"); 70 | } 71 | let template_data = segmented_ncc::prepare_template_picture(&template, &false, threshold); 72 | let template_data = match template_data { 73 | PreparedData::Segmented(data) => data, 74 | _ => panic!(), 75 | }; 76 | let mut dur = 0.0; 77 | let mut locations: Vec<(u32, u32, f32)> = Vec::new(); 78 | if template_path == "testing_images2/Split_template3.png" { 79 | } else { 80 | let start = std::time::Instant::now(); 81 | locations = 82 | segmented_ncc::fast_ncc_template_match(&main_image, 0.95, &template_data, &false); 83 | dur = start.elapsed().as_secs_f32(); 84 | } 85 | let mut first_location = (0, 0, 0.0); 86 | 87 | if locations.len() > 0 { 88 | first_location = locations[0]; 89 | println!( 90 | "Segmented {insert_str}: Location found at {}, {} and corr {}, time: {} ", 91 | first_location.0, first_location.1, locations[0].2, dur 92 | ); 93 | } 94 | assert!( 95 | first_location.0 == target_positions.0 as u32 96 | && first_location.1 == target_positions.1 as u32 97 | ); 98 | } 99 | 100 | #[cfg(feature = "opencl")] 101 | fn ocl_run( 102 | template: &image::ImageBuffer, Vec>, 103 | main_image: &image::ImageBuffer, Vec>, 104 | target_positions: (i32, i32), 105 | custom: bool, 106 | ocl_v: OclVersion, 107 | image_width: u32, 108 | image_height: u32, 109 | template_width: u32, 110 | template_height: u32, 111 | ) { 112 | use rustautogui::RustAutoGui; 113 | 114 | let data = RustAutoGui::dev_setup_opencl(Some(1)).unwrap(); 115 | 116 | let mut threshold = None; 117 | let mut insert_str = String::new(); 118 | let mut v_string = String::new(); 119 | if custom { 120 | threshold = Some(0.7); 121 | insert_str.push_str("custom"); 122 | } 123 | match ocl_v { 124 | OclVersion::V1 => v_string.push('1'), 125 | OclVersion::V2 => v_string.push('2'), 126 | } 127 | 128 | // Create a context for that device 129 | 130 | let queue = data.ocl_queue; 131 | let program = data.ocl_program; 132 | 133 | //////////////////////////////////////////////////////////////////////// OPENCL V1 134 | 135 | let template_data = segmented_ncc::prepare_template_picture(&template, &false, threshold); 136 | let template_data = match template_data { 137 | PreparedData::Segmented(x) => x, 138 | _ => panic!(), 139 | }; 140 | let gpu_pointers = GpuMemoryPointers::new( 141 | image_width, 142 | image_height, 143 | template_width, 144 | template_height, 145 | &queue, 146 | &template_data.template_segments_slow, 147 | &template_data.template_segments_fast, 148 | ) 149 | .unwrap(); 150 | let kernels = KernelStorage::new( 151 | &gpu_pointers, 152 | &program, 153 | &queue, 154 | image_width, 155 | image_height, 156 | template_width, 157 | template_height, 158 | template_data.template_segments_fast.len() as u32, 159 | template_data.template_segments_slow.len() as u32, 160 | template_data.segments_mean_fast, 161 | template_data.segments_mean_slow, 162 | template_data.segment_sum_squared_deviations_fast, 163 | template_data.segment_sum_squared_deviations_slow, 164 | template_data.expected_corr_fast, 165 | 256, 166 | ) 167 | .unwrap(); 168 | let start = std::time::Instant::now(); 169 | let locations = open_cl::gui_opencl_ncc_template_match( 170 | &queue, 171 | &program, 172 | 256, 173 | &kernels, 174 | &gpu_pointers, 175 | 0.95, 176 | &main_image, 177 | &template_data, 178 | ocl_v, 179 | ) 180 | .unwrap(); 181 | let mut first_location = (0, 0, 0.0); 182 | if locations.len() > 0 { 183 | first_location = locations[0]; 184 | println!( 185 | "OCL V{v_string} {insert_str}: Location found at {:?}, time: {}", 186 | first_location, 187 | start.elapsed().as_secs_f32() 188 | ); 189 | } 190 | assert!( 191 | first_location.0 == target_positions.0 as u32 192 | && first_location.1 == target_positions.1 as u32 193 | ); 194 | } 195 | 196 | fn fft_run( 197 | template: &image::ImageBuffer, Vec>, 198 | main_image: &image::ImageBuffer, Vec>, 199 | target_positions: (i32, i32), 200 | image_width: u32, 201 | image_height: u32, 202 | ) { 203 | // fft corr 204 | let template_data = fft_ncc::prepare_template_picture(&template, image_width, image_height); 205 | let start = std::time::Instant::now(); 206 | let locations = fft_ncc::fft_ncc(&main_image, 0.90, &template_data); 207 | 208 | let mut first_location = (0, 0, 0.0); 209 | if locations.len() > 0 { 210 | first_location.0 = locations[0].0; 211 | first_location.1 = locations[0].1; 212 | println!( 213 | "FFT: Location found at {}, {} and corr {} , time: {}", 214 | first_location.0, 215 | first_location.1, 216 | locations[0].2, 217 | start.elapsed().as_secs_f32() 218 | ); 219 | } 220 | assert!( 221 | first_location.0 == target_positions.0 as u32 222 | && first_location.1 == target_positions.1 as u32 223 | ); 224 | println!("\n"); 225 | } 226 | 227 | fn testing_run(image_path: &str, template_path: &str, target_positions: (i32, i32)) { 228 | let template: image::ImageBuffer, Vec> = 229 | imgtools::load_image_bw(template_path).unwrap(); 230 | let main_image: image::ImageBuffer, Vec> = 231 | imgtools::load_image_bw(image_path).unwrap(); 232 | let (image_width, image_height) = main_image.dimensions(); 233 | let (template_width, template_height) = template.dimensions(); 234 | 235 | println!("Image: {}, template:{}", image_path, template_path); 236 | 237 | segmented_run( 238 | &template, 239 | &main_image, 240 | target_positions, 241 | template_path, 242 | false, 243 | ); // default cpu 244 | segmented_run( 245 | &template, 246 | &main_image, 247 | target_positions, 248 | template_path, 249 | true, 250 | ); // cpu custom 251 | ocl_run( 252 | &template, 253 | &main_image, 254 | target_positions, 255 | false, 256 | OclVersion::V1, 257 | image_width, 258 | image_height, 259 | template_width, 260 | template_height, 261 | ); 262 | ocl_run( 263 | &template, 264 | &main_image, 265 | target_positions, 266 | false, 267 | OclVersion::V2, 268 | image_width, 269 | image_height, 270 | template_width, 271 | template_height, 272 | ); 273 | ocl_run( 274 | &template, 275 | &main_image, 276 | target_positions, 277 | true, 278 | OclVersion::V1, 279 | image_width, 280 | image_height, 281 | template_width, 282 | template_height, 283 | ); 284 | ocl_run( 285 | &template, 286 | &main_image, 287 | target_positions, 288 | true, 289 | OclVersion::V2, 290 | image_width, 291 | image_height, 292 | template_width, 293 | template_height, 294 | ); 295 | fft_run( 296 | &template, 297 | &main_image, 298 | target_positions, 299 | image_width, 300 | image_height, 301 | ); 302 | } 303 | } 304 | -------------------------------------------------------------------------------- /testspeed.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DavorMar/rustautogui/863b47af9751af9285d4c17dfe87699feb2bcc1c/testspeed.gif --------------------------------------------------------------------------------