├── .cargo └── config.toml ├── .github └── workflows │ ├── check_everything.yml │ └── create_release.yml ├── .gitignore ├── .vscode ├── launch.json └── settings.json ├── Cargo.toml ├── LICENSE ├── README.md ├── assets ├── 128x128.png ├── 32x32.png ├── NotoSans-Regular.ttf ├── index.html ├── logo.icns ├── logo.ico └── logo.png ├── build.rs ├── crates └── plugovr_types │ ├── Cargo.toml │ └── src │ └── lib.rs ├── debian └── postinst ├── deploy_osx.sh ├── deploy_osx_x86_64.sh ├── entitlements.plist └── src ├── llm.rs ├── main.rs ├── ui.rs ├── ui ├── answer_analyser.rs ├── assistance_window.rs ├── diff_view.rs ├── main_window.rs ├── screen_dimensions.rs ├── shortcut_window.rs ├── show_form_fields.rs ├── template_editor.rs └── user_interface.rs ├── usecase_editor.rs ├── usecase_recorder.rs ├── usecase_replay.rs ├── usecase_webserver.rs ├── version_check.rs └── window_handling.rs /.cargo/config.toml: -------------------------------------------------------------------------------- 1 | [target.x86_64-unknown-linux-gnu] 2 | rustflags = [ 3 | "-C", "link-arg=-std=c++14" 4 | ] 5 | -------------------------------------------------------------------------------- /.github/workflows/check_everything.yml: -------------------------------------------------------------------------------- 1 | on: 2 | push: 3 | pull_request: 4 | workflow_dispatch: 5 | 6 | name: build and check 7 | env: 8 | CARGO_INCREMENTAL: 0 9 | jobs: 10 | 11 | build: 12 | name: build 13 | 14 | runs-on: ${{matrix.os}} 15 | strategy: 16 | matrix: 17 | include: 18 | - os: ubuntu-24.04 19 | - os: windows-latest 20 | - os: macos-latest 21 | steps: 22 | - name: Checkout 23 | uses: actions/checkout@v4 24 | with: 25 | fetch-depth: 0 26 | ref: main 27 | token: ${{ secrets.PRIVATE_REPO_TOKEN }} 28 | submodules: false 29 | 30 | 31 | - name: Install LLVM and Clang 32 | uses: KyleMayes/install-llvm-action@v2 33 | with: 34 | version: "16.0" 35 | 36 | - name: Install Rust toolchain 37 | uses: actions-rs/toolchain@v1 38 | with: 39 | profile: minimal 40 | toolchain: stable 41 | override: true 42 | 43 | - name: Rust Cache 44 | uses: Swatinem/rust-cache@v1 45 | 46 | - name: Install dependencies # for glfw and rfd 47 | if: startsWith(matrix.os, 'ubuntu') 48 | run: sudo apt update && sudo apt install --no-install-recommends cmake build-essential libssl3 libdbus-1-3 libglfw3-dev libgtk-3-dev libxcb1-dev libxcb-render0-dev libxcb-shape0-dev libxcb-xfixes0-dev libxdo-dev 49 | 50 | - name: Fmt 51 | run: cargo fmt --check 52 | 53 | - name: Check 54 | run: cargo check 55 | 56 | - name: Clippy 57 | run: cargo clippy 58 | 59 | - name: Build Debug 60 | run: cargo build 61 | 62 | - name: Build Release 63 | run: cargo build --profile release-lto 64 | 65 | - name: Build with computeruse features 66 | run: cargo build --profile release-lto --features computeruse 67 | 68 | - name: Run tests 69 | run: cargo test --all --features computeruse 70 | -------------------------------------------------------------------------------- /.github/workflows/create_release.yml: -------------------------------------------------------------------------------- 1 | on: 2 | workflow_dispatch: 3 | 4 | name: create release 5 | env: 6 | CARGO_INCREMENTAL: 0 7 | jobs: 8 | bump_version: 9 | name: Bump version 10 | runs-on: ubuntu-24.04 11 | steps: 12 | - name: Checkout 13 | uses: actions/checkout@v2 14 | with: 15 | fetch-depth: 0 16 | 17 | - name: Install Rust toolchain 18 | uses: actions-rs/toolchain@v1 19 | with: 20 | profile: minimal 21 | toolchain: stable 22 | override: true 23 | 24 | 25 | - name: Install cargo-bump 26 | run: cargo install cargo-bump 27 | 28 | - name: Configure Git 29 | run: | 30 | git config user.name github-actions 31 | git config user.email github-actions@github.com 32 | 33 | - name: Update version, create tag, and push changes 34 | id: update_version 35 | env: 36 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 37 | run: | 38 | cargo bump patch 39 | NEW_VERSION=$(cargo pkgid | sed -E 's/.*[#@]([0-9]+\.[0-9]+\.[0-9]+)$/\1/') 40 | git add . 41 | git commit -m "Bump version to $NEW_VERSION" 42 | git tag -a "v$NEW_VERSION" -m "Release $NEW_VERSION" 43 | git push origin HEAD:${{ github.ref }} 44 | git push origin "v$NEW_VERSION" 45 | echo "NEW_VERSION=$NEW_VERSION" >> $GITHUB_OUTPUT 46 | echo "::set-output name=NEW_VERSION::$NEW_VERSION" # Correct output syntax 47 | echo "{\"version\": \"$NEW_VERSION\"}" > latest_version.json 48 | 49 | #- name: Configure AWS credentials 50 | # uses: aws-actions/configure-aws-credentials@v1 51 | # with: 52 | # aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} 53 | # aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} 54 | # aws-region: eu-central-1 55 | 56 | # - name: Copy latest_version.json to S3 57 | # run: aws s3 cp latest_version.json s3://plugovr.ai/latest_version.json 58 | 59 | - name: Create Release 60 | uses: actions/create-release@v1 61 | env: 62 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 63 | with: 64 | tag_name: v${{ steps.update_version.outputs.NEW_VERSION }} # Use the created tag 65 | release_name: Release v${{ steps.update_version.outputs.NEW_VERSION }} # Use the same tag for the release name 66 | draft: false 67 | prerelease: false 68 | 69 | 70 | - name: Sync deploy with main 71 | run: | 72 | git fetch origin main 73 | git checkout deploy 74 | git merge origin/main --no-edit 75 | git push origin deploy 76 | 77 | outputs: 78 | new_version: ${{ steps.update_version.outputs.NEW_VERSION }} 79 | 80 | 81 | build: 82 | name: build 83 | needs: bump_version 84 | runs-on: ${{matrix.os}} 85 | strategy: 86 | matrix: 87 | include: 88 | # - os: ubuntu-22.04 89 | - os: ubuntu-24.04 90 | - os: windows-latest 91 | steps: 92 | - name: Checkout 93 | uses: actions/checkout@v4 94 | with: 95 | fetch-depth: 0 96 | ref: deploy 97 | token: ${{ secrets.PRIVATE_REPO_TOKEN }} 98 | submodules: true 99 | 100 | 101 | - name: Install LLVM and Clang 102 | uses: KyleMayes/install-llvm-action@v2 103 | with: 104 | version: "16.0" 105 | 106 | - name: Install Rust toolchain 107 | uses: actions-rs/toolchain@v1 108 | with: 109 | profile: minimal 110 | toolchain: stable 111 | override: true 112 | 113 | - name: Rust Cache 114 | uses: Swatinem/rust-cache@v1 115 | 116 | - name: Install dependencies # for glfw and rfd 117 | if: startsWith(matrix.os, 'ubuntu') 118 | run: sudo apt update && sudo apt install --no-install-recommends cmake build-essential libssl3 libdbus-1-3 libglfw3-dev libgtk-3-dev libxcb1-dev libxcb-render0-dev libxcb-shape0-dev libxcb-xfixes0-dev libxdo-dev 119 | 120 | - name: Fmt 121 | run: cargo fmt --check 122 | 123 | - name: Check 124 | run: cargo check 125 | 126 | - name: Clippy 127 | run: cargo clippy 128 | 129 | - name: Build 130 | if: startsWith(matrix.os, 'macos') 131 | run: cargo build --profile release-lto --features glow --no-default-features 132 | 133 | - name: Build 134 | if: startsWith(matrix.os, 'ubuntu') || startsWith(matrix.os, 'windows') 135 | run: cargo build --profile release-lto 136 | 137 | - name: Create Debian package 138 | if: startsWith(matrix.os, 'ubuntu') 139 | run: | 140 | cargo install cargo-deb 141 | cargo deb 142 | 143 | - name: Create installer MacOS 144 | if: startsWith(matrix.os, 'macos') 145 | run: | 146 | cargo install cargo-bundle 147 | cargo bundle --profile release-lto 148 | - name: Create installer Windows 149 | if: startsWith(matrix.os, 'windows') 150 | run: | 151 | cargo install cargo-packager 152 | cargo packager --profile release-lto 153 | 154 | - name: Sign files with Trusted Signing 155 | if: startsWith(matrix.os, 'windows') 156 | uses: azure/trusted-signing-action@v0.5.0 157 | with: 158 | azure-tenant-id: ${{ secrets.AZURE_TENANT_ID }} 159 | azure-client-id: ${{ secrets.AZURE_CLIENT_ID }} 160 | azure-client-secret: ${{ secrets.AZURE_CLIENT_SECRET }} 161 | endpoint: https://weu.codesigning.azure.net/ 162 | trusted-signing-account-name: corneliuswefelscheid 163 | certificate-profile-name: PlugOvr 164 | files-folder: ${{ github.workspace }}\target\release\ 165 | files-folder-filter: exe,dll 166 | file-digest: SHA256 167 | timestamp-rfc3161: http://timestamp.acs.microsoft.com 168 | timestamp-digest: SHA256 169 | 170 | 171 | # - name: Upload signed executable 172 | # if: startsWith(matrix.os, 'windows') 173 | # uses: actions/upload-artifact@v2 174 | # with: 175 | # name: signed-executable 176 | # path: target\release\PlugOvr_${{ needs.bump_version.outputs.new_version }}_x64-setup.exe 177 | 178 | 179 | - name: Upload artifact for Linux 180 | if: startsWith(matrix.os, 'ubuntu') 181 | uses: actions/upload-artifact@v4 182 | with: 183 | name: PlugOvr_linux_${{ matrix.os }} 184 | path: target/debian/*.deb 185 | 186 | - name: Upload artifact for Windows 187 | if: startsWith(matrix.os, 'windows') 188 | uses: actions/upload-artifact@v4 189 | with: 190 | name: PlugOvr_win 191 | path: | 192 | target/release/*.exe 193 | 194 | # - name: Upload artifact for MacOS 195 | # if: startsWith(matrix.os, 'macos') 196 | # uses: actions/upload-artifact@v4 197 | # with: 198 | # name: PlugOvr_macos 199 | # path: | 200 | # target/release/PlugOvr 201 | # target/release/bundle/osx/*.app 202 | 203 | - name: Configure AWS credentials 204 | uses: aws-actions/configure-aws-credentials@v1 205 | with: 206 | aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} 207 | aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} 208 | aws-region: eu-central-1 209 | 210 | - name: Copy artifacts to S3 Linux 211 | if: startsWith(matrix.os, 'ubuntu') 212 | run: | 213 | aws s3 cp target/debian/*.deb s3://plugovr.ai/plugovr_${{ needs.bump_version.outputs.new_version }}_amd64_${{ matrix.os }}.deb 214 | 215 | - name: Copy artifacts to S3 Windows 216 | if: startsWith(matrix.os, 'windows') 217 | run: | 218 | aws s3 cp target/release/PlugOvr_${{ needs.bump_version.outputs.new_version }}_x64-setup.exe s3://plugovr.ai/ 219 | 220 | - name: Copy artifacts to S3 MacOS 221 | if: startsWith(matrix.os, 'macos') 222 | run: | 223 | aws s3 cp target/release/PlugOvr s3://plugovr.ai/PlugOvr 224 | aws s3 cp target/release/bundle/osx/*.app s3://plugovr.ai/ 225 | 226 | 227 | #- name: Audit 228 | # run: cargo audit 229 | 230 | update_latest_version: 231 | name: Update latest version 232 | needs: [bump_version, build] 233 | runs-on: ubuntu-latest 234 | steps: 235 | - name: Checkout 236 | uses: actions/checkout@v2 237 | with: 238 | fetch-depth: 0 239 | 240 | - name: Configure AWS credentials 241 | uses: aws-actions/configure-aws-credentials@v1 242 | with: 243 | aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} 244 | aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} 245 | aws-region: eu-central-1 246 | 247 | - name: Copy latest_version.json to S3 248 | run: | 249 | echo "{\"version\": \"${{ needs.bump_version.outputs.new_version }}\"}" > latest_version.json 250 | aws s3 cp latest_version.json s3://plugovr.ai/latest_version.json 251 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Generated by Cargo 2 | # will have compiled files and executables 3 | debug/ 4 | target/ 5 | 6 | # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries 7 | # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html 8 | Cargo.lock 9 | 10 | # These are backup files generated by rustfmt 11 | **/*.rs.bk 12 | 13 | # MSVC Windows builds of rustc generate these, which store debugging information 14 | *.pdb 15 | 16 | *.json 17 | 18 | # Added by cargo 19 | 20 | /target 21 | /PlugOvr.app 22 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "type": "lldb", 9 | "request": "launch", 10 | "name": "Debug executable 'PlugOvr'", 11 | "cargo": { 12 | "args": [ 13 | "build", 14 | "--bin=PlugOvr", 15 | "--package=PlugOvr" 16 | ], 17 | "filter": { 18 | "name": "PlugOvr", 19 | "kind": "bin" 20 | } 21 | }, 22 | "args": [], 23 | "cwd": "${workspaceFolder}" 24 | }, 25 | { 26 | "type": "lldb", 27 | "request": "launch", 28 | "name": "Debug unit tests in executable 'PlugOvr'", 29 | "cargo": { 30 | "args": [ 31 | "test", 32 | "--no-run", 33 | "--bin=PlugOvr", 34 | "--package=PlugOvr" 35 | ], 36 | "filter": { 37 | "name": "PlugOvr", 38 | "kind": "bin" 39 | } 40 | }, 41 | "args": [], 42 | "cwd": "${workspaceFolder}" 43 | } 44 | ] 45 | } -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "rust-analyzer.editor.defaultFormatter": "rust-lang.rust-analyzer", 3 | "rust-analyzer.editor.formatOnSave": true, 4 | "lldb.displayFormat": "auto", 5 | "lldb.showDisassembly": "auto", 6 | "lldb.dereferencePointers": true, 7 | "lldb.consoleMode": "commands", 8 | "rust-analyzer.checkOnSave.command": "clippy" 9 | } -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "PlugOvr" 3 | version = "0.2.12" 4 | edition = "2024" 5 | license-file = "LICENSE" 6 | authors = ["Cornelius Wefelscheid "] 7 | description = "PlugOvr provides AI assistance to every program." 8 | homepage = "https://plugovr.ai" 9 | #maintainers = ["Cornelius Wefelscheid "] 10 | repository = "https://github.com/PlugOvr-ai/PlugOvr" 11 | publish = false 12 | 13 | [package.metadata.deb] 14 | extended-description = """\ 15 | PlugOvr""" 16 | depends = "$auto" 17 | maintainer-scripts = "debian" 18 | 19 | 20 | [package.metadata.bundle] 21 | name = "PlugOvr" 22 | identifier = "ai.plugovr" 23 | icon = ["assets/logo.icns"] 24 | copyright = "Copyright (c) PlugOvr.ai2024. All rights reserved." 25 | category = "public.app-category.utilities" 26 | short_description = "Bring AI to every application." 27 | long_description = """ 28 | AI right at your fingertips. 29 | Select text as context for the AI. 30 | Get your assistance by pressing ctrl + alt + i. 31 | """ 32 | deb_depends = [ 33 | "libglfw3-dev", 34 | "libgtk-3-dev", 35 | "libxcb1-dev", 36 | "libxcb-render0-dev", 37 | "libxcb-shape0-dev", 38 | "libxcb-xfixes0-dev", 39 | "libxdo-dev", 40 | "libgl1-mesa-glx", 41 | "libsdl2-2.0-0 (>= 2.0.5)", 42 | ] 43 | 44 | osx_url_schemes = ["ai.plugovr"] 45 | [package.metadata.bundle.privacy] 46 | NSInputMonitoringUsageDescription = "PlugOvr needs input monitoring permissions to function properly." 47 | NSAppleEventsUsageDescription = "PlugOvr needs automation permissions to interact with other applications." 48 | NSScreenCaptureUsageDescription = "PlugOvr needs screen recording permissions to capture and display your screen." 49 | 50 | 51 | [package.metadata.packager] 52 | before-packaging-command = "cargo build --release" 53 | product-name = "PlugOvr" 54 | identifier = "ai.plugovr" 55 | #resources = ["Cargo.toml", "src", "assets"] 56 | icons = [ 57 | "assets/32x32.png", 58 | "assets/128x128.png", 59 | "assets/logo.ico", 60 | "assets/logo.icns", 61 | ] 62 | homepage = "https://plugovr.ai" 63 | copyright = "Copyright (c) PlugOvr.ai 2024. All rights reserved." 64 | 65 | 66 | [features] 67 | default = ["three_d"] 68 | three_d = ["egui_overlay/three_d"] 69 | wgpu = ["egui_overlay/wgpu"] 70 | glow = ["egui_overlay/glow"] 71 | cuda = ["kalosm/cuda"] 72 | metal = ["kalosm/metal"] 73 | cs = [] 74 | computeruse = [ 75 | "computeruse_replay", 76 | "computeruse_record", 77 | "computeruse_editor", 78 | "computeruse_remote", 79 | ] 80 | computeruse_replay = ["computeruse_record"] 81 | computeruse_record = [] 82 | computeruse_editor = [] 83 | computeruse_remote = [] 84 | 85 | [dependencies] 86 | #plugovr_cs = { path = "crates/plugovr_cs" } 87 | plugovr_types = { path = "crates/plugovr_types" } 88 | 89 | egui_overlay = { version = "0.9.0", git = "https://github.com/PlugOvr-ai/egui_overlay.git", default-features = false, features = [ 90 | "egui_default", 91 | "glfw_default", 92 | ] } 93 | tracing-subscriber = { version = "0.3", features = ["env-filter"] } 94 | #egui_window_glfw_passthrough = "0.8.1" 95 | #egui_render_three_d = { workspace = true } 96 | screenshots = { version = "0.8.10" } 97 | egui = { version = "0.31.1", default-features = false } 98 | bytemuck = { version = "1", default-features = false } 99 | raw-window-handle = { version = "0.6" } 100 | three-d = "0.18.2" 101 | three-d-asset = "0.9.2" 102 | rdev = { version = "0.5.3", git = "https://github.com/AlexKnauth/rdev.git", rev = "18cd3ac7ddbbce7a4b33e4a5f1dc901418336342", features = [ 103 | "serialize", 104 | ] } 105 | serde_json = "1.0.120" 106 | tokio = { version = "1.38.1", features = ["full", "macros"] } 107 | reqwest = { version = "0.12.7", features = ["blocking", "multipart", "json"] } 108 | itertools = "0.14.0" 109 | 110 | egui_autocomplete = { version = "9.1.0", git = "https://github.com/PlugOvr-ai/egui_autocomplete.git" } 111 | serde = { version = "1.0.210", features = ["derive"] } 112 | dirs = "6.0.0" 113 | 114 | kalosm = { version = "0.4.0", features = ["language"] } 115 | 116 | strum = { version = "0.27.0", features = ["derive"] } 117 | strum_macros = "0.27.0" 118 | similar = "2.6.0" 119 | image = { version = "0.25.5", features = ["png"] } 120 | image_24 = { package = "image", version = "0.24.9" } 121 | base64 = { version = "0.22.1" } 122 | arboard = "3.4.1" 123 | ollama-rs = { version = "0.2.1", features = ["stream"] } 124 | regex = "1.11.1" 125 | tray-icon = "0.19.2" 126 | webbrowser = "1.0.3" 127 | xcap = "0.3.1" 128 | uuid = { version = "1.11.0", features = ["v4"] } 129 | futures = "0.3.31" 130 | repair_json = "0.1.0" 131 | json-fixer = { version = "0.1.0", features = ["serde"] } 132 | openai_dive = "0.7" 133 | #rfd = "0.15.2" 134 | egui-file-dialog = "0.9.0" 135 | tokio-tungstenite = { version = "0.26.2", features = ["stream"] } 136 | tower-http = { version = "0.6.2", features = ["fs"] } 137 | axum = { version = "0.8.1", features = ["ws"] } 138 | clap = { version = "4.5.3", features = ["derive"] } 139 | rand = "0.8.5" 140 | 141 | [target.'cfg(target_os = "linux")'.dependencies] 142 | x11-clipboard = "0.9.2" 143 | x11rb = "0.13.1" 144 | enigo = "0.3.0" 145 | gtk = "0.18.2" 146 | glib = "0.20.6" 147 | 148 | [target.'cfg(target_os = "windows")'.dependencies] 149 | winapi = { version = "0.3", features = [ 150 | "winuser", 151 | "winbase", 152 | "processthreadsapi", 153 | ] } 154 | enigo = "0.3.0" 155 | 156 | [target.'cfg(target_os = "macos")'.dependencies] 157 | objc-foundation = "0.1" 158 | objc_id = "0.1" 159 | cocoa = "0.26.0" 160 | objc = "0.2.7" 161 | core-graphics = "0.24.0" 162 | core-foundation = "0.10.0" 163 | core-foundation-sys = "0.8" 164 | active-win-pos-rs = "0.8.3" 165 | 166 | [build-dependencies] 167 | winresource = "0.1.17" 168 | 169 | [profile.release-lto] 170 | inherits = "release" 171 | lto = true 172 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 PlugOvr.ai 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # PlugOvr 2 | 3 | [![CI Status](https://github.com/PlugOvr-ai/PlugOvr/actions/workflows/check_everything.yml/badge.svg)](https://github.com/PlugOvr-ai/PlugOvr/actions) 4 | 5 | PlugOvr is a Rust-based application for AI Assistance that integrates with your favorite applications. With one shortcut you can access PlugOvr from any application. PlugOvr is cross-platform and works on Linux, Windows and MacOS. 6 | 7 | Select the text you want to use, write your own instructions or use your favorite templates. 8 | 9 | ![shortcuts](https://plugovr.ai/images/shortcuts.jpg) 10 | 11 | ## Features 12 | 13 | - Create your own prompts 14 | - Choose for each template the LLM that performs best. 15 | - Integrates Ollama Models 16 | 17 | ## How to use 18 | 19 | - Download PlugOvr from [PlugOvr.ai](https://plugovr.ai) 20 | - Install PlugOvr 21 | - select the text you want to use 22 | - press Ctrl + Alt + I or Ctrl + I write your own instructions. 23 | - - Ctrl + I is enough but might conflict with shortcuts from other application e.g. making text italic in gmail. 24 | 25 | - or use your favorite templates with Ctrl + Space 26 | - select Replace, Extend or Ignore 27 | - accept or reject the AI answer 28 | 29 | ## compile from source 30 | 31 | ### dependencies 32 | 33 | Linux: 34 | ```bash 35 | sudo apt install --no-install-recommends cmake build-essential libssl3 libdbus-1-3 libglfw3-dev libgtk-3-dev libxcb1-dev libxcb-render0-dev libxcb-shape0-dev libxcb-xfixes0-dev libxdo-dev 36 | ``` 37 | 38 | 39 | ## build and run from source 40 | 41 | ```bash 42 | cargo run --release 43 | ``` 44 | 45 | 46 | ## ComputerUse 47 | 48 | Plugovr implements a ComputerUse Interface using Qwen2.5VL-7B. 49 | 50 | spin up your local llm server for Qwen2.5VL-7B 51 | 52 | ```bash 53 | vllm serve "Qwen/Qwen2.5-VL-7B-Instruct" 54 | ``` 55 | 56 | then run PlugOvr with the computeruse feature 57 | 58 | ```bash 59 | cargo run --release --features computeruse 60 | ``` 61 | 62 | There are two shortcuts defined for computeruse: 63 | 64 | F4: to get a dialog to enter the instruction 65 | 66 | F2: to proceed the next action 67 | 68 | THIS IS A BETA FEATURE AND MIGHT NOT WORK AS EXPECTED. 69 | 70 | This is why we have no AUTO mode for computeruse yet. 71 | 72 | Its all about getting experience about the capabilities of Qwen2.5VL-7B. 73 | 74 | The video below is a combination of Qwen2.5-VL-7B for planning and a fine-tuned Florence2 for detection. 75 | Reach out if you like to know more. 76 | 77 | https://github.com/user-attachments/assets/54ccd427-aec8-45dd-b7f5-7e54d42a7070 78 | 79 | 80 | -------------------------------------------------------------------------------- /assets/128x128.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PlugOvr-ai/PlugOvr/273d7ea0f00a725db5b40838e497bd3ecfe2c95e/assets/128x128.png -------------------------------------------------------------------------------- /assets/32x32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PlugOvr-ai/PlugOvr/273d7ea0f00a725db5b40838e497bd3ecfe2c95e/assets/32x32.png -------------------------------------------------------------------------------- /assets/NotoSans-Regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PlugOvr-ai/PlugOvr/273d7ea0f00a725db5b40838e497bd3ecfe2c95e/assets/NotoSans-Regular.ttf -------------------------------------------------------------------------------- /assets/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | PlugOvr Remote Control 8 | 9 | 189 | 190 | 191 | 192 |
193 |

PlugOvr Remote Control

194 |
195 |
196 | 197 | 198 | 199 |
200 | 201 | 202 |
203 | 204 | Ready 205 | 206 | 208 | 210 | 212 |   213 | URL Settings 214 | 215 |
216 | 217 |
218 |
219 |
220 |
221 | Planning URL 222 | 224 | 225 |
226 |
227 |
228 |
229 | Execution URL 230 | 232 | 233 |
234 |
235 |
236 |
237 |
238 | 239 |
240 |
241 | Screenshot 242 |
243 |
244 |
245 |
246 |
Action Plan
247 |
248 |
249 | 250 |
251 |
252 |
253 |
254 |
255 | 256 | 617 | 618 | 619 | -------------------------------------------------------------------------------- /assets/logo.icns: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PlugOvr-ai/PlugOvr/273d7ea0f00a725db5b40838e497bd3ecfe2c95e/assets/logo.icns -------------------------------------------------------------------------------- /assets/logo.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PlugOvr-ai/PlugOvr/273d7ea0f00a725db5b40838e497bd3ecfe2c95e/assets/logo.ico -------------------------------------------------------------------------------- /assets/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PlugOvr-ai/PlugOvr/273d7ea0f00a725db5b40838e497bd3ecfe2c95e/assets/logo.png -------------------------------------------------------------------------------- /build.rs: -------------------------------------------------------------------------------- 1 | use { 2 | std::{env, io}, 3 | winresource::WindowsResource, 4 | }; 5 | 6 | fn main() -> io::Result<()> { 7 | if env::var_os("CARGO_CFG_WINDOWS").is_some() { 8 | WindowsResource::new() 9 | // This path can be absolute, or relative to your crate root. 10 | .set_icon("assets/logo.ico") 11 | .compile()?; 12 | } 13 | Ok(()) 14 | } 15 | -------------------------------------------------------------------------------- /crates/plugovr_types/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "plugovr_types" 3 | version = "0.1.73" 4 | edition = "2021" 5 | license-file = "LICENSE" 6 | authors = ["Cornelius Wefelscheid "] 7 | description = "PlugOvr provides AI assistance to every program." 8 | homepage = "https://plugovr.ai" 9 | 10 | repository = "https://github.com/PlugOvr-ai/PlugOvr_types" 11 | #maintainers = ["Cornelius Wefelscheid "] 12 | publish = false 13 | 14 | 15 | [dependencies] 16 | 17 | image = "0.24.9" 18 | serde = "1.0.216" 19 | egui = "0.31.1" 20 | -------------------------------------------------------------------------------- /crates/plugovr_types/src/lib.rs: -------------------------------------------------------------------------------- 1 | use egui::Pos2; 2 | use image::{ImageBuffer, Rgba}; 3 | use serde::Deserialize; 4 | 5 | #[derive(Clone, Debug, Deserialize, PartialEq)] 6 | pub struct UserInfo { 7 | pub username: Option, 8 | pub nickname: Option, 9 | pub name: Option, 10 | pub email: String, 11 | pub access_token: Option, 12 | pub refresh_token: Option, 13 | pub subscription_status: Option, 14 | pub subscription_name: Option, 15 | pub subscription_end_date: Option, 16 | } 17 | 18 | pub type Screenshots = Vec<(ImageBuffer, Vec>, Pos2)>; 19 | -------------------------------------------------------------------------------- /debian/postinst: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # Add user to input group 3 | if [ -z "$SUDO_USER" ]; then 4 | echo "Warning: SUDO_USER is not set. Unable to add user to input group." 5 | else 6 | if getent group input > /dev/null; then 7 | usermod -a -G input "$SUDO_USER" 8 | echo "Added user $SUDO_USER to input group." 9 | else 10 | echo "Warning: input group does not exist. Unable to add user to input group." 11 | fi 12 | fi 13 | -------------------------------------------------------------------------------- /deploy_osx.sh: -------------------------------------------------------------------------------- 1 | cargo build --release --features metal 2 | cargo bundle --release --features metal 3 | 4 | # Add privacy permissions after bundle creation 5 | /usr/libexec/PlistBuddy -c "Add :NSInputMonitoringUsageDescription string 'PlugOvr needs input monitoring permissions to function properly.'" \ 6 | "target/release/bundle/osx/PlugOvr.app/Contents/Info.plist" 7 | /usr/libexec/PlistBuddy -c "Add :NSAppleEventsUsageDescription string 'PlugOvr needs automation permissions to interact with other applications.'" \ 8 | "target/release/bundle/osx/PlugOvr.app/Contents/Info.plist" 9 | /usr/libexec/PlistBuddy -c "Add :NSScreenCaptureUsageDescription string 'PlugOvr needs screen recording permissions to capture and display your screen.'" \ 10 | "target/release/bundle/osx/PlugOvr.app/Contents/Info.plist" 11 | /usr/libexec/PlistBuddy -c "Add :NSAccessibilityUsageDescription string 'PlugOvr needs accessibility permissions to function properly.'" \ 12 | "target/release/bundle/osx/PlugOvr.app/Contents/Info.plist" 13 | 14 | xattr -cr target/release/bundle/osx/PlugOvr.app 15 | codesign -s $DEV_ID_APP --entitlements entitlements.plist --deep --force --options runtime target/release/bundle/osx/PlugOvr.app/Contents/MacOS/* 16 | codesign -s $DEV_ID_APP --entitlements entitlements.plist --deep --force --options runtime target/release/bundle/osx/PlugOvr.app --entitlements entitlements.plist 17 | 18 | #ditto -c -k --keepParent "target/release/bundle/osx/PlugOvr.app" "PlugOvr.zip" 19 | 20 | # Create a temporary directory for DMG contents 21 | TMP_DMG_DIR="tmp_dmg" 22 | mkdir -p "${TMP_DMG_DIR}" 23 | 24 | # Create Applications folder symlink 25 | ln -s /Applications "${TMP_DMG_DIR}/Applications" 26 | 27 | # Copy the app to the temporary directory 28 | cp -r "target/release/bundle/osx/PlugOvr.app" "${TMP_DMG_DIR}/" 29 | 30 | # Create DMG with background and positioning 31 | hdiutil create -volname "PlugOvr" \ 32 | -srcfolder "${TMP_DMG_DIR}" \ 33 | -ov -format UDZO \ 34 | -fs HFS+ \ 35 | -size 200m \ 36 | "target/PlugOvr.dmg" 37 | 38 | # Clean up 39 | rm -rf "${TMP_DMG_DIR}" 40 | 41 | # Get version from Cargo.toml 42 | VERSION=$(grep '^version =' Cargo.toml | head -n1 | cut -d'"' -f2) 43 | 44 | # Rename DMG with version 45 | mv target/PlugOvr.dmg "target/PlugOvr_${VERSION}_aarch64.dmg" 46 | 47 | xcrun notarytool submit target/PlugOvr_${VERSION}_aarch64.dmg --apple-id $APPLE_ID --password $APPLE_ID_PASSWORD --team-id $TEAM_ID --wait 48 | 49 | xcrun stapler staple target/PlugOvr_${VERSION}_aarch64.dmg 50 | 51 | aws s3 cp target/PlugOvr_${VERSION}_aarch64.dmg s3://plugovr.ai/ 52 | 53 | echo "{\"version\": \"${VERSION}\"}" > target/latest_osx_aarch64_version.json 54 | aws s3 cp target/latest_osx_aarch64_version.json s3://plugovr.ai/latest_osx_aarch64_version.json 55 | 56 | #codesign -s "Developer ID Application: %(id)" --force --options runtime ./target/release/bundle/osx/RustDesk.app/Contents/MacOS/* 57 | #codesign -s "Developer ID Application: %(id)" --force --options runtime ./target/release/bundle/osx/RustDesk.app 58 | -------------------------------------------------------------------------------- /deploy_osx_x86_64.sh: -------------------------------------------------------------------------------- 1 | cargo build --target x86_64-apple-darwin --release --features metal 2 | cargo bundle --target x86_64-apple-darwin --release --features metal 3 | 4 | # Add privacy permissions after bundle creation 5 | /usr/libexec/PlistBuddy -c "Add :NSInputMonitoringUsageDescription string 'PlugOvr needs input monitoring permissions to function properly.'" \ 6 | "target/x86_64-apple-darwin/release/bundle/osx/PlugOvr.app/Contents/Info.plist" 7 | /usr/libexec/PlistBuddy -c "Add :NSAppleEventsUsageDescription string 'PlugOvr needs automation permissions to interact with other applications.'" \ 8 | "target/x86_64-apple-darwin/release/bundle/osx/PlugOvr.app/Contents/Info.plist" 9 | /usr/libexec/PlistBuddy -c "Add :NSScreenCaptureUsageDescription string 'PlugOvr needs screen recording permissions to capture and display your screen.'" \ 10 | "target/x86_64-apple-darwin/release/bundle/osx/PlugOvr.app/Contents/Info.plist" 11 | /usr/libexec/PlistBuddy -c "Add :NSAccessibilityUsageDescription string 'PlugOvr needs accessibility permissions to function properly.'" \ 12 | "target/x86_64-apple-darwin/release/bundle/osx/PlugOvr.app/Contents/Info.plist" 13 | 14 | xattr -cr target/x86_64-apple-darwin/release/bundle/osx/PlugOvr.app 15 | codesign -s $DEV_ID_APP --entitlements entitlements.plist --deep --force --options runtime target/x86_64-apple-darwin/release/bundle/osx/PlugOvr.app/Contents/MacOS/* 16 | codesign -s $DEV_ID_APP --entitlements entitlements.plist --deep --force --options runtime target/x86_64-apple-darwin/release/bundle/osx/PlugOvr.app --entitlements entitlements.plist 17 | 18 | #ditto -c -k --keepParent "target/release/bundle/osx/PlugOvr.app" "PlugOvr.zip" 19 | 20 | # Create a temporary directory for DMG contents 21 | TMP_DMG_DIR="tmp_dmg" 22 | mkdir -p "${TMP_DMG_DIR}" 23 | 24 | # Create Applications folder symlink 25 | ln -s /Applications "${TMP_DMG_DIR}/Applications" 26 | 27 | # Copy the app to the temporary directory 28 | cp -r "target/x86_64-apple-darwin/release/bundle/osx/PlugOvr.app" "${TMP_DMG_DIR}/" 29 | 30 | # Create DMG with background and positioning 31 | hdiutil create -volname "PlugOvr" \ 32 | -srcfolder "${TMP_DMG_DIR}" \ 33 | -ov -format UDZO \ 34 | -fs HFS+ \ 35 | -size 200m \ 36 | "target/PlugOvr.dmg" 37 | 38 | # Clean up 39 | rm -rf "${TMP_DMG_DIR}" 40 | 41 | # Get version from Cargo.toml 42 | VERSION=$(grep '^version =' Cargo.toml | head -n1 | cut -d'"' -f2) 43 | 44 | # Rename DMG with version 45 | mv target/PlugOvr.dmg "target/PlugOvr_${VERSION}_x86_64.dmg" 46 | 47 | xcrun notarytool submit target/PlugOvr_${VERSION}_x86_64.dmg --apple-id $APPLE_ID --password $APPLE_ID_PASSWORD --team-id $TEAM_ID --wait 48 | 49 | xcrun stapler staple target/PlugOvr_${VERSION}_x86_64.dmg 50 | 51 | aws s3 cp target/PlugOvr_${VERSION}_x86_64.dmg s3://plugovr.ai/ 52 | 53 | echo "{\"version\": \"${VERSION}\"}" > target/latest_osx_x86_64_version.json 54 | aws s3 cp target/latest_osx_x86_64_version.json s3://plugovr.ai/latest_osx_x86_64_version.json 55 | 56 | #codesign -s "Developer ID Application: %(id)" --force --options runtime ./target/release/bundle/osx/RustDesk.app/Contents/MacOS/* 57 | #codesign -s "Developer ID Application: %(id)" --force --options runtime ./target/release/bundle/osx/RustDesk.app 58 | -------------------------------------------------------------------------------- /entitlements.plist: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | com.apple.security.device.input-monitoring 6 | 7 | com.apple.security.automation.apple-events 8 | 9 | com.apple.security.screen-recording 10 | 11 | com.apple.security.accessibility 12 | 13 | 14 | -------------------------------------------------------------------------------- /src/llm.rs: -------------------------------------------------------------------------------- 1 | // Add these imports at the top of the file 2 | use kalosm::language::*; 3 | #[cfg(feature = "cs")] 4 | use plugovr_cs::cloud_llm::call_aws_lambda; 5 | use plugovr_types::{Screenshots, UserInfo}; 6 | use std::error::Error; 7 | use std::io::Write; 8 | use std::sync::{Arc, Mutex}; 9 | 10 | #[cfg(feature = "cs")] 11 | use plugovr_cs::user_management::get_user; 12 | 13 | use egui::{Context, Window}; 14 | 15 | use image_24::{ImageBuffer, Rgba}; 16 | use ollama_rs::{ 17 | Ollama, 18 | generation::chat::{ChatMessage, MessageRole, request::ChatMessageRequest}, 19 | generation::images::Image, 20 | generation::options::GenerationOptions, 21 | }; 22 | use serde::{Deserialize, Serialize}; 23 | use std::fmt; 24 | use std::fs::File; 25 | use std::io::Read; 26 | use strum::EnumIter; 27 | 28 | async fn call_ollama( 29 | ollama: Arc>>, 30 | model: String, 31 | input: String, 32 | _context: String, 33 | _instruction: String, 34 | ai_answer: Arc>, 35 | screenshots: &Screenshots, 36 | ) -> Result> { 37 | use base64::{Engine as _, engine::general_purpose}; 38 | 39 | let screenshot_base64 = screenshots 40 | .iter() 41 | .map(|img| { 42 | let mut buf = vec![]; 43 | img.0 44 | .write_to( 45 | &mut std::io::Cursor::new(&mut buf), 46 | image_24::ImageOutputFormat::Png, 47 | ) 48 | .unwrap(); 49 | general_purpose::STANDARD.encode(&buf) 50 | }) 51 | .collect::>(); 52 | let images = screenshot_base64 53 | .iter() 54 | .map(|base64| Image::from_base64(base64)) 55 | .collect::>(); 56 | 57 | let options = GenerationOptions::default() 58 | .temperature(0.2) 59 | .repeat_penalty(1.1) 60 | .top_k(40) 61 | .top_p(0.5) 62 | .num_predict(500); 63 | 64 | // let mut stream = ollama 65 | // .as_ref() 66 | // .lock() 67 | // .unwrap() 68 | // .as_ref() 69 | // .unwrap() 70 | // .generate_stream( 71 | // GenerationRequest::new(model, input) 72 | // .options(options) 73 | // .images(images), 74 | // ) 75 | // .await 76 | // .unwrap(); 77 | let message = ChatMessage::new(MessageRole::User, input.to_string()); 78 | let messages = vec![message.clone().with_images(images)]; 79 | let ollama_instance = { 80 | let guard = ollama.lock().unwrap(); 81 | guard.as_ref().unwrap().clone() 82 | }; 83 | 84 | let stream = ollama_instance 85 | .send_chat_messages_stream(ChatMessageRequest::new(model, messages).options(options)) 86 | .await; 87 | let mut response = String::new(); 88 | match stream { 89 | Err(e) => { 90 | *ai_answer.lock().unwrap() = format!("Error: {}", e); 91 | response = format!("Error: {}", e); 92 | Ok(response) 93 | } 94 | Ok(mut stream) => { 95 | while let Some(Ok(res)) = stream.next().await { 96 | let assistant_message = res.message; 97 | response += assistant_message.content.as_str(); 98 | *ai_answer.lock().unwrap() = response.clone(); 99 | } 100 | Ok(response) 101 | } 102 | } 103 | } 104 | async fn call_local_llm( 105 | _input: String, 106 | context: String, 107 | instruction: String, 108 | ai_answer: Arc>, 109 | model: Arc>>, 110 | ) -> Result> { 111 | let model = model.try_lock(); 112 | if model.is_err() { 113 | eprintln!("Model is not locked"); 114 | return Err(Box::new(std::io::Error::new( 115 | std::io::ErrorKind::Other, 116 | "Model is not locked", 117 | ))); 118 | } 119 | 120 | let prompt = format!("Context: {} Instruction: {}", context, instruction); 121 | 122 | let model_instance = { 123 | let mut guard = model.unwrap(); 124 | guard.as_mut().unwrap().clone() 125 | }; 126 | let mut stream = model_instance(&prompt); 127 | 128 | let mut response = String::new(); 129 | 130 | while let Some(token) = stream.next().await { 131 | response.push_str(&token); 132 | 133 | // Update ai_answer with the current response 134 | *ai_answer.lock().unwrap() = response.clone(); 135 | } 136 | Ok(response) 137 | } 138 | 139 | #[derive(Clone, PartialEq, Serialize, Deserialize)] 140 | pub enum LLMType { 141 | Cloud(CloudModel), 142 | Local(LocalModel), 143 | Ollama(String), 144 | } 145 | use strum::IntoEnumIterator; 146 | 147 | #[derive(Clone, Copy, PartialEq, EnumIter, Serialize, Deserialize)] 148 | pub enum CloudModel { 149 | AnthropicHaiku, 150 | AnthropicSonnet3_5, 151 | } 152 | impl CloudModel { 153 | pub fn description(&self) -> String { 154 | match self { 155 | CloudModel::AnthropicHaiku => "Anthropic Haiku".to_string(), 156 | CloudModel::AnthropicSonnet3_5 => "Anthropic Sonnet 3.5".to_string(), 157 | } 158 | } 159 | } 160 | 161 | #[derive(Clone, Copy, PartialEq, EnumIter, Serialize, Deserialize)] 162 | pub enum LocalModel { 163 | Llama32S1bChat, 164 | Llama32S3bChat, 165 | } 166 | impl LocalModel { 167 | pub fn description(&self) -> String { 168 | match self { 169 | LocalModel::Llama32S1bChat => "Llama 3.2 1B Chat".to_string(), 170 | LocalModel::Llama32S3bChat => "Llama 3.2 3B Chat".to_string(), 171 | } 172 | } 173 | } 174 | impl LLMType { 175 | pub fn description(&self) -> String { 176 | match self { 177 | LLMType::Cloud(cloud_model) => format!("{} - Cloud", cloud_model.description()), 178 | LLMType::Local(local_model) => format!("{} - Local", local_model.description()), 179 | LLMType::Ollama(model) => format!("Ollama - {}", model), 180 | } 181 | } 182 | } 183 | impl LocalModel { 184 | pub fn source(&self) -> LlamaSource { 185 | match self { 186 | LocalModel::Llama32S1bChat => LlamaSource::llama_3_2_1b_chat(), 187 | LocalModel::Llama32S3bChat => LlamaSource::llama_3_2_3b_chat(), 188 | } 189 | } 190 | } 191 | 192 | pub struct LLMSelector { 193 | llm_type: LLMType, 194 | model: Arc>>, 195 | show_window: bool, 196 | download_progress: Arc>, 197 | download_error: Arc>>, 198 | pub user_info: Arc>>, 199 | ollama: Arc>>, 200 | pub ollama_models: Arc>>>, 201 | } 202 | 203 | impl LLMSelector { 204 | pub fn new(user_info: Arc>>) -> Self { 205 | let llm_type = load_llm_type().unwrap_or(LLMType::Cloud(CloudModel::AnthropicHaiku)); 206 | let ollama = Ollama::default(); 207 | let ollama_models = Arc::new(Mutex::new(None)); 208 | { 209 | let ollama_models = ollama_models.clone(); 210 | tokio::task::spawn(async move { 211 | if let Ok(models) = ollama.list_local_models().await { 212 | *ollama_models.lock().unwrap() = Some(models); 213 | } 214 | }); 215 | } 216 | let ollama = Ollama::default(); 217 | LLMSelector { 218 | llm_type, 219 | model: Arc::new(Mutex::new(None)), 220 | show_window: false, 221 | download_progress: Arc::new(Mutex::new(0.0)), 222 | download_error: Arc::new(Mutex::new(None)), 223 | user_info, 224 | ollama: Arc::new(Mutex::new(Some(ollama))), 225 | ollama_models, 226 | } 227 | } 228 | 229 | pub async fn load_model(&self) { 230 | let mut model = self.model.lock().unwrap(); 231 | *model = match &self.llm_type { 232 | LLMType::Local(LocalModel::Llama32S1bChat) => Some( 233 | Llama::builder() 234 | .with_source(LlamaSource::llama_3_2_1b_chat()) 235 | .build() 236 | .await 237 | .unwrap(), 238 | ), 239 | LLMType::Local(LocalModel::Llama32S3bChat) => Some( 240 | Llama::builder() 241 | .with_source(LlamaSource::llama_3_2_3b_chat()) 242 | .build() 243 | .await 244 | .unwrap(), 245 | ), 246 | LLMType::Cloud(CloudModel::AnthropicHaiku) => None, 247 | LLMType::Cloud(CloudModel::AnthropicSonnet3_5) => None, 248 | LLMType::Ollama(_model) => None, 249 | }; 250 | } 251 | 252 | pub fn process_input( 253 | &self, 254 | prompt: String, 255 | context: String, 256 | screenshots: Vec<(ImageBuffer, Vec>, egui::Pos2)>, 257 | instruction: String, 258 | ai_answer: Arc>, 259 | max_tokens_reached: Arc>, 260 | spinner: Arc>, 261 | llm_from_template: Option, 262 | ) -> Result, Box> { 263 | let mut llm_type = self.llm_type.clone(); 264 | if let Some(llm_from_template) = llm_from_template { 265 | llm_type = llm_from_template; 266 | } 267 | 268 | let model = self.model.clone(); 269 | let spinner_clone = spinner.clone(); 270 | let user_info = self.user_info.clone(); 271 | if (llm_type == LLMType::Cloud(CloudModel::AnthropicHaiku) 272 | || llm_type == LLMType::Cloud(CloudModel::AnthropicSonnet3_5)) 273 | && user_info.lock().unwrap().is_none() 274 | { 275 | *ai_answer.lock().unwrap() = 276 | "Please login to use cloud LLM or switch to local LLM".to_string(); 277 | return Err(Box::new(std::io::Error::new( 278 | std::io::ErrorKind::Other, 279 | "Please login to use cloud LLM or switch to local LLM", 280 | ))); 281 | } 282 | let ollama = self.ollama.clone(); 283 | 284 | let handle = tokio::task::spawn_blocking(move || { 285 | *spinner_clone.lock().unwrap() = true; 286 | let llm_type_clone = llm_type.clone(); 287 | let result: Result<(String, bool), Box> = match llm_type { 288 | LLMType::Cloud(CloudModel::AnthropicHaiku) 289 | | LLMType::Cloud(CloudModel::AnthropicSonnet3_5) => { 290 | let user_info = user_info.lock().unwrap().as_ref().unwrap().clone(); 291 | let model = llm_type.clone().to_string(); 292 | #[cfg(feature = "cs")] 293 | { 294 | Ok(call_aws_lambda( 295 | user_info, 296 | prompt.clone(), 297 | model, 298 | &screenshots, 299 | )) 300 | } 301 | #[cfg(not(feature = "cs"))] 302 | { 303 | Ok(( 304 | "Download PlugOvr from https://plugovr.ai to use cloud LLM".to_string(), 305 | false, 306 | )) 307 | } 308 | } 309 | LLMType::Local(_) => { 310 | use tokio::runtime::Runtime; 311 | let rt = Runtime::new().unwrap(); 312 | rt.block_on(async { 313 | let result = call_local_llm( 314 | prompt.clone(), 315 | context, 316 | instruction, 317 | ai_answer.clone(), 318 | model, 319 | ) 320 | .await; 321 | Ok((result?, false)) 322 | }) 323 | } 324 | LLMType::Ollama(model) => { 325 | use tokio::runtime::Runtime; 326 | let rt = Runtime::new().unwrap(); 327 | let model_clone = model.clone(); 328 | rt.block_on(async { 329 | let result = call_ollama( 330 | ollama, 331 | model_clone, 332 | prompt.clone(), 333 | context, 334 | instruction, 335 | ai_answer.clone(), 336 | &screenshots, 337 | ) 338 | .await; 339 | Ok((result?, false)) 340 | }) 341 | } 342 | }; 343 | 344 | let mut result = result.unwrap(); 345 | #[cfg(feature = "cs")] 346 | if result.0.contains("Access token expired") { 347 | match get_user() { 348 | Ok(user_info_tmp) => { 349 | *user_info.lock().unwrap() = Some(user_info_tmp); 350 | 351 | if llm_type_clone == LLMType::Cloud(CloudModel::AnthropicHaiku) 352 | || llm_type_clone == LLMType::Cloud(CloudModel::AnthropicSonnet3_5) 353 | { 354 | let user_info = user_info.lock().unwrap().as_ref().unwrap().clone(); 355 | let model = llm_type_clone.clone().to_string(); 356 | result = call_aws_lambda(user_info, prompt, model, &screenshots); 357 | } 358 | } 359 | Err(_e) => { 360 | *user_info.lock().unwrap() = None; 361 | } 362 | } 363 | } 364 | *ai_answer.lock().unwrap() = result.0; 365 | *max_tokens_reached.lock().unwrap() = result.1; 366 | *spinner.lock().unwrap() = false; 367 | }); 368 | Ok(handle) 369 | } 370 | 371 | pub fn show_selection_window(&mut self, ctx: &Context) { 372 | Window::new("LLM Selection") 373 | .open(&mut self.show_window) 374 | .collapsible(false) 375 | .max_width(400.0) 376 | .show(ctx, |ui| { 377 | ui.heading("Cloud Models"); 378 | let cloud_models = CloudModel::iter().collect::>(); 379 | for cloud_model in cloud_models { 380 | if ui 381 | .radio_value( 382 | &mut self.llm_type, 383 | LLMType::Cloud(cloud_model), 384 | LLMType::Cloud(cloud_model).description(), 385 | ) 386 | .changed() 387 | { 388 | save_llm_type(LLMType::Cloud(cloud_model)) 389 | .unwrap_or_else(|e| eprintln!("Failed to save LLM type: {}", e)); 390 | } 391 | } 392 | 393 | ui.heading("Local Models"); 394 | for local_model in LocalModel::iter() { 395 | let requires_download = Llama::builder() 396 | .with_source(local_model.source()) 397 | .requires_download(); 398 | 399 | ui.horizontal(|ui| { 400 | if ui 401 | .add_enabled( 402 | !requires_download, 403 | egui::RadioButton::new( 404 | self.llm_type == LLMType::Local(local_model), 405 | LLMType::Local(local_model).description(), 406 | ), 407 | ) 408 | .clicked() 409 | { 410 | self.llm_type = LLMType::Local(local_model); 411 | save_llm_type(LLMType::Local(local_model)) 412 | .unwrap_or_else(|e| eprintln!("Failed to save LLM type: {}", e)); 413 | 414 | let model = self.model.clone(); 415 | let llama_source = local_model.source(); 416 | 417 | tokio::spawn(async move { 418 | let llama = Llama::builder() 419 | .with_source(llama_source) 420 | .build() 421 | .await 422 | .unwrap(); 423 | if let Ok(mut model_guard) = model.lock() { 424 | *model_guard = Some(llama); 425 | } else { 426 | eprintln!("Failed to acquire lock on model"); 427 | } 428 | }); 429 | } 430 | if !requires_download { 431 | ui.label("Downloaded"); 432 | } else if ui.button("Download").clicked() { 433 | let llama_source = local_model.source(); 434 | let download_progress = self.download_progress.clone(); 435 | let download_error = self.download_error.clone(); 436 | tokio::spawn(async move { 437 | let download_progress = download_progress.clone(); 438 | let _llama = match Llama::builder() 439 | .with_source(llama_source) 440 | .build_with_loading_handler(move |x| match x.clone() { 441 | ModelLoadingProgress::Downloading { .. } => { 442 | *download_progress.lock().unwrap() = x.progress() 443 | } 444 | ModelLoadingProgress::Loading { progress } => { 445 | *download_progress.lock().unwrap() = progress; 446 | } 447 | }) 448 | .await 449 | { 450 | Ok(llama) => llama, 451 | Err(e) => { 452 | eprintln!("Failed to download/load model: {}", e); 453 | *download_error.lock().unwrap() = Some(e.to_string()); 454 | return; 455 | } 456 | }; 457 | }); 458 | } 459 | }); 460 | } 461 | if let Some(ollama_models) = self.ollama_models.lock().unwrap().as_ref() { 462 | ui.heading("Ollama Models"); 463 | let ollama_models = ollama_models.clone(); 464 | if ollama_models.is_empty() { 465 | ui.label( 466 | "No ollama models found, pull some models with e.g. ollama pull llama3.2:1b" 467 | ); 468 | } else { 469 | for ollama_model in ollama_models { 470 | if ui 471 | .radio_value( 472 | &mut self.llm_type, 473 | LLMType::Ollama(ollama_model.name.clone()), 474 | LLMType::Ollama(ollama_model.name.clone()).description(), 475 | ) 476 | .changed() 477 | { 478 | save_llm_type(LLMType::Ollama(ollama_model.name.clone())) 479 | .unwrap_or_else(|e| eprintln!("Failed to save LLM type: {}", e)); 480 | } 481 | } 482 | } 483 | } else { 484 | ui.heading("Ollama not installed"); 485 | } 486 | 487 | if *self.download_progress.lock().unwrap() > 0.0 { 488 | ui.label("Download progress"); 489 | ui.add( 490 | egui::ProgressBar::new(*self.download_progress.lock().unwrap()) 491 | .show_percentage(), 492 | ); 493 | } 494 | if let Some(error) = self.download_error.lock().unwrap().as_ref() { 495 | ui.colored_label(ui.visuals().error_fg_color, error); 496 | } 497 | }); 498 | } 499 | 500 | pub fn toggle_window(&mut self) { 501 | self.show_window = !self.show_window; 502 | } 503 | 504 | pub fn get_llm_type(&self) -> LLMType { 505 | self.llm_type.clone() 506 | } 507 | } 508 | // Add this new function to save the LLMType 509 | fn save_llm_type(llm_type: LLMType) -> std::io::Result<()> { 510 | let mut path = dirs::home_dir().expect("Unable to get home directory"); 511 | path.push(".plugovr"); 512 | std::fs::create_dir_all(&path)?; 513 | path.push("llm_type.json"); 514 | 515 | let serialized = serde_json::to_string(&llm_type)?; 516 | let mut file = File::create(path)?; 517 | file.write_all(serialized.as_bytes())?; 518 | Ok(()) 519 | } 520 | 521 | // Add this new function to load the LLMType 522 | fn load_llm_type() -> std::io::Result { 523 | let mut path = dirs::home_dir().expect("Unable to get home directory"); 524 | path.push(".plugovr"); 525 | path.push("llm_type.json"); 526 | 527 | let mut file = File::open(path)?; 528 | let mut contents = String::new(); 529 | file.read_to_string(&mut contents)?; 530 | let llm_type: LLMType = serde_json::from_str(&contents)?; 531 | Ok(llm_type) 532 | } 533 | 534 | impl fmt::Display for LLMType { 535 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { 536 | match self { 537 | LLMType::Cloud(CloudModel::AnthropicHaiku) => write!(f, "AnthropicHaiku"), 538 | LLMType::Cloud(CloudModel::AnthropicSonnet3_5) => write!(f, "AnthropicSonnet3_5"), 539 | LLMType::Local(LocalModel::Llama32S1bChat) => write!(f, "Llama32S1bChat"), 540 | LLMType::Local(LocalModel::Llama32S3bChat) => write!(f, "Llama32S3bChat"), 541 | LLMType::Ollama(model) => write!(f, "{}", model), 542 | } 543 | } 544 | } 545 | -------------------------------------------------------------------------------- /src/ui.rs: -------------------------------------------------------------------------------- 1 | pub mod answer_analyser; 2 | pub mod assistance_window; 3 | pub mod diff_view; 4 | pub mod main_window; 5 | pub mod screen_dimensions; 6 | pub mod shortcut_window; 7 | pub mod show_form_fields; 8 | pub mod template_editor; 9 | pub mod user_interface; 10 | -------------------------------------------------------------------------------- /src/ui/answer_analyser.rs: -------------------------------------------------------------------------------- 1 | #[derive(Debug)] 2 | #[allow(dead_code)] 3 | pub struct Coords { 4 | pub x1: i32, 5 | pub y1: i32, 6 | pub x2: i32, 7 | pub y2: i32, 8 | } 9 | 10 | #[derive(Debug)] 11 | #[allow(dead_code)] 12 | pub struct FormFields { 13 | pub field_name: String, 14 | pub field_value: String, 15 | pub field_coords: Coords, 16 | } 17 | 18 | pub fn analyse_answer(ai_answer: &str) -> Option> { 19 | let mut form_fields: Vec = Vec::new(); 20 | 21 | // Find JSON content with proper bracket matching 22 | if let Some(start) = ai_answer.find('[') { 23 | let mut bracket_count = 0; 24 | let mut end = start; 25 | 26 | for (i, c) in ai_answer[start..].char_indices() { 27 | match c { 28 | '[' => bracket_count += 1, 29 | ']' => { 30 | bracket_count -= 1; 31 | if bracket_count == 0 { 32 | end = start + i; 33 | break; 34 | } 35 | } 36 | _ => {} 37 | } 38 | } 39 | 40 | if bracket_count == 0 { 41 | let json_str = &ai_answer[start..=end]; 42 | 43 | // Try to parse the extracted JSON 44 | if let Ok(json) = serde_json::from_str::(json_str) { 45 | // Check if it's an object 46 | if let Some(obj) = json.as_array() { 47 | // Validate expected structure 48 | for field_obj in obj { 49 | if let Some(field_obj) = field_obj.as_object() { 50 | // Check if required fields exist 51 | let has_content = field_obj.contains_key("content"); 52 | let has_caption = field_obj.contains_key("caption"); 53 | let has_coords = field_obj.contains_key("coordinates"); 54 | 55 | if !has_content || !has_caption || !has_coords { 56 | return None; 57 | } 58 | 59 | // Extract coordinates 60 | let coords_str = field_obj["coordinates"].as_str().unwrap_or("[]"); 61 | // Remove brackets and split by comma 62 | let coords: Vec = coords_str 63 | .trim_matches(|p| p == '[' || p == ']') 64 | .split(',') 65 | .filter_map(|s| s.trim().parse().ok()) 66 | .collect(); 67 | 68 | if coords.len() == 4 { 69 | let field_coords = Coords { 70 | x1: coords[0], 71 | y1: coords[1], 72 | x2: coords[2], 73 | y2: coords[3], 74 | }; 75 | 76 | // Create and push FormFields 77 | form_fields.push(FormFields { 78 | field_name: field_obj["caption"] 79 | .as_str() 80 | .unwrap_or("") 81 | .to_string(), 82 | field_value: field_obj["content"] 83 | .as_str() 84 | .unwrap_or("") 85 | .to_string(), 86 | field_coords, 87 | }); 88 | } 89 | } else { 90 | return None; 91 | } 92 | } 93 | } 94 | } 95 | } 96 | } 97 | Some(form_fields) 98 | } 99 | -------------------------------------------------------------------------------- /src/ui/diff_view.rs: -------------------------------------------------------------------------------- 1 | use similar::{ChangeTag, TextDiff}; 2 | 3 | pub fn display_diff(ui: &mut egui::Ui, old_text: &str, new_text: &str) { 4 | let diff = TextDiff::from_chars(old_text, new_text); 5 | 6 | let mut job = egui::text::LayoutJob::default(); 7 | for change in diff.iter_all_changes() { 8 | match change.tag() { 9 | ChangeTag::Delete => job.append( 10 | &change 11 | .to_string() 12 | .chars() 13 | .next() 14 | .unwrap_or_default() 15 | .to_string(), 16 | 0.0, 17 | egui::TextFormat { 18 | color: ui.style().visuals.text_color(), 19 | background: egui::Color32::from_rgb(255, 0, 0), 20 | font_id: ui 21 | .style() 22 | .text_styles 23 | .get(&egui::TextStyle::Body) 24 | .expect("Body style not found") 25 | .clone(), 26 | ..Default::default() 27 | }, 28 | ), 29 | ChangeTag::Insert => job.append( 30 | &change 31 | .to_string() 32 | .chars() 33 | .next() 34 | .unwrap_or_default() 35 | .to_string(), 36 | 0.0, 37 | egui::TextFormat { 38 | color: ui.style().visuals.text_color(), 39 | background: egui::Color32::from_rgb(0, 255, 0), 40 | font_id: ui 41 | .style() 42 | .text_styles 43 | .get(&egui::TextStyle::Body) 44 | .expect("Body style not found") 45 | .clone(), 46 | ..Default::default() 47 | }, 48 | ), 49 | ChangeTag::Equal => job.append( 50 | &change 51 | .to_string() 52 | .chars() 53 | .next() 54 | .unwrap_or_default() 55 | .to_string(), 56 | 0.0, 57 | egui::TextFormat { 58 | color: ui.style().visuals.text_color(), 59 | font_id: ui 60 | .style() 61 | .text_styles 62 | .get(&egui::TextStyle::Body) 63 | .expect("Body style not found") 64 | .clone(), 65 | ..Default::default() 66 | }, 67 | ), 68 | }; 69 | } 70 | ui.label(job); 71 | } 72 | -------------------------------------------------------------------------------- /src/ui/main_window.rs: -------------------------------------------------------------------------------- 1 | use crate::llm::LLMSelector; 2 | use crate::ui::template_editor::TemplateEditor; 3 | use crate::ui::template_editor::TemplateMap; 4 | 5 | #[cfg(feature = "cs")] 6 | use plugovr_cs::login_window::LoginWindow; 7 | 8 | use plugovr_types::UserInfo; 9 | use std::collections::HashMap; 10 | use std::sync::{Arc, Mutex}; 11 | use webbrowser; 12 | 13 | #[cfg(feature = "computeruse_editor")] 14 | use crate::usecase_editor::UsecaseEditor; 15 | 16 | pub struct MainWindow { 17 | #[cfg(feature = "cs")] 18 | login_window: LoginWindow, 19 | template_editor: TemplateEditor, 20 | window_pos_initialized: bool, 21 | pub user_info: Arc>>, 22 | pub is_loading_user_info: Arc>, 23 | pub version_msg: Arc>, 24 | pub version_msg_old: Arc>, 25 | pub llm_selector: Arc>, 26 | show_template_editor: Arc>, 27 | show_llm_selector: Arc>, 28 | show_login_window: Arc>, 29 | pub menu_map: Arc>>>, 30 | #[cfg(feature = "computeruse_editor")] 31 | show_usecase_editor: Arc>, 32 | #[cfg(feature = "computeruse_editor")] 33 | usecase_editor: Arc>, 34 | } 35 | impl MainWindow { 36 | pub fn new( 37 | user_info: Arc>>, 38 | is_loading_user_info: Arc>, 39 | prompt_templates: TemplateMap, 40 | llm_selector: Arc>, 41 | version_msg: Arc>, 42 | #[cfg(feature = "computeruse_editor")] usecase_editor: Arc>, 43 | ) -> Self { 44 | use tray_icon::menu::MenuEvent; 45 | let show_login_window = Arc::new(Mutex::new(false)); 46 | let show_template_editor = Arc::new(Mutex::new(false)); 47 | let show_llm_selector = Arc::new(Mutex::new(false)); 48 | #[cfg(feature = "computeruse_editor")] 49 | let show_usecase_editor = Arc::new(Mutex::new(false)); 50 | #[cfg(feature = "cs")] 51 | let login_window = LoginWindow::new(user_info.clone(), is_loading_user_info.clone()); 52 | let template_editor = TemplateEditor::new(prompt_templates.clone()); 53 | let menu_map = Arc::new(Mutex::new(Option::>::None)); 54 | let menu_channel = MenuEvent::receiver(); 55 | 56 | { 57 | let show_login_window = show_login_window.clone(); 58 | let show_template_editor = show_template_editor.clone(); 59 | let show_llm_selector = show_llm_selector.clone(); 60 | let user_info = user_info.clone(); 61 | let menu_map = menu_map.clone(); 62 | #[cfg(feature = "computeruse_editor")] 63 | let show_usecase_editor = show_usecase_editor.clone(); 64 | std::thread::spawn(move || { 65 | while let Ok(recv) = menu_channel.recv() { 66 | let id = recv.id().0.to_string(); 67 | let menu_map = menu_map.lock().unwrap().clone(); 68 | if let Some(menu_map) = menu_map { 69 | #[cfg(feature = "cs")] 70 | if id == *menu_map.get("Login").unwrap_or(&"".to_string()) { 71 | if user_info.lock().unwrap().is_some() { 72 | *user_info.lock().unwrap() = None; 73 | _ = plugovr_cs::user_management::logout(); 74 | } else { 75 | *show_login_window.lock().unwrap() = true; 76 | } 77 | } 78 | #[cfg(feature = "computeruse_editor")] 79 | if id == *menu_map.get("Usecase Editor").unwrap_or(&"".to_string()) { 80 | println!("Usecase Editor"); 81 | *show_usecase_editor.lock().unwrap() = true; 82 | } 83 | if id == *menu_map.get("Template Editor").unwrap_or(&"".to_string()) { 84 | println!("Template Editor"); 85 | *show_template_editor.lock().unwrap() = true; 86 | } 87 | if id == *menu_map.get("LLM Selector").unwrap_or(&"".to_string()) { 88 | println!("LLM Selector"); 89 | *show_llm_selector.lock().unwrap() = true; 90 | } 91 | if id == *menu_map.get("Updater").unwrap_or(&"".to_string()) { 92 | let _ = webbrowser::open("https://plugovr.ai/download").is_ok(); 93 | } 94 | if id == *menu_map.get("Quit").unwrap_or(&"".to_string()) { 95 | println!("Quit"); 96 | std::process::exit(0); 97 | } 98 | } 99 | } 100 | }); 101 | } 102 | 103 | Self { 104 | #[cfg(feature = "cs")] 105 | login_window, 106 | template_editor, 107 | window_pos_initialized: false, 108 | user_info: user_info.clone(), 109 | is_loading_user_info, 110 | version_msg, 111 | version_msg_old: Arc::new(Mutex::new("".to_string())), 112 | llm_selector, 113 | show_template_editor, 114 | show_llm_selector, 115 | show_login_window, 116 | menu_map, 117 | #[cfg(feature = "computeruse_editor")] 118 | show_usecase_editor, 119 | #[cfg(feature = "computeruse_editor")] 120 | usecase_editor, 121 | } 122 | } 123 | pub fn show(&mut self, egui_context: &egui::Context) { 124 | #[cfg(feature = "cs")] 125 | if *self.show_login_window.lock().unwrap() { 126 | self.login_window.show_login_window = true; 127 | *self.show_login_window.lock().unwrap() = false; 128 | } 129 | #[cfg(feature = "cs")] 130 | if self.login_window.show_login_window { 131 | self.login_window.show(egui_context); 132 | } 133 | if *self.show_template_editor.lock().unwrap() { 134 | println!("Show Template Editor"); 135 | self.template_editor.show = true; 136 | *self.show_template_editor.lock().unwrap() = false; 137 | } 138 | if self.template_editor.show { 139 | self.template_editor.show_template_editor(egui_context); 140 | } 141 | #[cfg(feature = "computeruse_editor")] 142 | if *self.show_usecase_editor.lock().unwrap() { 143 | *self.show_usecase_editor.lock().unwrap() = self 144 | .usecase_editor 145 | .lock() 146 | .unwrap() 147 | .show_editor(egui_context); 148 | } 149 | if *self.show_llm_selector.lock().unwrap() { 150 | self.llm_selector.lock().unwrap().toggle_window(); 151 | *self.show_llm_selector.lock().unwrap() = false; 152 | } 153 | 154 | self.llm_selector 155 | .lock() 156 | .expect("Failed to lock llm_selector POISON") 157 | .show_selection_window(egui_context); 158 | } 159 | } 160 | -------------------------------------------------------------------------------- /src/ui/screen_dimensions.rs: -------------------------------------------------------------------------------- 1 | #[cfg(target_os = "linux")] 2 | use x11rb::connection::Connection; 3 | #[cfg(target_os = "macos")] 4 | pub fn get_screen_dimensions() -> (u16, u16) { 5 | use core_graphics::display::CGDisplay; 6 | 7 | let main_display = CGDisplay::main(); 8 | let width = main_display.pixels_wide() as u16; 9 | let height = main_display.pixels_high() as u16; 10 | (width, height) 11 | } 12 | #[cfg(target_os = "linux")] 13 | pub fn get_screen_dimensions() -> (u16, u16) { 14 | if let Ok((conn, _)) = x11rb::connect(None) { 15 | let screens = &conn.setup().roots; 16 | 17 | let (total_width, max_height) = screens.iter().fold((0, 0), |(w, h), screen| { 18 | (w + screen.width_in_pixels, h.max(screen.height_in_pixels)) 19 | }); 20 | 21 | (total_width, max_height) 22 | } else { 23 | eprintln!("Failed to connect to X11 assuming 1920x1080"); 24 | (1920, 1080) 25 | } 26 | } 27 | 28 | #[cfg(target_os = "windows")] 29 | pub fn get_screen_dimensions() -> (u16, u16) { 30 | use winapi::um::winuser::{GetSystemMetrics, SM_CXVIRTUALSCREEN, SM_CYVIRTUALSCREEN}; 31 | unsafe { 32 | let width = GetSystemMetrics(SM_CXVIRTUALSCREEN) as u16; 33 | let height = GetSystemMetrics(SM_CYVIRTUALSCREEN) as u16; 34 | (width - 1, height - 1) 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/ui/shortcut_window.rs: -------------------------------------------------------------------------------- 1 | use super::user_interface::PlugOvr; 2 | use itertools::Itertools; 3 | 4 | impl PlugOvr { 5 | pub fn show_shortcut_window(&mut self, egui_context: &egui::Context, scale: f32) { 6 | let mut window = egui::Window::new("PlugOvr Shortcuts") 7 | .movable(true) 8 | .drag_to_scroll(true) 9 | .interactable(true) 10 | .title_bar(true) 11 | .collapsible(false); 12 | let text_entryfield_position = self 13 | .text_entryfield_position 14 | .lock() 15 | .expect("Failed to lock text_entryfield_position POISON"); 16 | let x = text_entryfield_position.0 as f32 / scale; 17 | let y = text_entryfield_position.1 as f32 / scale; 18 | window = window.current_pos(egui::pos2(x, y)); 19 | let templates = self 20 | .prompt_templates 21 | .lock() 22 | .expect("Failed to lock prompt_templates POISON"); 23 | window.show(egui_context, |ui| { 24 | // Calculate the maximum width needed for any button 25 | let max_width = templates 26 | .iter() 27 | .filter(|(_, (_, _, is_shortcut))| *is_shortcut) 28 | .map(|(key, _)| ui.text_style_height(&egui::TextStyle::Body) * key.len() as f32) 29 | .max_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)) 30 | .unwrap_or(100.0); 31 | ui.label("AI Context:"); 32 | ui.label( 33 | self.assistance_window 34 | .ai_context 35 | .lock() 36 | .expect("Failed to lock ai_context POISON") 37 | .clone(), 38 | ); 39 | for (key, _) in templates 40 | .iter() 41 | .filter(|(_, (_, _, is_shortcut))| *is_shortcut) 42 | .sorted_by(|a, b| a.0.cmp(b.0)) 43 | { 44 | let button = egui::Button::new(key) 45 | .min_size(egui::vec2(max_width, ui.spacing().interact_size.y)); 46 | 47 | if ui.add(button).clicked() { 48 | self.assistance_window.text = key.clone(); 49 | *self 50 | .text_entry 51 | .lock() 52 | .expect("Failed to lock text_entry POISON") = true; 53 | self.assistance_window.shortcut_clicked = true; 54 | self.assistance_window.text_entry_changed = false; 55 | self.assistance_window.small_window = true; 56 | *self 57 | .shortcut_window 58 | .lock() 59 | .expect("Failed to lock shortcut_window POISON") = false; 60 | } 61 | } 62 | }); 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/ui/show_form_fields.rs: -------------------------------------------------------------------------------- 1 | use crate::ui::answer_analyser::FormFields; 2 | 3 | //#[cfg(any(target_os = "linux", target_os = "windows"))] 4 | //use enigo::Direction; 5 | //#[cfg(any(target_os = "linux", target_os = "windows"))] 6 | //use enigo::Mouse; 7 | //#[cfg(target_os = "macos")] 8 | //use enigo::{Enigo, Key, Keyboard, Settings, }; 9 | //#[cfg(any(target_os = "linux", target_os = "windows"))] 10 | use rdev::{Button, EventType, simulate}; 11 | use std::collections::HashSet; 12 | use std::error::Error; 13 | use std::sync::{Arc, Mutex}; 14 | use std::{thread, time}; 15 | #[cfg(target_os = "macos")] 16 | fn send_cmd_v() -> Result<(), Box> { 17 | use std::process::Command; 18 | 19 | Command::new("osascript") 20 | .arg("-e") 21 | .arg(r#"tell application "System Events" to keystroke "v" using command down"#) 22 | .output()?; 23 | 24 | Ok(()) 25 | } 26 | #[cfg(target_os = "macos")] 27 | fn send_fill_command( 28 | x: i32, 29 | y: i32, 30 | _mouse_position_x: i32, 31 | _mouse_position_y: i32, 32 | field_value: &str, 33 | ) -> Result<(), Box> { 34 | // Set clipboard content 35 | arboard::Clipboard::new()?.set_text(field_value)?; 36 | 37 | thread::spawn(move || { 38 | simulate(&EventType::MouseMove { 39 | x: x as f64, 40 | y: y as f64, 41 | }) 42 | .unwrap(); 43 | let delay = time::Duration::from_millis(40); 44 | thread::sleep(time::Duration::from_millis(40)); 45 | simulate(&EventType::ButtonPress(Button::Left)).unwrap(); 46 | thread::sleep(delay); 47 | simulate(&EventType::ButtonRelease(Button::Left)).unwrap(); 48 | 49 | thread::sleep(time::Duration::from_millis(40)); 50 | simulate(&EventType::ButtonPress(Button::Left)).unwrap(); 51 | thread::sleep(delay); 52 | simulate(&EventType::ButtonRelease(Button::Left)).unwrap(); 53 | 54 | thread::sleep(time::Duration::from_millis(40)); 55 | simulate(&EventType::ButtonPress(Button::Left)).unwrap(); 56 | thread::sleep(delay); 57 | simulate(&EventType::ButtonRelease(Button::Left)).unwrap(); 58 | 59 | // Send Command+V to paste the clipboard content (simulates the Cmd+V keyboard press) 60 | send_cmd_v().unwrap(); 61 | thread::sleep(time::Duration::from_millis(40)); 62 | arboard::Clipboard::new().unwrap().set_text("").unwrap(); 63 | }); 64 | Ok(()) 65 | } 66 | 67 | #[cfg(not(target_os = "macos"))] 68 | fn send_fill_command( 69 | x: i32, 70 | y: i32, 71 | mouse_position_x: i32, 72 | mouse_position_y: i32, 73 | field_value: &str, 74 | ) -> Result<(), Box> { 75 | //fill clipboard with field_value 76 | arboard::Clipboard::new()?.set_text(field_value)?; 77 | //send mouse click to (x,y) 78 | 79 | // if let Err(e) = activate_window(active_window) { 80 | // println!("Failed to activate window '{}': {:?}", active_window.0, e); 81 | //} else { 82 | // println!("Activated window '{}'", active_window.0); 83 | //} 84 | #[cfg(any(target_os = "linux", target_os = "windows"))] 85 | thread::spawn(move || { 86 | //println!("send_fill_command"); 87 | use rdev::Key; 88 | let delay = time::Duration::from_millis(40); 89 | simulate(&EventType::MouseMove { 90 | x: x as f64, 91 | y: y as f64, 92 | }) 93 | .unwrap(); 94 | 95 | thread::sleep(time::Duration::from_millis(40)); 96 | simulate(&EventType::ButtonPress(Button::Left)).unwrap(); 97 | thread::sleep(delay); 98 | simulate(&EventType::ButtonRelease(Button::Left)).unwrap(); 99 | 100 | thread::sleep(time::Duration::from_millis(40)); 101 | simulate(&EventType::ButtonPress(Button::Left)).unwrap(); 102 | thread::sleep(delay); 103 | simulate(&EventType::ButtonRelease(Button::Left)).unwrap(); 104 | 105 | thread::sleep(time::Duration::from_millis(40)); 106 | simulate(&EventType::ButtonPress(Button::Left)).unwrap(); 107 | thread::sleep(delay); 108 | simulate(&EventType::ButtonRelease(Button::Left)).unwrap(); 109 | 110 | thread::sleep(time::Duration::from_millis(40)); 111 | simulate(&EventType::KeyPress(Key::ControlLeft)).unwrap(); 112 | thread::sleep(delay); 113 | simulate(&EventType::KeyPress(Key::KeyV)).unwrap(); 114 | 115 | thread::sleep(delay); 116 | simulate(&EventType::KeyRelease(Key::KeyV)).unwrap(); 117 | thread::sleep(delay); 118 | simulate(&EventType::KeyRelease(Key::ControlLeft)).unwrap(); 119 | simulate(&EventType::MouseMove { 120 | x: mouse_position_x as f64, 121 | y: mouse_position_y as f64, 122 | }) 123 | .unwrap(); 124 | thread::sleep(time::Duration::from_millis(40)); 125 | arboard::Clipboard::new().unwrap().set_text("").unwrap(); 126 | }); 127 | // thread::spawn(move || { 128 | // let delay = time::Duration::from_millis(400); 129 | // thread::sleep(delay); 130 | // let mut enigo = Enigo::new(&Settings::default()).unwrap(); 131 | // enigo 132 | // .move_mouse(x as i32 - 10, y as i32, enigo::Coordinate::Abs) 133 | // .unwrap(); 134 | // enigo 135 | // .button(enigo::Button::Left, enigo::Direction::Click) 136 | // .unwrap(); 137 | // enigo 138 | // .button(enigo::Button::Left, enigo::Direction::Click) 139 | // .unwrap(); 140 | // enigo.key(Key::Control, Direction::Press).unwrap(); 141 | // enigo.key(Key::Unicode('v'), Direction::Press).unwrap(); 142 | // enigo.key(Key::Unicode('v'), Direction::Release).unwrap(); 143 | // enigo.key(Key::Control, Direction::Release).unwrap(); 144 | // }); 145 | 146 | Ok(()) 147 | } 148 | 149 | pub struct FormFieldsOverlay { 150 | pub form_fields: Option>, 151 | pub hidden_fields: Arc>>, 152 | pub mouse_position: Arc>, 153 | } 154 | impl FormFieldsOverlay { 155 | pub fn new(mouse_position: Arc>) -> Self { 156 | Self { 157 | form_fields: None, 158 | hidden_fields: Arc::new(Mutex::new(HashSet::new())), 159 | mouse_position, 160 | } 161 | } 162 | pub fn show(&self, egui_context: &egui::Context, screenshot_pos: Option<&egui::Pos2>) { 163 | egui::Window::new("Overlay") 164 | .interactable(false) 165 | .title_bar(false) 166 | .default_pos(egui::Pos2::new(0.0, 0.0)) 167 | .auto_sized() 168 | .frame(egui::Frame { 169 | fill: egui::Color32::TRANSPARENT, 170 | ..Default::default() 171 | }) 172 | .show(egui_context, |_ui| { 173 | if let Some(form_fields) = &self.form_fields { 174 | for (i, field) in form_fields.iter().enumerate() { 175 | // Skip if field is marked as hidden 176 | if self 177 | .hidden_fields 178 | .lock() 179 | .expect("Failed to lock hidden_fields POISON") 180 | .contains(&i) 181 | { 182 | continue; 183 | } 184 | //draw rect to check how precise the coordinates are 185 | /*ui.painter().rect( 186 | egui::Rect::from_min_max( 187 | egui::pos2( 188 | field.field_coords.x1 as f32 189 | + screenshot_pos.unwrap_or(&egui::Pos2::ZERO).x, 190 | field.field_coords.y1 as f32 191 | + screenshot_pos.unwrap_or(&egui::Pos2::ZERO).y, 192 | ), 193 | egui::pos2( 194 | field.field_coords.x2 as f32 195 | + screenshot_pos.unwrap_or(&egui::Pos2::ZERO).x, 196 | field.field_coords.y2 as f32 197 | + screenshot_pos.unwrap_or(&egui::Pos2::ZERO).y, 198 | ), 199 | ), 200 | 0.0, 201 | egui::Color32::from_rgba_premultiplied(150, 150, 150, 100), 202 | egui::Stroke::new(1.0, egui::Color32::BLACK), 203 | );*/ 204 | egui::Area::new(egui::Id::new(i.to_string())) 205 | .fixed_pos(egui::pos2( 206 | field.field_coords.x1 as f32 207 | + screenshot_pos.unwrap_or(&egui::Pos2::ZERO).x 208 | + 20.0, 209 | (field.field_coords.y1 as f32 + field.field_coords.y2 as f32 210 | - 15.0) 211 | / 2.0 212 | + screenshot_pos.unwrap_or(&egui::Pos2::ZERO).y, 213 | )) 214 | .interactable(true) 215 | .default_width(1000.0) 216 | .show(egui_context, |ui| { 217 | ui.add(egui::Label::new(egui::RichText::new( 218 | field.field_value.as_str(), 219 | ))); 220 | if field.field_value.as_str() != "" 221 | && ui.button("fill").interact(egui::Sense::click()).clicked() 222 | { 223 | println!("fill: {}", field.field_value.as_str()); 224 | let mouse_position_x = self 225 | .mouse_position 226 | .lock() 227 | .expect("Failed to lock mouse_position POISON") 228 | .0; 229 | let mouse_position_y = self 230 | .mouse_position 231 | .lock() 232 | .expect("Failed to lock mouse_position POISON") 233 | .1; 234 | let _ = send_fill_command( 235 | field.field_coords.x1 236 | + screenshot_pos.unwrap_or(&egui::Pos2::ZERO).x as i32 237 | + 10, 238 | (field.field_coords.y1 239 | + screenshot_pos.unwrap_or(&egui::Pos2::ZERO).y as i32 240 | + field.field_coords.y2 241 | + screenshot_pos.unwrap_or(&egui::Pos2::ZERO).y as i32) 242 | / 2, 243 | mouse_position_x, 244 | mouse_position_y, 245 | field.field_value.as_str(), 246 | ); 247 | // Instead of removing, add to hidden fields 248 | self.hidden_fields.lock().unwrap().insert(i); 249 | } 250 | }); 251 | } 252 | } 253 | }); 254 | } 255 | } 256 | -------------------------------------------------------------------------------- /src/ui/template_editor.rs: -------------------------------------------------------------------------------- 1 | use crate::llm::CloudModel; // Add this line 2 | use crate::llm::LLMSelector; 3 | use crate::llm::LLMType; 4 | use crate::llm::LocalModel; 5 | use std::collections::HashMap; 6 | use std::sync::{Arc, Mutex}; 7 | use strum::IntoEnumIterator; 8 | 9 | pub type TemplateMap = Arc, bool)>>>; 10 | 11 | pub struct TemplateEditor { 12 | pub show: bool, 13 | prompt_templates: TemplateMap, 14 | new_template_key: String, 15 | new_template_value: String, 16 | reset_templates_confirmation: String, 17 | llm_selector: Arc>, 18 | } 19 | 20 | impl TemplateEditor { 21 | pub fn new(prompt_templates: TemplateMap) -> Self { 22 | Self { 23 | show: false, 24 | prompt_templates, 25 | new_template_key: String::new(), 26 | new_template_value: String::new(), 27 | reset_templates_confirmation: String::new(), 28 | llm_selector: Arc::new(Mutex::new(LLMSelector::new(Arc::new(Mutex::new(None))))), 29 | } 30 | } 31 | pub fn show_template_editor(&mut self, egui_context: &egui::Context) { 32 | let mut show_window = self.show; // Create local copy 33 | egui::Window::new("Template Editor") 34 | .resizable(true) 35 | .collapsible(false) 36 | .open(&mut show_window) // Use local copy 37 | .show(egui_context, |ui| { 38 | ui.group(|ui| { 39 | let mut templates_to_remove = Vec::new(); 40 | let mut templates_to_add = Vec::new(); 41 | 42 | // Clone the templates to avoid holding the lock 43 | let templates_clone = self.prompt_templates.lock().unwrap().clone(); 44 | // Convert HashMap to Vec and sort 45 | let mut templates_vec: Vec<_> = templates_clone.into_iter().collect(); 46 | templates_vec.sort_by(|a, b| a.0.cmp(&b.0)); 47 | 48 | egui::Grid::new("template_grid") 49 | .num_columns(4) 50 | .max_col_width(400.0) 51 | .min_col_width(50.) 52 | .striped(false) 53 | .show(ui, |ui| { 54 | ui.label("Template Name"); 55 | ui.label("Instruction"); 56 | ui.label("AI Model"); 57 | ui.label("Shortcut"); 58 | 59 | ui.end_row(); 60 | 61 | // Use templates_vec instead of templates_clone in the loop 62 | for (key, value) in &templates_vec { 63 | let mut local_value = value.clone(); 64 | ui.label(key); 65 | let value_edit = ui.text_edit_singleline(&mut local_value.0); 66 | let mut selected_llm = local_value.1.clone(); 67 | egui::ComboBox::from_id_source(format!("llm_type_{}", key)) 68 | .selected_text(if let Some(llm_type) = &local_value.1 { 69 | format!("{:?}", llm_type.description()) 70 | } else { 71 | "default".to_string() 72 | }) 73 | .show_ui(ui, |ui| { 74 | ui.selectable_value(&mut selected_llm, None, "default"); 75 | for cloud_model in CloudModel::iter() { 76 | ui.selectable_value( 77 | &mut selected_llm, 78 | Some(LLMType::Cloud(cloud_model)), 79 | cloud_model.description(), 80 | ); 81 | } 82 | for local_model in LocalModel::iter() { 83 | ui.selectable_value( 84 | &mut selected_llm, 85 | Some(LLMType::Local(local_model)), 86 | local_model.description(), 87 | ); 88 | } 89 | let ollama_models = self 90 | .llm_selector 91 | .lock() 92 | .unwrap() 93 | .ollama_models 94 | .lock() 95 | .unwrap() 96 | .clone(); 97 | if let Some(models) = ollama_models { 98 | for model in models { 99 | let llm_type = LLMType::Ollama(model.name.clone()); 100 | ui.selectable_value( 101 | &mut selected_llm, 102 | Some(llm_type), 103 | model.name.clone(), 104 | ); 105 | } 106 | } 107 | 108 | if selected_llm != local_value.1.clone() { 109 | local_value.1 = selected_llm; 110 | templates_to_add 111 | .push((key.clone(), local_value.clone())); 112 | } 113 | }); 114 | 115 | if ui.checkbox(&mut local_value.2, "").changed() { 116 | templates_to_add.push((key.clone(), local_value.clone())); 117 | } 118 | 119 | ui.horizontal(|ui| { 120 | if ui 121 | .button("Default") 122 | .on_hover_text("Resets to default value if existed.") 123 | .clicked() 124 | { 125 | if let Some(default_value) = 126 | create_prompt_templates().get(key) 127 | { 128 | let local_value = default_value.clone(); 129 | templates_to_add 130 | .push((key.clone(), local_value.clone())); 131 | } 132 | } 133 | if ui.button("Remove").clicked() { 134 | templates_to_remove.push(key.clone()); 135 | } 136 | }); 137 | 138 | if value_edit.changed() { 139 | templates_to_add.push((key.clone(), local_value)); 140 | } 141 | 142 | ui.end_row(); 143 | } 144 | 145 | ui.end_row(); 146 | ui.label("Add new template"); 147 | ui.end_row(); 148 | let key = self.new_template_key.clone(); 149 | ui.add( 150 | egui::TextEdit::singleline(&mut self.new_template_key) 151 | .desired_width(150.0) 152 | .hint_text("@...") 153 | .text_color_opt(if key.starts_with('@') { 154 | None 155 | } else { 156 | Some(egui::Color32::RED) 157 | }), 158 | ); 159 | ui.text_edit_singleline(&mut self.new_template_value); 160 | if ui.button("Add").clicked() 161 | && !self.new_template_key.is_empty() 162 | && !self.new_template_value.is_empty() 163 | && key.starts_with('@') 164 | { 165 | self.prompt_templates.lock().unwrap().insert( 166 | self.new_template_key.clone(), 167 | (self.new_template_value.clone(), None, false), 168 | ); 169 | self.save_templates(); 170 | self.new_template_key.clear(); 171 | self.new_template_value.clear(); 172 | } 173 | }); 174 | 175 | ui.horizontal(|ui| { 176 | ui.label("Reset all templates to default:"); 177 | ui.add( 178 | egui::TextEdit::singleline(&mut self.reset_templates_confirmation) 179 | .hint_text("type 'reset' to confirm"), 180 | ); 181 | if ui.button("Reset all").clicked() 182 | && self.reset_templates_confirmation == "reset" 183 | { 184 | self.reset_templates(); 185 | self.reset_templates_confirmation.clear(); 186 | } 187 | }); 188 | // Handle additions and removals 189 | let mut templates = self.prompt_templates.lock().unwrap(); 190 | for key in &templates_to_remove { 191 | templates.remove(key); 192 | } 193 | for (key, value) in &templates_to_add { 194 | templates.insert(key.clone(), value.clone()); 195 | } 196 | 197 | if !templates_to_remove.is_empty() || !templates_to_add.is_empty() { 198 | drop(templates); // Release the lock before saving 199 | self.save_templates(); 200 | } 201 | 202 | ui.add_space(10.0); 203 | }); 204 | }); 205 | self.show = show_window; // Update original value 206 | } 207 | 208 | pub fn save_templates(&self) { 209 | let templates = self.prompt_templates.lock().unwrap(); 210 | let templates_json = 211 | serde_json::to_string_pretty(&*templates).expect("Failed to serialize templates"); 212 | let home_dir = dirs::home_dir().expect("Unable to find home directory"); 213 | let config_dir = home_dir.join(".plugovr"); 214 | std::fs::create_dir_all(&config_dir).expect("Failed to create config directory"); 215 | let config_file = config_dir.join("templates.json"); 216 | std::fs::write(config_file, templates_json).expect("Failed to write templates to file"); 217 | } 218 | 219 | pub fn load_templates(&mut self) { 220 | let home_dir = dirs::home_dir().expect("Unable to find home directory"); 221 | let config_file = home_dir.join(".plugovr").join("templates.json"); 222 | if let Ok(templates_json) = std::fs::read_to_string(config_file) { 223 | if let Ok(loaded_templates) = serde_json::from_str(&templates_json) { 224 | *self.prompt_templates.lock().unwrap() = loaded_templates; 225 | } 226 | } 227 | } 228 | pub fn reset_templates(&mut self) { 229 | let home_dir = dirs::home_dir().expect("Unable to find home directory"); 230 | let config_file = home_dir.join(".plugovr").join("templates.json"); 231 | match std::fs::remove_file(config_file) { 232 | Ok(_) => println!("Templates reset successfully"), 233 | Err(e) => println!("Failed to reset templates: {}", e), 234 | } 235 | *self.prompt_templates.lock().unwrap() = create_prompt_templates(); 236 | } 237 | } 238 | pub fn create_prompt_templates() -> HashMap, bool)> { 239 | let mut templates: HashMap, bool)> = HashMap::new(); 240 | templates.insert( 241 | "@correct".to_string(), 242 | ( 243 | "Correct the text without explanation".to_string(), 244 | None, 245 | true, 246 | ), 247 | ); 248 | templates.insert( 249 | "@translate(english)".to_string(), 250 | ( 251 | "Translate the text to english without explanation".to_string(), 252 | None, 253 | true, 254 | ), 255 | ); 256 | templates.insert( 257 | "@translate(german)".to_string(), 258 | ( 259 | "Translate the text to german without explanation".to_string(), 260 | None, 261 | true, 262 | ), 263 | ); 264 | templates.insert( 265 | "@translate(spanish)".to_string(), 266 | ( 267 | "Translate the text to spanish without explanation".to_string(), 268 | None, 269 | true, 270 | ), 271 | ); 272 | templates.insert( 273 | "@summarize".to_string(), 274 | ( 275 | "Provide a short summary of the text".to_string(), 276 | None, 277 | true, 278 | ), 279 | ); 280 | 281 | templates.insert( 282 | "@improve".to_string(), 283 | ( 284 | "Suggest improvements or enhancements for the given text without explanation" 285 | .to_string(), 286 | None, 287 | false, 288 | ), 289 | ); 290 | templates.insert( 291 | "@format".to_string(), 292 | ( 293 | "Format the text for better readability without explanation".to_string(), 294 | None, 295 | true, 296 | ), 297 | ); 298 | 299 | templates.insert( 300 | "@simplify".to_string(), 301 | ("Simplify complex text or concepts".to_string(), None, false), 302 | ); 303 | templates.insert( 304 | "@extend".to_string(), 305 | ( 306 | "continue the text without explanation".to_string(), 307 | None, 308 | true, 309 | ), 310 | ); 311 | templates.insert( 312 | "@filename".to_string(), 313 | ("propose filename for document: structure:date(year_month_day )_topic_company. Output only filename".to_string(), None, false), 314 | ); 315 | // templates.insert( 316 | // "@fillfields".to_string(), 317 | // ("Extract caption and coordinates from all textboxes in image 2. For each textbox, find the corresponding content in image 1. Output the information in format of json. [{ \"content_image_1\": \"content\", \"caption_textbox_image_2\": \"caption\", \"coordinates_textbox_image_2\": \"[x1,y1,x2,y2]\" }]".to_string(), 318 | // None), 319 | // ); 320 | templates.insert( 321 | "@fillform".to_string(), 322 | ("#computeruse Output the coordinates for each input field / textbox in json format from image 1 (screenshot) and fill with information from images starting from image 2. The original textbox should be empty before we fill it. Json format: [{ \"caption\": \"\", \"content\": \"\", \"coordinates\": \"[x1, y1, x2, y2]\" }]".to_string(), 323 | Some(LLMType::Cloud(CloudModel::AnthropicSonnet3_5)), false), 324 | ); 325 | templates 326 | } 327 | -------------------------------------------------------------------------------- /src/usecase_editor.rs: -------------------------------------------------------------------------------- 1 | use crate::usecase_recorder::{EventType, Point, UseCase}; 2 | use egui_file_dialog::FileDialog; 3 | use image::GenericImageView; 4 | //use rfd; 5 | 6 | use serde::{Deserialize, Serialize}; 7 | use std::fs; 8 | use std::path::PathBuf; 9 | 10 | #[derive(Default)] 11 | pub struct UsecaseEditor { 12 | usecase: Option, 13 | current_step: usize, 14 | file_dialog: FileDialog, 15 | picked_file: Option, 16 | cached_textures: std::collections::HashMap, 17 | } 18 | 19 | impl UsecaseEditor { 20 | pub fn new() -> Self { 21 | Default::default() 22 | } 23 | 24 | pub fn show_editor(&mut self, ctx: &egui::Context) -> bool { 25 | let mut show = true; 26 | egui::Window::new("Usecase Editor") 27 | .default_size([800.0, 600.0]) 28 | .open(&mut show) 29 | .show(ctx, |ui| { 30 | self.show_menu_bar(ui); 31 | self.show_content(ui); 32 | // Update the dialog 33 | self.file_dialog.update(ctx); 34 | 35 | // Check if the user picked a file. 36 | if let Some(path) = self.file_dialog.take_picked() { 37 | self.picked_file = Some(path.to_path_buf()); 38 | if let Some(path) = &self.picked_file { 39 | if let Ok(contents) = fs::read_to_string(path) { 40 | if let Ok(usecase) = serde_json::from_str::(&contents) { 41 | self.usecase = Some(usecase); 42 | self.current_step = 0; 43 | // Clear existing cached textures 44 | self.cached_textures.clear(); 45 | 46 | // Pre-cache textures for all images in the usecase 47 | 48 | for (step_index, step) in self 49 | .usecase 50 | .as_ref() 51 | .unwrap() 52 | .usecase_steps 53 | .iter() 54 | .enumerate() 55 | { 56 | match step { 57 | EventType::Monitor1(data) => { 58 | if let Ok(image_data) = base64::decode(data) { 59 | if let Ok(image) = 60 | image::load_from_memory(&image_data) 61 | { 62 | display_step_image( 63 | ui, 64 | data, 65 | &format!("image_{}", step_index), 66 | (-1, -1), 67 | &mut self.cached_textures, 68 | false, 69 | 2.0, 70 | ); 71 | display_step_image( 72 | ui, 73 | data, 74 | &format!("image_thump_{}", step_index), 75 | (-1, -1), 76 | &mut self.cached_textures, 77 | false, 78 | 8.0, 79 | ); 80 | } 81 | } 82 | } 83 | EventType::Monitor2(data) => { 84 | display_step_image( 85 | ui, 86 | data, 87 | &format!("image_{}", step_index), 88 | (-1, -1), 89 | &mut self.cached_textures, 90 | false, 91 | 2.0, 92 | ); 93 | display_step_image( 94 | ui, 95 | data, 96 | &format!("image_thump_{}", step_index), 97 | (-1, -1), 98 | &mut self.cached_textures, 99 | false, 100 | 8.0, 101 | ); 102 | } 103 | _ => {} 104 | } 105 | } 106 | } 107 | } 108 | } 109 | } 110 | }); 111 | show 112 | } 113 | 114 | fn show_menu_bar(&mut self, ui: &mut egui::Ui) { 115 | egui::menu::bar(ui, |ui| { 116 | egui::menu::menu_button(ui, "File", |ui| { 117 | if ui.button("Open").clicked() { 118 | self.file_dialog.pick_file(); 119 | } 120 | if ui.button("Save").clicked() { 121 | if let Some(usecase) = &mut self.usecase { 122 | if let Some(path) = &self.picked_file { 123 | if let Ok(contents) = serde_json::to_string_pretty(usecase) { 124 | fs::write(path, contents).unwrap(); 125 | } 126 | } 127 | } 128 | } 129 | }); 130 | }); 131 | } 132 | 133 | fn show_content(&mut self, ui: &mut egui::Ui) { 134 | if let Some(usecase) = &mut self.usecase { 135 | // Make name and instructions editable 136 | ui.text_edit_singleline(&mut usecase.usecase_name); 137 | ui.text_edit_multiline(&mut usecase.usecase_instructions); 138 | 139 | ui.add_space(10.0); 140 | 141 | // Navigation controls 142 | ui.horizontal(|ui| { 143 | if ui.button("Previous").clicked() && self.current_step > 0 { 144 | self.current_step -= 1; 145 | } 146 | ui.label(format!( 147 | "Step {} of {}", 148 | self.current_step + 1, 149 | usecase.usecase_steps.len() 150 | )); 151 | if ui.button("Next").clicked() 152 | && self.current_step < usecase.usecase_steps.len() - 1 153 | { 154 | self.current_step += 1; 155 | } 156 | }); 157 | 158 | ui.add_space(10.0); 159 | 160 | // Show current step 161 | let current_step = self.current_step; 162 | let (prev_steps, current_and_after) = usecase.usecase_steps.split_at_mut(current_step); 163 | 164 | // Collect after images before match 165 | let (step, after_images) = current_and_after.split_first_mut().unwrap(); 166 | let after_images: Vec<_> = after_images 167 | .iter() 168 | .enumerate() 169 | .filter_map(|(i, step)| { 170 | if let EventType::Monitor1(data) = step { 171 | Some((i + current_step + 1, data)) 172 | } else { 173 | None 174 | } 175 | }) 176 | .collect(); 177 | 178 | match step { 179 | EventType::Click(point, desc, ids) => { 180 | ui.label(format!("Click coordinates: ({}, {})", point.x, point.y)); 181 | ui.label("Description:"); 182 | let mut desc_clone = desc.clone(); 183 | if ui.text_edit_singleline(&mut desc_clone).changed() { 184 | *desc = desc_clone; 185 | } 186 | // Search backwards for the most recent Monitor1 event 187 | if let Some((monitor_index, monitor_data)) = 188 | prev_steps.iter().enumerate().rev().find_map(|(i, step)| { 189 | if let EventType::Monitor1(data) = step { 190 | Some((i, data)) 191 | } else { 192 | None 193 | } 194 | }) 195 | { 196 | let _ = display_step_image( 197 | ui, 198 | monitor_data, 199 | &format!("image_{}", monitor_index), 200 | (point.x as i32, point.y as i32), 201 | &mut self.cached_textures, 202 | true, 203 | 2.0, 204 | ); 205 | } 206 | let point = (point.x as i32, point.y as i32); 207 | 208 | let mut before_images = prev_steps 209 | .iter() 210 | .enumerate() 211 | .filter_map(|(i, step)| { 212 | if let EventType::Monitor1(data) = step { 213 | Some((i, data)) 214 | } else { 215 | None 216 | } 217 | }) 218 | .collect::>(); 219 | let mut ids_tmp = ids.clone(); 220 | egui::ScrollArea::horizontal().show(ui, |ui| { 221 | ui.horizontal(|ui| { 222 | let thumbnail_size = egui::Vec2::new(100.0, 60.0); 223 | 224 | // Show before images 225 | for (i, (monitor_index, data)) in before_images.iter().enumerate() { 226 | let offset = -(before_images.len() as i32) + i as i32 + 1; 227 | ui.vertical(|ui| { 228 | ui.label(format!("T{}", offset)); 229 | // Add checkbox 230 | let is_selected = ids_tmp.contains(monitor_index); 231 | let mut checked = is_selected; 232 | if ui.checkbox(&mut checked, "").changed() { 233 | if checked { 234 | if !ids.contains(monitor_index) { 235 | ids.push(*monitor_index); 236 | } 237 | } else { 238 | if let Some(pos) = 239 | ids.iter().position(|x| x == monitor_index) 240 | { 241 | ids.remove(pos); 242 | } 243 | } 244 | } 245 | // Add hover functionality to thumbnail 246 | let response = display_step_image( 247 | ui, 248 | data, 249 | &format!("image_thump_{}", monitor_index), 250 | (point.0, point.1), 251 | &mut self.cached_textures, 252 | true, 253 | 8.0, 254 | ); 255 | 256 | // Show larger image on hover 257 | if response.hovered() { 258 | egui::Window::new(format!( 259 | "hover_preview_{}", 260 | monitor_index 261 | )) 262 | .fixed_pos(ui.input(|i| { 263 | let pos = i.pointer.hover_pos().unwrap_or_default(); 264 | pos + egui::vec2(20.0, 20.0) // Offset from cursor 265 | })) 266 | .title_bar(false) 267 | .frame(egui::Frame::none()) 268 | .auto_sized() 269 | .show( 270 | ui.ctx(), 271 | |ui| { 272 | display_step_image( 273 | ui, 274 | data, 275 | &format!("image_{}", monitor_index), 276 | (point.0, point.1), 277 | &mut self.cached_textures, 278 | true, 279 | 2.0, 280 | ); 281 | }, 282 | ); 283 | } 284 | }); 285 | } 286 | 287 | // Show after images 288 | for (i, (monitor_index, data)) in after_images.iter().enumerate() { 289 | ui.vertical(|ui| { 290 | ui.label(format!("T+{}", i + 1)); 291 | // Add checkbox 292 | let is_selected = ids.contains(monitor_index); 293 | let mut checked = is_selected; 294 | if ui.checkbox(&mut checked, "").changed() { 295 | if checked { 296 | if !ids.contains(monitor_index) { 297 | ids.push(*monitor_index); 298 | } 299 | } else { 300 | if let Some(pos) = 301 | ids.iter().position(|x| x == monitor_index) 302 | { 303 | ids.remove(pos); 304 | } 305 | } 306 | } 307 | // Add hover functionality to thumbnail 308 | let response = display_step_image( 309 | ui, 310 | data, 311 | &format!("image_thump_{}", monitor_index), 312 | (point.0, point.1), 313 | &mut self.cached_textures, 314 | true, 315 | 8.0, 316 | ); 317 | 318 | // Show larger image on hover 319 | if response.hovered() { 320 | egui::Window::new(format!( 321 | "hover_preview_{}", 322 | monitor_index 323 | )) 324 | .fixed_pos(ui.input(|i| { 325 | let pos = i.pointer.hover_pos().unwrap_or_default(); 326 | pos + egui::vec2(20.0, 20.0) // Offset from cursor 327 | })) 328 | .title_bar(false) 329 | .frame(egui::Frame::none()) 330 | .auto_sized() 331 | .show( 332 | ui.ctx(), 333 | |ui| { 334 | display_step_image( 335 | ui, 336 | data, 337 | &format!("image_{}", monitor_index), 338 | (point.0, point.1), 339 | &mut self.cached_textures, 340 | true, 341 | 2.0, 342 | ); 343 | }, 344 | ); 345 | } 346 | }); 347 | } 348 | }); 349 | }); 350 | } 351 | EventType::Monitor1(data) => { 352 | ui.label("Monitor1"); 353 | display_step_image( 354 | ui, 355 | data, 356 | &format!("image_{}", current_step), 357 | (-1, -1), 358 | &mut self.cached_textures, 359 | true, 360 | 2.0, 361 | ); 362 | } 363 | EventType::Monitor2(data) => { 364 | ui.label("Monitor2"); 365 | display_step_image( 366 | ui, 367 | data, 368 | &format!("image_{}", current_step), 369 | (-1, -1), 370 | &mut self.cached_textures, 371 | true, 372 | 2.0, 373 | ); 374 | } 375 | EventType::Monitor3(data) => { 376 | ui.label("Monitor3"); 377 | display_step_image( 378 | ui, 379 | data, 380 | &format!("image_{}", current_step), 381 | (-1, -1), 382 | &mut self.cached_textures, 383 | true, 384 | 2.0, 385 | ); 386 | } 387 | EventType::Text(text) => { 388 | ui.label("Text"); 389 | let mut text_clone = text.clone(); 390 | if ui.text_edit_singleline(&mut text_clone).changed() { 391 | *text = text_clone; 392 | } 393 | } 394 | EventType::KeyDown(key) => { 395 | ui.label("KeyDown"); 396 | let mut key_clone = key.clone(); 397 | if ui.text_edit_singleline(&mut key_clone).changed() { 398 | *key = key_clone; 399 | } 400 | } 401 | EventType::KeyUp(key) => { 402 | ui.label("KeyUp"); 403 | let mut key_clone = key.clone(); 404 | if ui.text_edit_singleline(&mut key_clone).changed() { 405 | *key = key_clone; 406 | } 407 | } 408 | // Handle other event types as needed 409 | _ => {} 410 | } 411 | } else { 412 | ui.centered_and_justified(|ui| { 413 | ui.label("Open a usecase file using the File menu"); 414 | }); 415 | } 416 | } 417 | } 418 | 419 | // Helper function to display step images 420 | fn display_step_image( 421 | ui: &mut egui::Ui, 422 | monitor_data: &str, 423 | texture_id: &str, 424 | coords: (i32, i32), 425 | cached_textures: &mut std::collections::HashMap, 426 | show_image: bool, 427 | scale: f32, 428 | ) -> egui::Response { 429 | if let Some(texture) = cached_textures.get(texture_id) { 430 | if show_image { 431 | let before_rect = ui.cursor(); 432 | let response = ui.add(egui::Image::new(texture)); 433 | let after_rect = ui.cursor(); 434 | // Draw circle at cursor position 435 | if coords.0 != -1 && coords.1 != -1 { 436 | // Scale click coordinates to match displayed image size 437 | let circle_x = before_rect.min.x + (coords.0 as f32 / scale); 438 | let circle_y = before_rect.min.y + (coords.1 as f32 / scale); 439 | 440 | let circle_pos = egui::pos2(circle_x, circle_y); 441 | let circle_radius = 5.0; 442 | 443 | // Draw red circle 444 | ui.painter() 445 | .circle_filled(circle_pos, circle_radius, egui::Color32::RED); 446 | } 447 | return response; 448 | } 449 | return ui.allocate_response(egui::Vec2::ZERO, egui::Sense::hover()); 450 | } 451 | 452 | if let Ok(image_data) = base64::decode(monitor_data) { 453 | if let Ok(image) = image::load_from_memory(&image_data) { 454 | // Draw a red circle at click coordinates 455 | let mut image = image.to_rgba8(); 456 | // let radius = 10; 457 | // let color = image::Rgba([255, 0, 0, 255]); // Red circle 458 | 459 | // // Scale coords to match resized image dimensions 460 | // let scaled_x = coords.0; 461 | // let scaled_y = coords.1; 462 | // if scaled_x != -1 && scaled_y != -1 { 463 | // // Draw circle by iterating over pixels in bounding box 464 | // for y in -radius..=radius { 465 | // for x in -radius..=radius { 466 | // // Check if point is within circle using distance formula 467 | // if x * x + y * y <= radius * radius { 468 | // let px = scaled_x + x; 469 | // let py = scaled_y + y; 470 | 471 | // // Only draw if within image bounds 472 | // if px >= 0 473 | // && px < image.width() as i32 474 | // && py >= 0 475 | // && py < image.height() as i32 476 | // { 477 | // image.put_pixel(px as u32, py as u32, color); 478 | // } 479 | // } 480 | // } 481 | // } 482 | // } 483 | 484 | let image = image::DynamicImage::ImageRgba8(image); 485 | 486 | let image = image::imageops::resize( 487 | &image, 488 | image.width() / scale as u32, 489 | image.height() / scale as u32, 490 | image::imageops::FilterType::CatmullRom, 491 | ); 492 | let size = image.dimensions(); 493 | let image = 494 | egui::ColorImage::from_rgba_unmultiplied([size.0 as _, size.1 as _], &image); 495 | 496 | // Create and cache the texture 497 | let texture = ui 498 | .ctx() 499 | .load_texture(texture_id, image, egui::TextureOptions::default()); 500 | cached_textures.insert(texture_id.to_string(), texture.clone()); 501 | if show_image { 502 | ui.image(&texture); 503 | } 504 | } 505 | } 506 | 507 | // Return a default response if we didn't show an image 508 | ui.allocate_response(egui::Vec2::ZERO, egui::Sense::hover()) 509 | } 510 | -------------------------------------------------------------------------------- /src/usecase_recorder.rs: -------------------------------------------------------------------------------- 1 | use egui::Context; 2 | use std::sync::{Arc, Mutex}; 3 | use std::thread; 4 | use std::time::{Duration, SystemTime}; 5 | use xcap::Monitor; 6 | pub struct UseCaseRecorder { 7 | usecase: Option, 8 | usecase_name: String, 9 | usecase_instructions: String, 10 | pub recording: Arc>, 11 | pub show: bool, 12 | pub add_image: bool, 13 | pub add_image_delay: Option, 14 | pub add_image_now: Option, 15 | pub screenshot_buffer1: Arc>>, 16 | pub screenshot_buffer2: Arc>>, 17 | pub screenshot_buffer3: Arc>>, 18 | } 19 | 20 | use image::{ImageBuffer, Rgba}; 21 | use serde::{Deserialize, Serialize}; 22 | use std::fs::File; 23 | #[derive(Debug, Serialize, Deserialize)] 24 | pub struct Point { 25 | pub x: f32, 26 | pub y: f32, 27 | } 28 | #[derive(Debug, Serialize, Deserialize)] 29 | pub enum EventType { 30 | Click(Point, String, #[serde(default)] Vec), 31 | KeyDown(String), 32 | KeyUp(String), 33 | Monitor1(String), 34 | Monitor2(String), 35 | Monitor3(String), 36 | Text(String), 37 | } 38 | #[derive(Debug, Serialize, Deserialize)] 39 | pub struct UseCase { 40 | pub usecase_id: String, 41 | pub usecase_name: String, 42 | pub usecase_instructions: String, 43 | pub usecase_steps: Vec, 44 | } 45 | pub fn buffer_screenshots( 46 | screenshot_buffer1: Arc>>, 47 | screenshot_buffer2: Arc>>, 48 | screenshot_buffer3: Arc>>, 49 | recording: Arc>, 50 | ) { 51 | let monitors = Monitor::all().unwrap(); 52 | while *recording.lock().unwrap() { 53 | for (i, monitor) in monitors.iter().enumerate() { 54 | let image: ImageBuffer, Vec> = monitor.capture_image().unwrap(); 55 | // Resize image to half size 56 | // #[cfg(target_os = "macos")] 57 | // let image = image::imageops::resize( 58 | // &image, 59 | // image.width() / 2, 60 | // image.height() / 2, 61 | // image::imageops::FilterType::Lanczos3, 62 | // ); 63 | let base64 = UseCaseRecorder::image_buffer2base64(image); 64 | if i == 0 { 65 | screenshot_buffer1.lock().unwrap().replace(base64); 66 | } else if i == 1 { 67 | screenshot_buffer2.lock().unwrap().replace(base64); 68 | } else if i == 2 { 69 | screenshot_buffer3.lock().unwrap().replace(base64); 70 | } 71 | } 72 | thread::sleep(Duration::from_millis(100)); 73 | } 74 | } 75 | 76 | impl UseCaseRecorder { 77 | pub fn new() -> Self { 78 | let recording = Arc::new(Mutex::new(false)); 79 | let instance = Self { 80 | usecase: None, 81 | usecase_name: String::new(), 82 | usecase_instructions: String::new(), 83 | recording: recording.clone(), 84 | show: false, 85 | add_image: false, 86 | add_image_delay: None, 87 | add_image_now: None, 88 | screenshot_buffer1: Arc::new(Mutex::new(None)), 89 | screenshot_buffer2: Arc::new(Mutex::new(None)), 90 | screenshot_buffer3: Arc::new(Mutex::new(None)), 91 | }; 92 | 93 | instance 94 | } 95 | pub fn show_window(&mut self, ctx: &Context) { 96 | if self.show { 97 | let mut show = self.show; 98 | egui::Window::new("Use Case Recorder") 99 | .open(&mut show) 100 | .show(ctx, |ui| { 101 | ui.label("Use Case Recorder"); 102 | ui.label("Filename"); 103 | ui.add(egui::TextEdit::multiline(&mut self.usecase_name)); 104 | ui.label("Instructions"); 105 | ui.add(egui::TextEdit::multiline(&mut self.usecase_instructions)); 106 | 107 | if ui.button("Record").clicked() { 108 | self.start_recording(); 109 | self.show = false; 110 | } 111 | if ui.button("Stop").clicked() { 112 | self.stop_recording(); 113 | } 114 | }); 115 | if self.show { 116 | self.show = show; 117 | } 118 | } 119 | } 120 | pub fn image_buffer2base64(image_buffer: ImageBuffer, Vec>) -> String { 121 | use base64::Engine as _; 122 | let mut buf = vec![]; 123 | #[cfg(target_os = "macos")] 124 | let image_buffer = image::imageops::resize( 125 | &image_buffer, 126 | image_buffer.width() / 2, 127 | image_buffer.height() / 2, 128 | image::imageops::FilterType::Triangle, 129 | ); 130 | 131 | image_buffer 132 | .write_to(&mut std::io::Cursor::new(&mut buf), image::ImageFormat::Png) 133 | .unwrap(); 134 | base64::engine::general_purpose::STANDARD.encode(&buf) 135 | } 136 | 137 | pub fn add_screenshot(&mut self) { 138 | println!("Adding screenshot"); 139 | let monitors = Monitor::all().unwrap(); 140 | 141 | for (i, monitor) in monitors.iter().enumerate() { 142 | let image = monitor.capture_image().unwrap(); 143 | let base64 = Self::image_buffer2base64(image); 144 | if i == 0 { 145 | self.add_event(EventType::Monitor1(base64)); 146 | } else if i == 1 { 147 | self.add_event(EventType::Monitor2(base64)); 148 | } else if i == 2 { 149 | self.add_event(EventType::Monitor3(base64)); 150 | } 151 | } 152 | } 153 | pub fn add_event(&mut self, event: EventType) { 154 | if let EventType::Monitor1(ref _base64) = event { 155 | println!("Adding monitor1 image"); 156 | } else if let EventType::Monitor2(ref _base64) = event { 157 | println!("Adding monitor2 image"); 158 | } else if let EventType::Monitor3(ref _base64) = event { 159 | println!("Adding monitor3 image"); 160 | } else { 161 | println!("Adding event: {:?}", event); 162 | } 163 | if let EventType::KeyDown(ref key) = event { 164 | if key == "Escape" { 165 | self.stop_recording(); 166 | } 167 | if key == "Enter" { 168 | //self.add_screenshot(); 169 | //self.add_image = true; 170 | //let now = SystemTime::now(); 171 | //self.add_image_delay = Some(Duration::from_secs(1)); 172 | //self.add_image_now = Some(now); 173 | } 174 | } 175 | if let EventType::Click(ref _point, ref _op, ref _ids) = event { 176 | //self.add_image = true; 177 | //let now = SystemTime::now(); 178 | //self.add_image_delay = Some(Duration::from_secs(1)); 179 | //self.add_image_now = Some(now); 180 | let screenshot1 = self.screenshot_buffer1.lock().unwrap().clone(); 181 | if let Some(screenshot) = screenshot1 { 182 | self.add_event(EventType::Monitor1(screenshot)); 183 | } 184 | let screenshot2 = self.screenshot_buffer2.lock().unwrap().clone(); 185 | if let Some(screenshot) = screenshot2 { 186 | self.add_event(EventType::Monitor2(screenshot)); 187 | } 188 | let screenshot3 = self.screenshot_buffer3.lock().unwrap().clone(); 189 | if let Some(screenshot) = screenshot3 { 190 | self.add_event(EventType::Monitor3(screenshot)); 191 | } 192 | } 193 | 194 | self.usecase.as_mut().unwrap().usecase_steps.push(event); 195 | } 196 | fn start_recording(&mut self) { 197 | self.usecase = Some(UseCase { 198 | usecase_id: uuid::Uuid::new_v4().to_string(), 199 | usecase_name: self.usecase_name.clone(), 200 | usecase_instructions: self.usecase_instructions.clone(), 201 | usecase_steps: Vec::new(), 202 | }); 203 | println!("Starting recording"); 204 | *self.recording.lock().unwrap() = true; 205 | self.show = false; 206 | 207 | let screenshot_buffer1 = self.screenshot_buffer1.clone(); 208 | let screenshot_buffer2 = self.screenshot_buffer2.clone(); 209 | let screenshot_buffer3 = self.screenshot_buffer3.clone(); 210 | let recording = self.recording.clone(); 211 | thread::spawn(move || { 212 | buffer_screenshots( 213 | screenshot_buffer1, 214 | screenshot_buffer2, 215 | screenshot_buffer3, 216 | recording, 217 | ); 218 | }); 219 | 220 | // let screenshot1 = self.screenshot_buffer1.lock().unwrap().clone(); 221 | // if let Some(screenshot) = screenshot1 { 222 | // self.add_event(EventType::Monitor1(screenshot)); 223 | // } 224 | // let screenshot2 = self.screenshot_buffer2.lock().unwrap().clone(); 225 | // if let Some(screenshot) = screenshot2 { 226 | // self.add_event(EventType::Monitor2(screenshot)); 227 | // } 228 | // let screenshot3 = self.screenshot_buffer3.lock().unwrap().clone(); 229 | // if let Some(screenshot) = screenshot3 { 230 | // self.add_event(EventType::Monitor3(screenshot)); 231 | // } 232 | // self.add_image = true; 233 | // let now = SystemTime::now(); 234 | // self.add_image_delay = Some(Duration::from_secs(0)); 235 | // self.add_image_now = Some(now); 236 | // //std::thread::sleep(std::time::Duration::from_secs(1)); 237 | //self.add_screenshot(); 238 | } 239 | 240 | fn stop_recording(&mut self) { 241 | println!("Stopping recording"); 242 | *self.recording.lock().unwrap() = false; 243 | 244 | // Compress keyboard events before saving 245 | if let Some(usecase) = &mut self.usecase { 246 | let mut compressed_steps = Vec::new(); 247 | let mut keyboard_events = Vec::new(); 248 | 249 | // Process all events 250 | for event in usecase.usecase_steps.drain(..) { 251 | match event { 252 | EventType::KeyDown(ref key) | EventType::KeyUp(ref key) if key != "Return" => { 253 | keyboard_events.push(event); 254 | } 255 | other_event => { 256 | // If we have pending keyboard events, compress them first 257 | if !keyboard_events.is_empty() { 258 | let text = Self::compress_keyboard_events(&keyboard_events); 259 | if !text.is_empty() { 260 | compressed_steps.push(EventType::Text(text)); 261 | } 262 | keyboard_events.clear(); 263 | } 264 | compressed_steps.push(other_event); 265 | } 266 | } 267 | } 268 | 269 | // Handle any remaining keyboard events 270 | if !keyboard_events.is_empty() { 271 | let text = Self::compress_keyboard_events(&keyboard_events); 272 | if !text.is_empty() { 273 | compressed_steps.push(EventType::Text(text)); 274 | } 275 | } 276 | 277 | usecase.usecase_steps = compressed_steps; 278 | } 279 | 280 | // Save recording to file 281 | let file_name = format!("{}.json", self.usecase_name); 282 | let file = File::create(file_name).unwrap(); 283 | serde_json::to_writer_pretty(file, &self.usecase.as_ref().unwrap()).unwrap(); 284 | } 285 | 286 | pub fn compress_keyboard_events(events: &[EventType]) -> String { 287 | let mut result = String::new(); 288 | let mut shift_pressed = false; 289 | let mut altgr_pressed = false; 290 | 291 | for event in events { 292 | match event { 293 | EventType::KeyDown(key) => match key.as_str() { 294 | "ShiftLeft" => shift_pressed = true, 295 | "AltGr" => altgr_pressed = true, 296 | "Space" => result.push(' '), 297 | "Backspace" => { 298 | result.pop(); // Remove the last character if any 299 | } 300 | key => { 301 | if altgr_pressed && key == "Q" { 302 | result.push('@'); 303 | } else { 304 | let mut c = key.chars().next().unwrap_or_default(); 305 | if shift_pressed { 306 | c = c.to_uppercase().next().unwrap_or(c); 307 | } else { 308 | c = c.to_lowercase().next().unwrap_or(c); 309 | } 310 | if shift_pressed { 311 | if c == '.' { 312 | c = ':'; 313 | } 314 | if c == ',' { 315 | c = ';'; 316 | } 317 | // if c == '/' { 318 | // c = '?'; 319 | // } 320 | // if c == '-' { 321 | // c = '_'; 322 | // } 323 | // if c == '=' { 324 | // c = '+'; 325 | // } 326 | // if c == '[' { 327 | // c = '{'; 328 | // } 329 | } 330 | result.push(c); 331 | } 332 | } 333 | }, 334 | EventType::KeyUp(key) => match key.as_str() { 335 | "ShiftLeft" => shift_pressed = false, 336 | "AltGr" => altgr_pressed = false, 337 | _ => {} 338 | }, 339 | _ => {} 340 | } 341 | } 342 | 343 | result 344 | } 345 | } 346 | -------------------------------------------------------------------------------- /src/usecase_webserver.rs: -------------------------------------------------------------------------------- 1 | use axum::extract::ws::{Message, WebSocket, WebSocketUpgrade}; 2 | use axum::{ 3 | Json, Router, 4 | extract::State, 5 | http::{Request, StatusCode, header}, 6 | middleware::{self, Next}, 7 | response::{Html, IntoResponse, Response}, 8 | routing::{get, post}, 9 | }; 10 | use base64::{Engine, engine::general_purpose::STANDARD}; 11 | use futures::{sink::SinkExt, stream::StreamExt}; 12 | use serde::{Deserialize, Serialize}; 13 | use std::{ 14 | collections::HashMap, 15 | sync::{Arc, Mutex}, 16 | }; 17 | use tokio::sync::broadcast; 18 | use tower_http::services::ServeDir; 19 | 20 | use crate::usecase_replay::UseCaseReplay; 21 | 22 | #[derive(Clone)] 23 | struct WebServerState { 24 | usecase_replay: Arc>, 25 | clients: Arc>>>, 26 | password: Option, 27 | } 28 | 29 | #[derive(Serialize, Deserialize)] 30 | struct WebCommand { 31 | command: String, 32 | instruction: Option, 33 | url: Option, 34 | enabled: Option, 35 | } 36 | 37 | #[derive(Serialize, Deserialize)] 38 | struct UrlResponse { 39 | planning_url: String, 40 | execution_url: String, 41 | } 42 | 43 | // Middleware to check authentication 44 | async fn auth_middleware( 45 | State(state): State, 46 | req: Request, 47 | next: Next, 48 | ) -> Result { 49 | // Skip authentication for login-related routes 50 | let path = req.uri().path(); 51 | if path == "/login" || path == "/auth" || path.starts_with("/assets/") { 52 | return Ok(next.run(req).await); 53 | } 54 | 55 | // If no password is set, allow access 56 | if state.password.is_none() { 57 | return Ok(next.run(req).await); 58 | } 59 | 60 | // Check for authentication cookie 61 | if let Some(cookie) = req.headers().get(header::COOKIE) { 62 | if let Ok(cookie_str) = cookie.to_str() { 63 | if cookie_str.contains("plugovr_auth=true") { 64 | // User is authenticated 65 | return Ok(next.run(req).await); 66 | } 67 | } 68 | } 69 | 70 | // For API routes, check for authentication 71 | // In a real app, you'd use cookies or JWT tokens 72 | // This is a simplified version for demonstration 73 | if path.starts_with("/ws") || path.starts_with("/command") || path.starts_with("/urls") { 74 | // For now, we'll just allow these routes without checking auth 75 | // In a real app, you'd verify a token here 76 | return Ok(next.run(req).await); 77 | } 78 | 79 | // For the main page, redirect to login 80 | if path == "/" { 81 | // Create a redirect response instead of returning 401 82 | let redirect = axum::response::Redirect::to("/login"); 83 | return Ok(redirect.into_response()); 84 | } 85 | 86 | Ok(next.run(req).await) 87 | } 88 | 89 | pub async fn start_server(usecase_replay: Arc>, password: Option) { 90 | let state = WebServerState { 91 | usecase_replay, 92 | clients: Arc::new(Mutex::new(HashMap::new())), 93 | password, 94 | }; 95 | 96 | let state_clone = state.clone(); 97 | 98 | let app = Router::new() 99 | .route("/login", get(login_handler)) 100 | .route("/auth", post(auth_handler)) 101 | .route("/", get(index_handler)) 102 | .route("/ws", get(ws_handler)) 103 | .route("/command", post(command_handler)) 104 | .route("/urls", get(get_urls_handler)) 105 | .route("/urls/planning", post(set_planning_url_handler)) 106 | .route("/urls/execution", post(set_execution_url_handler)) 107 | .nest_service("/assets", ServeDir::new("assets")) 108 | .layer(middleware::from_fn_with_state( 109 | state_clone.clone(), 110 | auth_middleware, 111 | )) 112 | .with_state(state_clone); 113 | 114 | if let Some(pwd) = &state.password { 115 | println!("Starting password-protected webserver on http://localhost:3000"); 116 | println!("Password: {}", pwd); 117 | } else { 118 | println!("Starting webserver on http://localhost:3000"); 119 | } 120 | 121 | let listener = tokio::net::TcpListener::bind("0.0.0.0:3000").await.unwrap(); 122 | axum::serve(listener, app).await.unwrap(); 123 | } 124 | 125 | async fn index_handler() -> impl IntoResponse { 126 | Html(include_str!("../assets/index.html")) 127 | } 128 | 129 | async fn login_handler() -> impl IntoResponse { 130 | Html( 131 | r#" 132 | 133 | 134 | 135 | PlugOvr - Login 136 | 193 | 194 | 195 | 203 | 248 | 249 | 250 | "#, 251 | ) 252 | } 253 | 254 | #[derive(Deserialize)] 255 | struct AuthRequest { 256 | password: String, 257 | } 258 | 259 | async fn auth_handler( 260 | State(state): State, 261 | Json(auth): Json, 262 | ) -> impl IntoResponse { 263 | if let Some(correct_password) = &state.password { 264 | if &auth.password == correct_password { 265 | // Password is correct 266 | println!("Authentication successful for password: {}", auth.password); 267 | 268 | // Create a response with a cookie 269 | let mut response = axum::response::Response::builder() 270 | .status(StatusCode::OK) 271 | .header(header::CONTENT_TYPE, "text/plain") 272 | .header( 273 | header::SET_COOKIE, 274 | "plugovr_auth=true; Path=/; HttpOnly; SameSite=Strict", 275 | ) 276 | .body("Authentication successful".into()) 277 | .unwrap(); 278 | 279 | response 280 | } else { 281 | // Password is incorrect 282 | println!("Authentication failed for password: {}", auth.password); 283 | (StatusCode::UNAUTHORIZED, "Invalid password").into_response() 284 | } 285 | } else { 286 | // No password required 287 | println!("No password required, authentication successful"); 288 | (StatusCode::OK, "No authentication required").into_response() 289 | } 290 | } 291 | 292 | async fn get_urls_handler(State(state): State) -> impl IntoResponse { 293 | let usecase_replay = state.usecase_replay.lock().unwrap(); 294 | let response = UrlResponse { 295 | planning_url: usecase_replay.server_url_planning.clone(), 296 | execution_url: usecase_replay.server_url_execution.clone(), 297 | }; 298 | Json(response) 299 | } 300 | 301 | async fn set_planning_url_handler( 302 | State(state): State, 303 | Json(command): Json, 304 | ) -> impl IntoResponse { 305 | if let Some(url) = command.url { 306 | let mut usecase_replay = state.usecase_replay.lock().unwrap(); 307 | usecase_replay.server_url_planning = url.clone(); 308 | 309 | // Use the existing save function 310 | if let Err(e) = crate::usecase_replay::save_server_url_planning(&url) { 311 | println!("Error saving planning URL: {}", e); 312 | } 313 | 314 | // Broadcast the URL change to all clients 315 | let update = serde_json::json!({ 316 | "type": "url_update", 317 | "planning_url": url, 318 | }); 319 | for (_, tx) in state.clients.lock().unwrap().iter() { 320 | let _ = tx.send(update.to_string()); 321 | } 322 | "Planning URL updated".to_string() 323 | } else { 324 | "No URL provided".to_string() 325 | } 326 | } 327 | 328 | async fn set_execution_url_handler( 329 | State(state): State, 330 | Json(command): Json, 331 | ) -> impl IntoResponse { 332 | if let Some(url) = command.url { 333 | let mut usecase_replay = state.usecase_replay.lock().unwrap(); 334 | usecase_replay.server_url_execution = url.clone(); 335 | 336 | // Use the existing save function 337 | if let Err(e) = crate::usecase_replay::save_server_url_execution(&url) { 338 | println!("Error saving execution URL: {}", e); 339 | } 340 | 341 | // Broadcast the URL change to all clients 342 | let update = serde_json::json!({ 343 | "type": "url_update", 344 | "execution_url": url, 345 | }); 346 | for (_, tx) in state.clients.lock().unwrap().iter() { 347 | let _ = tx.send(update.to_string()); 348 | } 349 | "Execution URL updated".to_string() 350 | } else { 351 | "No URL provided".to_string() 352 | } 353 | } 354 | 355 | async fn command_handler( 356 | State(state): State, 357 | Json(command): Json, 358 | ) -> impl IntoResponse { 359 | match command.command.as_str() { 360 | "next" => { 361 | state.usecase_replay.lock().unwrap().step(); 362 | "Next action triggered".to_string() 363 | } 364 | "new_instruction" => { 365 | if let Some(instruction) = command.instruction { 366 | state 367 | .usecase_replay 368 | .lock() 369 | .unwrap() 370 | .vec_instructions 371 | .lock() 372 | .unwrap() 373 | .clear(); 374 | state 375 | .usecase_replay 376 | .lock() 377 | .unwrap() 378 | .execute_usecase(instruction); 379 | state.usecase_replay.lock().unwrap().show = true; 380 | *state 381 | .usecase_replay 382 | .lock() 383 | .unwrap() 384 | .index_action 385 | .lock() 386 | .unwrap() = 0; 387 | *state 388 | .usecase_replay 389 | .lock() 390 | .unwrap() 391 | .index_instruction 392 | .lock() 393 | .unwrap() = 0; 394 | "New instruction executed".to_string() 395 | } else { 396 | "No instruction provided".to_string() 397 | } 398 | } 399 | "set_auto_mode" => { 400 | if let Some(enabled) = command.enabled { 401 | let mut usecase_replay = state.usecase_replay.lock().unwrap(); 402 | usecase_replay.set_auto_mode(enabled); 403 | 404 | // Send dedicated auto mode update to all clients 405 | let update = serde_json::json!({ 406 | "type": "auto_mode_update", 407 | "enabled": enabled 408 | }); 409 | for (_, tx) in state.clients.lock().unwrap().iter() { 410 | let _ = tx.send(update.to_string()); 411 | } 412 | 413 | "Auto mode updated".to_string() 414 | } else { 415 | "No auto mode state provided".to_string() 416 | } 417 | } 418 | _ => "Unknown command".to_string(), 419 | } 420 | } 421 | 422 | async fn ws_handler( 423 | ws: WebSocketUpgrade, 424 | State(state): State, 425 | ) -> impl IntoResponse { 426 | ws.on_upgrade(|socket| handle_socket(socket, state)) 427 | } 428 | 429 | async fn handle_socket(socket: WebSocket, state: WebServerState) { 430 | let (mut sender, mut receiver) = socket.split(); 431 | 432 | let client_id = uuid::Uuid::new_v4().to_string(); 433 | let (tx, mut rx) = broadcast::channel(100); 434 | state 435 | .clients 436 | .lock() 437 | .unwrap() 438 | .insert(client_id.clone(), tx.clone()); 439 | 440 | // Spawn task to send screenshots and overlay data 441 | let state_clone = state.clone(); 442 | let tx_clone = tx.clone(); 443 | 444 | // Spawn a task to forward broadcast messages to the WebSocket 445 | let send_task = tokio::spawn(async move { 446 | while let Ok(msg) = rx.recv().await { 447 | if sender.send(Message::Text(msg.into())).await.is_err() { 448 | break; 449 | } 450 | } 451 | }); 452 | 453 | // Spawn task to capture and send updates 454 | let update_task = tokio::spawn(async move { 455 | let mut last_planning_url = String::new(); 456 | let mut last_execution_url = String::new(); 457 | 458 | loop { 459 | tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; 460 | let mut usecase_replay = state_clone.usecase_replay.lock().unwrap(); 461 | usecase_replay.grab_screenshot(); 462 | if let Some(screenshot) = &usecase_replay.monitor1 { 463 | let mut buffer = Vec::new(); 464 | screenshot 465 | .write_to( 466 | &mut std::io::Cursor::new(&mut buffer), 467 | image::ImageFormat::Png, 468 | ) 469 | .unwrap(); 470 | let base64_image = STANDARD.encode(&buffer); 471 | 472 | let index_instruction = *usecase_replay.index_instruction.lock().unwrap(); 473 | let index_action = *usecase_replay.index_action.lock().unwrap(); 474 | 475 | let current_action = if let Some(actions) = usecase_replay 476 | .vec_instructions 477 | .lock() 478 | .unwrap() 479 | .get(index_instruction) 480 | { 481 | if index_action < actions.actions.len() { 482 | Some(actions.actions[index_action].clone()) 483 | } else { 484 | None 485 | } 486 | } else { 487 | None 488 | }; 489 | 490 | // Get the full action plan and mark executed actions 491 | let action_plan = if let Some(actions) = usecase_replay 492 | .vec_instructions 493 | .lock() 494 | .unwrap() 495 | .get(index_instruction) 496 | { 497 | let mut plan_with_status: Vec = actions 498 | .actions 499 | .iter() 500 | .enumerate() 501 | .map(|(i, action)| { 502 | serde_json::json!({ 503 | "action": action, 504 | "executed": i < index_action, 505 | "current": i == index_action 506 | }) 507 | }) 508 | .collect(); 509 | Some(plan_with_status) 510 | } else { 511 | None 512 | }; 513 | 514 | // Only include URLs in the update if they've changed 515 | let mut update = serde_json::json!({ 516 | "type": "update", 517 | "screenshot": base64_image, 518 | "current_action": current_action, 519 | "action_plan": action_plan, 520 | "computing": *usecase_replay.computing_action.lock().unwrap(), 521 | "computing_plan": *usecase_replay.computing_plan.lock().unwrap(), 522 | "show": usecase_replay.show, 523 | }); 524 | 525 | // Only include URLs if they've changed 526 | if usecase_replay.server_url_planning != last_planning_url { 527 | update["planning_url"] = 528 | serde_json::Value::String(usecase_replay.server_url_planning.clone()); 529 | last_planning_url = usecase_replay.server_url_planning.clone(); 530 | } 531 | if usecase_replay.server_url_execution != last_execution_url { 532 | update["execution_url"] = 533 | serde_json::Value::String(usecase_replay.server_url_execution.clone()); 534 | last_execution_url = usecase_replay.server_url_execution.clone(); 535 | } 536 | 537 | if let Err(e) = tx_clone.send(update.to_string()) { 538 | println!("Error sending update: {}", e); 539 | break; 540 | } 541 | } 542 | } 543 | }); 544 | 545 | // Handle incoming messages 546 | while let Some(Ok(msg)) = receiver.next().await { 547 | if let Message::Text(text) = msg { 548 | println!("Received message: {}", text); 549 | } 550 | } 551 | 552 | // Clean up when client disconnects 553 | state.clients.lock().unwrap().remove(&client_id); 554 | update_task.abort(); 555 | send_task.abort(); 556 | } 557 | -------------------------------------------------------------------------------- /src/version_check.rs: -------------------------------------------------------------------------------- 1 | use reqwest::blocking::Client; 2 | use serde_json::json; 3 | use std::error::Error; 4 | 5 | const LAMBDA_UPDATE_CHECK: &str = "https://5cy8qwxk18.execute-api.eu-central-1.amazonaws.com/v1"; 6 | 7 | pub fn update_check() -> Result> { 8 | let client = Client::new(); 9 | 10 | let body = json!({ 11 | "version": env!("CARGO_PKG_VERSION"), 12 | }); 13 | 14 | let response = client.post(LAMBDA_UPDATE_CHECK).json(&body).send()?; 15 | if response.status().is_success() { 16 | let response_body: serde_json::Value = response.json()?; 17 | 18 | let msg = response_body["body"] 19 | .as_str() 20 | .ok_or("Invalid response format")? 21 | .to_string(); 22 | Ok(msg) 23 | } else { 24 | Ok(String::new()) 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/window_handling.rs: -------------------------------------------------------------------------------- 1 | use crate::ui::assistance_window::AiResponseAction; 2 | use arboard::Clipboard; 3 | use std::error::Error; 4 | use std::sync::{Arc, Mutex}; 5 | 6 | #[cfg(target_os = "linux")] 7 | use x11rb::connection::Connection; 8 | #[cfg(target_os = "linux")] 9 | use x11rb::protocol::xproto::ConnectionExt; 10 | #[cfg(target_os = "linux")] 11 | pub struct ActiveWindow(pub u32); 12 | #[cfg(target_os = "macos")] 13 | pub struct ActiveWindow(pub u64); 14 | 15 | #[cfg(target_os = "windows")] 16 | use winapi::shared::windef::HWND; 17 | 18 | #[cfg(target_os = "windows")] 19 | pub struct ActiveWindow(pub usize); 20 | 21 | #[cfg(target_os = "linux")] 22 | pub fn activate_window(window_title: &ActiveWindow) -> Result<(), Box> { 23 | use std::time::Duration; 24 | use x11rb::connection::Connection; 25 | use x11rb::protocol::Event; 26 | use x11rb::protocol::xproto::{ConnectionExt, EventMask, Window}; 27 | 28 | let (conn, screen_num) = x11rb::connect(None)?; 29 | let screen = &conn.setup().roots[screen_num]; 30 | let root = screen.root; 31 | let window_id = Window::from(window_title.0); 32 | 33 | // Prepare atoms 34 | let net_active_window = conn 35 | .intern_atom(false, b"_NET_ACTIVE_WINDOW")? 36 | .reply()? 37 | .atom; 38 | let net_wm_state = conn.intern_atom(false, b"_NET_WM_STATE")?.reply()?.atom; 39 | let net_wm_state_focused = conn 40 | .intern_atom(false, b"_NET_WM_STATE_FOCUSED")? 41 | .reply()? 42 | .atom; 43 | 44 | // Send _NET_ACTIVE_WINDOW message 45 | conn.send_event( 46 | false, 47 | root, 48 | EventMask::SUBSTRUCTURE_REDIRECT | EventMask::SUBSTRUCTURE_NOTIFY, 49 | x11rb::protocol::xproto::ClientMessageEvent::new( 50 | 32, 51 | window_id, 52 | net_active_window, 53 | [2, 0, 0, 0, 0], 54 | ), 55 | )?; 56 | 57 | // Send _NET_WM_STATE message to set focus 58 | conn.send_event( 59 | false, 60 | window_id, 61 | EventMask::PROPERTY_CHANGE, 62 | x11rb::protocol::xproto::ClientMessageEvent::new( 63 | 32, 64 | window_id, 65 | net_wm_state, 66 | [1, net_wm_state_focused, 0, 0, 0], 67 | ), 68 | )?; 69 | 70 | conn.flush()?; 71 | 72 | // Wait for the window to be activated 73 | let start_time = std::time::Instant::now(); 74 | let timeout = Duration::from_millis(10); 75 | while start_time.elapsed() < timeout { 76 | if let Some(Event::PropertyNotify(e)) = conn.poll_for_event()? { 77 | if e.window == window_id && e.atom == net_wm_state { 78 | break; 79 | } 80 | } 81 | std::thread::sleep(Duration::from_millis(10)); 82 | } 83 | 84 | Ok(()) 85 | } 86 | 87 | #[cfg(target_os = "macos")] 88 | use cocoa::appkit::NSApplicationActivateIgnoringOtherApps; 89 | #[cfg(target_os = "macos")] 90 | 91 | pub fn activate_window(id: &ActiveWindow) -> Result<(), Box> { 92 | //println!("activate_window: {:?}", id.0); 93 | use cocoa::base::id; 94 | use objc::{msg_send, sel, sel_impl}; 95 | 96 | unsafe { 97 | let shared_app: id = msg_send![class!(NSWorkspace), sharedWorkspace]; 98 | let running_apps: id = msg_send![shared_app, runningApplications]; 99 | let count: usize = msg_send![running_apps, count]; 100 | 101 | for i in 0..count { 102 | let app: id = msg_send![running_apps, objectAtIndex: i]; 103 | let pid: u64 = msg_send![app, processIdentifier]; 104 | 105 | if pid == id.0 { 106 | let _: () = 107 | msg_send![app, activateWithOptions:NSApplicationActivateIgnoringOtherApps]; 108 | return Ok(()); 109 | } 110 | } 111 | } 112 | 113 | Err("Window not found".into()) 114 | } 115 | #[cfg(target_os = "windows")] 116 | pub fn activate_window(hwnd_value: &ActiveWindow) -> Result<(), Box> { 117 | force_foreground_window(hwnd_value.0 as HWND); 118 | 119 | Ok(()) 120 | } 121 | 122 | #[cfg(target_os = "windows")] 123 | use winapi::shared::minwindef::DWORD; 124 | 125 | #[cfg(target_os = "windows")] 126 | use winapi::um::processthreadsapi::GetCurrentThreadId; 127 | #[cfg(target_os = "windows")] 128 | use winapi::um::winuser::{ 129 | AttachThreadInput, BringWindowToTop, GetForegroundWindow, GetWindowThreadProcessId, ShowWindow, 130 | }; 131 | #[cfg(target_os = "windows")] 132 | pub fn force_foreground_window(hwnd: HWND) { 133 | unsafe { 134 | let foreground_window = GetForegroundWindow(); 135 | let mut foreground_thread_id = 0; 136 | let window_thread_process_id = 137 | GetWindowThreadProcessId(foreground_window, &mut foreground_thread_id); 138 | let current_thread_id = GetCurrentThreadId(); 139 | const SW_SHOW: DWORD = 5; 140 | 141 | AttachThreadInput(window_thread_process_id, current_thread_id, 1); 142 | BringWindowToTop(hwnd); 143 | ShowWindow(hwnd, SW_SHOW as i32); 144 | AttachThreadInput(window_thread_process_id, current_thread_id, 0); 145 | } 146 | } 147 | #[cfg(target_os = "linux")] 148 | pub fn find_window_by_title(title: &str) -> Option { 149 | let (conn, screen_num) = x11rb::connect(None).unwrap(); 150 | let screen = &conn.setup().roots[screen_num]; 151 | let root = screen.root; 152 | 153 | let windows = conn.query_tree(root).unwrap().reply().unwrap().children; 154 | for window in windows { 155 | if let Ok(name) = conn 156 | .get_property( 157 | false, 158 | window, 159 | x11rb::protocol::xproto::AtomEnum::WM_NAME, 160 | x11rb::protocol::xproto::AtomEnum::STRING, 161 | 0, 162 | 1024, 163 | ) 164 | .unwrap() 165 | .reply() 166 | { 167 | if let Ok(window_title) = String::from_utf8(name.value) { 168 | if window_title == title { 169 | return Some(window); 170 | } 171 | } 172 | } 173 | } 174 | None 175 | } 176 | #[cfg(target_os = "macos")] 177 | pub fn find_window_by_title(title: &str) -> Option { 178 | use cocoa::base::id; 179 | use objc::{msg_send, sel, sel_impl}; 180 | 181 | unsafe { 182 | let shared_app: id = msg_send![class!(NSWorkspace), sharedWorkspace]; 183 | let running_apps: id = msg_send![shared_app, runningApplications]; 184 | let count: usize = msg_send![running_apps, count]; 185 | 186 | for i in 0..count { 187 | let app: id = msg_send![running_apps, objectAtIndex: i]; 188 | let app_name: id = msg_send![app, localizedName]; 189 | let name = cocoa::foundation::NSString::UTF8String(app_name); 190 | let name_str = std::ffi::CStr::from_ptr(name).to_string_lossy(); 191 | 192 | if name_str == title { 193 | let pid: u64 = msg_send![app, processIdentifier]; 194 | return Some(pid); 195 | } 196 | } 197 | None 198 | } 199 | } 200 | #[cfg(target_os = "windows")] 201 | use std::ptr::null; 202 | #[cfg(target_os = "windows")] 203 | pub fn activate_window_title(window_title: &String) -> Result<(), Box> { 204 | use winapi::um::winuser::FindWindowA; 205 | unsafe { 206 | let hwnd = FindWindowA(null(), window_title.as_ptr() as *const i8); 207 | //println!("hwnd: {:?}", hwnd); 208 | if !hwnd.is_null() { 209 | //SetForegroundWindow(hwnd); 210 | force_foreground_window(hwnd); 211 | //println!("Activated window '{}'", window_title); 212 | Ok(()) 213 | } else { 214 | Err(format!("Failed to activate window '{}'", window_title).into()) 215 | } 216 | } 217 | } 218 | 219 | #[cfg(target_os = "linux")] 220 | pub fn get_active_window() -> Option { 221 | let (conn, screen_num) = x11rb::connect(None).unwrap(); 222 | let screen = &conn.setup().roots[screen_num]; 223 | let root = screen.root; 224 | 225 | let atom_name = "_NET_ACTIVE_WINDOW"; 226 | let atom = conn 227 | .intern_atom(false, atom_name.as_bytes()) 228 | .unwrap() 229 | .reply() 230 | .unwrap() 231 | .atom; 232 | 233 | let active_window = conn 234 | .get_property( 235 | false, 236 | root, 237 | atom, 238 | x11rb::protocol::xproto::AtomEnum::WINDOW, 239 | 0, 240 | 1, 241 | ) 242 | .unwrap() 243 | .reply() 244 | .unwrap(); 245 | // Extract the window title from the active window 246 | 247 | active_window 248 | .value32() 249 | .and_then(|mut v| v.next()) 250 | .map(ActiveWindow) 251 | } 252 | 253 | /*#[cfg(target_os = "linux")] 254 | fn get_active_window2() -> Option { 255 | let (conn, screen_num) = x11rb::connect(None).unwrap(); 256 | let screen = &conn.setup().roots[screen_num]; 257 | let root = screen.root; 258 | 259 | let atom_name = "_NET_ACTIVE_WINDOW"; 260 | let atom = conn 261 | .intern_atom(false, atom_name.as_bytes()) 262 | .unwrap() 263 | .reply() 264 | .unwrap() 265 | .atom; 266 | 267 | let active_window = conn 268 | .get_property( 269 | false, 270 | root, 271 | atom, 272 | x11rb::protocol::xproto::AtomEnum::WINDOW, 273 | 0, 274 | 1, 275 | ) 276 | .unwrap() 277 | .reply() 278 | .unwrap(); 279 | // Extract the window title from the active window 280 | let active_window_title = 281 | if let Some(window_id) = active_window.value32().and_then(|mut v| v.next()) { 282 | let window = Window::from(window_id); 283 | let wm_name = conn 284 | .get_property( 285 | false, 286 | window, 287 | conn.intern_atom(false, b"_NET_WM_NAME") 288 | .unwrap() 289 | .reply() 290 | .unwrap() 291 | .atom, 292 | conn.intern_atom(false, b"UTF8_STRING") 293 | .unwrap() 294 | .reply() 295 | .unwrap() 296 | .atom, 297 | 0, 298 | u32::max_value(), 299 | ) 300 | .unwrap() 301 | .reply(); 302 | 303 | match wm_name { 304 | Ok(property) => String::from_utf8(property.value).ok(), 305 | Err(_) => None, 306 | } 307 | } else { 308 | None 309 | }; 310 | 311 | // Return the title if found, otherwise return None 312 | return Some(ActiveWindow(active_window_title.unwrap())); 313 | } 314 | 315 | #[cfg(target_os = "windows")] 316 | fn get_active_window1() -> Option { 317 | use std::ptr; 318 | use winapi::um::winuser::{GetForegroundWindow, GetWindowTextLengthW, GetWindowTextW}; 319 | 320 | unsafe { 321 | let hwnd = GetForegroundWindow(); 322 | if hwnd != ptr::null_mut() { 323 | let mut buffer = [0u16; 1024]; 324 | let mut text_length = GetWindowTextLengthW(hwnd); 325 | if text_length > 0 { 326 | text_length += 1; // Include null terminator 327 | GetWindowTextW( 328 | hwnd, 329 | &mut buffer as *mut u16 as *mut u16, 330 | text_length as i32, 331 | ); 332 | Some(String::from_utf16_lossy(&buffer).to_string()) 333 | } else { 334 | None 335 | } 336 | } else { 337 | None 338 | } 339 | } 340 | }*/ 341 | 342 | #[cfg(target_os = "macos")] 343 | pub fn get_active_window() -> Option { 344 | use active_win_pos_rs::get_active_window; 345 | let active_window = get_active_window().ok(); 346 | if let Some(active_window) = active_window { 347 | Some(ActiveWindow(active_window.process_id)) 348 | } else { 349 | None 350 | } 351 | } 352 | #[cfg(target_os = "windows")] 353 | pub fn get_active_window() -> Option { 354 | use winapi::um::winuser::{GetForegroundWindow, GetWindowTextLengthW, GetWindowTextW}; 355 | 356 | unsafe { 357 | let hwnd = GetForegroundWindow(); 358 | if !hwnd.is_null() { 359 | let mut buffer = [0u16; 1024]; 360 | let mut text_length = GetWindowTextLengthW(hwnd); 361 | if text_length > 0 { 362 | text_length += 1; // Include null terminator 363 | GetWindowTextW(hwnd, &mut buffer as *mut u16, text_length as i32); 364 | Some(ActiveWindow(hwnd as usize)) 365 | } else { 366 | None 367 | } 368 | } else { 369 | None 370 | } 371 | } 372 | } 373 | 374 | #[cfg(target_os = "windows")] 375 | pub fn find_window_by_title(title: &str) -> Option { 376 | use std::ffi::OsStr; 377 | use std::os::windows::ffi::OsStrExt; 378 | use winapi::um::winuser::FindWindowW; 379 | 380 | // Convert &str to wide string (UTF-16) 381 | let wide: Vec = OsStr::new(title) 382 | .encode_wide() 383 | .chain(std::iter::once(0)) 384 | .collect(); 385 | 386 | unsafe { 387 | let hwnd = FindWindowW(std::ptr::null(), wide.as_ptr()); 388 | if hwnd.is_null() { 389 | None 390 | } else { 391 | Some(hwnd as usize) 392 | } 393 | } 394 | } 395 | 396 | pub fn send_results( 397 | active_window: Arc>, 398 | ai_context: Arc>, 399 | ai_answer: Arc>, 400 | ai_resonde_action: AiResponseAction, 401 | ) -> Result<(), Box> { 402 | //println!("trigger action to take over answer and move focus back"); 403 | // let window_title = active_window.lock().unwrap().to_string(); 404 | //println!("activate {:}", window_title); 405 | if let Err(e) = activate_window(&active_window.lock().unwrap()) { 406 | eprintln!("Failed to activate window: {:?}", e); 407 | } 408 | // Get the AI answer 409 | let ai_answer = ai_answer.lock().unwrap().clone(); 410 | let ai_context = ai_context.lock().unwrap().clone(); 411 | 412 | // Copy AI answer to clipboard 413 | // use clipboard::{ClipboardContext, ClipboardProvider}; 414 | // let mut ctx: ClipboardContext = ClipboardProvider::new().unwrap(); 415 | let mut clipboard = Clipboard::new().unwrap(); 416 | 417 | match ai_resonde_action { 418 | AiResponseAction::Replace => { 419 | let _ = clipboard.set_text(ai_answer.to_owned()); 420 | } 421 | AiResponseAction::Extend => { 422 | let _ = clipboard.set_text(ai_context.to_owned() + " " + &ai_answer.to_owned()); 423 | } 424 | AiResponseAction::Ignore => { 425 | return Ok(()); 426 | } 427 | } 428 | 429 | // Send Ctrl+V to paste 430 | #[cfg(any(target_os = "linux", target_os = "windows"))] 431 | { 432 | use enigo::{Direction, Enigo, Key, Keyboard, Settings}; 433 | if let Ok(mut enigo) = Enigo::new(&Settings::default()) { 434 | enigo.key(Key::Control, Direction::Press)?; 435 | enigo.key(Key::Unicode('v'), Direction::Press)?; 436 | enigo.key(Key::Unicode('v'), Direction::Release)?; 437 | enigo.key(Key::Control, Direction::Release)?; 438 | } 439 | } 440 | 441 | #[cfg(target_os = "macos")] 442 | send_cmd_v()?; 443 | 444 | Ok(()) 445 | } 446 | #[cfg(target_os = "macos")] 447 | fn send_cmd_v() -> Result<(), Box> { 448 | use std::process::Command; 449 | 450 | Command::new("osascript") 451 | .arg("-e") 452 | .arg(r#"tell application "System Events" to keystroke "v" using command down"#) 453 | .output()?; 454 | 455 | Ok(()) 456 | } 457 | --------------------------------------------------------------------------------