├── .gitignore ├── Cargo.lock ├── Cargo.toml ├── IMPLEMENTATION_STATUS.md ├── README.md ├── __pycache__ └── simple_test_client.cpython-313.pyc ├── advanced_debug.py ├── basic_test.py ├── better_test.py ├── core_api.txt ├── debug_mcp_server.py ├── debug_paint.py ├── debug_test.py ├── direct_draw_test.py ├── direct_json_rpc_test.py ├── direct_paint_test.py ├── direct_test.py ├── echo_test.py ├── final_test.py ├── latest_server_log.txt ├── minimal_test.py ├── multi_shape_test.py ├── random_lines_test.py ├── rectangle_debug_test.py ├── rectangle_test.py ├── samples └── uia_test.rs ├── server_log.txt ├── server_stderr.log ├── simple_test.py ├── simple_test_client.py ├── simplest_test.py ├── specs ├── architecture_diagram.md ├── client_integration.md ├── mcp_protocol.md └── windows_integration.md ├── src ├── bin │ └── uia_test.rs ├── core.rs ├── error.rs ├── lib.rs ├── main.rs ├── protocol.rs ├── uia.rs └── windows.rs ├── super_simple_test.py └── test_client.py /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "mcp-server-microsoft-paint" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | [lib] 7 | crate-type = ["cdylib", "rlib"] # cdylib for potential FFI, rlib for Rust usage 8 | 9 | [dependencies] 10 | # MCP Core 11 | mcp_rust_sdk = "0.1.1" # Use the latest version found on crates.io (0.1.1) 12 | 13 | # Async Trait 14 | async-trait = "0.1" 15 | 16 | # Async Utilities 17 | futures = "0.3" 18 | 19 | # Async Runtime 20 | tokio = { version = "1", features = ["full"] } # Need tokio for mcp_rust_sdk examples 21 | 22 | # Serialization/Deserialization 23 | serde = { version = "1.0", features = ["derive"] } 24 | serde_json = "1.0" 25 | 26 | # Windows API Interaction 27 | windows-sys = { version = "0.52", features = [ 28 | "Win32_Foundation", 29 | "Win32_UI_WindowsAndMessaging", 30 | "Win32_System_Threading", # For CreateProcess 31 | "Win32_Graphics_Gdi", # For GetDeviceCaps, ClientToScreen etc. 32 | "Win32_UI_Input_KeyboardAndMouse", # For SendInput 33 | "Win32_System_SystemInformation", # For screen metrics if needed 34 | "Win32_Storage_FileSystem", # For file operations 35 | "Win32_Security", # Potentially needed for some operations 36 | "Win32_UI_Shell", # For ShellExecuteW 37 | # Add more features as needed 38 | ] } 39 | 40 | # Error Handling 41 | thiserror = "1.0" 42 | 43 | # Logging 44 | log = "0.4" 45 | # env_logger = "0.11" # Replace env_logger with simplelog 46 | simplelog = "0.12" # Add simplelog 47 | 48 | # Base64 Encoding/Decoding 49 | base64 = "0.22" 50 | uiautomation = { version = "0.17.3", features = ["log"] } 51 | 52 | # Optional: Add development dependencies for testing 53 | [dev-dependencies] 54 | # Add testing-specific crates here if needed, e.g., mocking libraries 55 | -------------------------------------------------------------------------------- /IMPLEMENTATION_STATUS.md: -------------------------------------------------------------------------------- 1 | # Implementation Status for MCP Server for Windows 11 Paint 2 | 3 | This document tracks the implementation progress based on the specifications in the `specs/` directory. 4 | 5 | ## Legend 6 | 7 | - `[ ]` Not Started 8 | - `[~]` In Progress 9 | - `[x]` Completed 10 | 11 | ## Core Protocol & Communication (STDIO JSON-RPC) 12 | 13 | - `[ ]` Establish STDIO communication channel 14 | - `[ ]` Implement JSON-RPC 2.0 message parsing (Requests) 15 | - `[ ]` Implement JSON-RPC 2.0 message serialization (Responses/Notifications) 16 | - `[ ]` Utilize `rust-mcp-sdk` for core communication handling 17 | 18 | ## Connection Management 19 | 20 | - `[x]` Implement `connect` command 21 | - `[x]` Handle `client_id` and `client_name` 22 | - `[x]` Return `paint_version`, initial `canvas_width`, `canvas_height` 23 | - `[x]` Implement `disconnect` command 24 | 25 | ## Windows 11 Paint Integration 26 | 27 | - `[x]` Paint Application Detection 28 | - `[x]` Enumerate windows (`EnumWindows`) for "MSPaintApp" class or "Paint" title 29 | - `[x]` Launch Paint (`CreateProcess` with "mspaint.exe") if not running 30 | - `[x]` Window Management 31 | - `[x]` Implement `activate_window` command 32 | - `[x]` Bring window to foreground (enhanced activation) 33 | - `[x]` Ensure window is not minimized (`ShowWindow`) 34 | - `[x]` Handle activation failures and retries 35 | - `[x]` Implement `get_canvas_dimensions` command 36 | - `[x]` Get window rectangle (`GetWindowRect`) 37 | - `[x]` Calculate canvas area based on Win11 UI layout 38 | - `[x]` Mouse Event Simulation (`SendInput`) 39 | - `[x]` Implement mouse movement 40 | - `[x]` Implement mouse clicks (left button) 41 | - `[x]` Implement coordinate translation (client to screen, `ClientToScreen`) 42 | - `[x]` Implement normalized coordinates for High-DPI support 43 | - `[x]` Keyboard Event Simulation (`SendInput`) 44 | - `[x]` Implement basic key presses 45 | - `[x]` Implement modifier keys (Ctrl, Alt, Shift) 46 | - `[x]` Implement special key codes (Enter, Tab, etc.) 47 | - `[x]` Handle key scan codes (`KEYEVENTF_SCANCODE`) 48 | - `[~]` UI Element Interaction (Win11 Modern UI) 49 | - `[~]` Locate toolbar elements 50 | - `[ ]` Locate property panels 51 | - `[ ]` Interact with buttons, sliders, dropdowns 52 | - `[ ]` Dialog Interaction 53 | - `[ ]` Identify dialog windows (class/title) 54 | - `[ ]` Implement navigation (Tab, Space, Enter) 55 | - `[ ]` Handle Font selection dialog 56 | - `[ ]` Handle Resize/Scale dialog 57 | - `[ ]` Handle Save/Open dialogs (if feasible) 58 | - `[ ]` Handle New Canvas dialog 59 | - `[ ]` Menu Interaction 60 | - `[ ]` Access main menu items (Image, Rotate, Flip) 61 | - `[ ]` Navigate submenus 62 | 63 | ## Drawing Operations 64 | 65 | - `[~]` Tool Selection (`select_tool` command) 66 | - `[x]` Pencil 67 | - `[x]` Brush 68 | - `[x]` Fill 69 | - `[x]` Text 70 | - `[x]` Eraser 71 | - `[x]` Select 72 | - `[x]` Shape (with `shape_type` parameter) 73 | - `[ ]` Rectangle 74 | - `[ ]` Ellipse 75 | - `[ ]` Line 76 | - `[ ]` Arrow 77 | - `[ ]` Triangle 78 | - `[ ]` Pentagon 79 | - `[ ]` Hexagon 80 | - `[~]` Color Selection (`set_color` command) 81 | - `[x]` Handle `#RRGGBB` format 82 | - `[~]` Interact with Win11 color panel/picker 83 | - `[~]` Thickness Selection (`set_thickness` command) 84 | - `[x]` Handle levels 1-5 85 | - `[~]` Interact with Win11 thickness controls 86 | - `[~]` Brush Size Configuration (`set_brush_size` command) 87 | - `[x]` Handle pixel size (1-30px) 88 | - `[~]` Interact with Win11 size slider/presets 89 | - `[ ]` Map requested size to available options 90 | - `[~]` Fill Type Selection (`set_fill` command for shapes) 91 | - `[x]` Handle `none|solid|outline` 92 | - `[~]` Interact with Win11 shape fill/outline options 93 | - `[x]` Draw Pixel (`draw_pixel` command) 94 | - `[x]` Select pencil tool (1px) 95 | - `[~]` Set color (optional) 96 | - `[x]` Perform single click at coordinates 97 | - `[ ]` Handle zoom for precision (optional) 98 | - `[x]` Draw Line (`draw_line` command) 99 | - `[x]` Simulate mouse drag 100 | - `[~]` Set color (optional) 101 | - `[~]` Set thickness (optional) 102 | - `[~]` Draw Shape (`draw_shape` command) 103 | - `[x]` Select shape tool and type 104 | - `[~]` Set color (optional) 105 | - `[~]` Set thickness (optional) 106 | - `[~]` Set fill type (optional) 107 | - `[x]` Simulate mouse drag from start to end 108 | - `[x]` Draw Polyline (`draw_polyline` command) 109 | - `[x]` Select pencil or brush tool 110 | - `[~]` Set color (optional) 111 | - `[~]` Set thickness (optional) 112 | - `[x]` Simulate sequence of mouse drags between points 113 | 114 | ## Text Operations 115 | 116 | - `[~]` Add Text (`add_text` command) 117 | - `[x]` Select text tool 118 | - `[x]` Click at position 119 | - `[x]` Simulate typing text content 120 | - `[x]` Handle finalization (click elsewhere / Enter) 121 | - `[~]` Set color (optional) 122 | - `[~]` Enhanced version parameters: 123 | - `[~]` Set font name (optional) 124 | - `[~]` Set font size (optional) 125 | - `[~]` Set font style (`regular|bold|italic|bold_italic`) (optional) 126 | - `[ ]` Interact with font selection dialog/panel 127 | 128 | ## Selection Operations 129 | 130 | - `[x]` Select Region (`select_region` command) 131 | - `[x]` Select selection tool 132 | - `[x]` Simulate mouse drag for rectangle 133 | - `[x]` Copy Selection (`copy_selection` command) 134 | - `[x]` Simulate Ctrl+C or menu command 135 | - `[x]` Paste (`paste` command) 136 | - `[x]` Simulate Ctrl+V or menu command 137 | - `[x]` Position pasted content (if possible via click at `x`, `y`) 138 | 139 | ## Canvas Management 140 | 141 | - `[x]` Clear Canvas (`clear_canvas` command) 142 | - `[x]` Select All (Ctrl+A) 143 | - `[x]` Press Delete key 144 | - `[~]` Create New Canvas (`create_canvas` command) 145 | - `[x]` Trigger New command (Ctrl+N / menu) 146 | - `[~]` Interact with New Canvas dialog 147 | - `[~]` Set width and height 148 | - `[~]` Handle background color fill (optional) 149 | - `[ ]` Save Canvas (`save` command) 150 | - `[ ]` Trigger Save command (Ctrl+S / menu) 151 | - `[ ]` Interact with Save dialog 152 | - `[ ]` Enter file path 153 | - `[ ]` Select format (`png|jpeg|bmp`) 154 | - `[ ]` Confirm save 155 | - `[ ]` Fetch Image (`fetch_image` command) 156 | - `[ ]` Verify file existence 157 | - `[ ]` Read file contents securely 158 | - `[ ]` Validate image format (e.g., PNG) 159 | - `[ ]` Base64 encode image data 160 | - `[ ]` Return response with data, format, dimensions 161 | 162 | ## Image Transformations (New) 163 | 164 | - `[ ]` Rotate Image (`rotate_image` command) 165 | - `[ ]` Select All (Ctrl+A) (if needed) 166 | - `[ ]` Trigger Rotate command via menu 167 | - `[ ]` Select direction (90°/180°/270°, clockwise/counter-clockwise) 168 | - `[ ]` Flip Image (`flip_image` command) 169 | - `[ ]` Select All (Ctrl+A) (if needed) 170 | - `[ ]` Trigger Flip command via menu 171 | - `[ ]` Select direction (horizontal/vertical) 172 | - `[ ]` Scale Image (`scale_image` command) 173 | - `[ ]` Trigger Resize command via menu/shortcut 174 | - `[ ]` Interact with Resize dialog 175 | - `[ ]` Set dimensions or percentage 176 | - `[ ]` Handle "Maintain aspect ratio" checkbox 177 | - `[ ]` Confirm resize 178 | - `[ ]` Crop Image (`crop_image` command) 179 | - `[ ]` Requires prior selection (`select_region`) 180 | - `[ ]` Trigger Crop command via menu 181 | 182 | ## Image Recreation 183 | 184 | - `[ ]` Implement `recreate_image` command 185 | - `[ ]` Decode base64 image data 186 | - `[ ]` Create new canvas (or clear existing) 187 | - `[ ]` Iterate through pixels (or segments) of source image 188 | - `[ ]` Use `draw_pixel` or optimized drawing method for each pixel/segment 189 | - `[ ]` Handle `max_detail_level` parameter (e.g., sampling) 190 | - `[ ]` Save result to `output_filename` (optional) 191 | 192 | ## Error Handling 193 | 194 | - `[ ]` Implement standard error response format (`status: "error"`, `error` message/code) 195 | - `[ ]` Define and use specific error codes (1000-1015+) 196 | - `[ ]` Implement timeouts for UI operations 197 | - `[ ]` Add logging for debugging 198 | 199 | ## Versioning 200 | 201 | - `[x]` Implement `get_version` command 202 | - `[x]` Return `protocol_version`, `server_version`, `paint_version` 203 | 204 | ## Client Library (`PaintMcpClient` Rust Example) 205 | 206 | *(This section tracks the conceptual mapping to the client library, not its implementation itself unless it's part of this specific project)* 207 | 208 | - `[ ]` Map `connect` to `PaintMcpClient::new()` and `client.connect()` 209 | - `[ ]` Map window commands to `activate_window`, `get_canvas_dimensions` 210 | - `[ ]` Map canvas commands to `clear_canvas`, `create_canvas` 211 | - `[ ]` Map tool selection to `select_tool` with `DrawingTool` enum 212 | - `[ ]` Map color/style commands to `set_color`, `set_thickness`, `set_brush_size`, `set_shape_fill` 213 | - `[ ]` Map drawing commands to `draw_line`, `draw_polyline`, `draw_pixel`, `draw_shape` 214 | - `[ ]` Map text commands to `add_text`, `add_text_with_options` 215 | - `[ ]` Map selection commands to `select_region`, `copy_selection`, `paste` 216 | - `[ ]` Map transformation commands to `rotate_image`, `flip_image`, `scale_image`, `crop_image` 217 | - `[ ]` Map file operations to `save_canvas`, `fetch_image`, `fetch_image_with_metadata` 218 | - `[ ]` Map image recreation to `recreate_image` 219 | - `[ ]` Map errors to `PaintMcpError` enum -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # MCP Server for Microsoft Paint 2 | 3 | A JSON-RPC 2.0 compatible server for controlling Microsoft Paint through the Microsoft Commandline Protocol (MCP). 4 | 5 | ## Features 6 | 7 | - Launch and connect to Microsoft Paint 8 | - Draw lines, shapes, and pixels 9 | - Set colors and tool properties 10 | - Control the Paint window 11 | 12 | ## Requirements 13 | 14 | - Windows 10/11 with Microsoft Paint installed 15 | - Rust (for building the server) 16 | - Python (for the test client examples) 17 | 18 | ## Building and Running 19 | 20 | To build the server: 21 | 22 | ``` 23 | cargo build --release 24 | ``` 25 | 26 | To run the server: 27 | 28 | ``` 29 | cargo run --release 30 | ``` 31 | 32 | The server accepts JSON-RPC 2.0 requests via stdin and responds via stdout. 33 | 34 | ## JSON-RPC Methods 35 | 36 | ### `initialize` 37 | 38 | Finds or launches Microsoft Paint. 39 | 40 | **Request:** 41 | ```json 42 | { 43 | "jsonrpc": "2.0", 44 | "id": 1, 45 | "method": "initialize", 46 | "params": {} 47 | } 48 | ``` 49 | 50 | ### `connect` 51 | 52 | Connects to an already running Paint window. 53 | 54 | **Request:** 55 | ```json 56 | { 57 | "jsonrpc": "2.0", 58 | "id": 2, 59 | "method": "connect", 60 | "params": { 61 | "client_id": "your-client-id", 62 | "client_name": "Your Client Name" 63 | } 64 | } 65 | ``` 66 | 67 | ### `draw_line` 68 | 69 | Draws a line from one point to another. 70 | 71 | **Request:** 72 | ```json 73 | { 74 | "jsonrpc": "2.0", 75 | "id": 3, 76 | "method": "draw_line", 77 | "params": { 78 | "start_x": 100, 79 | "start_y": 100, 80 | "end_x": 300, 81 | "end_y": 100, 82 | "color": "#FF0000", 83 | "thickness": 3 84 | } 85 | } 86 | ``` 87 | 88 | ### Other Methods 89 | 90 | - `activate_window` - Brings the Paint window to the foreground 91 | - `get_canvas_dimensions` - Returns the current canvas size 92 | - `draw_pixel` - Draws a single pixel 93 | - `draw_shape` - Draws a shape (rectangle, ellipse, etc.) 94 | - `select_tool` - Selects a drawing tool 95 | - `set_color` - Sets the current color 96 | - And more... 97 | 98 | ## Example Test Client 99 | 100 | A simple test client is provided in `final_test.py` to demonstrate how to use the server: 101 | 102 | ```bash 103 | python final_test.py 104 | ``` 105 | 106 | ## Troubleshooting 107 | 108 | If you encounter issues with the server connecting to Paint: 109 | 110 | 1. Make sure Microsoft Paint is installed and accessible 111 | 2. Try manually launching Paint before starting the server 112 | 3. Check the server logs for detailed error messages 113 | 114 | ## License 115 | 116 | This project is available under the MIT License. -------------------------------------------------------------------------------- /__pycache__/simple_test_client.cpython-313.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghuntley/mcp-server-microsoft-paint/bf6bf12da2f6c4200af1b8677e725ca07c170f55/__pycache__/simple_test_client.cpython-313.pyc -------------------------------------------------------------------------------- /basic_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import json 3 | import subprocess 4 | import sys 5 | import time 6 | import os 7 | 8 | def main(): 9 | # Kill any existing Paint processes 10 | os.system('taskkill /f /im mspaint.exe 2>nul') 11 | time.sleep(1) 12 | 13 | # Launch Paint 14 | subprocess.run(["start", "mspaint.exe"], shell=True) 15 | print("Launched MS Paint") 16 | time.sleep(3) # Wait for Paint to start 17 | 18 | # Start the MCP server 19 | print("Starting MCP server...") 20 | server_process = subprocess.Popen( 21 | ["cargo", "run", "--release"], 22 | stdin=subprocess.PIPE, 23 | stdout=subprocess.PIPE, 24 | stderr=subprocess.PIPE, 25 | text=True, 26 | bufsize=1 27 | ) 28 | 29 | try: 30 | # Wait for server to start 31 | time.sleep(3) 32 | 33 | # Connect request 34 | print("Sending connect request...") 35 | connect_request = { 36 | "jsonrpc": "2.0", 37 | "id": 1, 38 | "method": "connect", 39 | "params": { 40 | "client_id": "basic-test", 41 | "client_name": "Basic Test Client" 42 | } 43 | } 44 | 45 | # Send the request 46 | request_json = json.dumps(connect_request) + "\n" 47 | server_process.stdin.write(request_json) 48 | server_process.stdin.flush() 49 | 50 | # Read response with timeout 51 | response = "" 52 | start_time = time.time() 53 | while time.time() - start_time < 10: # 10 second timeout 54 | try: 55 | line = server_process.stdout.readline().strip() 56 | if line: 57 | print(f"Received: {line}") 58 | try: 59 | response = json.loads(line) 60 | break 61 | except json.JSONDecodeError: 62 | print(f"Not valid JSON: {line}") 63 | except Exception as e: 64 | print(f"Error reading response: {e}") 65 | time.sleep(0.1) 66 | 67 | if not response: 68 | print("No response received") 69 | stderr = server_process.stderr.read() 70 | if stderr: 71 | print(f"Server stderr: {stderr}") 72 | return 73 | 74 | print(f"Connect response: {response}") 75 | 76 | # Activate window request 77 | print("Sending activate_window request...") 78 | activate_request = { 79 | "jsonrpc": "2.0", 80 | "id": 2, 81 | "method": "activate_window", 82 | "params": {} 83 | } 84 | 85 | request_json = json.dumps(activate_request) + "\n" 86 | server_process.stdin.write(request_json) 87 | server_process.stdin.flush() 88 | 89 | # Read response 90 | response = "" 91 | start_time = time.time() 92 | while time.time() - start_time < 5: # 5 second timeout 93 | line = server_process.stdout.readline().strip() 94 | if line: 95 | print(f"Received: {line}") 96 | try: 97 | response = json.loads(line) 98 | break 99 | except json.JSONDecodeError: 100 | pass 101 | time.sleep(0.1) 102 | 103 | if not response: 104 | print("No response received for activate_window") 105 | return 106 | 107 | print(f"Activate response: {response}") 108 | 109 | # Draw a simple line 110 | print("Drawing a horizontal line...") 111 | line_request = { 112 | "jsonrpc": "2.0", 113 | "id": 3, 114 | "method": "draw_line", 115 | "params": { 116 | "start_x": 100, 117 | "start_y": 100, 118 | "end_x": 300, 119 | "end_y": 100, 120 | "color": "#FF0000", 121 | "thickness": 3 122 | } 123 | } 124 | 125 | request_json = json.dumps(line_request) + "\n" 126 | server_process.stdin.write(request_json) 127 | server_process.stdin.flush() 128 | 129 | # Read response 130 | response = "" 131 | start_time = time.time() 132 | while time.time() - start_time < 10: # 10 second timeout for drawing 133 | line = server_process.stdout.readline().strip() 134 | if line: 135 | print(f"Received: {line}") 136 | try: 137 | response = json.loads(line) 138 | break 139 | except json.JSONDecodeError: 140 | pass 141 | time.sleep(0.1) 142 | 143 | if not response: 144 | print("No response received for draw_line") 145 | return 146 | 147 | print(f"Draw line response: {response}") 148 | 149 | # Wait to observe the result 150 | print("Test completed. Wait 5 seconds before cleanup...") 151 | time.sleep(5) 152 | 153 | except Exception as e: 154 | print(f"Error during test: {e}") 155 | finally: 156 | server_process.terminate() 157 | print("Server terminated") 158 | 159 | if __name__ == "__main__": 160 | main() -------------------------------------------------------------------------------- /better_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import json 3 | import subprocess 4 | import sys 5 | import time 6 | import os 7 | 8 | def main(): 9 | # Kill any existing Paint processes 10 | os.system('taskkill /f /im mspaint.exe 2>nul') 11 | time.sleep(1) 12 | 13 | # Launch Paint 14 | subprocess.run(["start", "mspaint.exe"], shell=True) 15 | print("Launched MS Paint") 16 | time.sleep(3) # Wait for Paint to start 17 | 18 | # Start the MCP server 19 | print("Starting MCP server...") 20 | server_process = subprocess.Popen( 21 | ["cargo", "run", "--release"], 22 | stdin=subprocess.PIPE, 23 | stdout=subprocess.PIPE, 24 | stderr=open("server_stderr.log", "w"), # Redirect stderr to a file 25 | text=True, 26 | bufsize=1 27 | ) 28 | 29 | try: 30 | # Wait for server to start 31 | time.sleep(3) 32 | 33 | # Connect request 34 | print("Sending connect request...") 35 | connect_request = { 36 | "jsonrpc": "2.0", 37 | "id": 1, 38 | "method": "connect", 39 | "params": { 40 | "client_id": "better-test", 41 | "client_name": "Better Test Client" 42 | } 43 | } 44 | 45 | # Send the request 46 | request_json = json.dumps(connect_request) + "\n" 47 | print(f"Request: {request_json.strip()}") 48 | server_process.stdin.write(request_json) 49 | server_process.stdin.flush() 50 | 51 | # Read response with timeout 52 | response = "" 53 | start_time = time.time() 54 | while time.time() - start_time < 10: # 10 second timeout 55 | try: 56 | line = server_process.stdout.readline().strip() 57 | if line: 58 | print(f"Received: {line}") 59 | try: 60 | response = json.loads(line) 61 | break 62 | except json.JSONDecodeError: 63 | print(f"Not valid JSON: {line}") 64 | except Exception as e: 65 | print(f"Error reading response: {e}") 66 | time.sleep(0.1) 67 | 68 | if not response: 69 | print("No response received") 70 | return 71 | 72 | print(f"Connect response: {response}") 73 | 74 | # Activate window request 75 | print("Sending activate_window request...") 76 | activate_request = { 77 | "jsonrpc": "2.0", 78 | "id": 2, 79 | "method": "activate_window", 80 | "params": {} 81 | } 82 | 83 | request_json = json.dumps(activate_request) + "\n" 84 | print(f"Request: {request_json.strip()}") 85 | server_process.stdin.write(request_json) 86 | server_process.stdin.flush() 87 | 88 | # Read response 89 | response = "" 90 | start_time = time.time() 91 | while time.time() - start_time < 5: # 5 second timeout 92 | line = server_process.stdout.readline().strip() 93 | if line: 94 | print(f"Received: {line}") 95 | try: 96 | response = json.loads(line) 97 | break 98 | except json.JSONDecodeError: 99 | pass 100 | time.sleep(0.1) 101 | 102 | if not response: 103 | print("No response received for activate_window") 104 | return 105 | 106 | print(f"Activate response: {response}") 107 | 108 | # Draw a simple line 109 | print("Drawing a horizontal line...") 110 | line_request = { 111 | "jsonrpc": "2.0", 112 | "id": 3, 113 | "method": "draw_line", 114 | "params": { 115 | "start_x": 100, 116 | "start_y": 100, 117 | "end_x": 300, 118 | "end_y": 100, 119 | "color": "#FF0000", 120 | "thickness": 3 121 | } 122 | } 123 | 124 | request_json = json.dumps(line_request) + "\n" 125 | print(f"Request: {request_json.strip()}") 126 | server_process.stdin.write(request_json) 127 | server_process.stdin.flush() 128 | 129 | # Read response 130 | response = "" 131 | start_time = time.time() 132 | while time.time() - start_time < 10: # 10 second timeout for drawing 133 | line = server_process.stdout.readline().strip() 134 | if line: 135 | print(f"Received: {line}") 136 | try: 137 | response = json.loads(line) 138 | break 139 | except json.JSONDecodeError: 140 | pass 141 | time.sleep(0.1) 142 | 143 | if not response: 144 | print("No response received for draw_line") 145 | return 146 | 147 | print(f"Draw line response: {response}") 148 | 149 | # Wait to observe the result 150 | print("Test completed. Wait 5 seconds before cleanup...") 151 | time.sleep(5) 152 | 153 | except Exception as e: 154 | print(f"Error during test: {e}") 155 | finally: 156 | server_process.terminate() 157 | print("Server terminated") 158 | 159 | if __name__ == "__main__": 160 | main() -------------------------------------------------------------------------------- /core_api.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghuntley/mcp-server-microsoft-paint/bf6bf12da2f6c4200af1b8677e725ca07c170f55/core_api.txt -------------------------------------------------------------------------------- /debug_mcp_server.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | import time 3 | import os 4 | import sys 5 | import json 6 | import logging 7 | 8 | def setup_logging(): 9 | log_formatter = logging.Formatter("%(asctime)s [%(levelname)-5.5s] %(message)s") 10 | root_logger = logging.getLogger() 11 | root_logger.setLevel(logging.INFO) # Default to INFO for this utility 12 | 13 | # Console Handler 14 | console_handler = logging.StreamHandler(sys.stdout) 15 | console_handler.setFormatter(log_formatter) 16 | root_logger.addHandler(console_handler) 17 | logging.info("Logging initialized for debug_mcp_server.py") 18 | 19 | def print_divider(title=""): 20 | """Print a section divider using logging""" 21 | logging.info("\n" + "=" * 60) 22 | if title: 23 | logging.info(title.center(60)) 24 | logging.info("=" * 60) 25 | 26 | def build_server(): 27 | """Build the MCP server using cargo""" 28 | print_divider("Building MCP Server") 29 | try: 30 | logging.debug("Running 'cargo build --release'") 31 | result = subprocess.run(['cargo', 'build', '--release'], 32 | capture_output=True, 33 | text=True) 34 | if result.returncode == 0: 35 | logging.info("Build successful") 36 | if result.stdout: 37 | logging.debug(f"Build STDOUT:\n{result.stdout}") 38 | else: 39 | logging.error(f"Build failed with return code {result.returncode}") 40 | if result.stdout: 41 | logging.error(f"Build STDOUT:\n{result.stdout}") 42 | if result.stderr: 43 | logging.error(f"Build STDERR:\n{result.stderr}") 44 | return result.returncode == 0 45 | except FileNotFoundError: 46 | logging.error("'cargo' command not found. Make sure Rust and Cargo are installed and in your PATH.") 47 | return False 48 | except Exception as e: 49 | logging.error(f"Exception during build: {e}") 50 | return False 51 | 52 | def launch_paint(): 53 | """Attempt to launch MS Paint directly""" 54 | print_divider("Launching MS Paint directly") 55 | try: 56 | # Try multiple possible locations for mspaint.exe 57 | locations = [ 58 | "mspaint.exe", # Try PATH first 59 | "C:\\Windows\\System32\\mspaint.exe", 60 | "C:\\Windows\\mspaint.exe" 61 | ] 62 | 63 | for location in locations: 64 | logging.debug(f"Trying to launch from: {location}") 65 | try: 66 | process = subprocess.Popen([location], 67 | stdout=subprocess.PIPE, 68 | stderr=subprocess.PIPE, 69 | text=True) 70 | logging.debug(f"Launched process with PID: {process.pid}") 71 | time.sleep(2) # Wait for process to potentially start/exit 72 | 73 | # Check if it's still running 74 | exit_code = process.poll() 75 | if exit_code is None: 76 | logging.info(f"Successfully launched MS Paint from {location} (PID: {process.pid}). Terminating...") 77 | process.terminate() 78 | try: 79 | process.wait(timeout=3) 80 | logging.debug("Paint process terminated.") 81 | except subprocess.TimeoutExpired: 82 | logging.warning(f"Paint process {process.pid} did not terminate gracefully, killing.") 83 | process.kill() 84 | process.wait() 85 | logging.debug("Paint process killed.") 86 | return True 87 | else: 88 | logging.warning(f"Process from {location} exited quickly with code: {exit_code}") 89 | stdout, stderr = process.communicate() 90 | if stdout: 91 | logging.debug(f"STDOUT: {stdout}") 92 | if stderr: 93 | logging.warning(f"STDERR: {stderr}") 94 | except FileNotFoundError: 95 | logging.warning(f"Executable not found at: {location}") 96 | except Exception as e: 97 | logging.error(f"Error launching from {location}: {e}") 98 | 99 | logging.error("Failed to launch Paint from any known location.") 100 | return False 101 | except Exception as e: 102 | logging.error(f"Exception during Paint launch test: {e}") 103 | return False 104 | 105 | def test_manual_server_launch(): 106 | """Test launching the MCP server manually and passing a simple command""" 107 | print_divider("Testing Manual Server Launch") 108 | server_process = None 109 | try: 110 | server_path = os.path.join("target", "release", "mcp-server-microsoft-paint.exe") 111 | if not os.path.exists(server_path): 112 | logging.error(f"Server executable not found at {server_path}") 113 | return False 114 | 115 | logging.info(f"Launching server from {server_path}") 116 | 117 | # Launch the server process 118 | server_process = subprocess.Popen( 119 | [server_path], 120 | stdin=subprocess.PIPE, 121 | stdout=subprocess.PIPE, 122 | stderr=subprocess.PIPE, 123 | text=True, # Use text mode for direct interaction in this test 124 | encoding='utf-8' # Specify encoding 125 | ) 126 | logging.debug(f"Server process launched with PID: {server_process.pid}") 127 | 128 | # Wait for server to initialize 129 | logging.debug("Waiting for server initialization (1 second)...") 130 | time.sleep(1) 131 | 132 | # Send a get_version command 133 | request = json.dumps({ 134 | "jsonrpc": "2.0", 135 | "id": 1, 136 | "method": "get_version", 137 | "params": {} 138 | }) + "\n" 139 | 140 | logging.debug(f"Sending request: {request.strip()}") 141 | server_process.stdin.write(request) 142 | server_process.stdin.flush() 143 | 144 | # Read response with timeout 145 | output_lines = [] 146 | stderr_lines = [] 147 | 148 | # Non-blocking read with timeout 149 | start_time = time.time() 150 | timeout = 5 # seconds 151 | logging.debug(f"Waiting for response (timeout: {timeout}s)") 152 | 153 | response_received = False 154 | while time.time() - start_time < timeout: 155 | # Check if process is still running 156 | if server_process.poll() is not None: 157 | logging.warning(f"Server process exited prematurely with code: {server_process.poll()}") 158 | break 159 | 160 | # Check for output 161 | stdout_line = server_process.stdout.readline().strip() 162 | if stdout_line: 163 | logging.debug(f"Received stdout line: {stdout_line}") 164 | output_lines.append(stdout_line) 165 | # Assuming the first valid JSON is the response 166 | try: 167 | json.loads(stdout_line) 168 | response_received = True 169 | break # Got the response 170 | except json.JSONDecodeError: 171 | logging.warning(f"Received non-JSON line on stdout: {stdout_line}") 172 | # Continue reading, might be debug info 173 | 174 | # Check for stderr output (non-blocking) 175 | # This part is tricky with readline, might need select or threading for robust non-blocking stderr 176 | # For simplicity, we'll read stderr at the end 177 | 178 | # Brief pause to avoid CPU spin 179 | time.sleep(0.1) 180 | 181 | if not response_received and server_process.poll() is None: 182 | logging.warning("Timeout waiting for response.") 183 | 184 | # Clean up 185 | if server_process.poll() is None: 186 | logging.debug("Terminating server process...") 187 | server_process.terminate() 188 | try: 189 | stdout_rem, stderr_rem = server_process.communicate(timeout=3) 190 | logging.debug(f"Server terminated with code: {server_process.returncode}") 191 | if stdout_rem: 192 | logging.debug(f"Remaining stdout:\n{stdout_rem}") 193 | output_lines.append(stdout_rem) 194 | if stderr_rem: 195 | logging.warning(f"Remaining stderr:\n{stderr_rem}") 196 | stderr_lines.append(stderr_rem) 197 | except subprocess.TimeoutExpired: 198 | logging.warning("Server did not terminate gracefully, killing...") 199 | server_process.kill() 200 | stdout_rem, stderr_rem = server_process.communicate() 201 | logging.debug("Server killed.") 202 | if stdout_rem: 203 | logging.debug(f"Remaining stdout:\n{stdout_rem}") 204 | output_lines.append(stdout_rem) 205 | if stderr_rem: 206 | logging.warning(f"Remaining stderr:\n{stderr_rem}") 207 | stderr_lines.append(stderr_rem) 208 | else: # Server already exited 209 | logging.debug("Server already exited. Reading remaining output.") 210 | stdout_rem, stderr_rem = server_process.communicate() 211 | if stdout_rem: 212 | logging.debug(f"Remaining stdout:\n{stdout_rem}") 213 | output_lines.append(stdout_rem) 214 | if stderr_rem: 215 | logging.warning(f"Remaining stderr:\n{stderr_rem}") 216 | stderr_lines.append(stderr_rem) 217 | 218 | # Print the results 219 | if response_received: 220 | logging.info("Response received:") 221 | for line in output_lines: 222 | logging.info(line) # Log each line of output 223 | if stderr_lines: # Also log stderr if any was captured 224 | logging.warning("Stderr received:") 225 | for line in stderr_lines: 226 | logging.warning(line) 227 | return True 228 | else: 229 | logging.error("No valid JSON response received from server") 230 | if output_lines: 231 | logging.error("Raw stdout received:") 232 | for line in output_lines: 233 | logging.error(line) 234 | if stderr_lines: 235 | logging.error("Stderr received:") 236 | for line in stderr_lines: 237 | logging.error(line) 238 | return False 239 | 240 | except Exception as e: 241 | logging.error(f"Exception during server test: {e}") 242 | import traceback 243 | logging.error(traceback.format_exc()) 244 | # Ensure process termination if started 245 | if server_process and server_process.poll() is None: 246 | logging.warning("Terminating server process due to exception...") 247 | server_process.kill() 248 | server_process.wait() 249 | return False 250 | 251 | def main(): 252 | setup_logging() 253 | logging.info("MCP Server for Windows 11 Paint Debug Utility") 254 | logging.info(f"Python version: {sys.version}") 255 | logging.info(f"Current directory: {os.getcwd()}") 256 | 257 | # Run the tests 258 | build_ok = build_server() 259 | paint_ok = launch_paint() 260 | server_ok = test_manual_server_launch() 261 | 262 | # Summary 263 | print_divider("Summary") 264 | logging.info(f"Build successful: {build_ok}") 265 | logging.info(f"MS Paint launch successful: {paint_ok}") 266 | logging.info(f"Server test successful: {server_ok}") 267 | 268 | if not paint_ok: 269 | logging.warning("\nPossible issues with MS Paint:") 270 | logging.warning("1. MS Paint may not be installed on this system") 271 | logging.warning("2. MS Paint executable might be in a non-standard location") 272 | logging.warning("3. There might be permission issues launching MS Paint") 273 | 274 | if not server_ok: 275 | logging.warning("\nPossible issues with MCP Server:") 276 | logging.warning("1. Server might be failing to initialize") 277 | logging.warning("2. JSON-RPC handling might be incorrect") 278 | logging.warning("3. Server might be failing to find or launch MS Paint") 279 | logging.warning("4. Server output format might not match what we're expecting") 280 | logging.warning("Check server logs (stderr) for more details if possible.") 281 | 282 | if __name__ == "__main__": 283 | main() -------------------------------------------------------------------------------- /debug_paint.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | import time 3 | import os 4 | import sys 5 | import logging 6 | 7 | def setup_logging(): 8 | log_formatter = logging.Formatter("%(asctime)s [%(levelname)-5.5s] %(message)s") 9 | root_logger = logging.getLogger() 10 | root_logger.setLevel(logging.INFO) # Default to INFO for this utility 11 | 12 | # Console Handler 13 | console_handler = logging.StreamHandler(sys.stdout) 14 | console_handler.setFormatter(log_formatter) 15 | root_logger.addHandler(console_handler) 16 | logging.info("Logging initialized for debug_paint.py") 17 | 18 | def is_paint_running(): 19 | """Check if MS Paint is running using tasklist""" 20 | try: 21 | result = subprocess.run(['tasklist', '/FI', 'IMAGENAME eq mspaint.exe'], 22 | capture_output=True, text=True, check=False) 23 | is_running = 'mspaint.exe' in result.stdout 24 | logging.debug(f"tasklist check for mspaint.exe: stdout='{result.stdout[:50]}...', running={is_running}") 25 | return is_running 26 | except FileNotFoundError: 27 | logging.error("'tasklist' command not found. Cannot check if Paint is running.") 28 | return False 29 | except Exception as e: 30 | logging.error(f"Error checking if Paint is running: {e}") 31 | return False 32 | 33 | def main(): 34 | setup_logging() 35 | logging.info("Debugging MS Paint launch...") 36 | 37 | # Check if Paint is already running 38 | if is_paint_running(): 39 | logging.info("MS Paint is already running") 40 | else: 41 | logging.info("MS Paint is not running, attempting to launch...") 42 | 43 | process = None 44 | try: 45 | # Try to launch MS Paint 46 | logging.debug("Launching 'mspaint.exe'...") 47 | process = subprocess.Popen(['mspaint.exe'], 48 | stdout=subprocess.PIPE, 49 | stderr=subprocess.PIPE) 50 | logging.debug(f"Paint process started with PID: {process.pid}") 51 | 52 | # Wait a bit for Paint to start 53 | logging.debug("Waiting 2 seconds for Paint to initialize...") 54 | time.sleep(2) 55 | 56 | # Check if it's running now 57 | if is_paint_running(): 58 | logging.info("Successfully launched MS Paint") 59 | else: 60 | logging.error("Failed to launch MS Paint or it exited quickly!") 61 | 62 | # Get the exit code (should be None if still running) 63 | exit_code = process.poll() 64 | logging.info(f"Paint process exit code after 2s wait: {exit_code}") 65 | 66 | except FileNotFoundError: 67 | logging.error("'mspaint.exe' not found in PATH. Cannot launch Paint.") 68 | except Exception as e: 69 | logging.error(f"Error launching MS Paint: {e}") 70 | finally: 71 | # Try to terminate the process if it was started and might still be running 72 | if process and process.poll() is None: 73 | logging.debug("Terminating MS Paint process...") 74 | process.terminate() 75 | try: 76 | outs, errs = process.communicate(timeout=3) 77 | logging.debug(f"Paint process terminated with code: {process.returncode}") 78 | if outs: 79 | logging.debug(f"Paint stdout:\n{outs.decode('utf-8', errors='replace')}") 80 | if errs: 81 | logging.warning(f"Paint stderr:\n{errs.decode('utf-8', errors='replace')}") 82 | except subprocess.TimeoutExpired: 83 | logging.warning("Paint did not terminate gracefully, killing...") 84 | process.kill() 85 | outs, errs = process.communicate() 86 | logging.debug("Paint killed.") 87 | if outs: 88 | logging.debug(f"Paint stdout:\n{outs.decode('utf-8', errors='replace')}") 89 | if errs: 90 | logging.warning(f"Paint stderr:\n{errs.decode('utf-8', errors='replace')}") 91 | except Exception as term_err: 92 | logging.error(f"Error terminating Paint process: {term_err}") 93 | elif process: 94 | logging.debug(f"Paint process already exited with code: {process.poll()}") 95 | # Read remaining streams 96 | try: 97 | outs, errs = process.communicate() 98 | if outs: 99 | logging.debug(f"Paint stdout:\n{outs.decode('utf-8', errors='replace')}") 100 | if errs: 101 | logging.warning(f"Paint stderr:\n{errs.decode('utf-8', errors='replace')}") 102 | except Exception as comm_err: 103 | logging.error(f"Error reading Paint streams after exit: {comm_err}") 104 | 105 | logging.info("\nSystem info:") 106 | logging.info(f"Python version: {sys.version}") 107 | logging.info(f"Current directory: {os.getcwd()}") 108 | 109 | # List mspaint.exe in Windows directory 110 | logging.info("\nChecking for mspaint.exe in common Windows directories:") 111 | windows_dirs = [ 112 | os.environ.get('WINDIR', 'C:\\Windows'), 113 | os.path.join(os.environ.get('WINDIR', 'C:\\Windows'), 'System32'), 114 | # os.path.join(os.environ.get('PROGRAMFILES', 'C:\\Program Files'), 'WindowsApps') # Usually restricted access 115 | os.environ.get('SystemRoot', 'C:\\Windows') # Often same as WINDIR 116 | ] 117 | 118 | # Deduplicate paths 119 | checked_paths = set() 120 | for directory in windows_dirs: 121 | if not directory or not os.path.isdir(directory): 122 | logging.debug(f"Skipping invalid or inaccessible directory: {directory}") 123 | continue 124 | 125 | mspaint_path = os.path.join(directory, 'mspaint.exe') 126 | if mspaint_path in checked_paths: 127 | continue 128 | checked_paths.add(mspaint_path) 129 | 130 | if os.path.exists(mspaint_path): 131 | logging.info(f"Found mspaint.exe at: {mspaint_path}") 132 | else: 133 | logging.info(f"mspaint.exe not found at: {mspaint_path}") 134 | 135 | # Also check PATH 136 | logging.info("\nChecking if 'mspaint.exe' is in system PATH...") 137 | try: 138 | result = subprocess.run(['where', 'mspaint.exe'], capture_output=True, text=True, check=False) 139 | if result.returncode == 0 and result.stdout: 140 | logging.info(f"Found mspaint.exe via PATH: {result.stdout.strip()}") 141 | else: 142 | logging.warning("'mspaint.exe' not found in PATH.") 143 | except FileNotFoundError: 144 | logging.warning("'where' command not found. Cannot check PATH.") 145 | except Exception as e: 146 | logging.error(f"Error checking PATH for mspaint.exe: {e}") 147 | 148 | logging.info("\nPaint debug check finished.") 149 | 150 | if __name__ == "__main__": 151 | main() -------------------------------------------------------------------------------- /debug_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import json 3 | import subprocess 4 | import time 5 | import sys 6 | import os 7 | import threading 8 | 9 | def main(): 10 | # First, make sure Paint is not running 11 | os.system('taskkill /f /im mspaint.exe 2>nul') 12 | time.sleep(1) 13 | 14 | # Launch Paint 15 | print("Launching MS Paint...") 16 | paint_process = subprocess.Popen(["mspaint.exe"]) 17 | time.sleep(3) # Wait for Paint to start 18 | 19 | # Start the server in debug mode (not release) for more verbose output 20 | print("Starting MCP server...") 21 | server_process = subprocess.Popen( 22 | ["cargo", "run"], 23 | stdin=subprocess.PIPE, 24 | stdout=subprocess.PIPE, 25 | stderr=subprocess.PIPE, 26 | text=True, 27 | bufsize=1 28 | ) 29 | 30 | # Create a thread to continuously read and print stderr 31 | def read_stderr(): 32 | while True: 33 | line = server_process.stderr.readline() 34 | if not line: 35 | break 36 | print(f"SERVER STDERR: {line.strip()}") 37 | 38 | stderr_thread = threading.Thread(target=read_stderr, daemon=True) 39 | stderr_thread.start() 40 | 41 | try: 42 | # Step 1: Initialize 43 | print("Step 1: Sending initialize request...") 44 | init_request = { 45 | "jsonrpc": "2.0", 46 | "id": 1, 47 | "method": "initialize", 48 | "params": {} 49 | } 50 | 51 | response = send_request(server_process, init_request) 52 | if not response: 53 | print("ERROR: No response received for initialize request") 54 | 55 | # Step 2: Connect 56 | print("Step 2: Sending connect request...") 57 | connect_request = { 58 | "jsonrpc": "2.0", 59 | "id": 2, 60 | "method": "connect", 61 | "params": { 62 | "client_id": "debug_test", 63 | "client_name": "Debug Test Client" 64 | } 65 | } 66 | 67 | response = send_request(server_process, connect_request) 68 | if not response: 69 | print("ERROR: No response received for connect request") 70 | 71 | # Step 3: Draw a line 72 | print("Step 3: Sending draw_line request...") 73 | line_request = { 74 | "jsonrpc": "2.0", 75 | "id": 3, 76 | "method": "draw_line", 77 | "params": { 78 | "start_x": 100, 79 | "start_y": 200, 80 | "end_x": 400, 81 | "end_y": 200, 82 | "color": "#FF0000", # Red color 83 | "thickness": 5 # Thick line 84 | } 85 | } 86 | 87 | response = send_request(server_process, line_request) 88 | if not response: 89 | print("ERROR: No response received for draw_line request") 90 | 91 | print("Test completed! Check Paint to see if anything was drawn.") 92 | print("Press Enter to close the test and kill Paint...") 93 | input() 94 | 95 | except Exception as e: 96 | print(f"Test failed with error: {e}") 97 | finally: 98 | server_process.terminate() 99 | print("Server process terminated") 100 | paint_process.terminate() 101 | print("Paint process terminated") 102 | 103 | def send_request(process, request): 104 | """Send a request to the server and print the response.""" 105 | request_str = json.dumps(request) + "\n" 106 | print(f"SENDING: {request_str.strip()}") 107 | 108 | process.stdin.write(request_str) 109 | process.stdin.flush() 110 | print("Request sent and flushed") 111 | 112 | # Read response with timeout 113 | start_time = time.time() 114 | timeout = 10 # seconds 115 | 116 | while time.time() - start_time < timeout: 117 | print("Waiting for response...") 118 | line = process.stdout.readline().strip() 119 | if line: 120 | print(f"RESPONSE: {line}") 121 | try: 122 | return json.loads(line) 123 | except json.JSONDecodeError: 124 | print(f"WARNING: Received non-JSON response: {line}") 125 | time.sleep(0.5) 126 | 127 | print("WARNING: No response received within timeout") 128 | return None 129 | 130 | if __name__ == "__main__": 131 | main() -------------------------------------------------------------------------------- /direct_draw_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import subprocess 3 | import time 4 | import win32gui 5 | import win32con 6 | import win32api 7 | import ctypes 8 | import sys 9 | 10 | def main(): 11 | # Launch MS Paint 12 | print("Launching MS Paint...") 13 | subprocess.Popen(["mspaint.exe"]) 14 | time.sleep(3) # Wait for Paint to start 15 | 16 | # Find MS Paint window 17 | paint_hwnd = win32gui.FindWindow(None, "Untitled - Paint") 18 | if not paint_hwnd: 19 | print("Could not find MS Paint window.") 20 | return 21 | 22 | print(f"Found MS Paint window: {paint_hwnd}") 23 | 24 | # Activate MS Paint window 25 | win32gui.SetForegroundWindow(paint_hwnd) 26 | time.sleep(1) 27 | 28 | # Get window dimensions 29 | left, top, right, bottom = win32gui.GetClientRect(paint_hwnd) 30 | 31 | # Adjust for window borders and toolbars (approximate) 32 | toolbar_height = 150 # Adjust as needed for your Paint version 33 | drawing_area_top = top + toolbar_height 34 | 35 | # Calculate coordinates for a horizontal line 36 | start_x = 100 37 | start_y = 200 38 | end_x = 400 39 | end_y = 200 40 | 41 | # Convert client coordinates to screen coordinates 42 | start_screen_x, start_screen_y = win32gui.ClientToScreen(paint_hwnd, (start_x, start_y + toolbar_height)) 43 | end_screen_x, end_screen_y = win32gui.ClientToScreen(paint_hwnd, (end_x, end_y + toolbar_height)) 44 | 45 | print(f"Drawing line from ({start_screen_x}, {start_screen_y}) to ({end_screen_x}, {end_screen_y})") 46 | 47 | # Move mouse to starting position 48 | win32api.SetCursorPos((start_screen_x, start_screen_y)) 49 | time.sleep(0.5) 50 | 51 | # Press left mouse button 52 | win32api.mouse_event(win32con.MOUSEEVENTF_LEFTDOWN, 0, 0, 0, 0) 53 | time.sleep(0.2) 54 | 55 | # Move to ending position (draw the line) 56 | # Do this in small steps for smoother line 57 | steps = 10 58 | for i in range(1, steps + 1): 59 | x = start_screen_x + (end_screen_x - start_screen_x) * i // steps 60 | y = start_screen_y + (end_screen_y - start_screen_y) * i // steps 61 | win32api.SetCursorPos((x, y)) 62 | time.sleep(0.02) 63 | 64 | # Release left mouse button 65 | win32api.mouse_event(win32con.MOUSEEVENTF_LEFTUP, 0, 0, 0, 0) 66 | 67 | print("Line drawing completed. Paint window will remain open.") 68 | print("Press Enter to close this script (Paint will stay open)...") 69 | input() 70 | 71 | if __name__ == "__main__": 72 | main() -------------------------------------------------------------------------------- /direct_json_rpc_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import json 3 | import subprocess 4 | import time 5 | import sys 6 | 7 | def main(): 8 | # First manually launch Paint 9 | print("Launching MS Paint directly...") 10 | paint_process = subprocess.Popen(["mspaint.exe"]) 11 | time.sleep(3) # Give Paint time to start 12 | 13 | # Now start the MCP server 14 | print("Starting MCP server...") 15 | server_process = subprocess.Popen( 16 | ["cargo", "run", "--release"], 17 | stdin=subprocess.PIPE, 18 | stdout=subprocess.PIPE, 19 | stderr=open("server_stderr.log", "w"), # Capture stderr to file 20 | text=True, 21 | bufsize=1 22 | ) 23 | 24 | try: 25 | # Wait for server to start up 26 | time.sleep(3) 27 | 28 | # Send connect request 29 | print("Sending connect request...") 30 | connect_request = { 31 | "jsonrpc": "2.0", 32 | "id": 1, 33 | "method": "connect", 34 | "params": { 35 | "client_id": "direct-test", 36 | "client_name": "Direct JSON-RPC Test" 37 | } 38 | } 39 | 40 | request_json = json.dumps(connect_request) + "\n" 41 | print(f"Request: {request_json.strip()}") 42 | server_process.stdin.write(request_json) 43 | server_process.stdin.flush() 44 | 45 | # Read response with timeout 46 | response = "" 47 | start_time = time.time() 48 | while time.time() - start_time < 5: # 5 second timeout 49 | line = server_process.stdout.readline().strip() 50 | if line: 51 | print(f"Response: {line}") 52 | try: 53 | response = json.loads(line) 54 | break 55 | except json.JSONDecodeError: 56 | print(f"Not valid JSON: {line}") 57 | time.sleep(0.1) 58 | 59 | if not response: 60 | print("No response received") 61 | return 62 | 63 | # Send a draw_line command 64 | print("Sending draw_line request...") 65 | line_request = { 66 | "jsonrpc": "2.0", 67 | "id": 2, 68 | "method": "draw_line", 69 | "params": { 70 | "start_x": 100, 71 | "start_y": 100, 72 | "end_x": 300, 73 | "end_y": 100, 74 | "color": "#FF0000", 75 | "thickness": 3 76 | } 77 | } 78 | 79 | request_json = json.dumps(line_request) + "\n" 80 | print(f"Request: {request_json.strip()}") 81 | server_process.stdin.write(request_json) 82 | server_process.stdin.flush() 83 | 84 | # Read response 85 | response = "" 86 | start_time = time.time() 87 | while time.time() - start_time < 10: # 10 second timeout for drawing 88 | line = server_process.stdout.readline().strip() 89 | if line: 90 | print(f"Response: {line}") 91 | try: 92 | response = json.loads(line) 93 | break 94 | except json.JSONDecodeError: 95 | print(f"Not valid JSON: {line}") 96 | time.sleep(0.1) 97 | 98 | if not response: 99 | print("No response received for draw_line") 100 | return 101 | 102 | print("Test completed. Wait 10 seconds to observe the result...") 103 | time.sleep(10) 104 | 105 | except Exception as e: 106 | print(f"Error during test: {e}") 107 | finally: 108 | server_process.terminate() 109 | paint_process.terminate() 110 | print("Tests terminated and MS Paint closed") 111 | 112 | if __name__ == "__main__": 113 | main() -------------------------------------------------------------------------------- /direct_paint_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import time 3 | import sys 4 | import logging 5 | import os 6 | import subprocess 7 | import win32gui 8 | import win32process 9 | import win32con 10 | import win32api 11 | import psutil 12 | import ctypes 13 | from ctypes import wintypes 14 | 15 | def setup_logging(): 16 | logging.basicConfig( 17 | level=logging.INFO, 18 | format="%(asctime)s [%(levelname)-5.5s] %(message)s", 19 | handlers=[logging.StreamHandler(sys.stdout)] 20 | ) 21 | logging.info("Logging initialized for direct_paint_test.py") 22 | 23 | def launch_paint(): 24 | """Launch Paint and wait for it to start""" 25 | try: 26 | # Try to close any existing Paint instances first 27 | os.system('taskkill /f /im mspaint.exe 2>nul') 28 | time.sleep(1) 29 | 30 | # Launch Paint maximized 31 | subprocess.run(["start", "/max", "mspaint.exe"], shell=True) 32 | logging.info("Started MS Paint") 33 | 34 | # Wait for Paint to initialize 35 | time.sleep(3) # Longer wait to ensure Paint is ready 36 | return True 37 | except Exception as e: 38 | logging.error(f"Error launching MS Paint: {e}") 39 | return False 40 | 41 | def find_paint_window(): 42 | """Find the Paint window handle""" 43 | def callback(hwnd, results): 44 | if win32gui.IsWindowVisible(hwnd): 45 | title = win32gui.GetWindowText(hwnd) 46 | if "Paint" in title and "mcp-server" not in title: 47 | try: 48 | _, pid = win32process.GetWindowThreadProcessId(hwnd) 49 | proc = psutil.Process(pid) 50 | if "mspaint" in proc.name().lower(): 51 | results.append(hwnd) 52 | except Exception as e: 53 | logging.error(f"Error getting process info: {e}") 54 | return True 55 | 56 | results = [] 57 | win32gui.EnumWindows(callback, results) 58 | 59 | if results: 60 | hwnd = results[0] 61 | logging.info(f"Found Paint window with handle: {hwnd}, title: {win32gui.GetWindowText(hwnd)}") 62 | return hwnd 63 | else: 64 | logging.error("No Paint window found") 65 | return None 66 | 67 | def simple_draw_line(start_x, start_y, end_x, end_y): 68 | """Draw a line using simulated mouse events at absolute screen coordinates""" 69 | try: 70 | # Move mouse to start position (absolute screen coordinates) 71 | win32api.SetCursorPos((start_x, start_y)) 72 | time.sleep(0.5) 73 | 74 | # Press left button 75 | win32api.mouse_event(win32con.MOUSEEVENTF_LEFTDOWN, 0, 0, 0, 0) 76 | time.sleep(0.5) 77 | 78 | # Move to end position in small steps 79 | steps = 10 80 | dx = (end_x - start_x) / steps 81 | dy = (end_y - start_y) / steps 82 | 83 | for i in range(1, steps + 1): 84 | x = int(start_x + (dx * i)) 85 | y = int(start_y + (dy * i)) 86 | win32api.SetCursorPos((x, y)) 87 | time.sleep(0.05) 88 | 89 | # Ensure we're at the end position 90 | win32api.SetCursorPos((end_x, end_y)) 91 | time.sleep(0.5) 92 | 93 | # Release left button 94 | win32api.mouse_event(win32con.MOUSEEVENTF_LEFTUP, 0, 0, 0, 0) 95 | time.sleep(0.5) 96 | 97 | logging.info(f"Drew line from ({start_x}, {start_y}) to ({end_x}, {end_y})") 98 | return True 99 | except Exception as e: 100 | logging.error(f"Error drawing line: {e}") 101 | return False 102 | 103 | def main(): 104 | setup_logging() 105 | 106 | # Launch Paint 107 | if not launch_paint(): 108 | return 109 | 110 | # Find Paint window 111 | hwnd = find_paint_window() 112 | if not hwnd: 113 | return 114 | 115 | try: 116 | # Make sure Paint is visible (don't worry about activation) 117 | win32gui.ShowWindow(hwnd, win32con.SW_MAXIMIZE) 118 | time.sleep(1) 119 | 120 | # Get some coordinates to use 121 | rect = win32gui.GetWindowRect(hwnd) 122 | window_x, window_y, window_right, window_bottom = rect 123 | logging.info(f"Paint window rectangle: {rect}") 124 | 125 | # Calculate center points for drawing 126 | center_x = (window_x + window_right) // 2 127 | center_y = (window_y + window_bottom) // 2 128 | 129 | # Adjust for drawing area (skip the ribbon) 130 | drawing_y_offset = 150 # Approximate height of ribbon 131 | 132 | # Draw a horizontal line in the center 133 | start_x = center_x - 100 134 | start_y = center_y + drawing_y_offset 135 | end_x = center_x + 100 136 | end_y = center_y + drawing_y_offset 137 | 138 | logging.info(f"Drawing horizontal line from ({start_x}, {start_y}) to ({end_x}, {end_y})") 139 | simple_draw_line(start_x, start_y, end_x, end_y) 140 | 141 | # Wait a moment 142 | time.sleep(1) 143 | 144 | # Draw a vertical line intersecting the horizontal line 145 | start_x = center_x 146 | start_y = center_y + drawing_y_offset - 100 147 | end_x = center_x 148 | end_y = center_y + drawing_y_offset + 100 149 | 150 | logging.info(f"Drawing vertical line from ({start_x}, {start_y}) to ({end_x}, {end_y})") 151 | simple_draw_line(start_x, start_y, end_x, end_y) 152 | 153 | logging.info("Drawing operations completed") 154 | 155 | except Exception as e: 156 | logging.error(f"Error in main: {e}") 157 | 158 | if __name__ == "__main__": 159 | main() -------------------------------------------------------------------------------- /direct_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import time 3 | import sys 4 | import os 5 | import subprocess 6 | import win32gui 7 | import win32process 8 | import win32con 9 | import win32api 10 | import psutil 11 | 12 | def main(): 13 | # Kill any existing Paint processes 14 | os.system('taskkill /f /im mspaint.exe 2>nul') 15 | time.sleep(1) 16 | 17 | # Launch Paint 18 | subprocess.run(["start", "mspaint.exe"], shell=True) 19 | print("Launched MS Paint") 20 | time.sleep(3) # Wait for Paint to start 21 | 22 | # Find the Paint window 23 | hwnd = find_paint_window() 24 | if not hwnd: 25 | print("Could not find Paint window") 26 | return 27 | 28 | print(f"Found Paint window: {hwnd}") 29 | 30 | # Make sure Paint is visible 31 | win32gui.ShowWindow(hwnd, win32con.SW_MAXIMIZE) 32 | time.sleep(1) 33 | 34 | # Get window dimensions 35 | rect = win32gui.GetWindowRect(hwnd) 36 | window_x, window_y, window_right, window_bottom = rect 37 | print(f"Paint window rectangle: {rect}") 38 | 39 | # Calculate center points for drawing 40 | center_x = (window_x + window_right) // 2 41 | center_y = (window_y + window_bottom) // 2 42 | 43 | # Adjust for drawing area (skip the ribbon) 44 | drawing_y_offset = 150 # Approximate height of ribbon 45 | 46 | # Draw a horizontal line in the center 47 | start_x = center_x - 100 48 | start_y = center_y + drawing_y_offset 49 | end_x = center_x + 100 50 | end_y = center_y + drawing_y_offset 51 | 52 | print(f"Drawing horizontal line from ({start_x}, {start_y}) to ({end_x}, {end_y})") 53 | draw_line(start_x, start_y, end_x, end_y) 54 | 55 | # Wait a moment 56 | time.sleep(1) 57 | 58 | # Draw a vertical line intersecting the horizontal line 59 | start_x = center_x 60 | start_y = center_y + drawing_y_offset - 100 61 | end_x = center_x 62 | end_y = center_y + drawing_y_offset + 100 63 | 64 | print(f"Drawing vertical line from ({start_x}, {start_y}) to ({end_x}, {end_y})") 65 | draw_line(start_x, start_y, end_x, end_y) 66 | 67 | print("Drawing completed") 68 | 69 | # Keep the window open - user can close it manually 70 | input("Press Enter to exit the script...") 71 | 72 | def find_paint_window(): 73 | """Find the Paint window handle""" 74 | def callback(hwnd, results): 75 | if win32gui.IsWindowVisible(hwnd): 76 | title = win32gui.GetWindowText(hwnd) 77 | if "Paint" in title and "mcp-server" not in title: 78 | try: 79 | _, pid = win32process.GetWindowThreadProcessId(hwnd) 80 | proc = psutil.Process(pid) 81 | if "mspaint" in proc.name().lower(): 82 | results.append(hwnd) 83 | except Exception as e: 84 | print(f"Error getting process info: {e}") 85 | return True 86 | 87 | results = [] 88 | win32gui.EnumWindows(callback, results) 89 | 90 | if results: 91 | hwnd = results[0] 92 | return hwnd 93 | else: 94 | return None 95 | 96 | def draw_line(start_x, start_y, end_x, end_y): 97 | """Draw a line using simulated mouse events at absolute screen coordinates""" 98 | # Move mouse to start position 99 | win32api.SetCursorPos((start_x, start_y)) 100 | time.sleep(0.5) 101 | 102 | # Press left button 103 | win32api.mouse_event(win32con.MOUSEEVENTF_LEFTDOWN, 0, 0, 0, 0) 104 | time.sleep(0.5) 105 | 106 | # Move to end position in small steps 107 | steps = 10 108 | dx = (end_x - start_x) / steps 109 | dy = (end_y - start_y) / steps 110 | 111 | for i in range(1, steps + 1): 112 | x = int(start_x + (dx * i)) 113 | y = int(start_y + (dy * i)) 114 | win32api.SetCursorPos((x, y)) 115 | time.sleep(0.05) 116 | 117 | # Ensure we're at the end position 118 | win32api.SetCursorPos((end_x, end_y)) 119 | time.sleep(0.5) 120 | 121 | # Release left button 122 | win32api.mouse_event(win32con.MOUSEEVENTF_LEFTUP, 0, 0, 0, 0) 123 | time.sleep(0.5) 124 | 125 | if __name__ == "__main__": 126 | main() -------------------------------------------------------------------------------- /echo_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import subprocess 3 | import threading 4 | import sys 5 | import time 6 | 7 | def main(): 8 | # Start the server 9 | print("Starting MCP server...") 10 | server_process = subprocess.Popen( 11 | ["cargo", "run"], 12 | stdin=subprocess.PIPE, 13 | stdout=subprocess.PIPE, 14 | stderr=subprocess.PIPE, 15 | text=True, 16 | bufsize=1 17 | ) 18 | 19 | # Thread for reading stdout 20 | def read_stdout(): 21 | while True: 22 | line = server_process.stdout.readline() 23 | if not line: 24 | break 25 | print(f"SERVER STDOUT: {line.strip()}") 26 | 27 | # Thread for reading stderr 28 | def read_stderr(): 29 | while True: 30 | line = server_process.stderr.readline() 31 | if not line: 32 | break 33 | print(f"SERVER STDERR: {line.strip()}") 34 | 35 | # Start stdout and stderr reader threads 36 | stdout_thread = threading.Thread(target=read_stdout, daemon=True) 37 | stderr_thread = threading.Thread(target=read_stderr, daemon=True) 38 | stdout_thread.start() 39 | stderr_thread.start() 40 | 41 | # Wait for server to start 42 | print("Waiting for server to start...") 43 | time.sleep(3) 44 | 45 | print("\n===== MCP SERVER ECHO TEST =====") 46 | print("Type JSON-RPC requests to send to the server.") 47 | print("Each line will be sent as a single request.") 48 | print("Type 'exit' to quit.") 49 | print("==================================\n") 50 | 51 | try: 52 | while True: 53 | try: 54 | user_input = input("> ") 55 | if user_input.lower() == 'exit': 56 | break 57 | 58 | # Add newline to user input and send to server 59 | server_process.stdin.write(user_input + "\n") 60 | server_process.stdin.flush() 61 | print(f"Sent: {user_input}") 62 | 63 | except KeyboardInterrupt: 64 | break 65 | finally: 66 | print("Terminating server...") 67 | server_process.terminate() 68 | print("Server terminated.") 69 | 70 | if __name__ == "__main__": 71 | main() -------------------------------------------------------------------------------- /final_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import json 3 | import subprocess 4 | import time 5 | import sys 6 | 7 | def main(): 8 | # First launch MS Paint 9 | print("Launching MS Paint...") 10 | subprocess.Popen(["mspaint.exe"]) 11 | time.sleep(3) # Give Paint time to start 12 | 13 | # Start the MCP server with our custom implementation 14 | print("Starting MCP server...") 15 | server_process = subprocess.Popen( 16 | ["cargo", "run", "--release"], 17 | stdin=subprocess.PIPE, 18 | stdout=subprocess.PIPE, 19 | stderr=subprocess.PIPE, 20 | text=True, 21 | bufsize=1 22 | ) 23 | 24 | try: 25 | # Wait for server to start 26 | time.sleep(3) 27 | 28 | # First initialize and find Paint 29 | print("Sending initialize request...") 30 | init_request = { 31 | "jsonrpc": "2.0", 32 | "id": 1, 33 | "method": "initialize", 34 | "params": {} 35 | } 36 | 37 | request_json = json.dumps(init_request) + "\n" 38 | print(f"Request: {request_json.strip()}") 39 | server_process.stdin.write(request_json) 40 | server_process.stdin.flush() 41 | 42 | # Read response 43 | response = server_process.stdout.readline().strip() 44 | print(f"Response: {response}") 45 | 46 | if not response: 47 | print("No response received from initialize") 48 | stderr = server_process.stderr.read() 49 | print(f"Server stderr: {stderr}") 50 | return 51 | 52 | # Send connect request 53 | print("Sending connect request...") 54 | connect_request = { 55 | "jsonrpc": "2.0", 56 | "id": 2, 57 | "method": "connect", 58 | "params": { 59 | "client_id": "final-test", 60 | "client_name": "Final JSON-RPC Test" 61 | } 62 | } 63 | 64 | request_json = json.dumps(connect_request) + "\n" 65 | print(f"Request: {request_json.strip()}") 66 | server_process.stdin.write(request_json) 67 | server_process.stdin.flush() 68 | 69 | # Read response 70 | response = server_process.stdout.readline().strip() 71 | print(f"Response: {response}") 72 | 73 | if not response: 74 | print("No response received from connect") 75 | stderr = server_process.stderr.read() 76 | print(f"Server stderr: {stderr}") 77 | return 78 | 79 | # Send draw_line request 80 | print("Sending draw_line request...") 81 | line_request = { 82 | "jsonrpc": "2.0", 83 | "id": 3, 84 | "method": "draw_line", 85 | "params": { 86 | "start_x": 100, 87 | "start_y": 100, 88 | "end_x": 400, 89 | "end_y": 100, 90 | "color": "#FF0000", 91 | "thickness": 3 92 | } 93 | } 94 | 95 | request_json = json.dumps(line_request) + "\n" 96 | print(f"Request: {request_json.strip()}") 97 | server_process.stdin.write(request_json) 98 | server_process.stdin.flush() 99 | 100 | # Read response 101 | response = server_process.stdout.readline().strip() 102 | print(f"Response: {response}") 103 | 104 | if not response: 105 | print("No response received from draw_line") 106 | stderr = server_process.stderr.read() 107 | print(f"Server stderr: {stderr}") 108 | return 109 | 110 | # Wait a moment to see if the drawing succeeded 111 | print("Test completed. Keeping Paint open for 10 seconds to observe results...") 112 | time.sleep(10) 113 | 114 | except Exception as e: 115 | print(f"Error during test: {e}") 116 | import traceback 117 | traceback.print_exc() 118 | finally: 119 | # Clean up 120 | server_process.terminate() 121 | print("Server terminated") 122 | # Keep Paint open to see the results 123 | 124 | if __name__ == "__main__": 125 | main() -------------------------------------------------------------------------------- /latest_server_log.txt: -------------------------------------------------------------------------------- 1 | Blocking waiting for file lock on build directory 2 | -------------------------------------------------------------------------------- /minimal_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import json 3 | import subprocess 4 | import time 5 | import sys 6 | 7 | def main(): 8 | # Start the MCP server 9 | print("Starting MCP server...") 10 | server_process = subprocess.Popen( 11 | ["cargo", "run", "--release"], 12 | stdin=subprocess.PIPE, 13 | stdout=subprocess.PIPE, 14 | stderr=subprocess.PIPE, 15 | text=True, 16 | bufsize=1 17 | ) 18 | 19 | try: 20 | # Wait for server to initialize 21 | time.sleep(3) 22 | 23 | # Send a simple initialize method (this should find or launch Paint) 24 | print("Sending initialize request...") 25 | init_request = { 26 | "jsonrpc": "2.0", 27 | "id": 1, 28 | "method": "initialize", 29 | "params": {} 30 | } 31 | 32 | request_json = json.dumps(init_request) + "\n" 33 | print(f"Request: {request_json.strip()}") 34 | server_process.stdin.write(request_json) 35 | server_process.stdin.flush() 36 | 37 | # Read response 38 | print("Reading response...") 39 | response = server_process.stdout.readline().strip() 40 | print(f"Response: {response}") 41 | 42 | if not response: 43 | print("No response received") 44 | stderr = server_process.stderr.read() 45 | print(f"Server stderr: {stderr}") 46 | return 47 | 48 | # Send connect request 49 | print("Sending connect request...") 50 | connect_request = { 51 | "jsonrpc": "2.0", 52 | "id": 2, 53 | "method": "connect", 54 | "params": { 55 | "client_id": "minimal-test", 56 | "client_name": "Minimal Test Client" 57 | } 58 | } 59 | 60 | request_json = json.dumps(connect_request) + "\n" 61 | print(f"Request: {request_json.strip()}") 62 | server_process.stdin.write(request_json) 63 | server_process.stdin.flush() 64 | 65 | # Read response 66 | print("Reading response...") 67 | response = server_process.stdout.readline().strip() 68 | print(f"Response: {response}") 69 | 70 | if not response: 71 | print("No response received") 72 | stderr = server_process.stderr.read() 73 | print(f"Server stderr: {stderr}") 74 | return 75 | 76 | # Send draw_line request 77 | print("Sending draw_line request...") 78 | line_request = { 79 | "jsonrpc": "2.0", 80 | "id": 3, 81 | "method": "draw_line", 82 | "params": { 83 | "start_x": 100, 84 | "start_y": 100, 85 | "end_x": 300, 86 | "end_y": 100, 87 | "color": "#FF0000", 88 | "thickness": 3 89 | } 90 | } 91 | 92 | request_json = json.dumps(line_request) + "\n" 93 | print(f"Request: {request_json.strip()}") 94 | server_process.stdin.write(request_json) 95 | server_process.stdin.flush() 96 | 97 | # Read response 98 | print("Reading response...") 99 | response = server_process.stdout.readline().strip() 100 | print(f"Response: {response}") 101 | 102 | if not response: 103 | print("No response received") 104 | stderr = server_process.stderr.read() 105 | print(f"Server stderr: {stderr}") 106 | return 107 | 108 | print("Test completed successfully!") 109 | 110 | except Exception as e: 111 | print(f"Error during test: {e}") 112 | finally: 113 | # Give some time to observe the result 114 | time.sleep(5) 115 | server_process.terminate() 116 | print("Server terminated") 117 | 118 | if __name__ == "__main__": 119 | main() -------------------------------------------------------------------------------- /multi_shape_test.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /random_lines_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import json 3 | import subprocess 4 | import time 5 | import sys 6 | import os 7 | import threading 8 | import traceback 9 | import random # Added for random line generation 10 | 11 | def main(): 12 | # First, make sure Paint is not running 13 | os.system('taskkill /f /im mspaint.exe 2>nul') 14 | time.sleep(1) 15 | 16 | # Launch Paint 17 | print("Launching MS Paint...") 18 | paint_process = subprocess.Popen(["mspaint.exe"]) 19 | time.sleep(3) # Wait for Paint to start 20 | 21 | # Start the server 22 | print("Starting MCP server...") 23 | server_process = subprocess.Popen( 24 | ["cargo", "run", "--bin", "mcp-server-microsoft-paint"], 25 | stdin=subprocess.PIPE, 26 | stdout=subprocess.PIPE, 27 | stderr=subprocess.PIPE, 28 | text=True, 29 | bufsize=1 30 | ) 31 | 32 | # Keep track of if the server is still running 33 | server_alive = True 34 | 35 | # Create a thread to continuously read and print stderr 36 | def read_stderr(): 37 | nonlocal server_alive 38 | while server_alive: 39 | try: 40 | line = server_process.stderr.readline() 41 | if not line: 42 | print("SERVER STDERR: End of stderr stream") 43 | break 44 | print(f"SERVER STDERR: {line.strip()}") 45 | except Exception as e: 46 | print(f"Error reading stderr: {e}") 47 | break 48 | 49 | stderr_thread = threading.Thread(target=read_stderr, daemon=True) 50 | stderr_thread.start() 51 | 52 | # Create a thread to continuously check if the server is running 53 | def check_server_alive(): 54 | nonlocal server_alive 55 | while server_alive: 56 | if server_process.poll() is not None: 57 | server_alive = False 58 | print(f"SERVER PROCESS TERMINATED with return code {server_process.returncode}") 59 | break 60 | time.sleep(0.5) 61 | 62 | alive_thread = threading.Thread(target=check_server_alive, daemon=True) 63 | alive_thread.start() 64 | 65 | try: 66 | # Step 1: Initialize 67 | print("Step 1: Sending initialize request...") 68 | init_request = { 69 | "jsonrpc": "2.0", 70 | "id": 1, 71 | "method": "initialize", 72 | "params": {} 73 | } 74 | 75 | response = send_request(server_process, init_request, server_alive) 76 | if not response: 77 | print("ERROR: No response received for initialize request") 78 | if not server_alive: 79 | print("SERVER TERMINATED during or after initialize request") 80 | return 81 | else: 82 | print(f"Initialize response received: {response}") 83 | 84 | # Verify server still running 85 | if not server_alive or server_process.poll() is not None: 86 | print("SERVER TERMINATED after initialize request") 87 | return 88 | 89 | # Step 2: Connect 90 | print("Step 2: Sending connect request...") 91 | connect_request = { 92 | "jsonrpc": "2.0", 93 | "id": 2, 94 | "method": "connect", 95 | "params": { 96 | "client_id": "random_lines_test", 97 | "client_name": "Random Lines Test Client" 98 | } 99 | } 100 | 101 | try: 102 | response = send_request(server_process, connect_request, server_alive) 103 | if not response: 104 | print("ERROR: No response received for connect request") 105 | else: 106 | print(f"Connect response received: {response}") 107 | 108 | # Define center of the screen 109 | center_x, center_y = 300, 250 110 | 111 | # Draw first random line 112 | print("Step 3: Sending draw_shape request (first random line)...") 113 | 114 | # Generate random offsets for line endpoints (-150 to 150 pixels from center) 115 | offset_x1 = random.randint(-150, 150) 116 | offset_y1 = random.randint(-150, 150) 117 | 118 | line1_request = { 119 | "jsonrpc": "2.0", 120 | "id": 3, 121 | "method": "draw_shape", 122 | "params": { 123 | "shape_type": "line", 124 | "start_x": center_x, 125 | "start_y": center_y, 126 | "end_x": center_x + offset_x1, 127 | "end_y": center_y + offset_y1, 128 | "thickness": 5 129 | } 130 | } 131 | 132 | response = send_request(server_process, line1_request, server_alive) 133 | if not response: 134 | print("ERROR: No response received for first line draw_shape request") 135 | 136 | # Wait a moment between drawing lines 137 | time.sleep(1) 138 | 139 | # Draw second random line 140 | print("Step 4: Sending draw_shape request (second random line)...") 141 | 142 | # Generate random offsets for second line 143 | offset_x2 = random.randint(-150, 150) 144 | offset_y2 = random.randint(-150, 150) 145 | 146 | line2_request = { 147 | "jsonrpc": "2.0", 148 | "id": 4, 149 | "method": "draw_shape", 150 | "params": { 151 | "shape_type": "line", 152 | "start_x": center_x, 153 | "start_y": center_y, 154 | "end_x": center_x + offset_x2, 155 | "end_y": center_y + offset_y2, 156 | "thickness": 5 157 | } 158 | } 159 | 160 | response = send_request(server_process, line2_request, server_alive) 161 | if not response: 162 | print("ERROR: No response received for second line draw_shape request") 163 | except BrokenPipeError: 164 | print("ERROR: Server pipe closed before or during connect request") 165 | except OSError as e: 166 | print(f"ERROR: OSError during connect request: {e}") 167 | traceback.print_exc() 168 | 169 | print("Test completed! Check Paint to see if random lines were drawn.") 170 | print("Press Enter to close the test and kill Paint...") 171 | input() 172 | 173 | except Exception as e: 174 | print(f"Test failed with error: {type(e).__name__}: {e}") 175 | traceback.print_exc() 176 | finally: 177 | server_alive = False 178 | if server_process.poll() is None: 179 | print("Terminating server process...") 180 | server_process.terminate() 181 | print("Server process terminated") 182 | 183 | if paint_process.poll() is None: 184 | print("Terminating Paint process...") 185 | paint_process.terminate() 186 | print("Paint process terminated") 187 | 188 | def send_request(process, request, server_alive): 189 | """Send a request to the server and print the response.""" 190 | if not server_alive or process.poll() is not None: 191 | print("Cannot send request - server process is not running") 192 | return None 193 | 194 | request_str = json.dumps(request) + "\n" 195 | print(f"SENDING: {request_str.strip()}") 196 | 197 | try: 198 | process.stdin.write(request_str) 199 | process.stdin.flush() 200 | print("Request sent and flushed") 201 | except BrokenPipeError: 202 | print("ERROR: Broken pipe when trying to send request") 203 | raise 204 | except OSError as e: 205 | print(f"ERROR: OSError when trying to send request: {e}") 206 | raise 207 | 208 | # Read response with timeout 209 | start_time = time.time() 210 | timeout = 10 # seconds 211 | 212 | while time.time() - start_time < timeout and server_alive and process.poll() is None: 213 | try: 214 | print("Waiting for response...") 215 | line = process.stdout.readline().strip() 216 | if line: 217 | print(f"RESPONSE: {line}") 218 | try: 219 | return json.loads(line) 220 | except json.JSONDecodeError: 221 | print(f"WARNING: Received non-JSON response: {line}") 222 | except Exception as e: 223 | print(f"Error reading response: {e}") 224 | return None 225 | time.sleep(0.5) 226 | 227 | if not server_alive or process.poll() is not None: 228 | print("WARNING: Server terminated while waiting for response") 229 | else: 230 | print("WARNING: No response received within timeout") 231 | return None 232 | 233 | if __name__ == "__main__": 234 | main() -------------------------------------------------------------------------------- /rectangle_debug_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import json 3 | import subprocess 4 | import time 5 | import sys 6 | import os 7 | import threading 8 | 9 | def main(): 10 | # First, make sure Paint is not running 11 | os.system('taskkill /f /im mspaint.exe 2>nul') 12 | time.sleep(1) 13 | 14 | # Launch Paint 15 | print("Launching MS Paint...") 16 | paint_process = subprocess.Popen(["mspaint.exe"]) 17 | time.sleep(3) # Wait for Paint to start 18 | 19 | # Start the server with higher log level 20 | print("Starting MCP server...") 21 | server_process = subprocess.Popen( 22 | ["cargo", "run"], 23 | stdin=subprocess.PIPE, 24 | stdout=subprocess.PIPE, 25 | stderr=subprocess.PIPE, 26 | text=True, 27 | bufsize=1 28 | ) 29 | 30 | # Create threads to continuously read and print output 31 | def read_stdout(): 32 | while True: 33 | line = server_process.stdout.readline() 34 | if not line: 35 | break 36 | print(f"SERVER STDOUT: {line.strip()}") 37 | 38 | def read_stderr(): 39 | while True: 40 | line = server_process.stderr.readline() 41 | if not line: 42 | break 43 | print(f"SERVER STDERR: {line.strip()}") 44 | 45 | stdout_thread = threading.Thread(target=read_stdout, daemon=True) 46 | stderr_thread = threading.Thread(target=read_stderr, daemon=True) 47 | 48 | stdout_thread.start() 49 | stderr_thread.start() 50 | 51 | try: 52 | # Step 1: Initialize 53 | print("\n=== STEP 1: Sending initialize request ===") 54 | init_request = { 55 | "jsonrpc": "2.0", 56 | "id": 1, 57 | "method": "initialize", 58 | "params": {} 59 | } 60 | 61 | response = send_request(server_process, init_request) 62 | print(f"Initialize response: {json.dumps(response, indent=2)}") 63 | 64 | # Step 2: Connect 65 | print("\n=== STEP 2: Sending connect request ===") 66 | connect_request = { 67 | "jsonrpc": "2.0", 68 | "id": 2, 69 | "method": "connect", 70 | "params": { 71 | "client_id": "rectangle_test", 72 | "client_name": "Rectangle Test Client" 73 | } 74 | } 75 | 76 | response = send_request(server_process, connect_request) 77 | print(f"Connect response: {json.dumps(response, indent=2)}") 78 | 79 | # Make sure the window is properly maximized 80 | time.sleep(2) 81 | 82 | # Step 3: Draw a rectangle 83 | print("\n=== STEP 3: Drawing a rectangle ===") 84 | rectangle_request = { 85 | "jsonrpc": "2.0", 86 | "id": 3, 87 | "method": "draw_shape", 88 | "params": { 89 | "shape_type": "rectangle", 90 | "start_x": 100, 91 | "start_y": 100, 92 | "end_x": 400, 93 | "end_y": 300 94 | } 95 | } 96 | 97 | response = send_request(server_process, rectangle_request) 98 | print(f"Rectangle response: {json.dumps(response, indent=2)}") 99 | 100 | print("\nTest completed! Check Paint to see if the rectangle was drawn correctly.") 101 | print("Press Enter to close the test and kill Paint...") 102 | input() 103 | 104 | except Exception as e: 105 | print(f"Test failed with error: {e}") 106 | finally: 107 | # Terminate the server process 108 | server_process.terminate() 109 | print("Server process terminated") 110 | 111 | # Allow time for process to terminate 112 | time.sleep(1) 113 | 114 | # Terminate Paint 115 | paint_process.terminate() 116 | print("Paint process terminated") 117 | 118 | def send_request(process, request): 119 | """Send a request to the server and print the response.""" 120 | request_str = json.dumps(request) + "\n" 121 | print(f"SENDING: {request_str.strip()}") 122 | 123 | process.stdin.write(request_str) 124 | process.stdin.flush() 125 | print("Request sent and flushed") 126 | 127 | # Read response with timeout 128 | start_time = time.time() 129 | timeout = 30 # seconds - increased for more debugging time 130 | 131 | while time.time() - start_time < timeout: 132 | print("Waiting for response...") 133 | line = process.stdout.readline().strip() 134 | if line: 135 | print(f"RESPONSE: {line}") 136 | try: 137 | return json.loads(line) 138 | except json.JSONDecodeError: 139 | print(f"WARNING: Received non-JSON response: {line}") 140 | time.sleep(0.5) 141 | 142 | print("WARNING: No response received within timeout") 143 | return None 144 | 145 | if __name__ == "__main__": 146 | main() -------------------------------------------------------------------------------- /rectangle_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import json 3 | import subprocess 4 | import time 5 | import sys 6 | import os 7 | import threading 8 | import traceback 9 | 10 | def main(): 11 | # First, make sure Paint is not running 12 | os.system('taskkill /f /im mspaint.exe 2>nul') 13 | time.sleep(1) 14 | 15 | # Launch Paint 16 | print("Launching MS Paint...") 17 | paint_process = subprocess.Popen(["mspaint.exe"]) 18 | time.sleep(3) # Wait for Paint to start 19 | 20 | # Start the server 21 | print("Starting MCP server...") 22 | server_process = subprocess.Popen( 23 | ["cargo", "run", "--bin", "mcp-server-microsoft-paint"], 24 | stdin=subprocess.PIPE, 25 | stdout=subprocess.PIPE, 26 | stderr=subprocess.PIPE, 27 | text=True, 28 | bufsize=1 29 | ) 30 | 31 | # Keep track of if the server is still running 32 | server_alive = True 33 | 34 | # Create a thread to continuously read and print stderr 35 | def read_stderr(): 36 | nonlocal server_alive 37 | while server_alive: 38 | try: 39 | line = server_process.stderr.readline() 40 | if not line: 41 | print("SERVER STDERR: End of stderr stream") 42 | break 43 | print(f"SERVER STDERR: {line.strip()}") 44 | except Exception as e: 45 | print(f"Error reading stderr: {e}") 46 | break 47 | 48 | stderr_thread = threading.Thread(target=read_stderr, daemon=True) 49 | stderr_thread.start() 50 | 51 | # Create a thread to continuously check if the server is running 52 | def check_server_alive(): 53 | nonlocal server_alive 54 | while server_alive: 55 | if server_process.poll() is not None: 56 | server_alive = False 57 | print(f"SERVER PROCESS TERMINATED with return code {server_process.returncode}") 58 | break 59 | time.sleep(0.5) 60 | 61 | alive_thread = threading.Thread(target=check_server_alive, daemon=True) 62 | alive_thread.start() 63 | 64 | try: 65 | # Step 1: Initialize 66 | print("Step 1: Sending initialize request...") 67 | init_request = { 68 | "jsonrpc": "2.0", 69 | "id": 1, 70 | "method": "initialize", 71 | "params": {} 72 | } 73 | 74 | response = send_request(server_process, init_request, server_alive) 75 | if not response: 76 | print("ERROR: No response received for initialize request") 77 | if not server_alive: 78 | print("SERVER TERMINATED during or after initialize request") 79 | return 80 | else: 81 | print(f"Initialize response received: {response}") 82 | 83 | # Verify server still running 84 | if not server_alive or server_process.poll() is not None: 85 | print("SERVER TERMINATED after initialize request") 86 | return 87 | 88 | # Step 2: Connect 89 | print("Step 2: Sending connect request...") 90 | connect_request = { 91 | "jsonrpc": "2.0", 92 | "id": 2, 93 | "method": "connect", 94 | "params": { 95 | "client_id": "rectangle_test", 96 | "client_name": "Rectangle Test Client" 97 | } 98 | } 99 | 100 | try: 101 | response = send_request(server_process, connect_request, server_alive) 102 | if not response: 103 | print("ERROR: No response received for connect request") 104 | else: 105 | print(f"Connect response received: {response}") 106 | 107 | # Step 3: Draw a rectangle directly 108 | print("Step 3: Sending draw_shape request (rectangle)...") 109 | rectangle_request = { 110 | "jsonrpc": "2.0", 111 | "id": 4, 112 | "method": "draw_shape", 113 | "params": { 114 | "shape_type": "rectangle", 115 | "start_x": 100, 116 | "start_y": 100, 117 | "end_x": 500, 118 | "end_y": 400, 119 | "thickness": 3, 120 | "fill_type": "outline" # Just the outline, not filled 121 | } 122 | } 123 | 124 | response = send_request(server_process, rectangle_request, server_alive) 125 | if not response: 126 | print("ERROR: No response received for draw_shape request") 127 | except BrokenPipeError: 128 | print("ERROR: Server pipe closed before or during connect request") 129 | except OSError as e: 130 | print(f"ERROR: OSError during connect request: {e}") 131 | traceback.print_exc() 132 | 133 | print("Test completed! Check Paint to see if a rectangle was drawn.") 134 | print("Press Enter to close the test and kill Paint...") 135 | input() 136 | 137 | except Exception as e: 138 | print(f"Test failed with error: {type(e).__name__}: {e}") 139 | traceback.print_exc() 140 | finally: 141 | server_alive = False 142 | if server_process.poll() is None: 143 | print("Terminating server process...") 144 | server_process.terminate() 145 | print("Server process terminated") 146 | 147 | if paint_process.poll() is None: 148 | print("Terminating Paint process...") 149 | paint_process.terminate() 150 | print("Paint process terminated") 151 | 152 | def send_request(process, request, server_alive): 153 | """Send a request to the server and print the response.""" 154 | if not server_alive or process.poll() is not None: 155 | print("Cannot send request - server process is not running") 156 | return None 157 | 158 | request_str = json.dumps(request) + "\n" 159 | print(f"SENDING: {request_str.strip()}") 160 | 161 | try: 162 | process.stdin.write(request_str) 163 | process.stdin.flush() 164 | print("Request sent and flushed") 165 | except BrokenPipeError: 166 | print("ERROR: Broken pipe when trying to send request") 167 | raise 168 | except OSError as e: 169 | print(f"ERROR: OSError when trying to send request: {e}") 170 | raise 171 | 172 | # Read response with timeout 173 | start_time = time.time() 174 | timeout = 10 # seconds 175 | 176 | while time.time() - start_time < timeout and server_alive and process.poll() is None: 177 | try: 178 | print("Waiting for response...") 179 | line = process.stdout.readline().strip() 180 | if line: 181 | print(f"RESPONSE: {line}") 182 | try: 183 | return json.loads(line) 184 | except json.JSONDecodeError: 185 | print(f"WARNING: Received non-JSON response: {line}") 186 | except Exception as e: 187 | print(f"Error reading response: {e}") 188 | return None 189 | time.sleep(0.5) 190 | 191 | if not server_alive or process.poll() is not None: 192 | print("WARNING: Server terminated while waiting for response") 193 | else: 194 | print("WARNING: No response received within timeout") 195 | return None 196 | 197 | if __name__ == "__main__": 198 | main() -------------------------------------------------------------------------------- /samples/uia_test.rs: -------------------------------------------------------------------------------- 1 | // Sample file to test uiautomation crate 2 | use uiautomation::UIAutomation; 3 | use std::thread::sleep; 4 | use std::time::Duration; 5 | 6 | fn main() -> Result<(), Box> { 7 | println!("Testing uiautomation with MS Paint"); 8 | 9 | // Launch MS Paint 10 | let proc = std::process::Command::new("mspaint.exe").spawn()?; 11 | 12 | // Wait for Paint to start 13 | sleep(Duration::from_secs(2)); 14 | 15 | // Initialize UI Automation 16 | let automation = UIAutomation::new()?; 17 | let root = automation.get_root_element()?; 18 | 19 | // Find Paint window 20 | let matcher = automation.create_matcher() 21 | .from(root) 22 | .timeout(10000) 23 | .classname("MSPaintApp"); 24 | 25 | match matcher.find_first() { 26 | Ok(paint_window) => { 27 | println!("Found Paint window: {} - {}", 28 | paint_window.get_name()?, 29 | paint_window.get_classname()?); 30 | 31 | // Try to find toolbar elements 32 | if let Ok(buttons) = paint_window.find_all_by_control_type(&automation, uiautomation::ElementType::Button) { 33 | println!("Found {} buttons in Paint", buttons.len()); 34 | 35 | // List button names 36 | for button in buttons { 37 | if let Ok(name) = button.get_name() { 38 | println!("Button: {}", name); 39 | } 40 | } 41 | } 42 | 43 | // Close Paint 44 | if let Ok(control) = paint_window.get_pattern::(&automation) { 45 | control.close()?; 46 | } 47 | }, 48 | Err(err) => { 49 | println!("Could not find Paint window: {}", err); 50 | } 51 | } 52 | 53 | Ok(()) 54 | } -------------------------------------------------------------------------------- /server_log.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ghuntley/mcp-server-microsoft-paint/bf6bf12da2f6c4200af1b8677e725ca07c170f55/server_log.txt -------------------------------------------------------------------------------- /server_stderr.log: -------------------------------------------------------------------------------- 1 | warning: output filename collision. 2 | The bin target `mcp-server-microsoft-paint` in package `mcp-server-microsoft-paint v0.1.0 (C:\Users\ghuntley\Desktop\mcp-server-microsoft-paint)` has the same output filename as the lib target `mcp_server_microsoft_paint` in package `mcp-server-microsoft-paint v0.1.0 (C:\Users\ghuntley\Desktop\mcp-server-microsoft-paint)`. 3 | Colliding filename is: C:\Users\ghuntley\Desktop\mcp-server-microsoft-paint\target\release\deps\mcp_server_microsoft_paint.pdb 4 | The output filenames should be unique. 5 | Consider changing their names to be unique or compiling them separately. 6 | This may become a hard error in the future; see . 7 | If this looks unexpected, it may be a bug in Cargo. Please file a bug report at 8 | https://github.com/rust-lang/cargo/issues/ with as much information as you 9 | can provide. 10 | cargo 1.85.1 (d73d2caf9 2024-12-31) running on `x86_64-pc-windows-msvc` target `x86_64-pc-windows-msvc` 11 | First unit: Unit { pkg: Package { id: PackageId { name: "mcp-server-microsoft-paint", version: "0.1.0", source: "C:\\Users\\ghuntley\\Desktop\\mcp-server-microsoft-paint" }, ..: ".." }, target: TargetInner { name: "mcp-server-microsoft-paint", doc: true, ..: with_path("C:\\Users\\ghuntley\\Desktop\\mcp-server-microsoft-paint\\src\\main.rs", Edition2021) }, profile: Profile { strip: Resolved(Named("debuginfo")), ..: default_release() }, kind: Host, mode: Build, features: [], rustflags: [], rustdocflags: [], links_overrides: {}, artifact: false, artifact_target_for_features: None, is_std: false, dep_hash: 392317476812791307 } 12 | Second unit: Unit { pkg: Package { id: PackageId { name: "mcp-server-microsoft-paint", version: "0.1.0", source: "C:\\Users\\ghuntley\\Desktop\\mcp-server-microsoft-paint" }, ..: ".." }, target: TargetInner { name_inferred: true, ..: lib_target("mcp_server_microsoft_paint", ["cdylib", "rlib"], "C:\\Users\\ghuntley\\Desktop\\mcp-server-microsoft-paint\\src\\lib.rs", Edition2021) }, profile: Profile { strip: Resolved(Named("debuginfo")), ..: default_release() }, kind: Host, mode: Build, features: [], rustflags: [], rustdocflags: [], links_overrides: {}, artifact: false, artifact_target_for_features: None, is_std: false, dep_hash: 2017330607264117138 } 13 | warning: output filename collision. 14 | The bin target `mcp-server-microsoft-paint` in package `mcp-server-microsoft-paint v0.1.0 (C:\Users\ghuntley\Desktop\mcp-server-microsoft-paint)` has the same output filename as the lib target `mcp_server_microsoft_paint` in package `mcp-server-microsoft-paint v0.1.0 (C:\Users\ghuntley\Desktop\mcp-server-microsoft-paint)`. 15 | Colliding filename is: C:\Users\ghuntley\Desktop\mcp-server-microsoft-paint\target\release\mcp_server_microsoft_paint.pdb 16 | The output filenames should be unique. 17 | Consider changing their names to be unique or compiling them separately. 18 | This may become a hard error in the future; see . 19 | If this looks unexpected, it may be a bug in Cargo. Please file a bug report at 20 | https://github.com/rust-lang/cargo/issues/ with as much information as you 21 | can provide. 22 | cargo 1.85.1 (d73d2caf9 2024-12-31) running on `x86_64-pc-windows-msvc` target `x86_64-pc-windows-msvc` 23 | First unit: Unit { pkg: Package { id: PackageId { name: "mcp-server-microsoft-paint", version: "0.1.0", source: "C:\\Users\\ghuntley\\Desktop\\mcp-server-microsoft-paint" }, ..: ".." }, target: TargetInner { name: "mcp-server-microsoft-paint", doc: true, ..: with_path("C:\\Users\\ghuntley\\Desktop\\mcp-server-microsoft-paint\\src\\main.rs", Edition2021) }, profile: Profile { strip: Resolved(Named("debuginfo")), ..: default_release() }, kind: Host, mode: Build, features: [], rustflags: [], rustdocflags: [], links_overrides: {}, artifact: false, artifact_target_for_features: None, is_std: false, dep_hash: 392317476812791307 } 24 | Second unit: Unit { pkg: Package { id: PackageId { name: "mcp-server-microsoft-paint", version: "0.1.0", source: "C:\\Users\\ghuntley\\Desktop\\mcp-server-microsoft-paint" }, ..: ".." }, target: TargetInner { name_inferred: true, ..: lib_target("mcp_server_microsoft_paint", ["cdylib", "rlib"], "C:\\Users\\ghuntley\\Desktop\\mcp-server-microsoft-paint\\src\\lib.rs", Edition2021) }, profile: Profile { strip: Resolved(Named("debuginfo")), ..: default_release() }, kind: Host, mode: Build, features: [], rustflags: [], rustdocflags: [], links_overrides: {}, artifact: false, artifact_target_for_features: None, is_std: false, dep_hash: 2017330607264117138 } 25 | Compiling mcp-server-microsoft-paint v0.1.0 (C:\Users\ghuntley\Desktop\mcp-server-microsoft-paint) 26 | warning: unused import: `LevelFilter` 27 | --> src\lib.rs:7:24 28 | | 29 | 7 | use log::{info, error, LevelFilter, debug, warn}; 30 | | ^^^^^^^^^^^ 31 | | 32 | = note: `#[warn(unused_imports)]` on by default 33 | 34 | warning: unused imports: `Write` and `self` 35 | --> src\lib.rs:13:15 36 | | 37 | 13 | use std::io::{self, Write}; 38 | | ^^^^ ^^^^^ 39 | 40 | warning: unused import: `std::ptr` 41 | --> src\windows.rs:6:5 42 | | 43 | 6 | use std::ptr; 44 | | ^^^^^^^^ 45 | 46 | warning: unused imports: `CreateProcessW`, `PROCESS_INFORMATION`, and `STARTUPINFOW` 47 | --> src\windows.rs:8:45 48 | | 49 | 8 | use windows_sys::Win32::System::Threading::{CreateProcessW, PROCESS_INFORMATION, STARTUPINFOW}; 50 | | ^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^^^^^ ^^^^^^^^^^^^ 51 | 52 | warning: unused imports: `VK_BACK`, `VK_DOWN`, `VK_LEFT`, `VK_MENU`, `VK_RIGHT`, and `VK_UP` 53 | --> src\windows.rs:20:80 54 | | 55 | 20 | INPUT_KEYBOARD, KEYEVENTF_KEYUP, KEYEVENTF_SCANCODE, VK_CONTROL, VK_SHIFT, VK_MENU, 56 | | ^^^^^^^ 57 | 21 | VK_RETURN, VK_TAB, VK_ESCAPE, VK_DELETE, VK_BACK, VK_SPACE, VK_LEFT, VK_RIGHT, VK_UP, VK_DOWN, 58 | | ^^^^^^^ ^^^^^^^ ^^^^^^^^ ^^^^^ ^^^^^^^ 59 | 60 | warning: unused import: `windows_sys::Win32::UI::Input::KeyboardAndMouse::MOUSEINPUT` 61 | --> src\windows.rs:25:5 62 | | 63 | 25 | use windows_sys::Win32::UI::Input::KeyboardAndMouse::MOUSEINPUT; 64 | | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 65 | 66 | warning: unused import: `ConnectResponse` 67 | --> src\core.rs:4:38 68 | | 69 | 4 | use crate::protocol::{ConnectParams, ConnectResponse, success_response, DrawPixelParams, DrawLineParams, DrawShapeParams, DrawPolylinePar... 70 | | ^^^^^^^^^^^^^^^ 71 | 72 | warning: unused import: `get_paint_hwnd` 73 | --> src\core.rs:6:22 74 | | 75 | 6 | use crate::windows::{get_paint_hwnd, get_initial_canvas_dimensions, activate_paint_window, get_canvas_dimensions, draw_pixel_at, draw_lin... 76 | | ^^^^^^^^^^^^^^ 77 | 78 | warning: unused imports: `debug` and `warn` 79 | --> src\core.rs:8:17 80 | | 81 | 8 | use log::{info, warn, error, debug}; 82 | | ^^^^ ^^^^^ 83 | 84 | warning: unused import: `std::time` 85 | --> src\core.rs:10:5 86 | | 87 | 10 | use std::time; 88 | | ^^^^^^^^^ 89 | 90 | warning: unused import: `tokio` 91 | --> src\core.rs:11:5 92 | | 93 | 11 | use tokio; 94 | | ^^^^^ 95 | 96 | warning: unnecessary `unsafe` block 97 | --> src\windows.rs:313:5 98 | | 99 | 313 | unsafe { 100 | | ^^^^^^ unnecessary `unsafe` block 101 | | 102 | = note: `#[warn(unused_unsafe)]` on by default 103 | 104 | warning: unused variable: `potential_hwnd` 105 | --> src\windows.rs:583:17 106 | | 107 | 583 | let mut potential_hwnd = 0; 108 | | ^^^^^^^^^^^^^^ help: if this is intentional, prefix it with an underscore: `_potential_hwnd` 109 | | 110 | = note: `#[warn(unused_variables)]` on by default 111 | 112 | warning: variable does not need to be mutable 113 | --> src\windows.rs:583:13 114 | | 115 | 583 | let mut potential_hwnd = 0; 116 | | ----^^^^^^^^^^^^^^ 117 | | | 118 | | help: remove this `mut` 119 | | 120 | = note: `#[warn(unused_mut)]` on by default 121 | 122 | warning: unused variable: `font_name` 123 | --> src\windows.rs:1764:5 124 | | 125 | 1764 | font_name: Option<&str>, 126 | | ^^^^^^^^^ help: if this is intentional, prefix it with an underscore: `_font_name` 127 | 128 | warning: unused variable: `font_size` 129 | --> src\windows.rs:1765:5 130 | | 131 | 1765 | font_size: Option, 132 | | ^^^^^^^^^ help: if this is intentional, prefix it with an underscore: `_font_size` 133 | 134 | warning: unused variable: `font_style` 135 | --> src\windows.rs:1766:5 136 | | 137 | 1766 | font_style: Option<&str> 138 | | ^^^^^^^^^^ help: if this is intentional, prefix it with an underscore: `_font_style` 139 | 140 | warning: constant `PAINT_CLASS_NAME` is never used 141 | --> src\windows.rs:31:7 142 | | 143 | 31 | const PAINT_CLASS_NAME: &str = "MSPaintApp"; 144 | | ^^^^^^^^^^^^^^^^ 145 | | 146 | = note: `#[warn(dead_code)]` on by default 147 | 148 | warning: constant `PAINT_WINDOW_TITLE_SUBSTRING` is never used 149 | --> src\windows.rs:32:7 150 | | 151 | 32 | const PAINT_WINDOW_TITLE_SUBSTRING: &str = "Paint"; 152 | | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ 153 | 154 | warning: `mcp-server-microsoft-paint` (lib) generated 19 warnings (run `cargo fix --lib -p mcp-server-microsoft-paint` to apply 12 suggestions) 155 | warning: unused import: `std::path::PathBuf` 156 | --> src\main.rs:7:5 157 | | 158 | 7 | use std::path::PathBuf; 159 | | ^^^^^^^^^^^^^^^^^^ 160 | | 161 | = note: `#[warn(unused_imports)]` on by default 162 | 163 | warning: `mcp-server-microsoft-paint` (bin "mcp-server-microsoft-paint") generated 1 warning (run `cargo fix --bin "mcp-server-microsoft-paint"` to apply 1 suggestion) 164 | Finished `release` profile [optimized] target(s) in 3.22s 165 | Running `target\release\mcp-server-microsoft-paint.exe` 166 | 2025-04-03T18:27:49.1365433Z [INFO] Logging initialized. Debug logs writing to: "C:\\Users\\ghuntley\\AppData\\Local\\Temp\\mcp_server_debug.log" 167 | 2025-04-03T18:27:49.1367866Z [INFO] Starting MCP Server for Windows 11 Paint... 168 | 2025-04-03T18:27:49.1368355Z [INFO] Starting MCP Server for Windows 11 Paint (Async Version)... 169 | 2025-04-03T18:27:49.1378011Z [INFO] MCP Server starting run loop... 170 | 2025-04-03T18:27:49.1382312Z [ERROR] MCP Server run failed: Serialization error: missing field `type` at line 1 column 127 171 | error: process didn't exit successfully: `target\release\mcp-server-microsoft-paint.exe` (exit code: 0xc000013a, STATUS_CONTROL_C_EXIT) 172 | -------------------------------------------------------------------------------- /simple_test.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | import time 3 | import json 4 | import sys 5 | import logging 6 | 7 | def setup_logging(): 8 | log_formatter = logging.Formatter("%(asctime)s [%(levelname)-5.5s] %(message)s") 9 | root_logger = logging.getLogger() 10 | root_logger.setLevel(logging.DEBUG) 11 | 12 | # Console Handler 13 | console_handler = logging.StreamHandler(sys.stdout) 14 | console_handler.setFormatter(log_formatter) 15 | root_logger.addHandler(console_handler) 16 | logging.info("Logging initialized for simple_test.py") 17 | 18 | def run_minimal_test(): 19 | setup_logging() 20 | logging.info("Starting minimal Paint MCP server test...") 21 | 22 | # Launch the server with binary mode for pipes 23 | server_process = None 24 | try: 25 | logging.debug("Launching MCP server process...") 26 | server_process = subprocess.Popen( 27 | ["target/release/mcp-server-microsoft-paint.exe"], 28 | stdin=subprocess.PIPE, 29 | stdout=subprocess.PIPE, 30 | stderr=subprocess.PIPE, 31 | text=False # Use binary mode 32 | ) 33 | logging.debug(f"Server process launched with PID: {server_process.pid}") 34 | except Exception as e: 35 | logging.error(f"Failed to launch server process: {e}") 36 | return 37 | 38 | # Wait for server to initialize 39 | logging.debug("Waiting for server initialization (2 seconds)...") 40 | time.sleep(2) 41 | 42 | # Define a simple test with just connect and get_version 43 | tests = [ 44 | { 45 | "name": "Get Version", 46 | "request": { 47 | "jsonrpc": "2.0", 48 | "id": 1, 49 | "method": "get_version", 50 | "params": {} 51 | } 52 | }, 53 | { 54 | "name": "Connect", 55 | "request": { 56 | "jsonrpc": "2.0", 57 | "id": 2, 58 | "method": "connect", 59 | "params": { 60 | "client_id": "test-client", 61 | "client_name": "Simple Test" 62 | } 63 | } 64 | }, 65 | { 66 | "name": "Activate Window", 67 | "request": { 68 | "jsonrpc": "2.0", 69 | "id": 3, 70 | "method": "activate_window", 71 | "params": {} 72 | } 73 | } 74 | ] 75 | 76 | try: 77 | # Run each test 78 | for test in tests: 79 | logging.info(f"\nRunning test: {test['name']}") 80 | request = test["request"] 81 | request_json = json.dumps(request) 82 | logging.debug(f"Sending: {request_json}") 83 | 84 | try: 85 | # Send request to server as bytes 86 | request_bytes = (request_json + "\n").encode('utf-8') 87 | server_process.stdin.write(request_bytes) 88 | server_process.stdin.flush() 89 | logging.debug(f"Request sent ({len(request_bytes)} bytes)") 90 | 91 | # Read response (with timeout) 92 | start_time = time.time() 93 | timeout = 10 # seconds 94 | response = None 95 | logging.debug(f"Waiting for response (timeout: {timeout}s)...") 96 | 97 | while time.time() - start_time < timeout: 98 | # Check if server has exited 99 | if server_process.poll() is not None: 100 | logging.warning(f"Server exited prematurely with code: {server_process.poll()}") 101 | break 102 | 103 | # Read line as bytes and decode 104 | line_bytes = server_process.stdout.readline() 105 | if not line_bytes: 106 | time.sleep(0.1) # Avoid busy-waiting 107 | continue 108 | 109 | line = line_bytes.decode('utf-8').strip() 110 | logging.debug(f"Received raw line: {line}") 111 | if line: 112 | try: 113 | response = json.loads(line) 114 | logging.debug("Parsed JSON response successfully") 115 | break 116 | except json.JSONDecodeError as je: 117 | logging.error(f"Invalid JSON response: {line}, Error: {je}") 118 | # Potentially continue reading if it's just debug output? 119 | # For now, we assume one line per response. 120 | # If the server mixes debug and JSON, this needs adjustment. 121 | 122 | if response: 123 | logging.info(f"Response: {json.dumps(response, indent=2)}") 124 | else: 125 | logging.warning("No valid response received (timeout or server exit)") 126 | # Attempt to read stderr after timeout/exit 127 | break 128 | 129 | except Exception as e: 130 | logging.error(f"Error during test '{test['name']}': {e}") 131 | import traceback 132 | logging.error(traceback.format_exc()) 133 | break 134 | 135 | # Wait for a moment to see if Paint stays open 136 | logging.info("\nTests completed. Waiting 5 seconds to see if Paint stays open...") 137 | time.sleep(5) 138 | 139 | finally: 140 | # Clean up the server process 141 | if server_process and server_process.poll() is None: 142 | logging.info("\nTerminating server process...") 143 | try: 144 | server_process.terminate() 145 | outs, errs = server_process.communicate(timeout=3) 146 | logging.debug(f"Server terminated with code: {server_process.returncode}") 147 | if outs: 148 | logging.debug(f"Final server stdout:\n{outs.decode('utf-8', errors='replace')}") 149 | if errs: 150 | logging.warning(f"Final server stderr:\n{errs.decode('utf-8', errors='replace')}") 151 | except subprocess.TimeoutExpired: 152 | logging.warning("Server did not terminate gracefully, killing...") 153 | server_process.kill() 154 | outs, errs = server_process.communicate() 155 | logging.debug("Server killed.") 156 | if outs: 157 | logging.debug(f"Final server stdout:\n{outs.decode('utf-8', errors='replace')}") 158 | if errs: 159 | logging.warning(f"Final server stderr:\n{errs.decode('utf-8', errors='replace')}") 160 | except Exception as e: 161 | logging.error(f"Error during termination: {e}") 162 | elif server_process: 163 | logging.info(f"Server process already exited with code: {server_process.poll()}") 164 | # Read remaining output/error 165 | try: 166 | outs, errs = server_process.communicate() 167 | if outs: 168 | logging.debug(f"Remaining server stdout:\n{outs.decode('utf-8', errors='replace')}") 169 | if errs: 170 | logging.warning(f"Remaining server stderr:\n{errs.decode('utf-8', errors='replace')}") 171 | except Exception as e: 172 | logging.error(f"Error reading remaining server output: {e}") 173 | else: 174 | logging.info("Server process was not running or failed to start.") 175 | 176 | logging.info("Minimal test finished.") 177 | 178 | if __name__ == "__main__": 179 | run_minimal_test() -------------------------------------------------------------------------------- /simple_test_client.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import json 3 | import subprocess 4 | import sys 5 | import time 6 | import logging 7 | import os 8 | 9 | def setup_logging(): 10 | logging.basicConfig( 11 | level=logging.DEBUG, 12 | format="%(asctime)s [%(levelname)-5.5s] %(message)s", 13 | handlers=[logging.StreamHandler(sys.stdout)] 14 | ) 15 | logging.info("Logging initialized for simple_test_client.py") 16 | 17 | def send_request(server_process, request): 18 | """Send a JSON-RPC request to the server and return the response.""" 19 | request_json = json.dumps(request) + "\n" 20 | try: 21 | server_process.stdin.write(request_json) 22 | server_process.stdin.flush() 23 | logging.info(f"Sent request: {request['method']}") 24 | except Exception as e: 25 | logging.error(f"Failed to send request: {e}") 26 | return None 27 | 28 | # Read response 29 | start_time = time.time() 30 | timeout = 30 31 | 32 | # Capture log lines for debugging 33 | log_lines = [] 34 | 35 | while time.time() - start_time < timeout: 36 | try: 37 | # Check if there's anything to read from stdout 38 | line = server_process.stdout.readline().strip() 39 | if line: 40 | logging.debug(f"Received raw line: {line}") 41 | 42 | # Skip lines that are clearly log output 43 | if any(marker in line for marker in ["[INFO]", "[DEBUG]", "[ERROR]", "[WARN]"]) or line.startswith("20"): 44 | log_lines.append(line) 45 | continue 46 | 47 | # Try to parse line as JSON 48 | try: 49 | if line.startswith('{') or line.startswith('['): 50 | response = json.loads(line) 51 | logging.info(f"Received response: {response}") 52 | return response 53 | except json.JSONDecodeError as e: 54 | logging.error(f"Failed to parse JSON: {line}, Error: {e}") 55 | else: 56 | # No data available, short sleep to prevent CPU spinning 57 | time.sleep(0.1) 58 | 59 | # Check if the process is still running 60 | if server_process.poll() is not None: 61 | logging.error(f"Server process terminated with exit code: {server_process.poll()}") 62 | return None 63 | 64 | except Exception as e: 65 | logging.error(f"Error reading response: {e}") 66 | time.sleep(0.1) 67 | 68 | # If we get here, we timed out 69 | logging.error(f"No response received within {timeout} seconds") 70 | if log_lines: 71 | logging.error("Last 10 log lines from server:") 72 | for log in log_lines[-10:]: 73 | logging.error(f" {log}") 74 | 75 | # Try to read any error output 76 | stderr_output = server_process.stderr.read() 77 | if stderr_output: 78 | logging.error(f"Server stderr output: {stderr_output}") 79 | 80 | return None 81 | 82 | def launch_paint(): 83 | """Launch MS Paint""" 84 | try: 85 | subprocess.run(["taskkill", "/f", "/im", "mspaint.exe"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) 86 | time.sleep(1) 87 | subprocess.run(["start", "mspaint.exe"], shell=True) 88 | logging.info("Launched MS Paint") 89 | time.sleep(3) # Wait for Paint to start 90 | return True 91 | except Exception as e: 92 | logging.error(f"Failed to launch Paint: {e}") 93 | return False 94 | 95 | def main(): 96 | setup_logging() 97 | 98 | # Launch Paint first 99 | launch_paint() 100 | 101 | # Start MCP server with stderr redirected to a file for debugging 102 | log_file_path = "server_stderr.log" 103 | with open(log_file_path, "w") as stderr_file: 104 | logging.info("Starting MCP server...") 105 | server_process = subprocess.Popen( 106 | ["cargo", "run", "--release"], 107 | stdin=subprocess.PIPE, 108 | stdout=subprocess.PIPE, 109 | stderr=stderr_file, 110 | text=True, 111 | bufsize=1, 112 | env=dict(os.environ, RUST_LOG="info,debug") 113 | ) 114 | 115 | try: 116 | # Wait for server to start 117 | logging.info("Waiting for server to initialize...") 118 | time.sleep(3) 119 | 120 | # Connect to server 121 | connect_request = { 122 | "jsonrpc": "2.0", 123 | "id": 1, 124 | "method": "connect", 125 | "params": { 126 | "client_id": "simple-test-client", 127 | "client_name": "Simple Test Client" 128 | } 129 | } 130 | 131 | connect_response = send_request(server_process, connect_request) 132 | if not connect_response: 133 | logging.error("No response received from connect request") 134 | with open(log_file_path, "r") as f: 135 | logging.error(f"Server stderr log:\n{f.read()}") 136 | return 137 | 138 | if connect_response.get("status") != "success": 139 | logging.error(f"Failed to connect to server: {connect_response}") 140 | return 141 | 142 | logging.info(f"Connected to server. Response: {connect_response}") 143 | canvas_width = connect_response.get('canvas_width', 800) 144 | canvas_height = connect_response.get('canvas_height', 600) 145 | 146 | # Activate Paint window 147 | logging.info("Sending activate_window request...") 148 | activate_request = { 149 | "jsonrpc": "2.0", 150 | "id": 2, 151 | "method": "activate_window", 152 | "params": {} 153 | } 154 | 155 | activate_response = send_request(server_process, activate_request) 156 | if not activate_response: 157 | logging.error("No response received from activate_window request") 158 | return 159 | 160 | if activate_response.get("status") != "success": 161 | logging.warning(f"Failed to activate window, but continuing anyway: {activate_response}") 162 | else: 163 | logging.info(f"Window activated successfully: {activate_response}") 164 | 165 | # Get canvas dimensions (alternative method) 166 | logging.info("Sending get_canvas_dimensions request...") 167 | dimensions_request = { 168 | "jsonrpc": "2.0", 169 | "id": 3, 170 | "method": "get_canvas_dimensions", 171 | "params": {} 172 | } 173 | 174 | dimensions_response = send_request(server_process, dimensions_request) 175 | if not dimensions_response: 176 | logging.error("No response received from get_canvas_dimensions request") 177 | with open(log_file_path, "r") as f: 178 | logging.error(f"Server stderr log:\n{f.read()}") 179 | return 180 | 181 | if dimensions_response.get("status") != "success": 182 | logging.warning(f"Failed to get canvas dimensions: {dimensions_response}") 183 | canvas_width = 800 184 | canvas_height = 600 185 | logging.warning(f"Using default canvas dimensions: {canvas_width}x{canvas_height}") 186 | else: 187 | canvas_width = dimensions_response.get("width", 800) 188 | canvas_height = dimensions_response.get("height", 600) 189 | logging.info(f"Canvas dimensions: {canvas_width}x{canvas_height}") 190 | 191 | # Calculate center points for drawing 192 | center_x = canvas_width // 2 193 | center_y = canvas_height // 2 194 | 195 | # Adjust for drawing area (skip the ribbon) 196 | drawing_y_offset = 0 # No needed for MCP since we're using canvas coordinates 197 | 198 | # Draw a horizontal line in the center 199 | start_x = center_x - 100 200 | start_y = center_y + drawing_y_offset 201 | end_x = center_x + 100 202 | end_y = center_y + drawing_y_offset 203 | 204 | logging.info(f"Drawing horizontal line from ({start_x}, {start_y}) to ({end_x}, {end_y})") 205 | draw_line_request = { 206 | "jsonrpc": "2.0", 207 | "id": 4, 208 | "method": "draw_line", 209 | "params": { 210 | "start_x": start_x, 211 | "start_y": start_y, 212 | "end_x": end_x, 213 | "end_y": end_y, 214 | "color": "#FF0000", # Red color 215 | "thickness": 3 216 | } 217 | } 218 | 219 | logging.info("Sending draw_line request for horizontal line...") 220 | line_response = send_request(server_process, draw_line_request) 221 | if not line_response: 222 | logging.error("No response received from horizontal line draw_line request") 223 | with open(log_file_path, "r") as f: 224 | logging.error(f"Server stderr log:\n{f.read()}") 225 | return 226 | 227 | if line_response.get("status") != "success": 228 | logging.error(f"Failed to draw horizontal line: {line_response}") 229 | return 230 | 231 | logging.info("Successfully drew horizontal line") 232 | 233 | # Wait a moment between drawing operations (like in direct_paint_test) 234 | time.sleep(1) 235 | 236 | # Draw a vertical line intersecting the horizontal line 237 | start_x = center_x 238 | start_y = center_y + drawing_y_offset - 100 239 | end_x = center_x 240 | end_y = center_y + drawing_y_offset + 100 241 | 242 | logging.info(f"Drawing vertical line from ({start_x}, {start_y}) to ({end_x}, {end_y})") 243 | draw_vert_request = { 244 | "jsonrpc": "2.0", 245 | "id": 5, 246 | "method": "draw_line", 247 | "params": { 248 | "start_x": start_x, 249 | "start_y": start_y, 250 | "end_x": end_x, 251 | "end_y": end_y, 252 | "color": "#0000FF", # Blue color 253 | "thickness": 3 254 | } 255 | } 256 | 257 | logging.info("Sending draw_line request for vertical line...") 258 | vert_response = send_request(server_process, draw_vert_request) 259 | if not vert_response: 260 | logging.error("No response received from vertical line draw_line request") 261 | with open(log_file_path, "r") as f: 262 | logging.error(f"Server stderr log:\n{f.read()}") 263 | return 264 | 265 | if vert_response.get("status") != "success": 266 | logging.error(f"Failed to draw vertical line: {vert_response}") 267 | return 268 | 269 | logging.info("Successfully drew vertical line") 270 | 271 | # Test is complete 272 | logging.info("Test completed successfully") 273 | 274 | except Exception as e: 275 | logging.error(f"Error during test: {e}", exc_info=True) 276 | finally: 277 | # Terminate the server process 278 | try: 279 | server_process.terminate() 280 | server_process.wait(timeout=5) 281 | logging.info("Server terminated") 282 | except: 283 | logging.warning("Could not cleanly terminate the server process") 284 | 285 | # Output server logs for review 286 | logging.info(f"Server stderr log available at: {log_file_path}") 287 | with open(log_file_path, "r") as f: 288 | server_logs = f.read() 289 | if len(server_logs) > 0: 290 | logging.info(f"Last 200 characters of server logs: {server_logs[-200:]}") 291 | 292 | if __name__ == "__main__": 293 | main() -------------------------------------------------------------------------------- /simplest_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import json 3 | import subprocess 4 | import time 5 | import sys 6 | import os 7 | import threading 8 | 9 | def main(): 10 | # First, make sure Paint is not running 11 | os.system('taskkill /f /im mspaint.exe 2>nul') 12 | time.sleep(1) 13 | 14 | # Launch Paint 15 | print("Launching MS Paint...") 16 | paint_process = subprocess.Popen(["mspaint.exe"]) 17 | time.sleep(3) # Wait for Paint to start 18 | 19 | # Start the server 20 | print("Starting MCP server...") 21 | server_process = subprocess.Popen( 22 | ["cargo", "run"], 23 | stdin=subprocess.PIPE, 24 | stdout=subprocess.PIPE, 25 | stderr=None, # Don't capture stderr so it appears in console directly 26 | text=True, 27 | bufsize=1 28 | ) 29 | 30 | try: 31 | # Step 1: Initialize 32 | print("Sending initialize request...") 33 | init_request = { 34 | "jsonrpc": "2.0", 35 | "id": 1, 36 | "method": "initialize", 37 | "params": {} 38 | } 39 | 40 | send_request_and_wait(server_process, init_request, 3) 41 | 42 | # Step 2: Connect 43 | print("Sending connect request...") 44 | connect_request = { 45 | "jsonrpc": "2.0", 46 | "id": 2, 47 | "method": "connect", 48 | "params": { 49 | "client_id": "simplest_test", 50 | "client_name": "Simplest Test Client" 51 | } 52 | } 53 | 54 | send_request_and_wait(server_process, connect_request, 3) 55 | 56 | # Step 3: Draw a large rectangle with lots of time for each step 57 | print("Drawing a large rectangle...") 58 | rectangle_request = { 59 | "jsonrpc": "2.0", 60 | "id": 3, 61 | "method": "draw_shape", 62 | "params": { 63 | "shape_type": "rectangle", 64 | "start_x": 50, 65 | "start_y": 50, 66 | "end_x": 500, 67 | "end_y": 400 68 | } 69 | } 70 | 71 | # Send request and wait longer for rectangle drawing to complete 72 | send_request_and_wait(server_process, rectangle_request, 10) 73 | 74 | print("Test completed. Rectangle should be drawn.") 75 | print("Press Enter to close test and Paint...") 76 | input() 77 | 78 | except Exception as e: 79 | print(f"Test failed with error: {e}") 80 | finally: 81 | # Terminate the server process 82 | server_process.terminate() 83 | print("Server process terminated") 84 | 85 | # Terminate Paint 86 | paint_process.terminate() 87 | print("Paint process terminated") 88 | 89 | def send_request_and_wait(process, request, wait_time=5): 90 | """Send a request to the server and wait a fixed time.""" 91 | request_str = json.dumps(request) + "\n" 92 | print(f"SENDING: {request_str.strip()}") 93 | 94 | process.stdin.write(request_str) 95 | process.stdin.flush() 96 | 97 | # Wait for response - simple approach with fixed wait time 98 | print(f"Waiting {wait_time} seconds for operation to complete...") 99 | 100 | # Read response 101 | response_line = process.stdout.readline().strip() 102 | if response_line: 103 | print(f"RESPONSE: {response_line}") 104 | 105 | # Wait additional time after response to ensure operation completes 106 | time.sleep(wait_time) 107 | 108 | return response_line 109 | 110 | if __name__ == "__main__": 111 | main() -------------------------------------------------------------------------------- /specs/architecture_diagram.md: -------------------------------------------------------------------------------- 1 | # Paint MCP Architecture Diagram (Windows 11 Edition) 2 | 3 | ``` 4 | ┌─────────────────────────────────────────────────────────────────┐ 5 | │ │ 6 | │ Client Applications │ 7 | │ (Launches Server Process) │ 8 | └───────────────────────────────┬─────────────────────────────────┘ 9 | │ <-- STDIO Communication --> 10 | ▼ 11 | ┌─────────────────────────────────────────────────────────────────┐ 12 | │ │ 13 | │ MCP Server Process │ 14 | │ (Entry Point / Client Interface) │ 15 | └───────────────────────────────┬─────────────────────────────────┘ 16 | │ 17 | ▼ 18 | ┌─────────────────────────────────────────────────────────────────┐ 19 | │ │ 20 | │ MCP Core Library │ 21 | │ (Implemented using rust-mcp-sdk) │ 22 | │ ┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ │ 23 | │ │ │ │ │ │ │ │ 24 | │ │ Command Layer │ │ Paint Layer │ │ Event Layer │ │ 25 | │ │ │ │ │ │ │ │ 26 | │ └────────┬────────┘ └────────┬────────┘ └────────┬────────┘ │ 27 | │ │ │ │ │ 28 | │ ┌────────▼────────┐ ┌────────▼────────┐ ┌────────▼────────┐ │ 29 | │ │ │ │ │ │ │ │ 30 | │ │ Text Manager │ │ Transform Mgr │ │ Canvas Manager │ │ 31 | │ │ │ │ │ │ │ │ 32 | │ └─────────────────┘ └─────────────────┘ └─────────────────┘ │ 33 | │ │ 34 | └───────────┬────────────────────┬────────────────────┬───────────┘ 35 | │ │ │ 36 | ▼ ▼ ▼ 37 | ┌─────────────────────────────────────────────────────────────────┐ 38 | │ │ 39 | │ Windows 11 Integration │ 40 | │ │ 41 | │ ┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ │ 42 | │ │ │ │ │ │ │ │ 43 | │ │ Window Manager │ │ UI Manager │ │ Input Manager │ │ 44 | │ │ │ │ │ │ │ │ 45 | │ └─────────────────┘ └─────────────────┘ └─────────────────┘ │ 46 | │ │ 47 | │ ┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ │ 48 | │ │ │ │ │ │ │ │ 49 | │ │ Dialog Manager │ │ Menu Manager │ │ Keyboard Manager│ │ 50 | │ │ │ │ │ │ │ │ 51 | │ └─────────────────┘ └─────────────────┘ └─────────────────┘ │ 52 | │ │ 53 | └───────────────────────────────┬─────────────────────────────────┘ 54 | │ 55 | ▼ 56 | ┌─────────────────────────────────────────────────────────────────┐ 57 | │ │ 58 | │ Windows 11 Paint Application │ 59 | │ │ 60 | └─────────────────────────────────────────────────────────────────┘ 61 | ``` 62 | 63 | ## Architecture Components 64 | 65 | ### Client Applications 66 | 67 | Client applications use the Paint MCP library to control Windows 11 Paint. These can be: 68 | 69 | * Custom applications 70 | * Automation scripts 71 | * Drawing utilities 72 | * Test frameworks 73 | 74 | ### MCP Client Interface 75 | 76 | This represents the entry point to the MCP Server process, handling the STDIO communication with the client application. It provides: 77 | 78 | * JSON-RPC message parsing/serialization (via rust-mcp-sdk) 79 | * Dispatching requests to the MCP Core Library 80 | * Sending responses/notifications back to the client via STDOUT 81 | * Asynchronous operations 82 | 83 | ### MCP Core Library 84 | 85 | The core library implements the MCP protocol and will be built using the `rust-mcp-sdk` crate ([https://crates.io/crates/rust-mcp-sdk](https://crates.io/crates/rust-mcp-sdk)). It provides: 86 | 87 | #### Command Layer 88 | * Translates high-level commands into Windows API calls 89 | * Validates parameters 90 | * Manages command sequencing 91 | * Handles timeouts and retries 92 | 93 | #### Paint Layer 94 | * Contains Windows 11 Paint-specific knowledge 95 | * Maps abstract commands to Paint UI interactions 96 | * Handles tool selection, color setting, and drawing operations 97 | * Enables pixel-precise drawing through optimized mouse operations 98 | * Provides fine-grained control over brush sizes and styles 99 | * Understands the Windows 11 Paint canvas coordinate system 100 | 101 | #### Text Manager 102 | * Manages text input and formatting 103 | * Controls font selection, size, and style options 104 | * Handles text positioning and rendering 105 | * Interacts with Windows 11 Paint's text formatting dialog 106 | * Converts text parameters to appropriate UI interactions 107 | 108 | #### Transform Manager 109 | * Handles image transformation operations 110 | * Implements rotation, flipping, scaling, and cropping 111 | * Manages coordinate transformations for transformed images 112 | * Interacts with Windows 11 Paint's transformation dialogs and menus 113 | * Provides error handling for transformation limits 114 | 115 | #### Canvas Manager 116 | * Controls canvas creation and configuration 117 | * Manages canvas dimensions and background settings 118 | * Handles clearing and resetting canvas state 119 | * Provides canvas property information 120 | * Ensures proper setup before drawing operations 121 | 122 | #### Event Layer 123 | * Handles asynchronous events 124 | * Provides status updates and operation completion notifications 125 | * Monitors for unexpected Paint dialogs or errors 126 | 127 | #### File Operations Layer 128 | * Manages saving images through Paint 129 | * Retrieves saved images from disk 130 | * Processes image data for transport 131 | * Extracts image metadata 132 | * Handles image format conversions 133 | 134 | ### Windows 11 Integration 135 | 136 | This layer contains the Windows-specific implementation: 137 | 138 | #### Window Manager 139 | * Finds and activates the Paint window 140 | * Gets window dimensions and position 141 | * Monitors window state changes 142 | * Handles window activation and focus 143 | 144 | #### UI Manager 145 | * Interacts with Windows 11 Paint's modern UI 146 | * Locates and manipulates toolbar elements 147 | * Finds canvas area and panels 148 | * Adapts to UI changes based on window size 149 | 150 | #### Input Manager 151 | * Simulates mouse and keyboard input 152 | * Translates coordinates between client and screen space 153 | * Handles input timing and synchronization 154 | * Manages input device state 155 | 156 | #### Dialog Manager 157 | * Identifies and interacts with Paint dialogs 158 | * Handles font selection, resize/scale, and other configuration dialogs 159 | * Provides navigation between dialog controls 160 | * Manages dialog confirmation and cancellation 161 | * Times operations to ensure dialogs are fully loaded before interaction 162 | 163 | #### Menu Manager 164 | * Accesses and navigates Paint's menu system 165 | * Triggers commands like rotate, flip, and crop 166 | * Handles submenu navigation 167 | * Provides consistent menu access across different UI states 168 | 169 | #### Keyboard Manager 170 | * Simulates complex keyboard interactions 171 | * Manages keyboard shortcuts for commands 172 | * Handles text input for dialogs and text tool 173 | * Ensures proper key press/release sequencing 174 | 175 | ### Windows 11 Paint Application 176 | 177 | The Microsoft Paint application that ships with Windows 11: 178 | 179 | * Modern UI design 180 | * Canvas for drawing 181 | * Toolbars and panels 182 | * File operations 183 | * Text formatting capabilities 184 | * Image transformation features 185 | * Canvas creation and management 186 | 187 | ## Data Flow 188 | 189 | 1. Client applications launch the MCP Server process and send JSON-RPC requests via STDIN. 190 | 2. The MCP Server process (Client Interface layer) receives requests, parses them (using `rust-mcp-sdk`), and dispatches them to the MCP Core Library. 191 | 3. The Command Layer validates and processes these calls. 192 | 4. Specialized managers handle specific functionality (Text, Transform, Canvas). 193 | 5. The Paint Layer translates commands to Paint-specific operations. 194 | 6. The Windows Integration layer interacts with Windows 11 APIs. 195 | 7. Input events are sent to the Windows 11 Paint application. 196 | 8. The Event Layer monitors results and provides feedback. 197 | 9. Responses/notifications are sent back to the client application via STDOUT. 198 | 199 | ## Implementation Details 200 | 201 | ### Windows 11 Paint-Specific Features 202 | 203 | * Modern UI toolbar with simplified layout 204 | * Property panels on the right side 205 | * Enhanced shape tools with fill/outline options 206 | * Cleaner canvas area with adjusted margins 207 | * Text formatting with font options 208 | * Image transformation menu options 209 | * Canvas creation dialog with size options 210 | 211 | ### Key Technical Components 212 | 213 | * Windows API for window management 214 | * SendInput for mouse/keyboard simulation 215 | * DPI awareness for proper coordinate translation 216 | * Window activation techniques optimized for Windows 11 217 | * Dialog interaction for text and canvas operations 218 | * Menu interaction for image transformations 219 | * Keyboard shortcuts for enhanced productivity 220 | 221 | ## Error Handling 222 | 223 | * Robust error detection and reporting 224 | * Recovery mechanisms for common failures 225 | * Timeout handling for operations 226 | * Logging for troubleshooting 227 | * Specialized error types for text, transformation, and canvas operations 228 | 229 | ## Future Architecture Extensions 230 | 231 | * Direct bitmap manipulation for faster drawing 232 | * Enhanced shape and text handling 233 | * Support for new Windows 11 Paint features as they evolve 234 | * Advanced text formatting and effects 235 | * More sophisticated image transformations 236 | * Layer management support if added to Paint -------------------------------------------------------------------------------- /specs/client_integration.md: -------------------------------------------------------------------------------- 1 | # Paint MCP Client Integration Specification 2 | 3 | ## Overview 4 | 5 | The Paint MCP client library provides a programmatic interface for controlling Windows 11 Paint. It communicates with a separate MCP server process (also implemented using `rust-mcp-sdk`) via **STDIO (Standard Input/Output)**. The client library handles launching the server process and managing the communication channel. This specification outlines the integration requirements for client applications using this library. 6 | 7 | ## Client API 8 | 9 | ### Initialization 10 | 11 | To begin interacting with Paint, the client application first creates an instance of the client library and then establishes a connection. The connection process typically involves locating the MCP server executable, launching it as a child process, and establishing communication over its standard input and standard output streams. 12 | 13 | ```rust 14 | // Create a new Paint MCP client instance 15 | // This might involve specifying the path to the server executable 16 | let client = PaintMcpClient::new(/* potentially server path */)?; 17 | 18 | // Connect to Windows 11 Paint 19 | // This launches the server process and sets up STDIO communication. 20 | // It likely performs the initial MCP handshake (initialize request/result). 21 | client.connect()?; 22 | ``` 23 | 24 | **Note:** The client library internally uses the `rust-mcp-sdk` crate ([https://crates.io/crates/rust-mcp-sdk](https://crates.io/crates/rust-mcp-sdk)) to manage the JSON-RPC 2.0 messages exchanged over STDIO with the server process. 25 | 26 | ### Basic Operations 27 | 28 | ```rust 29 | // Ensure Paint is visible and in foreground 30 | client.activate_window()?; 31 | 32 | // Get canvas dimensions 33 | let (width, height) = client.get_canvas_dimensions()?; 34 | 35 | // Clear canvas (creates new document) 36 | client.clear_canvas()?; 37 | 38 | // Create a new canvas with specific dimensions 39 | client.create_canvas(1024, 768, Some("#FFFFFF"))?; 40 | ``` 41 | 42 | ### Drawing Tools 43 | 44 | ```rust 45 | // Available tools for Windows 11 Paint 46 | pub enum DrawingTool { 47 | Pencil, 48 | Brush, 49 | Fill, 50 | Text, 51 | Eraser, 52 | Select, 53 | Shape(ShapeType), 54 | } 55 | 56 | // Select a drawing tool 57 | client.select_tool(DrawingTool::Pencil)?; 58 | client.select_tool(DrawingTool::Shape(ShapeType::Rectangle))?; 59 | ``` 60 | 61 | ### Shape Drawing 62 | 63 | ```rust 64 | // Available shape types in Windows 11 Paint 65 | pub enum ShapeType { 66 | Rectangle, 67 | Ellipse, 68 | Line, 69 | Arrow, 70 | Triangle, 71 | Pentagon, 72 | Hexagon, 73 | // Other shapes supported by Windows 11 Paint 74 | } 75 | 76 | // Draw shapes with specific dimensions 77 | client.draw_shape(ShapeType::Rectangle, x1, y1, x2, y2)?; 78 | 79 | // Shape fill options 80 | pub enum FillType { 81 | None, 82 | Solid, 83 | Outline, 84 | } 85 | 86 | // Set shape fill type 87 | client.set_shape_fill(FillType::Solid)?; 88 | ``` 89 | 90 | ### Color and Style 91 | 92 | ```rust 93 | // Set active color with hex color code 94 | client.set_color("#FF5733")?; 95 | 96 | // Set brush/line thickness (1-5) 97 | client.set_thickness(3)?; 98 | 99 | // Set precise brush size in pixels (1-30px depending on tool) 100 | client.set_brush_size(8)?; 101 | 102 | // Set brush size with specific tool 103 | client.set_brush_size_for_tool(12, DrawingTool::Brush)?; 104 | ``` 105 | 106 | ### Freeform Drawing 107 | 108 | ```rust 109 | // Draw a line from one point to another 110 | client.draw_line(x1, y1, x2, y2)?; 111 | 112 | // Draw a series of connected lines 113 | client.draw_polyline(vec![(x1, y1), (x2, y2), (x3, y3)])?; 114 | 115 | // Draw a single pixel at specific coordinates 116 | client.draw_pixel(x, y, "#FF0000")?; // With optional color parameter 117 | ``` 118 | 119 | ### Text Operations 120 | 121 | ```rust 122 | // Add simple text at specific position 123 | client.add_text(x, y, "Hello World")?; 124 | 125 | // Add text with enhanced font options 126 | client.add_text_with_options( 127 | x, y, 128 | "Hello World", 129 | Some("Arial"), // font name 130 | Some(24), // font size 131 | Some(FontStyle::Bold), // font style 132 | Some("#FF0000") // text color 133 | )?; 134 | 135 | // Font style options 136 | pub enum FontStyle { 137 | Regular, 138 | Bold, 139 | Italic, 140 | BoldItalic, 141 | } 142 | ``` 143 | 144 | ### Selection Operations 145 | 146 | ```rust 147 | // Select a region 148 | client.select_region(x1, y1, x2, y2)?; 149 | 150 | // Copy selected region 151 | client.copy_selection()?; 152 | 153 | // Paste at position 154 | client.paste(x, y)?; 155 | ``` 156 | 157 | ### Image Transformations 158 | 159 | ```rust 160 | // Rotate the image 161 | client.rotate_image(90, true)?; // 90 degrees clockwise 162 | 163 | // Flip the image 164 | client.flip_image(FlipDirection::Horizontal)?; 165 | // or 166 | client.flip_image(FlipDirection::Vertical)?; 167 | 168 | // Resize/scale the image 169 | client.scale_image( 170 | Some(800), // new width 171 | Some(600), // new height 172 | Some(true), // maintain aspect ratio 173 | None // no percentage scaling 174 | )?; 175 | 176 | // Alternatively, scale by percentage 177 | client.scale_image( 178 | None, // no fixed width 179 | None, // no fixed height 180 | None, // aspect ratio not applicable 181 | Some(50.0) // scale to 50% 182 | )?; 183 | 184 | // Crop the image 185 | client.crop_image(x, y, width, height)?; 186 | 187 | // Flip direction options 188 | pub enum FlipDirection { 189 | Horizontal, 190 | Vertical, 191 | } 192 | ``` 193 | 194 | ### File Operations 195 | 196 | ```rust 197 | // Save canvas to a file 198 | client.save_canvas("C:\\path\\to\\image.png", ImageFormat::Png)?; 199 | 200 | // Fetch a saved image as bytes 201 | let image_data: Vec = client.fetch_image("C:\\path\\to\\image.png")?; 202 | 203 | // Fetch a saved image with metadata 204 | let image_result = client.fetch_image_with_metadata("C:\\path\\to\\image.png")?; 205 | println!("Image format: {}", image_result.format); 206 | println!("Image dimensions: {}x{}", image_result.width, image_result.height); 207 | println!("Image data length: {} bytes", image_result.data.len()); 208 | ``` 209 | 210 | ### Image Recreation 211 | 212 | ```rust 213 | // Load an image from disk 214 | let img_path = "C:\\path\\to\\source_image.jpg"; 215 | let img_data = std::fs::read(img_path)?; 216 | let base64_data = base64::encode(&img_data); 217 | 218 | // Recreate the image in Paint 219 | // max_detail_level controls the level of detail - higher values (1-200) mean more detail but slower processing 220 | client.recreate_image(&base64_data, Some("C:\\path\\to\\output.png"), Some(100))?; 221 | 222 | // Or recreate without saving the result 223 | client.recreate_image(&base64_data, None, None)?; 224 | ``` 225 | 226 | ### Integration with Cursor 227 | 228 | ```rust 229 | // When integrating with Cursor, you can use this pattern to handle images from the clipboard: 230 | 231 | // 1. Get base64 image data from Cursor 232 | let base64_image = cursor.get_clipboard_image()?; 233 | 234 | // 2. Recreate the image in Paint 235 | paint_client.connect()?; 236 | paint_client.recreate_image(&base64_image, Some("C:\\path\\to\\recreated.png"), Some(150))?; 237 | 238 | // 3. Optionally, apply additional edits to the recreated image 239 | paint_client.select_tool(DrawingTool::Brush)?; 240 | paint_client.set_color("#FF0000")?; 241 | paint_client.draw_line(100, 100, 300, 300)?; 242 | ``` 243 | 244 | ## Windows 11 Paint Integration Details 245 | 246 | ### Paint Version Detection 247 | 248 | The client will detect and only support the Windows 11 version of Paint: 249 | 250 | ```rust 251 | // Windows 11 Paint detection 252 | let version = client.detect_paint_version()?; 253 | assert_eq!(version, PaintVersion::Modern); 254 | ``` 255 | 256 | ### Window Activation 257 | 258 | Reliable window activation is crucial for consistent operation: 259 | 260 | ```rust 261 | // Ensure Paint window is active 262 | client.activate_window()?; 263 | ``` 264 | 265 | ### Mouse Coordinate Mapping 266 | 267 | Client coordinates (0,0 at top-left of canvas) are mapped to screen coordinates: 268 | 269 | ```rust 270 | // Map client coordinates to screen coordinates 271 | let screen_x = canvas_x + canvas_rect.left; 272 | let screen_y = canvas_y + canvas_rect.top; 273 | ``` 274 | 275 | ### Error Handling 276 | 277 | All operations return a `Result` type with descriptive errors: 278 | 279 | ```rust 280 | pub enum PaintMcpError { 281 | WindowNotFound, 282 | ActivationFailed, 283 | OperationTimeout, 284 | InvalidColor, 285 | UnsupportedOperation, 286 | TextInputFailed, 287 | FontSelectionFailed, 288 | TransformationFailed, 289 | CanvasCreationFailed, 290 | // Other error types 291 | } 292 | ``` 293 | 294 | ## Implementation Constraints 295 | 296 | 1. **Windows 11 Specific**: This implementation only supports Windows 11 Paint. 297 | 2. **Coordinate System**: All coordinates are relative to the canvas (0,0 at top-left). 298 | 3. **UI Interaction**: Operations use UI automation and may be affected by Paint window focus. 299 | 4. **Timing Constraints**: Operations include reasonable timeouts to handle UI responsiveness. 300 | 5. **Resolution Independence**: Functions work across different screen resolutions. 301 | 6. **Font Availability**: Text operations depend on fonts installed on the system. 302 | 7. **Transformation Limitations**: Some transformations may be limited by Paint's capabilities. 303 | 304 | ## Security Considerations 305 | 306 | 1. Client requires appropriate permissions to interact with Windows UI. 307 | 2. Screen content may be visible during automation operations. 308 | 3. Clipboard operations may affect system clipboard content. 309 | 310 | ## Performance Guidelines 311 | 312 | 1. Operations should complete within reasonable timeframes (typically < 500ms). 313 | 2. Complex operations (like polyline drawing) may take longer. 314 | 3. Clients should implement appropriate timeout handling. 315 | 4. Window activation adds overhead to operations. 316 | 5. Image transformations and canvas operations may take longer on larger images. -------------------------------------------------------------------------------- /specs/mcp_protocol.md: -------------------------------------------------------------------------------- 1 | # MCP Protocol Specification for Windows 11 Paint 2 | 3 | ## Overview 4 | 5 | The Microsoft Paint Control Protocol (MCP) provides a standardized interface for programmatically controlling Windows 11 Paint. This specification defines the communication protocol between client applications and the Paint MCP server. 6 | 7 | ## Protocol Design 8 | 9 | The MCP protocol is a JSON-based RPC protocol communicated over **STDIO (Standard Input/Output)**. The client launches the server process and interacts with it by writing JSON-RPC request messages to the server's standard input and reading JSON-RPC response/notification messages from the server's standard output. All commands and responses are formatted as JSON objects, adhering to the JSON-RPC 2.0 specification where applicable. 10 | 11 | **Implementation Note:** The underlying implementation will utilize the `rust-mcp-sdk` crate ([https://crates.io/crates/rust-mcp-sdk](https://crates.io/crates/rust-mcp-sdk)), a toolkit for building MCP servers and clients, which supports STDIO transport. While this specification defines Paint-specific commands, leveraging this SDK provides a robust foundation for handling the JSON-RPC communication, serialization, and potentially standard MCP messages. 12 | 13 | ## Connection Management 14 | 15 | ### Connect Request 16 | 17 | ```json 18 | { 19 | "command": "connect", 20 | "params": { 21 | "client_id": "unique-client-identifier", 22 | "client_name": "Sample App" 23 | } 24 | } 25 | ``` 26 | 27 | ### Connect Response 28 | 29 | ```json 30 | { 31 | "status": "success", 32 | "paint_version": "windows11", 33 | "canvas_width": 800, 34 | "canvas_height": 600 35 | } 36 | ``` 37 | 38 | ### Disconnect Request 39 | 40 | ```json 41 | { 42 | "command": "disconnect" 43 | } 44 | ``` 45 | 46 | ### Disconnect Response 47 | 48 | ```json 49 | { 50 | "status": "success" 51 | } 52 | ``` 53 | 54 | ## Drawing Operations 55 | 56 | ### Select Tool 57 | 58 | ```json 59 | { 60 | "command": "select_tool", 61 | "params": { 62 | "tool": "pencil|brush|fill|text|eraser|select|shape", 63 | "shape_type": "rectangle|ellipse|line|arrow|triangle|pentagon|hexagon" 64 | } 65 | } 66 | ``` 67 | 68 | ### Set Color 69 | 70 | ```json 71 | { 72 | "command": "set_color", 73 | "params": { 74 | "color": "#RRGGBB" 75 | } 76 | } 77 | ``` 78 | 79 | ### Set Thickness 80 | 81 | ```json 82 | { 83 | "command": "set_thickness", 84 | "params": { 85 | "level": 1-5 86 | } 87 | } 88 | ``` 89 | 90 | ### Set Brush Size 91 | 92 | ```json 93 | { 94 | "command": "set_brush_size", 95 | "params": { 96 | "size": 1-30, // Pixel size (1-30px depending on tool) 97 | "tool": "pencil|brush" // Optional - defaults to current tool 98 | } 99 | } 100 | ``` 101 | 102 | ### Set Fill Type 103 | 104 | ```json 105 | { 106 | "command": "set_fill", 107 | "params": { 108 | "type": "none|solid|outline" 109 | } 110 | } 111 | ``` 112 | 113 | ### Draw Line 114 | 115 | ```json 116 | { 117 | "command": "draw_line", 118 | "params": { 119 | "start_x": 100, 120 | "start_y": 100, 121 | "end_x": 200, 122 | "end_y": 200, 123 | "color": "#RRGGBB", // Optional 124 | "thickness": 2 // Optional 125 | } 126 | } 127 | ``` 128 | 129 | ### Draw Pixel 130 | 131 | ```json 132 | { 133 | "command": "draw_pixel", 134 | "params": { 135 | "x": 150, 136 | "y": 150, 137 | "color": "#RRGGBB" // Optional 138 | } 139 | } 140 | ``` 141 | 142 | ### Draw Shape 143 | 144 | ```json 145 | { 146 | "command": "draw_shape", 147 | "params": { 148 | "shape": "rectangle|ellipse|line|arrow|triangle|pentagon|hexagon", 149 | "start_x": 100, 150 | "start_y": 100, 151 | "end_x": 300, 152 | "end_y": 200, 153 | "color": "#RRGGBB", // Optional 154 | "thickness": 2, // Optional 155 | "fill_type": "none|solid|outline" // Optional 156 | } 157 | } 158 | ``` 159 | 160 | ### Draw Polyline 161 | 162 | ```json 163 | { 164 | "command": "draw_polyline", 165 | "params": { 166 | "points": [ 167 | {"x": 100, "y": 100}, 168 | {"x": 150, "y": 50}, 169 | {"x": 200, "y": 100} 170 | ], 171 | "color": "#RRGGBB", // Optional 172 | "thickness": 2 // Optional 173 | } 174 | } 175 | ``` 176 | 177 | ### Add Text 178 | 179 | ```json 180 | { 181 | "command": "add_text", 182 | "params": { 183 | "x": 100, 184 | "y": 100, 185 | "text": "Hello World", 186 | "color": "#RRGGBB" // Optional 187 | } 188 | } 189 | ``` 190 | 191 | ### Enhanced Add Text (New) 192 | 193 | ```json 194 | { 195 | "command": "add_text", 196 | "params": { 197 | "x": 100, 198 | "y": 100, 199 | "text": "Hello World", 200 | "font_name": "Arial", // Optional 201 | "font_size": 24, // Optional 202 | "font_style": "bold", // Optional: regular|bold|italic|bold_italic 203 | "color": "#RRGGBB" // Optional 204 | } 205 | } 206 | ``` 207 | 208 | ## Selection Operations 209 | 210 | ### Select Region 211 | 212 | ```json 213 | { 214 | "command": "select_region", 215 | "params": { 216 | "start_x": 100, 217 | "start_y": 100, 218 | "end_x": 300, 219 | "end_y": 200 220 | } 221 | } 222 | ``` 223 | 224 | ### Copy Selection 225 | 226 | ```json 227 | { 228 | "command": "copy_selection" 229 | } 230 | ``` 231 | 232 | ### Paste 233 | 234 | ```json 235 | { 236 | "command": "paste", 237 | "params": { 238 | "x": 150, 239 | "y": 150 240 | } 241 | } 242 | ``` 243 | 244 | ## Canvas Management 245 | 246 | ### Clear Canvas 247 | 248 | ```json 249 | { 250 | "command": "clear_canvas" 251 | } 252 | ``` 253 | 254 | ### Create New Canvas (New) 255 | 256 | ```json 257 | { 258 | "command": "create_canvas", 259 | "params": { 260 | "width": 1024, 261 | "height": 768, 262 | "background_color": "#FFFFFF" // Optional, defaults to white 263 | } 264 | } 265 | ``` 266 | 267 | ### Save Canvas 268 | 269 | ```json 270 | { 271 | "command": "save", 272 | "params": { 273 | "path": "C:\\path\\to\\image.png", 274 | "format": "png|jpeg|bmp" 275 | } 276 | } 277 | ``` 278 | 279 | ### Fetch Image 280 | 281 | ```json 282 | { 283 | "command": "fetch_image", 284 | "params": { 285 | "path": "C:\\path\\to\\image.png" 286 | } 287 | } 288 | ``` 289 | 290 | ### Fetch Image Response 291 | 292 | ```json 293 | { 294 | "status": "success", 295 | "data": "base64_encoded_image_data_here", 296 | "format": "png", 297 | "width": 800, 298 | "height": 600 299 | } 300 | ``` 301 | 302 | ## Image Transformations (New) 303 | 304 | ### Rotate Image 305 | 306 | ```json 307 | { 308 | "command": "rotate_image", 309 | "params": { 310 | "degrees": 90, // Typically 90, 180, or 270 311 | "clockwise": true // Optional, defaults to true 312 | } 313 | } 314 | ``` 315 | 316 | ### Flip Image 317 | 318 | ```json 319 | { 320 | "command": "flip_image", 321 | "params": { 322 | "direction": "horizontal" // horizontal|vertical 323 | } 324 | } 325 | ``` 326 | 327 | ### Scale Image 328 | 329 | ```json 330 | { 331 | "command": "scale_image", 332 | "params": { 333 | "width": 800, // Optional if percentage is provided 334 | "height": 600, // Optional if percentage is provided 335 | "maintain_aspect_ratio": true, // Optional, defaults to false 336 | "percentage": 50 // Optional, scale as percentage (e.g., 50 for 50%, 200 for 200%) 337 | } 338 | } 339 | ``` 340 | 341 | ### Crop Image 342 | 343 | ```json 344 | { 345 | "command": "crop_image", 346 | "params": { 347 | "start_x": 50, 348 | "start_y": 50, 349 | "width": 400, 350 | "height": 300 351 | } 352 | } 353 | ``` 354 | 355 | ## Image Recreation 356 | 357 | ### Recreate Image Request 358 | 359 | ```json 360 | { 361 | "command": "recreate_image", 362 | "params": { 363 | "image_base64": "base64_encoded_image_data_here", 364 | "output_filename": "C:\\path\\to\\output.png", // Optional 365 | "max_detail_level": 100 // Optional, 1-200 366 | } 367 | } 368 | ``` 369 | 370 | ### Recreate Image Response 371 | 372 | ```json 373 | { 374 | "status": "success", 375 | "error": null 376 | } 377 | ``` 378 | 379 | ## Error Response 380 | 381 | All operations may return an error response: 382 | 383 | ```json 384 | { 385 | "status": "error", 386 | "error": "Error message describing what went wrong" 387 | } 388 | ``` 389 | 390 | ## Window Management 391 | 392 | ### Activate Window 393 | 394 | ```json 395 | { 396 | "command": "activate_window" 397 | } 398 | ``` 399 | 400 | ### Get Canvas Dimensions 401 | 402 | ```json 403 | { 404 | "command": "get_canvas_dimensions" 405 | } 406 | ``` 407 | 408 | Response: 409 | 410 | ```json 411 | { 412 | "status": "success", 413 | "width": 800, 414 | "height": 600 415 | } 416 | ``` 417 | 418 | ## Error Handling 419 | 420 | All responses include a `status` field indicating success or failure. In case of failure, an `error` field provides details: 421 | 422 | ```json 423 | { 424 | "status": "error", 425 | "error": { 426 | "code": 1001, 427 | "message": "Paint window not found" 428 | } 429 | } 430 | ``` 431 | 432 | ### Error Codes 433 | 434 | | Code | Description | 435 | |------|-------------| 436 | | 1000 | General error | 437 | | 1001 | Paint window not found | 438 | | 1002 | Operation timeout | 439 | | 1003 | Invalid parameters | 440 | | 1004 | Invalid color format | 441 | | 1005 | Invalid tool | 442 | | 1006 | Invalid shape | 443 | | 1007 | Window activation failed | 444 | | 1008 | Operation not supported in Windows 11 Paint | 445 | | 1009 | File not found | 446 | | 1010 | Permission denied accessing file | 447 | | 1011 | Invalid image format | 448 | | 1012 | Text input failed | 449 | | 1013 | Font selection failed | 450 | | 1014 | Image transformation failed | 451 | | 1015 | Canvas creation failed | 452 | 453 | ## Protocol Extensions 454 | 455 | The MCP protocol may be extended with new commands as Windows 11 Paint evolves. Clients should gracefully handle unknown commands and parameters. 456 | 457 | ## Windows 11 Paint-Specific Considerations 458 | 459 | 1. All coordinates are relative to the canvas (0,0 at top-left) 460 | 2. Color values must be in the format `#RRGGBB` 461 | 3. Shape operations honor the current fill settings 462 | 4. Operations that require dialog interactions (save/open) may be limited 463 | 5. Image transformations work on the entire canvas/image or current selection 464 | 6. Font availability depends on what's installed on the system 465 | 466 | ## Versioning 467 | 468 | This protocol specification is versioned: 469 | 470 | ```json 471 | { 472 | "command": "get_version" 473 | } 474 | ``` 475 | 476 | Response: 477 | 478 | ```json 479 | { 480 | "status": "success", 481 | "protocol_version": "1.1", 482 | "server_version": "1.1.0", 483 | "paint_version": "windows11" 484 | } 485 | ``` -------------------------------------------------------------------------------- /specs/windows_integration.md: -------------------------------------------------------------------------------- 1 | # Windows 11 Integration Specification 2 | 3 | ## Paint Application Detection 4 | 5 | The MCP server will locate Windows 11 Paint (mspaint.exe) using the following methods: 6 | 7 | 1. Enumerate windows using `EnumWindows` to find windows with class name "MSPaintApp" or title containing "Paint" 8 | 2. If no instance is found, launch Paint using `CreateProcess` with "mspaint.exe" 9 | 10 | ## Window Manipulation 11 | 12 | Once the Paint window is located: 13 | 14 | 1. Bring the window to the foreground using enhanced activation methods 15 | 2. Ensure the window is not minimized using `ShowWindow` 16 | 3. Get window dimensions with `GetWindowRect` 17 | 4. Calculate the canvas area within the window based on Windows 11 Paint's modern UI layout 18 | 19 | ## Drawing Operations 20 | 21 | Drawing operations will be performed by: 22 | 23 | 1. Selecting the appropriate tool from Windows 11 Paint's modern toolbar using mouse events 24 | 2. Setting appropriate color and thickness parameters using the property panels 25 | 3. Simulating mouse movements and clicks to perform drawing actions 26 | 27 | #### Pixel Drawing 28 | 29 | To draw a single pixel: 30 | 31 | 1. The pencil tool is selected with the minimum thickness (1px) 32 | 2. If a specific color is specified, that color is set as the active color 33 | 3. A single mouse click is performed at the exact coordinates 34 | 4. For better precision, the view is temporarily zoomed in if needed 35 | 5. Care is taken to ensure the click doesn't cause any dragging motion 36 | 37 | ### Mouse Event Simulation 38 | 39 | Mouse events will be simulated using: 40 | 41 | - `SendInput` with `INPUT` structures for mouse movement 42 | - `MOUSEINPUT` structures for button clicks 43 | - Coordinate translation from client to screen coordinates using `ClientToScreen` 44 | - Normalized coordinates for accurate positioning across different screen resolutions 45 | 46 | ### Drawing Tools Selection 47 | 48 | Windows 11 Paint has a modern toolbar with these tool positions: 49 | 50 | | Tool | Position in Windows 11 UI | 51 | |------|---------------------------| 52 | | Pencil | Left side of toolbar | 53 | | Brush | Next to pencil tool | 54 | | Fill | Color fill tool in toolbar | 55 | | Text | Text tool in toolbar | 56 | | Eraser | Eraser tool in toolbar | 57 | | Select | Selection tool in toolbar | 58 | | Shapes | Shape tool with additional dropdown | 59 | 60 | ### Color Selection 61 | 62 | Colors in Windows 11 Paint will be selected by: 63 | 64 | 1. Clicking the color button in the toolbar to open the color panel 65 | 2. Selecting from the color grid in the panel 66 | 3. For custom colors, using the expanded color picker when needed 67 | 68 | ### Thickness Selection 69 | 70 | Line thickness in Windows 11 Paint is set via: 71 | 72 | 1. Accessing the properties panel on the right side 73 | 2. Clicking the thickness/size button 74 | 3. Selecting from available thickness options 75 | 76 | ### Brush Size Configuration 77 | 78 | Windows 11 Paint provides more granular control over brush sizes: 79 | 80 | 1. The MCP selects the appropriate tool (pencil or brush) 81 | 2. Accesses the properties panel on the right side 82 | 3. Locates the size slider control in the panel 83 | 4. Sets the precise pixel size by: 84 | - For predefined sizes (small/medium/large): Clicking the appropriate preset 85 | - For custom sizes: Using the slider control to set exact pixel values 86 | 5. For pixel-precise work (1px), the pencil tool with minimum size is used 87 | 6. Different tools support different size ranges: 88 | - Pencil: 1-8px 89 | - Brush: 2-30px 90 | - Specialized brushes: Various ranges 91 | 92 | The implementation maps the requested size to the closest available option in Windows 11 Paint's UI. 93 | 94 | ### Enhanced Text Support 95 | 96 | Windows 11 Paint's text tool provides rich formatting options: 97 | 98 | 1. Text is added by: 99 | - Selecting the text tool from the toolbar 100 | - Clicking at the desired position to create a text box 101 | - Typing the desired text content 102 | - Clicking elsewhere or pressing Enter to finalize 103 | 104 | 2. Font settings are configured through: 105 | - Opening the text format dialog (typically Ctrl+F or via the text properties panel) 106 | - Setting font name from the dropdown list of available system fonts 107 | - Setting font size from the size options 108 | - Selecting style options (regular, bold, italic, bold italic) 109 | - Selecting text color from the color picker 110 | - Confirming settings by clicking OK 111 | 112 | 3. Text box handling requires precise interaction: 113 | - The MCP ensures proper timing between text tool selection and click 114 | - For multi-line text, newline characters are converted to appropriate key events 115 | - The implementation handles text completion to prevent text remaining in edit mode 116 | 117 | ### Image Transformations 118 | 119 | Windows 11 Paint provides various image transformation capabilities: 120 | 121 | 1. Rotation is implemented by: 122 | - Selecting the entire canvas (Ctrl+A) 123 | - Accessing the Image or Rotate menu 124 | - Selecting the appropriate rotation option (90° clockwise/counterclockwise or 180°) 125 | - Waiting for the operation to complete 126 | 127 | 2. Flipping is implemented by: 128 | - Selecting the entire canvas (Ctrl+A) 129 | - Accessing the Image or Flip menu 130 | - Selecting horizontal or vertical flip option 131 | - Waiting for the operation to complete 132 | 133 | 3. Scaling/resizing is implemented by: 134 | - Accessing the Resize dialog through the Image menu or keyboard shortcut 135 | - Setting the desired dimensions or percentage 136 | - Setting or clearing the "Maintain aspect ratio" checkbox as needed 137 | - Confirming the resize operation 138 | - The implementation handles both pixel-based and percentage-based scaling 139 | 140 | 4. Cropping is implemented by: 141 | - Selecting the selection tool 142 | - Drawing a selection rectangle around the desired area 143 | - Triggering the crop command from the Image menu 144 | - Waiting for the operation to complete 145 | 146 | ### Canvas Management 147 | 148 | Managing the canvas in Windows 11 Paint involves: 149 | 150 | 1. Creating a new canvas by: 151 | - Sending Ctrl+N or accessing the New command from the menu 152 | - Setting dimensions in the new canvas dialog 153 | - Confirming the creation 154 | - If a background color other than white is specified: 155 | - Setting the active color to the desired background color 156 | - Selecting the fill tool 157 | - Clicking anywhere on the canvas to fill it 158 | 159 | 2. Clearing the canvas by: 160 | - Selecting all (Ctrl+A) 161 | - Pressing Delete or accessing the Clear command 162 | - This creates a blank white canvas 163 | 164 | ### File Operations 165 | 166 | #### Saving Files 167 | 168 | Paint files are saved by: 169 | 170 | 1. Sending keyboard shortcuts (Ctrl+S) or using automation to trigger the save dialog 171 | 2. Entering the file path in the save dialog 172 | 3. Selecting the appropriate file format from the dropdown 173 | 4. Confirming the save operation 174 | 175 | #### Fetching Images 176 | 177 | Saved images are retrieved using the following procedure: 178 | 179 | 1. The MCP server verifies the file exists at the specified path 180 | 2. The file is read using secure file I/O operations 181 | 3. For PNG files, the image is loaded and validated as a valid PNG 182 | 4. The image data is encoded as base64 for transfer via JSON 183 | 5. Optional metadata (dimensions, color depth) is extracted from the image header 184 | 185 | ## Technical Aspects of Windows 11 Integration 186 | 187 | ### Modern UI Layout 188 | 189 | Windows 11 Paint has a completely redesigned interface with: 190 | 191 | 1. A horizontal toolbar at the top of the window 192 | 2. Property panels that appear on the right side 193 | 3. Enhanced shape tools with fill/outline options 194 | 4. A cleaner canvas area with adjusted margins 195 | 5. Text formatting toolbar that appears when text tool is active 196 | 6. Image transformation options in the Image menu 197 | 198 | ### Handling High-DPI Displays 199 | 200 | Windows 11 has better support for high-DPI displays. The implementation: 201 | 202 | 1. Uses normalized mouse coordinates (0-65535 range) 203 | 2. Properly accounts for scaling factors 204 | 3. Adjusts click positions based on actual screen dimensions 205 | 206 | ### Improved Window Activation 207 | 208 | Windows 11 has stricter window activation policies. Our implementation: 209 | 210 | 1. Uses advanced activation techniques including Alt key simulation 211 | 2. Verifies active window status 212 | 3. Includes retry mechanisms with timeout 213 | 214 | ### Keyboard Simulation for UI Navigation 215 | 216 | Some operations require keyboard navigation: 217 | 218 | 1. Key combinations are sent using `SendInput` with `KEYEVENTF_SCANCODE` flags 219 | 2. Modifier keys (Ctrl, Alt, Shift) are properly handled using key down/up pairs 220 | 3. Special characters and menu navigation uses appropriate virtual key codes 221 | 4. Dialog interaction uses Tab, Space, and Enter for navigation and confirmation 222 | 223 | ## Dialog Interaction 224 | 225 | Dialog handling is critical for many operations: 226 | 227 | 1. Font selection dialog: 228 | - The dialog is identified by window class and title 229 | - Fields are accessed in sequence: font, style, size 230 | - Keyboard navigation moves between controls 231 | - Enter key confirms, or OK button is clicked 232 | 233 | 2. Resize canvas dialog: 234 | - Width and height fields are populated using keyboard input 235 | - Maintain aspect ratio checkbox is toggled if needed 236 | - Percentage vs. pixels mode is selected as appropriate 237 | - OK button is clicked or Enter key confirms 238 | 239 | ## Limitations 240 | 241 | 1. Operations requiring dialog interaction (like open/save) may be less reliable 242 | 2. Color matching is approximate as Paint has a predefined palette 243 | 3. Drawing complex shapes with precision may be challenging 244 | 4. Windows 11 security settings may prevent some automated interactions 245 | 5. Font availability depends on what's installed on the system 246 | 6. Some transformations may have limitations based on canvas size or available memory 247 | 7. Text alignment options may be limited by Paint's capabilities 248 | 249 | ## Future Enhancements 250 | 251 | 1. Direct bitmap manipulation for faster and more precise drawing 252 | 2. Support for Windows 11 Paint's enhanced features as they become available 253 | 3. Adaptation to Paint updates in future Windows 11 releases 254 | 4. Improved dialog handling for more robust interaction 255 | 5. Enhanced text formatting options including alignment and spacing 256 | 6. More advanced image transformations like skew and perspective -------------------------------------------------------------------------------- /src/bin/uia_test.rs: -------------------------------------------------------------------------------- 1 | // Sample file to test uiautomation crate 2 | use uiautomation::{UIAutomation, types::TreeScope}; 3 | use uiautomation::controls::{Control, ButtonControl}; 4 | use std::thread::sleep; 5 | use std::time::Duration; 6 | 7 | fn main() -> Result<(), Box> { 8 | println!("Testing uiautomation with MS Paint"); 9 | 10 | // Launch MS Paint 11 | let proc = std::process::Command::new("mspaint.exe").spawn()?; 12 | 13 | // Wait for Paint to start 14 | sleep(Duration::from_secs(2)); 15 | 16 | // Initialize UI Automation 17 | let automation = UIAutomation::new()?; 18 | let root = automation.get_root_element()?; 19 | 20 | // Find Paint window 21 | let matcher = automation.create_matcher() 22 | .from(root) 23 | .timeout(10000) 24 | .classname("MSPaintApp"); 25 | 26 | match matcher.find_first() { 27 | Ok(paint_window) => { 28 | println!("Found Paint window: {} - {}", 29 | paint_window.get_name()?, 30 | paint_window.get_classname()?); 31 | 32 | // Try to find toolbar elements 33 | // Create a condition for button type 34 | let true_condition = automation.create_true_condition()?; 35 | 36 | // Find all elements 37 | let all_elements = paint_window.find_all(TreeScope::Subtree, &true_condition)?; 38 | 39 | // Filter for button elements 40 | let buttons: Vec<_> = all_elements.into_iter() 41 | .filter(|el| { 42 | if let Ok(control_type) = el.get_control_type() { 43 | control_type == ButtonControl::TYPE 44 | } else { 45 | false 46 | } 47 | }) 48 | .collect(); 49 | 50 | println!("Found {} buttons in Paint", buttons.len()); 51 | 52 | // List button names 53 | for button in &buttons { 54 | if let Ok(name) = button.get_name() { 55 | println!("Button: {}", name); 56 | } 57 | } 58 | 59 | // Close Paint 60 | if let Ok(control) = paint_window.get_pattern::() { 61 | control.close()?; 62 | } 63 | }, 64 | Err(err) => { 65 | println!("Could not find Paint window: {}", err); 66 | } 67 | } 68 | 69 | Ok(()) 70 | } -------------------------------------------------------------------------------- /src/error.rs: -------------------------------------------------------------------------------- 1 | use thiserror::Error; 2 | 3 | #[derive(Error, Debug)] 4 | pub enum MspMcpError { 5 | #[error("General MCP Server Error: {0}")] 6 | General(String), 7 | 8 | #[error("Paint window not found")] 9 | WindowNotFound, // 1001 10 | 11 | #[error("Operation timed out: {0}")] 12 | OperationTimeout(String), // 1002 13 | 14 | #[error("Invalid parameters: {0}")] 15 | InvalidParameters(String), // 1003 16 | 17 | #[error("Invalid color format: {0}")] 18 | InvalidColorFormat(String), // 1004 19 | 20 | #[error("Invalid tool specified: {0}")] 21 | InvalidTool(String), // 1005 22 | 23 | #[error("Invalid shape specified: {0}")] 24 | InvalidShape(String), // 1006 25 | 26 | #[error("Window activation failed: {0}")] 27 | WindowActivationFailed(String), // 1007 28 | 29 | #[error("Operation not supported in Windows 11 Paint: {0}")] 30 | OperationNotSupported(String), // 1008 31 | 32 | #[error("File not found: {0}")] 33 | FileNotFound(String), // 1009 34 | 35 | #[error("Permission denied accessing file: {0}")] 36 | FilePermissionDenied(String), // 1010 37 | 38 | #[error("Invalid image format: {0}")] 39 | InvalidImageFormat(String), // 1011 40 | 41 | #[error("Text input failed: {0}")] 42 | TextInputFailed(String), // 1012 43 | 44 | #[error("Font selection failed: {0}")] 45 | FontSelectionFailed(String), // 1013 46 | 47 | #[error("Image transformation failed: {0}")] 48 | ImageTransformationFailed(String), // 1014 49 | 50 | #[error("Canvas creation failed: {0}")] 51 | CanvasCreationFailed(String), // 1015 52 | 53 | #[error("Element not found: {0}")] 54 | ElementNotFound(String), // 1016 55 | 56 | #[error("Windows API error: {0}")] 57 | WindowsApiError(String), 58 | 59 | #[error("UI Automation error: {0}")] 60 | UiAutomationError(String), 61 | 62 | #[error("IO error: {0}")] 63 | IoError(#[from] std::io::Error), 64 | 65 | #[error("JSON serialization/deserialization error: {0}")] 66 | JsonError(#[from] serde_json::Error), 67 | 68 | #[error("Base64 decoding error: {0}")] 69 | Base64DecodeError(#[from] base64::DecodeError), 70 | 71 | // Add more specific errors as needed 72 | } 73 | 74 | // Optional: Map errors to MCP error codes 75 | impl MspMcpError { 76 | pub fn code(&self) -> i32 { 77 | match self { 78 | MspMcpError::General(_) => 1000, 79 | MspMcpError::WindowNotFound => 1001, 80 | MspMcpError::OperationTimeout(_) => 1002, 81 | MspMcpError::InvalidParameters(_) => 1003, 82 | MspMcpError::InvalidColorFormat(_) => 1004, 83 | MspMcpError::InvalidTool(_) => 1005, 84 | MspMcpError::InvalidShape(_) => 1006, 85 | MspMcpError::WindowActivationFailed(_) => 1007, 86 | MspMcpError::OperationNotSupported(_) => 1008, 87 | MspMcpError::FileNotFound(_) => 1009, 88 | MspMcpError::FilePermissionDenied(_) => 1010, 89 | MspMcpError::InvalidImageFormat(_) => 1011, 90 | MspMcpError::TextInputFailed(_) => 1012, 91 | MspMcpError::FontSelectionFailed(_) => 1013, 92 | MspMcpError::ImageTransformationFailed(_) => 1014, 93 | MspMcpError::CanvasCreationFailed(_) => 1015, 94 | MspMcpError::ElementNotFound(_) => 1016, 95 | // Internal errors might map to a general code or have specific ones if needed 96 | MspMcpError::WindowsApiError(_) => 1000, 97 | MspMcpError::UiAutomationError(_) => 1000, 98 | MspMcpError::IoError(_) => 1000, 99 | MspMcpError::JsonError(_) => 1000, 100 | MspMcpError::Base64DecodeError(_) => 1003, // Map to invalid params maybe? 101 | } 102 | } 103 | } 104 | 105 | // Implement From for UIAutomation errors 106 | impl From for MspMcpError { 107 | fn from(err: uiautomation::Error) -> Self { 108 | MspMcpError::UiAutomationError(format!("{}", err)) 109 | } 110 | } 111 | 112 | pub type Result = std::result::Result; -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | use mcp_rust_sdk::{ 2 | error::{Error as SdkError, ErrorCode}, 3 | server::{ServerHandler, Server}, 4 | transport::stdio::StdioTransport, 5 | types::{ClientCapabilities, ServerCapabilities, Implementation}, 6 | }; 7 | use log::{info, error, LevelFilter, debug, warn}; 8 | use tokio::runtime::Runtime; 9 | use std::sync::Arc; 10 | use std::sync::Mutex; 11 | use windows_sys::Win32::Foundation::HWND; 12 | use std::process::Command; 13 | use std::io::{self, Write}; 14 | 15 | // Define modules 16 | pub mod error; 17 | pub mod protocol; 18 | pub mod windows; 19 | pub mod core; 20 | pub mod uia; 21 | 22 | use crate::error::{Result, MspMcpError}; 23 | 24 | // Helper function to log process tree (Windows specific for now) 25 | fn log_process_tree(label: &str) { 26 | if cfg!(target_os = "windows") { 27 | debug!("Capturing process tree ({}) using tasklist...", label); 28 | match Command::new("tasklist").arg("/V").output() { 29 | Ok(output) => { 30 | if output.status.success() { 31 | match String::from_utf8(output.stdout) { 32 | Ok(stdout_str) => debug!("Process Tree ({}):\n{}", label, stdout_str), 33 | Err(e) => warn!("Failed to decode tasklist stdout: {}", e), 34 | } 35 | } else { 36 | warn!("tasklist command failed with status: {}", output.status); 37 | if let Ok(stderr_str) = String::from_utf8(output.stderr) { 38 | warn!("tasklist stderr:\n{}", stderr_str); 39 | } 40 | } 41 | } 42 | Err(e) => { 43 | warn!("Failed to execute tasklist command: {}", e); 44 | } 45 | } 46 | } else { 47 | debug!("Process tree logging not implemented for this OS."); 48 | } 49 | } 50 | 51 | // Define a struct to hold our server state 52 | #[derive(Clone)] 53 | pub struct PaintServerState { 54 | pub paint_hwnd: Arc>>, // Store HWND in Arc 55 | } 56 | 57 | impl PaintServerState { 58 | pub fn new() -> Self { 59 | PaintServerState { 60 | paint_hwnd: Arc::new(Mutex::new(None)), 61 | } 62 | } 63 | } 64 | 65 | // Implement the server handler trait from mcp_rust_sdk 66 | #[async_trait::async_trait] 67 | impl ServerHandler for PaintServerState { 68 | 69 | // Required method: initialize 70 | async fn initialize(&self, _implementation: Implementation, _client_capabilities: ClientCapabilities) -> std::result::Result { 71 | info!("Server received initialize request. Finding/Launching Paint..."); 72 | 73 | // --- Log process tree BEFORE attempting launch --- 74 | log_process_tree("Before Paint Find/Launch"); 75 | // ----------------------------------------------- 76 | 77 | // --- Start: Logic moved from handle_connect --- 78 | match crate::windows::get_paint_hwnd() { 79 | Ok(hwnd) => { 80 | // Store the HWND in the shared state 81 | let mut hwnd_state = self.paint_hwnd.lock() 82 | .map_err(|_| SdkError::protocol(ErrorCode::InternalError, "Failed to lock HWND state".to_string()))?; 83 | *hwnd_state = Some(hwnd); 84 | info!("Stored Paint HWND: {}", hwnd); 85 | 86 | // --- Log process tree AFTER successful find/launch --- 87 | log_process_tree("After Paint Find/Launch"); 88 | // ----------------------------------------------------- 89 | } 90 | Err(e) => { 91 | // Log process tree on failure too 92 | log_process_tree("After Paint Find/Launch Failure"); 93 | 94 | // If we can't get the HWND during init, it's a fatal error for this server 95 | let error_msg = format!("Failed to find or launch Paint during initialization: {}", e); 96 | error!("{}", error_msg); 97 | // Convert our error to an SdkError for the initialize response 98 | return Err(SdkError::protocol(ErrorCode::InternalError, error_msg)); 99 | } 100 | } 101 | // --- End: Logic moved from handle_connect --- 102 | 103 | // Return default capabilities (or customize later if needed) 104 | info!("Paint found/launched. Initialization successful."); 105 | Ok(ServerCapabilities::default()) 106 | } 107 | 108 | // Required method: shutdown 109 | async fn shutdown(&self) -> std::result::Result<(), SdkError> { 110 | info!("Server received shutdown request."); 111 | // TODO: Perform cleanup if necessary 112 | Ok(()) 113 | } 114 | 115 | // Required method: handle_method 116 | async fn handle_method(&self, method: &str, params: Option) -> std::result::Result { 117 | info!("Handling method: {} with params: {:?}", method, params); 118 | 119 | // Route request to appropriate async handler in `core` module 120 | // Pass the cloned state to the handler 121 | let result: std::result::Result = match method { 122 | "initialize" => { 123 | core::handle_initialize(self.clone(), params).await 124 | } 125 | "connect" => { 126 | core::handle_connect(self.clone(), params).await 127 | } 128 | "disconnect" => { 129 | core::handle_disconnect(self.clone(), params).await 130 | } 131 | "get_version" => { 132 | core::handle_get_version(self.clone(), params).await 133 | } 134 | "activate_window" => { 135 | core::handle_activate_window(self.clone(), params).await 136 | } 137 | "get_canvas_dimensions" => { 138 | core::handle_get_canvas_dimensions(self.clone(), params).await 139 | } 140 | "draw_pixel" => { 141 | core::handle_draw_pixel(self.clone(), params).await 142 | } 143 | "draw_line" => { 144 | core::handle_draw_line(self.clone(), params).await 145 | } 146 | "draw_shape" => { 147 | core::handle_draw_shape(self.clone(), params).await 148 | } 149 | "draw_polyline" => { 150 | core::handle_draw_polyline(self.clone(), params).await 151 | } 152 | "set_color" => { 153 | core::handle_set_color(self.clone(), params).await 154 | } 155 | "set_thickness" => { 156 | core::handle_set_thickness(self.clone(), params).await 157 | } 158 | "set_fill" => { 159 | core::handle_set_fill(self.clone(), params).await 160 | } 161 | "select_tool" => { 162 | core::handle_select_tool(self.clone(), params).await 163 | } 164 | // Add other method handlers here, calling functions in core.rs 165 | _ => { 166 | Err(MspMcpError::OperationNotSupported(format!("Method '{}' not implemented", method))) 167 | } 168 | }; 169 | 170 | // Convert our Result to Result 171 | match result { 172 | Ok(value) => { 173 | // Just return the value since the SDK should handle adding jsonrpc and id 174 | Ok(value) 175 | }, 176 | Err(msp_error) => { 177 | let code = msp_error.code(); // Keep our internal code for logging 178 | let message = msp_error.to_string(); 179 | error!("Error processing method '{}': Code {}, Message: {}", method, code, message); 180 | 181 | // Convert to a SdkError which the SDK will format as a proper JSON-RPC error 182 | Err(SdkError::Protocol { 183 | code: ErrorCode::InternalError, 184 | message: message, 185 | data: None, 186 | }) 187 | } 188 | } 189 | } 190 | } 191 | 192 | // Main entry point function 193 | pub fn run_server() -> Result<()> { 194 | // Remove the env_logger initialization since we're using simplelog now 195 | // env_logger::Builder::from_default_env() 196 | // .filter_level(LevelFilter::Info) 197 | // .try_init().map_err(|e| MspMcpError::General(format!("Failed to init logger: {}", e)))?; 198 | 199 | info!("Starting MCP Server for Windows 11 Paint (Async Version)..."); 200 | 201 | let rt = Runtime::new().map_err(|e| MspMcpError::IoError(e))?; 202 | 203 | rt.block_on(async { 204 | let initial_state = PaintServerState::new(); 205 | let (transport, _handler_connection) = StdioTransport::new(); // handler_connection might not be needed here 206 | 207 | let handler = Arc::new(initial_state); 208 | let transport_arc = Arc::new(transport); 209 | 210 | // Correct Server::new call (takes transport and handler) 211 | let server = Server::new(transport_arc.clone(), handler.clone()); 212 | 213 | info!("MCP Server starting run loop..."); 214 | 215 | // Use server.start() to run the server loop 216 | if let Err(e) = server.start().await { 217 | error!("MCP Server run failed: {}", e); 218 | // Attempt to downcast the SDK error or format it 219 | let error_message = format!("Server run failed: {}", e); 220 | return Err(MspMcpError::General(error_message)); 221 | } 222 | 223 | info!("MCP Server finished."); 224 | Ok(()) 225 | }) 226 | } 227 | 228 | 229 | #[cfg(test)] 230 | mod tests { 231 | // use super::*; // Keep this if testing functions within lib.rs directly 232 | 233 | #[test] 234 | fn it_works() { 235 | let result = 2 + 2; 236 | assert_eq!(result, 4); 237 | } 238 | 239 | // Tests for async functions require #[tokio::test] 240 | // Add tests for core handlers in core.rs 241 | // Add tests for protocol structs in protocol.rs 242 | // Add tests for windows functions in windows.rs 243 | } 244 | -------------------------------------------------------------------------------- /src/main.rs: -------------------------------------------------------------------------------- 1 | use mcp_server_microsoft_paint::PaintServerState; 2 | use mcp_rust_sdk::server::ServerHandler; 3 | use mcp_rust_sdk::transport::stdio::StdioTransport; 4 | use std::process; 5 | use log::{info, error, debug}; 6 | use simplelog::{CombinedLogger, Config, ConfigBuilder, TermLogger, WriteLogger, TerminalMode, ColorChoice, LevelFilter}; 7 | use std::fs::File; 8 | use std::sync::Once; 9 | use std::path::PathBuf; 10 | use std::env; 11 | use std::io; 12 | use serde_json; 13 | 14 | // Use a Once to ensure we only initialize the logger once 15 | static LOGGER_INIT: Once = Once::new(); 16 | 17 | #[tokio::main] 18 | async fn main() -> Result<(), Box> { 19 | // Initialize the logger 20 | init_logger(); 21 | 22 | info!("Starting MCP Server for Windows 11 Paint..."); 23 | 24 | // Print version information 25 | let version = env!("CARGO_PKG_VERSION"); 26 | info!("MCP Server version: {}", version); 27 | 28 | // Run the JSON-RPC server 29 | run_server_async().await?; 30 | 31 | info!("MCP Server shutting down"); 32 | Ok(()) 33 | } 34 | 35 | // The main run loop for the server 36 | async fn run_server_async() -> Result<(), Box> { 37 | info!("MCP Server starting run loop..."); 38 | 39 | // Create the Paint server state 40 | let paint_server = PaintServerState::new(); 41 | 42 | let mut buffer = String::new(); 43 | 44 | loop { 45 | // Reset the buffer for the next request 46 | buffer.clear(); 47 | 48 | // Read a line from stdin 49 | match io::stdin().read_line(&mut buffer) { 50 | Ok(0) => { 51 | // End of input (Ctrl+D or stream closed) 52 | info!("End of input - server shutting down"); 53 | break; 54 | } 55 | Ok(_) => { 56 | // Process the received JSON-RPC request 57 | if let Some(parsed_request) = parse_json_rpc_request(&buffer) { 58 | // If parsing successful, handle the request 59 | info!("Received request: {}", parsed_request.trim()); 60 | 61 | // Extract method and params 62 | match extract_method_and_params(&parsed_request) { 63 | Ok((method, params, id)) => { 64 | // Handle the method call 65 | debug!("Handling method: {}, params: {:?}", method, params); 66 | 67 | let result = paint_server.clone().handle_method(&method, params).await; 68 | 69 | // Send the result back as a JSON-RPC response 70 | match result { 71 | Ok(response) => { 72 | // Make sure the response has the correct ID 73 | let mut response_obj = response.as_object().unwrap_or(&serde_json::Map::new()).clone(); 74 | response_obj.insert("id".to_string(), id); 75 | 76 | if !response_obj.contains_key("jsonrpc") { 77 | response_obj.insert("jsonrpc".to_string(), serde_json::Value::String("2.0".to_string())); 78 | } 79 | 80 | let response_json = serde_json::to_string(&response_obj)?; 81 | println!("{}", response_json); 82 | } 83 | Err(e) => { 84 | let error_response = serde_json::json!({ 85 | "jsonrpc": "2.0", 86 | "id": id, 87 | "error": { 88 | "code": -32603, // Internal error 89 | "message": e.to_string() 90 | } 91 | }); 92 | println!("{}", serde_json::to_string(&error_response)?); 93 | } 94 | } 95 | } 96 | Err(e) => { 97 | let error_response = serde_json::json!({ 98 | "jsonrpc": "2.0", 99 | "id": null, 100 | "error": { 101 | "code": -32600, // Invalid request 102 | "message": e 103 | } 104 | }); 105 | println!("{}", serde_json::to_string(&error_response)?); 106 | } 107 | } 108 | } 109 | } 110 | Err(e) => { 111 | // Handle read errors 112 | error!("Error reading from stdin: {}", e); 113 | break; 114 | } 115 | } 116 | } 117 | 118 | Ok(()) 119 | } 120 | 121 | // Parse a string as a JSON-RPC request 122 | fn parse_json_rpc_request(input: &str) -> Option { 123 | let trimmed = input.trim(); 124 | if trimmed.is_empty() { 125 | return None; 126 | } 127 | 128 | match serde_json::from_str::(trimmed) { 129 | Ok(json) => { 130 | // Just verify this is an object - more detailed checking 131 | // happens in extract_method_and_params 132 | if json.is_object() { 133 | Some(trimmed.to_string()) 134 | } else { 135 | error!("Invalid JSON-RPC request: Not an object"); 136 | None 137 | } 138 | } 139 | Err(e) => { 140 | error!("Failed to parse JSON-RPC request: {}", e); 141 | None 142 | } 143 | } 144 | } 145 | 146 | // Extract method and params from JSON-RPC request 147 | fn extract_method_and_params(request_str: &str) -> Result<(String, Option, serde_json::Value), String> { 148 | // Parse the request 149 | let request: serde_json::Value = serde_json::from_str(request_str) 150 | .map_err(|e| format!("Invalid JSON: {}", e))?; 151 | 152 | // Check this is a JSON-RPC 2.0 request object 153 | let obj = request.as_object() 154 | .ok_or_else(|| "Request must be a JSON object".to_string())?; 155 | 156 | // Extract the JSON-RPC version (optional check) 157 | if let Some(version) = obj.get("jsonrpc") { 158 | if version != "2.0" { 159 | return Err("Only JSON-RPC 2.0 is supported".to_string()); 160 | } 161 | } 162 | 163 | // Extract the method 164 | let method = obj.get("method") 165 | .ok_or_else(|| "Missing 'method' field".to_string())? 166 | .as_str() 167 | .ok_or_else(|| "'method' must be a string".to_string())? 168 | .to_string(); 169 | 170 | // Extract the params (optional) 171 | let params = obj.get("params").cloned(); 172 | 173 | // Extract the id (or use default) 174 | let id = obj.get("id").unwrap_or(&serde_json::Value::Null).clone(); 175 | 176 | Ok((method, params, id)) 177 | } 178 | 179 | // Initialize the logger 180 | fn init_logger() { 181 | // Initialize logger exactly once 182 | LOGGER_INIT.call_once(|| { 183 | let log_level = LevelFilter::Debug; // Log debug level and above 184 | let log_file_path = env::temp_dir().join("mcp_server_debug.log"); 185 | 186 | if let Ok(log_file) = File::create(&log_file_path) { 187 | let config = ConfigBuilder::new() 188 | .set_time_format_rfc3339() 189 | .build(); 190 | 191 | let write_logger = WriteLogger::new( 192 | log_level, 193 | config.clone(), 194 | log_file 195 | ); 196 | 197 | // Log to stderr instead of stdout to avoid interfering with JSON-RPC 198 | let term_logger = TermLogger::new( 199 | LevelFilter::Info, 200 | config, 201 | TerminalMode::Stderr, // Use Stderr instead of Mixed mode 202 | ColorChoice::Auto 203 | ); 204 | 205 | if let Err(e) = CombinedLogger::init(vec![term_logger, write_logger]) { 206 | eprintln!("Failed to initialize combined logger: {}", e); // Fallback 207 | } 208 | 209 | info!("Logging initialized. Debug logs writing to: {:?}", log_file_path); 210 | 211 | } else { 212 | eprintln!("Failed to create log file at {:?}, logging to stderr only.", log_file_path); 213 | if let Err(e) = TermLogger::init( 214 | log_level, 215 | ConfigBuilder::new().build(), 216 | TerminalMode::Stderr, // Use Stderr instead of Mixed mode 217 | ColorChoice::Auto 218 | ) { 219 | eprintln!("Failed to initialize terminal logger: {}", e); // Fallback 220 | } 221 | } 222 | }); 223 | } -------------------------------------------------------------------------------- /src/protocol.rs: -------------------------------------------------------------------------------- 1 | use serde::{Deserialize, Serialize}; 2 | use serde_json::Value; 3 | use serde_json::json; 4 | use crate::error::Result; 5 | use crate::core; 6 | 7 | // Define handler type using Box to allow storing async functions 8 | // This avoids type issues with different impl Future types 9 | pub type MethodHandler = Box) -> 10 | futures::future::BoxFuture<'static, Result> + Send + Sync>; 11 | 12 | // Function to box the handlers properly to match the type 13 | fn box_handler(f: F) -> MethodHandler 14 | where 15 | F: Fn(crate::PaintServerState, Option) -> Fut + Send + Sync + 'static, 16 | Fut: futures::Future> + Send + 'static, 17 | { 18 | Box::new(move |state, value| Box::pin(f(state, value))) 19 | } 20 | 21 | // === Request Parameters === 22 | 23 | #[derive(Deserialize, Debug)] 24 | pub struct ConnectParams { 25 | pub client_id: String, 26 | pub client_name: String, 27 | } 28 | 29 | #[derive(Deserialize, Debug)] 30 | pub struct SelectToolParams { 31 | pub tool: String, // Consider using an enum later: "pencil|brush|fill|text|eraser|select|shape" 32 | pub shape_type: Option, // Consider enum: "rectangle|ellipse|line|..." 33 | } 34 | 35 | #[derive(Deserialize, Debug)] 36 | pub struct SetColorParams { 37 | pub color: String, // Expecting "#RRGGBB" 38 | } 39 | 40 | #[derive(Deserialize, Debug)] 41 | pub struct SetThicknessParams { 42 | pub level: u32, // Expecting 1-5 43 | } 44 | 45 | #[derive(Deserialize, Debug)] 46 | pub struct SetBrushSizeParams { 47 | pub size: u32, // Expecting 1-30 48 | pub tool: Option, // Consider enum: "pencil|brush" 49 | } 50 | 51 | #[derive(Deserialize, Debug)] 52 | pub struct SetFillParams { 53 | pub fill_type: String, // Expecting "none|solid|outline" 54 | } 55 | 56 | #[derive(Deserialize, Debug)] 57 | pub struct DrawPixelParams { 58 | pub x: i32, 59 | pub y: i32, 60 | pub color: Option, // Optional color in #RRGGBB format 61 | } 62 | 63 | #[derive(Deserialize, Debug)] 64 | pub struct DrawLineParams { 65 | pub start_x: i32, 66 | pub start_y: i32, 67 | pub end_x: i32, 68 | pub end_y: i32, 69 | pub color: Option, // Optional color in #RRGGBB format 70 | pub thickness: Option, // Optional thickness level (1-5) 71 | } 72 | 73 | #[derive(Deserialize, Debug)] 74 | pub struct DrawShapeParams { 75 | pub shape_type: String, // "rectangle|ellipse|line|arrow|triangle|pentagon|hexagon" 76 | pub start_x: i32, 77 | pub start_y: i32, 78 | pub end_x: i32, 79 | pub end_y: i32, 80 | pub color: Option, // Optional color in #RRGGBB format 81 | pub thickness: Option, // Optional thickness level (1-5) 82 | pub fill_type: Option, // Optional fill type "none|solid|outline" 83 | } 84 | 85 | #[derive(Deserialize, Debug)] 86 | pub struct DrawPolylineParams { 87 | pub points: Vec, // Series of points to connect 88 | pub color: Option, // Optional color in #RRGGBB format 89 | pub thickness: Option, // Optional thickness level (1-5) 90 | pub tool: Option, // Optional tool: "pencil" or "brush" 91 | } 92 | 93 | #[derive(Deserialize, Debug)] 94 | pub struct AddTextParams { 95 | pub x: i32, // X position to place text 96 | pub y: i32, // Y position to place text 97 | pub text: String, // Text content to add 98 | pub color: Option, // Optional color in #RRGGBB format 99 | pub font_name: Option, // Optional font name 100 | pub font_size: Option, // Optional font size 101 | pub font_style: Option, // Optional style: "regular", "bold", "italic", "bold_italic" 102 | } 103 | 104 | #[derive(Deserialize, Debug)] 105 | pub struct CreateCanvasParams { 106 | pub width: u32, // Canvas width in pixels 107 | pub height: u32, // Canvas height in pixels 108 | pub background_color: Option, // Optional background color in #RRGGBB format 109 | } 110 | 111 | #[derive(Deserialize, Debug)] 112 | pub struct SaveCanvasParams { 113 | pub file_path: String, // Path where to save the file 114 | pub format: String, // Format - "png", "jpeg", or "bmp" 115 | } 116 | 117 | #[derive(Deserialize, Debug)] 118 | pub struct Point { 119 | pub x: i32, 120 | pub y: i32, 121 | } 122 | 123 | // Add more request parameter structs here... 124 | // e.g., DrawLineParams, DrawPixelParams, AddTextParams, etc. 125 | 126 | // === Response Payloads === 127 | 128 | #[derive(Serialize, Debug)] 129 | pub struct SuccessResponse { 130 | pub status: String, // Always "success" 131 | } 132 | 133 | #[derive(Serialize, Debug)] 134 | pub struct ConnectResponse { 135 | pub status: String, // Always "success" 136 | pub paint_version: String, 137 | pub canvas_width: u32, 138 | pub canvas_height: u32, 139 | } 140 | 141 | #[derive(Serialize, Debug)] 142 | pub struct GetVersionResponse { 143 | pub status: String, // Always "success" 144 | pub protocol_version: String, 145 | pub server_version: String, 146 | pub paint_version: String, 147 | } 148 | 149 | #[derive(Serialize, Debug)] 150 | pub struct ErrorResponse { 151 | pub status: String, // Always "error" 152 | pub error: ErrorDetails, 153 | } 154 | 155 | #[derive(Serialize, Debug)] 156 | pub struct ErrorDetails { 157 | pub code: i32, 158 | pub message: String, 159 | } 160 | 161 | // Add more response structs here... 162 | // e.g., GetCanvasDimensionsResponse, FetchImageResponse, etc. 163 | 164 | 165 | // === Utility === 166 | 167 | // Helper function to create a standard success response 168 | pub fn success_response() -> serde_json::Value { 169 | json!({ 170 | "jsonrpc": "2.0", 171 | "id": 1, // Default id, should be overridden when needed 172 | "result": {} // Empty result object for simple success responses 173 | }) 174 | } 175 | 176 | // Helper function to create a standard error response 177 | pub fn error_response(code: i32, message: String) -> serde_json::Value { 178 | json!({ 179 | "jsonrpc": "2.0", 180 | "id": null, // Use null for errors where id is unknown 181 | "error": { 182 | "code": code, 183 | "message": message 184 | } 185 | }) 186 | } 187 | 188 | // Basic tests for struct serialization/deserialization 189 | #[cfg(test)] 190 | mod tests { 191 | use super::*; 192 | 193 | #[test] 194 | fn test_connect_params_deserialization() { 195 | let json = r#"{ 196 | "client_id": "test-client", 197 | "client_name": "Test App" 198 | }"#; 199 | let params: ConnectParams = serde_json::from_str(json).unwrap(); 200 | assert_eq!(params.client_id, "test-client"); 201 | assert_eq!(params.client_name, "Test App"); 202 | } 203 | 204 | #[test] 205 | fn test_connect_response_serialization() { 206 | let response = ConnectResponse { 207 | status: "success".to_string(), 208 | paint_version: "windows11".to_string(), 209 | canvas_width: 1024, 210 | canvas_height: 768, 211 | }; 212 | let json = serde_json::to_string(&response).unwrap(); 213 | assert!(json.contains("\"status\":\"success\"")); 214 | assert!(json.contains("\"paint_version\":\"windows11\"")); 215 | assert!(json.contains("\"canvas_width\":1024")); 216 | assert!(json.contains("\"canvas_height\":768")); 217 | } 218 | 219 | #[test] 220 | fn test_error_response_serialization() { 221 | let response_val = error_response(1001, "Window not found".to_string()); 222 | let json_string = serde_json::to_string(&response_val).unwrap(); 223 | assert!(json_string.contains("\"status\":\"error\"")); 224 | assert!(json_string.contains("\"code\":1001")); 225 | assert!(json_string.contains("\"message\":\"Window not found\"")); 226 | } 227 | 228 | #[test] 229 | fn test_draw_polyline_params_deserialization() { 230 | let json = r###"{ 231 | "points": [ 232 | {"x": 10, "y": 20}, 233 | {"x": 30, "y": 40}, 234 | {"x": 50, "y": 60} 235 | ], 236 | "color": "#FF0000", 237 | "thickness": 2, 238 | "tool": "pencil" 239 | }"###; 240 | 241 | let params: DrawPolylineParams = serde_json::from_str(json).unwrap(); 242 | 243 | assert_eq!(params.points.len(), 3); 244 | assert_eq!(params.points[0].x, 10); 245 | assert_eq!(params.points[0].y, 20); 246 | assert_eq!(params.points[1].x, 30); 247 | assert_eq!(params.points[1].y, 40); 248 | assert_eq!(params.points[2].x, 50); 249 | assert_eq!(params.points[2].y, 60); 250 | 251 | assert_eq!(params.color.as_deref(), Some("#FF0000")); 252 | assert_eq!(params.thickness, Some(2)); 253 | assert_eq!(params.tool.as_deref(), Some("pencil")); 254 | } 255 | 256 | // Add more tests for other structs... 257 | } 258 | 259 | // Map of method names to handler functions 260 | pub fn get_method_handler(method: &str) -> Option { 261 | match method { 262 | "initialize" => Some(box_handler(core::handle_initialize)), 263 | "connect" => Some(box_handler(core::handle_connect)), 264 | "activate_window" => Some(box_handler(core::handle_activate_window)), 265 | "get_canvas_dimensions" => Some(box_handler(core::handle_get_canvas_dimensions)), 266 | "disconnect" => Some(box_handler(core::handle_disconnect)), 267 | "get_version" => Some(box_handler(core::handle_get_version)), 268 | // Drawing commands 269 | "draw_pixel" => Some(box_handler(core::handle_draw_pixel)), 270 | "draw_line" => Some(box_handler(core::handle_draw_line)), 271 | "draw_shape" => Some(box_handler(core::handle_draw_shape)), 272 | "draw_polyline" => Some(box_handler(core::handle_draw_polyline)), 273 | // Text operations 274 | "add_text" => Some(box_handler(core::handle_add_text)), 275 | // Selection operations 276 | "select_region" => Some(box_handler(core::handle_select_region)), 277 | "copy_selection" => Some(box_handler(core::handle_copy_selection)), 278 | "paste" => Some(box_handler(core::handle_paste)), 279 | // Canvas operations 280 | "clear_canvas" => Some(box_handler(core::handle_clear_canvas)), 281 | "create_canvas" => Some(box_handler(core::handle_create_canvas)), 282 | // Tool settings 283 | "select_tool" => Some(box_handler(core::handle_select_tool)), 284 | "set_color" => Some(box_handler(core::handle_set_color)), 285 | "set_thickness" => Some(box_handler(core::handle_set_thickness)), 286 | "set_brush_size" => Some(box_handler(core::handle_set_brush_size)), 287 | "set_fill" => Some(box_handler(core::handle_set_fill)), 288 | // Unknown method 289 | _ => None, 290 | } 291 | } -------------------------------------------------------------------------------- /super_simple_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import json 3 | import subprocess 4 | import time 5 | import sys 6 | import os 7 | 8 | def main(): 9 | # First, make sure Paint is not running 10 | os.system('taskkill /f /im mspaint.exe 2>nul') 11 | time.sleep(1) 12 | 13 | # Launch Paint 14 | print("Launching MS Paint...") 15 | subprocess.Popen(["mspaint.exe"]) 16 | time.sleep(3) # Wait for Paint to start 17 | 18 | # Start the server in release mode 19 | print("Starting MCP server...") 20 | server_process = subprocess.Popen( 21 | ["cargo", "run", "--release"], 22 | stdin=subprocess.PIPE, 23 | stdout=subprocess.PIPE, 24 | stderr=open("latest_server_log.txt", "w"), 25 | text=True, 26 | bufsize=1 27 | ) 28 | 29 | try: 30 | # Step 1: Initialize 31 | print("Step 1: Sending initialize request...") 32 | init_request = { 33 | "jsonrpc": "2.0", 34 | "id": 1, 35 | "method": "initialize", 36 | "params": {} 37 | } 38 | 39 | send_request(server_process, init_request) 40 | 41 | # Step 2: Connect 42 | print("Step 2: Sending connect request...") 43 | connect_request = { 44 | "jsonrpc": "2.0", 45 | "id": 2, 46 | "method": "connect", 47 | "params": { 48 | "client_id": "super_simple_test", 49 | "client_name": "Super Simple Test" 50 | } 51 | } 52 | 53 | send_request(server_process, connect_request) 54 | 55 | # Step 3: Draw a line 56 | print("Step 3: Sending draw_line request...") 57 | line_request = { 58 | "jsonrpc": "2.0", 59 | "id": 3, 60 | "method": "draw_line", 61 | "params": { 62 | "start_x": 100, 63 | "start_y": 200, 64 | "end_x": 400, 65 | "end_y": 200, 66 | "color": "#FF0000", # Red color 67 | "thickness": 5 # Thick line 68 | } 69 | } 70 | 71 | send_request(server_process, line_request) 72 | 73 | print("Test completed! Keeping Paint open to observe results...") 74 | print("Press Enter to close the test...") 75 | input() 76 | 77 | except Exception as e: 78 | print(f"Test failed with error: {e}") 79 | finally: 80 | server_process.terminate() 81 | print("Server process terminated") 82 | 83 | def send_request(process, request): 84 | """Send a request to the server and print the response.""" 85 | request_str = json.dumps(request) + "\n" 86 | print(f"Sending: {request_str.strip()}") 87 | 88 | process.stdin.write(request_str) 89 | process.stdin.flush() 90 | 91 | # Read response with timeout 92 | start_time = time.time() 93 | timeout = 10 # seconds 94 | 95 | while time.time() - start_time < timeout: 96 | line = process.stdout.readline().strip() 97 | if line: 98 | print(f"Response: {line}") 99 | try: 100 | return json.loads(line) 101 | except json.JSONDecodeError: 102 | print(f"Warning: Received non-JSON response: {line}") 103 | time.sleep(0.1) 104 | 105 | print("Warning: No response received within timeout") 106 | return None 107 | 108 | if __name__ == "__main__": 109 | main() --------------------------------------------------------------------------------