├── .gitignore
├── command_wizard.talon
├── docs
    └── video-demo-thumb.png
├── command_wizard_open.talon
├── TODO
├── LICENSE
├── marker_ui_open.talon
├── marker_ui_actions.py
├── README.md
├── ui_widgets.py
├── marker_ui.py
├── blob_detector.py
├── command_wizard.py
├── mouse_helper.py
└── overlays.py


/.gitignore:
--------------------------------------------------------------------------------
1 | *.sw?
2 | 


--------------------------------------------------------------------------------
/command_wizard.talon:
--------------------------------------------------------------------------------
1 | command wizard show: user.command_wizard_show()
2 | 


--------------------------------------------------------------------------------
/docs/video-demo-thumb.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/splondike/talon_ui_helper/HEAD/docs/video-demo-thumb.png


--------------------------------------------------------------------------------
/command_wizard_open.talon:
--------------------------------------------------------------------------------
1 | tag: user.command_wizard_showing
2 | -
3 | choose <number_small>: user.command_wizard_choose_option(number_small)
4 | command wizard hide: user.command_wizard_hide()
5 | 


--------------------------------------------------------------------------------
/TODO:
--------------------------------------------------------------------------------
1 | * GUI macro mode, perhaps saves the macro as a talon file in a given location.
2 | * Voice command to open the most likely .talon file that you want to add a voice command to (based on the title).
3 | * Keyboard commands to adjust the minimum spacing in the blob detector
4 | * Keyboard commands to adjust the thresholding in the image selector
5 | * Action to attempt to automatically remove unused templates in the image_templates directory by parsing usage from .talon files.
6 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright (c) 2021 Stefan Schneider
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/marker_ui_open.talon:
--------------------------------------------------------------------------------
 1 | tag: user.marker_ui_showing
 2 | -
 3 | marker hide:
 4 |     user.marker_ui_hide()
 5 | 
 6 | jump <user.marker_ui_label>:
 7 |     user.marker_ui_mouse_move(marker_ui_label)
 8 |     user.marker_ui_hide()
 9 | 
10 | touch <user.marker_ui_label>:
11 |     user.marker_ui_mouse_move(marker_ui_label)
12 |     mouse_click(0)
13 |     user.marker_ui_hide()
14 | 
15 | touch <user.marker_ui_label> restore:
16 |     user.mouse_helper_position_save()
17 |     user.marker_ui_mouse_move(marker_ui_label)
18 |     mouse_click(0)
19 |     user.marker_ui_hide()
20 |     user.mouse_helper_position_restore()
21 | 
22 | righty <user.marker_ui_label>:
23 |     user.marker_ui_mouse_move(marker_ui_label)
24 |     mouse_click(1)
25 |     user.marker_ui_hide()
26 | 
27 | midclick <user.marker_ui_label>:
28 |     user.marker_ui_mouse_move(marker_ui_label)
29 |     mouse_click(2)
30 |     user.marker_ui_hide()
31 | 
32 | # Versions that don't close the overlay
33 | 
34 | jump <user.marker_ui_label> more:
35 |     user.marker_ui_mouse_move(marker_ui_label)
36 | 
37 | touch <user.marker_ui_label> more:
38 |     user.marker_ui_mouse_move(marker_ui_label)
39 |     mouse_click(0)
40 | 
41 | touch <user.marker_ui_label> more restore:
42 |     user.mouse_helper_position_save()
43 |     user.marker_ui_mouse_move(marker_ui_label)
44 |     mouse_click(0)
45 |     user.mouse_helper_position_restore()
46 | 
47 | righty <user.marker_ui_label> more:
48 |     user.marker_ui_mouse_move(marker_ui_label)
49 |     mouse_click(1)
50 | 
51 | midclick <user.marker_ui_label> more:
52 |     user.marker_ui_mouse_move(marker_ui_label)
53 |     mouse_click(2)
54 | 


--------------------------------------------------------------------------------
/marker_ui_actions.py:
--------------------------------------------------------------------------------
 1 | """
 2 | Exposes the Marker UI and associated functionality as actions.
 3 | """
 4 | 
 5 | from typing import List
 6 | 
 7 | from talon import Module, Context, actions, settings
 8 | from talon.types import Rect as TalonRect
 9 | 
10 | from .marker_ui import MarkerUi
11 | 
12 | 
13 | mod = Module()
14 | mod.tag("marker_ui_showing", desc="The marker UI labels are showing")
15 | setting_labels = mod.setting(
16 |     "marker_ui_labels",
17 |     type=str,
18 |     desc="Space separated labels to use in the marker UI. See also marker_ui_label capture if overwriting.",
19 |     default=" ".join("abcdefghijklmnopqrstuvwxyz0123456789")
20 | )
21 | 
22 | ctx = Context()
23 | 
24 | marker_ui = None
25 | 
26 | 
27 | @mod.capture(rule="<user.letter> | <user.number>")
28 | def marker_ui_label(m) -> str:
29 |     """
30 |     Capture for the labels used in the marker UI. See also marker_ui_labels setting if you want
31 |     to override it.
32 |     """
33 | 
34 |     return str(m)
35 | 
36 | 
37 | @mod.action_class
38 | class MarkerUiActions:
39 |     """
40 |     Actions related to showing, hiding, and using the marker UI interface.
41 |     """
42 | 
43 |     def marker_ui_show(rects: List[TalonRect]):
44 |         """
45 |         Shows the given markers in the Marker UI. They can then be clicked or moved
46 |         to using other actions in this class.
47 |         """
48 | 
49 |         global marker_ui
50 | 
51 |         if marker_ui is not None:
52 |             marker_ui.destroy()
53 | 
54 |         markers = [
55 |             MarkerUi.Marker(rect, label)
56 |             for rect, label in zip(
57 |                 rects, settings.get("user.marker_ui_labels").split(" ")
58 |             )
59 |         ]
60 | 
61 |         marker_ui = MarkerUi(markers)
62 | 
63 |         marker_ui.show()
64 |         ctx.tags = ["user.marker_ui_showing"]
65 | 
66 |     def marker_ui_hide():
67 |         """
68 |         Hides any visible marker UI
69 |         """
70 | 
71 |         global marker_ui
72 | 
73 |         if marker_ui is not None:
74 |             marker_ui.destroy()
75 | 
76 |         marker_ui = None
77 |         ctx.tags = []
78 | 
79 |     def marker_ui_mouse_move(label: str):
80 |         """
81 |         Moves the mouse cursor to the label corresponding to the given label
82 |         """
83 | 
84 |         global marker_ui
85 | 
86 |         if marker_ui is None:
87 |             return
88 | 
89 |         rect = marker_ui.find_rect(label)
90 | 
91 |         if rect is None:
92 |             return
93 | 
94 |         actions.mouse_move(
95 |             rect.x + rect.width / 2,
96 |             rect.y + rect.height / 2,
97 |         )
98 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | This is a project related to [Talon](https://talonvoice.com/). It is a set of utility actions and a command generation wizard UI aimed at making it much easier to build new mouse based voice commands for applications.
 2 | 
 3 | # Overview of features
 4 | 
 5 | There are several features provided by this package:
 6 | 
 7 | 1. A set of actions in `mouse_helper.py` that make it easier to build mouse based commands in Talon. For example there are actions to save and restore the mouse position, to move the mouse cursor relative to the active window, or to find a saved image on the screen.
 8 | 2. A UI for indicating positions on the screen in `marker_ui.py` and voice commands to move the cursor to them or click them.
 9 | 3. A voice command wizard in `command_wizard.py` which lets you more easily build voice commands based on the above features using a GUI.
10 | 
11 | # Usage
12 | 
13 | You can use the exposed actions directly if you like, otherwise you most likely want to use the command wizard. The video linked below shows an overview of how to do this.
14 | 
15 | [![Demonstration video](docs/video-demo-thumb.png)](https://talon-ui-helper.s3.ap-southeast-2.amazonaws.com/talon-ui-helper-demo.webm)
16 | 
17 | The video shows the following:
18 | 
19 | 1. The GUI can be brought up by saying "command wizard show". This then presents a menu of voice command templates you can build. The button text in the menu is also mapped as a voice command (e.g. "choose 1").
20 | 2. After selecting one of the menu options you are then shown a full screen overlay allowing you to select the relevant region of the screen to operate on. The overlay can be interracted with via the mouse or the keyboard shortcuts displayed on the screen. The mouse grid built in to knausj\_talon works quite well here also.
21 | 3. After confirming your selection with the enter key a command is added to your clipboard which you can then paste directly in to a .talon file. After that you only need to choose the actual voice command you want to use to trigger it.
22 | 
23 | ## Available builders
24 | 
25 | * Single image selection. This builder will allow you to choose a region of the screen to save as an image. This image will be clicked by your voice command.
26 | * Multi image selection. This builder draws a label on each of the matches it finds allowing you to move to the image in question using the 'jump <label>' (e.g. 'jump bat') voice command. 'touch <label>' and 'righty <label>' are also available.
27 | * Find items in a box/blob detector. This allows you to draw a rectangle which will then be searched for clickable regions. The first column of pixels is treated as background, and the rest is foreground. If the box is wider than high, then the first row are the background pixels. Note that the rectangle you define is relative to the currently focussed window, so if that window gets moved or resized, the voice command should still be able to find the relevant region.
28 | 
29 | ## Settings
30 | 
31 | The image selector command builder needs a place to save the image templates it generates. By default this is the `image_templates` directory in your Talon user folder. If you'd like to change where these are stored you can use the `user.mouse_helper_template_directory` setting.
32 | 


--------------------------------------------------------------------------------
/ui_widgets.py:
--------------------------------------------------------------------------------
 1 | """
 2 | Simple widget system for use with Talon's canvas.
 3 | """
 4 | 
 5 | from typing import Tuple, Dict, Any
 6 | from talon.skia.paint import Paint
 7 | 
 8 | def render_text(canvas, formatted_text, x, y):
 9 |     """
10 |     Renders formatted_text from layout_text() to canvas at the given coordinates. The
11 |     paint style is contained within the formatted_text data structure.
12 |     """
13 | 
14 |     for i, line in enumerate(formatted_text["output_lines"]):
15 |         calc_y = y + i * formatted_text["line_height"]
16 |         canvas.draw_text(line, x, calc_y, formatted_text["paint"])
17 | 
18 | def layout_text(
19 |         text: str,
20 |         paint: Paint,
21 |         max_width: int) -> Tuple[Tuple[float, float], Dict[str, Any]]:
22 |     """
23 |     Works out the layout of the given plain text (maybe with newlines)
24 |     rendered with the style described by paint constrained to fit within the given width.
25 | 
26 |     Returns the width/height bounding box that is used by the text.
27 |     This will be the same or a smaller width than max_width. Also returns a data
28 |     structure that can be used by render_text() to draw the text at an arbitrary position
29 |     on a canvas.
30 |     """
31 | 
32 |     paint = paint.clone()
33 |     line_height = int(paint.textsize * 1.2)
34 |     space_width = paint.measure_text(" ")[0]
35 | 
36 |     # Break newlines out into their own chunks
37 |     # IDEA: The output of this function could be made into a monoid of a width/height tuple
38 |     # and a class that knows how to render itself at a position. Then we could handle multi
39 |     # element layouts (e.g. headers, indented areas) easily, and also the newlines code here.
40 |     chunks = []
41 |     for chunk in text.split(" "):
42 |         if "\n" in chunk:
43 |             for bit in chunk.split("\n"):
44 |                 if bit == "":
45 |                     additions = ["\n"]
46 |                 else:
47 |                     additions = [bit, "\n"]
48 |                 chunks += additions
49 |             # Remove trailing newline
50 |             chunks = chunks[:-1]
51 | 
52 |         else:
53 |             chunks.append(chunk)
54 | 
55 |     max_output_width = 0
56 |     current_width = 0
57 |     output_lines = []
58 |     current_line = []
59 |     for chunk in chunks:
60 |         if chunk == "\n":
61 |             output_lines.append(" ".join(current_line))
62 |             current_line = []
63 |             current_width = 0
64 |             continue
65 | 
66 |         chunk_width = paint.measure_text(chunk)[0]
67 |         new_width = current_width + space_width + chunk_width
68 | 
69 |         if new_width > max_width:
70 |             output_lines.append(" ".join(current_line))
71 |             current_line = [chunk]
72 |             current_width = chunk_width
73 |         else:
74 |             current_line.append(chunk)
75 |             current_width = new_width
76 |             max_output_width = max(new_width, max_output_width)
77 | 
78 |     if len(current_line) > 0:
79 |         output_lines.append(" ".join(current_line))
80 | 
81 |     return (
82 |         (max_output_width, line_height * len(output_lines)),
83 |         {
84 |             "line_height": line_height,
85 |             "paint": paint,
86 |             "output_lines": output_lines
87 |         }
88 |     )
89 | 


--------------------------------------------------------------------------------
/marker_ui.py:
--------------------------------------------------------------------------------
  1 | """
  2 | Draws a set of tags as a floating overlay. Allows users to indicate a particular
  3 | point on the screen by name.
  4 | """
  5 | 
  6 | from typing import List, NamedTuple, Optional
  7 | 
  8 | import re
  9 | 
 10 | from talon import screen, canvas, ui, ctrl
 11 | from talon import cron
 12 | from talon.types import Rect
 13 | from talon.skia.bitmap import Bitmap
 14 | from talon.skia.typeface import Typeface
 15 | 
 16 | 
 17 | class MarkerUi:
 18 |     """
 19 |     Draws some markers pointing to particular locations on the screen
 20 |     """
 21 | 
 22 |     class Marker(NamedTuple):
 23 |         target_region: Rect
 24 |         label: str
 25 | 
 26 |     def __init__(self, markers: List[Marker]=[], screen_idx: Optional[int]=None):
 27 |         """
 28 |         Args:
 29 | 
 30 |             markers: List of marker locations and labels to show
 31 |             screen_idx: The Talon screen index we're showing the markers on. If
 32 |               None, then try to autoselect the active screen.
 33 |         """
 34 |         if screen_idx is None:
 35 |             active_window = ui.active_window()
 36 |             if active_window.id == -1:
 37 |                 rect = ui.main_screen().rect
 38 |             else:
 39 |                 rect = active_window.screen.rect
 40 | 
 41 |             # width -= 1 works around bug in Windows Talon, see: https://github.com/talonvoice/talon/issues/670
 42 |             rect.width -= 1
 43 |             self.can = canvas.Canvas.from_rect(rect)
 44 |         else:
 45 |             self.can = canvas.Canvas.from_screen(ui.screens()[screen_idx])
 46 | 
 47 |         self.markers = markers
 48 |         self.can.register("draw", self._draw)
 49 |         self.can.hide()
 50 |         self.visible = False
 51 | 
 52 |     def show(self):
 53 |         self.can.show()
 54 |         # Freeze stops draw being called at 60Hz and just uses the initial paint
 55 |         self.can.freeze()
 56 |         self.visible = True
 57 | 
 58 |     def hide(self):
 59 |         self.can.hide()
 60 |         self.visible = False
 61 | 
 62 |     def destroy(self):
 63 |         self.can.close()
 64 | 
 65 |     def find_rect(self, identifier: str) -> Optional[Rect]:
 66 |         """
 67 |         Finds the rectangle corresponding to the given identifier, or None if
 68 |         it couldn't be found.
 69 |         """
 70 | 
 71 |         for marker in self.markers:
 72 |             if marker.label == identifier:
 73 |                 return marker.target_region
 74 | 
 75 |         return None
 76 | 
 77 |     def _draw(self, canvas):
 78 |         self.draw_markers(canvas, self.markers)
 79 | 
 80 |     @staticmethod
 81 |     def draw_markers(canvas, markers):
 82 |         """
 83 |         Draws out the given set of markers on the given canvas. Split out so that it can be 
 84 |         applied to canvases managed by other classes.
 85 |         """
 86 |         paint = canvas.paint
 87 |         paint.textsize = 12
 88 |         paint.typeface = Typeface.from_name('monospace')
 89 |         min_width = 10
 90 |         min_height = 10
 91 |         for marker in markers:
 92 |             region = marker.target_region
 93 | 
 94 |             # trect.x and .y are the offsets the text is printed at
 95 |             _, trect = paint.measure_text(marker.label)
 96 | 
 97 |             # Draw the box
 98 |             height = max(trect.height, min_height) + 2
 99 |             width = max(trect.width, min_width) + 2
100 |             ypos = region.y + (region.height - height) // 2
101 |             bg_rect = Rect(
102 |                 region.x + (region.width - width) // 2,
103 |                 ypos,
104 |                 width,
105 |                 height
106 |             )
107 | 
108 |             paint.style = paint.Style.FILL
109 |             paint.color = 'aaffffff'
110 |             canvas.draw_rect(bg_rect)
111 |             paint.color = 'black'
112 | 
113 |             # Draw the label
114 |             paint.style = paint.Style.FILL
115 |             canvas.draw_text(
116 |                 marker.label,
117 |                 bg_rect.x - trect.x + (bg_rect.width - trect.width) / 2,
118 |                 bg_rect.y - trect.y + (bg_rect.height - trect.height) / 2
119 |             )
120 | 


--------------------------------------------------------------------------------
/blob_detector.py:
--------------------------------------------------------------------------------
  1 | """
  2 | Clickable blob detection logic
  3 | """
  4 | 
  5 | from typing import List
  6 | import numpy as np
  7 | 
  8 | from talon.types import Rect as TalonRect
  9 | 
 10 | 
 11 | def calculate_blob_rects(image: 'talon.skia.Image', region: TalonRect, min_gap_size=5) -> List[TalonRect]:
 12 |     """
 13 |     Finds screen relative rectangles corresponding to clickable blobs in the given image. Region
 14 |     is the position on the screen the image corresponds to.
 15 |     """
 16 | 
 17 |     def _offset_rect(rect, dx, dy):
 18 |         return TalonRect(
 19 |             rect.x + dx,
 20 |             rect.y + dy,
 21 |             rect.width,
 22 |             rect.height
 23 |         )
 24 | 
 25 |     image_array = np.array(image)
 26 |     rects = calculate_blob_rects_from_numpy(image_array, min_gap_size=min_gap_size)
 27 | 
 28 |     return [
 29 |         TalonRect(
 30 |             rect.x + region.x,
 31 |             rect.y + region.y,
 32 |             rect.width,
 33 |             rect.height
 34 |         )
 35 | 
 36 |         for rect in rects
 37 |     ]
 38 | 
 39 | def save_mask(mask, output_filename):
 40 |     """
 41 |     Utility function for saving a boolean bitmap to a black and white file
 42 |     """
 43 | 
 44 |     from talon.skia import Image
 45 | 
 46 |     Image.from_array(mask.astype("uint8")*255).write_file(output_filename)
 47 | 
 48 | 
 49 | def calculate_blob_rects_from_numpy(image_array: np.ndarray, min_gap_size=5) -> List[TalonRect]:
 50 |     """
 51 |     Finds likely blobs suitable for clicking in the given numpy array.
 52 |     """
 53 | 
 54 |     # First column or row of pixels are all considered background colors (depending on dimensions
 55 |     # of rectangle)
 56 |     if image_array.shape[0] > image_array.shape[1]:
 57 |         # Higher than wide, assume we're finding vertically stacked items
 58 |         background_colors = np.unique(image_array[:, 0, :], axis=0)
 59 |         rollup_axis = 1
 60 |         def _vert_rect_builder(blob):
 61 |             return TalonRect(
 62 |                 0,
 63 |                 blob[0],
 64 |                 image_array.shape[1],
 65 |                 blob[1] - blob[0]
 66 |             )
 67 |         rect_builder = _vert_rect_builder
 68 |     else:
 69 |         # Wider than high, assume we're finding horizontally stacked items
 70 |         background_colors = np.unique(image_array[0, :, :], axis=0)
 71 |         rollup_axis = 0
 72 |         def _horiz_rect_builder(blob):
 73 |             return TalonRect(
 74 |                 blob[0],
 75 |                 0,
 76 |                 blob[1] - blob[0],
 77 |                 image_array.shape[0]
 78 |             )
 79 |         rect_builder = _horiz_rect_builder
 80 | 
 81 |     height, width, _ = image_array.shape
 82 |     # Build a boolean grid where True is foreground and False is background
 83 |     mask_array = np.ones((height, width), np.uint8) == 1
 84 |     for color_array in background_colors:
 85 |         mask_array = mask_array & (image_array != color_array).any(axis=2)
 86 | 
 87 |     # Find which elements (rows or columns depending on orientation)
 88 |     # have any foreground pixels in them (make a 1d array)
 89 |     elements_with_foreground = np.any(mask_array, axis=rollup_axis)
 90 | 
 91 |     # Form the 1d array into spans
 92 |     spans = []
 93 |     start_span_i = 0
 94 |     end_span_i = 0
 95 |     state = "bg"
 96 |     bg_counter = 0
 97 |     for i, item in enumerate(elements_with_foreground):
 98 |         if item and state == "bg":
 99 |             start_span_i = i
100 |             state = "fg"
101 |         elif not item and state == "fg":
102 |             state = "fg_countdown"
103 |             bg_counter = 0
104 |         elif item and state == "fg_countdown":
105 |             # Gap was smaller than our min_gap_size and we just got a foreground,
106 |             # resume including in the span
107 |             state = "fg"
108 |         elif not item and state == "fg_countdown":
109 |             bg_counter += 1
110 |             if bg_counter >= min_gap_size:
111 |                 state = "bg"
112 |                 end_span_i = i - bg_counter
113 |                 spans.append((start_span_i, end_span_i))
114 |         else:
115 |             # Maintaining same state, pass
116 |             pass
117 |     if state == "fg":
118 |         end_span_i = len(elements_with_foreground)
119 |         spans.append((start_span_i, end_span_i))
120 |     elif state == "fg_countdown":
121 |         end_span_i = i - bg_counter
122 |         spans.append((start_span_i, end_span_i))
123 | 
124 |     return [
125 |         rect_builder(span)
126 | 
127 |         for span in spans
128 |     ]
129 | 


--------------------------------------------------------------------------------
/command_wizard.py:
--------------------------------------------------------------------------------
  1 | """
  2 | Actions related to building voice commands.
  3 | """
  4 | 
  5 | import os
  6 | import datetime
  7 | 
  8 | from talon import Module, Context, actions, cron, ui, imgui
  9 | 
 10 | from .overlays import ImageSelectorOverlay, BlobBoxOverlay
 11 | from .mouse_helper import get_image_template_directory
 12 | 
 13 | 
 14 | mod = Module()
 15 | mod.tag("command_wizard_showing", desc="The command wizard is showing")
 16 | ctx = Context()
 17 | 
 18 | 
 19 | def save_image_template(image):
 20 |     """
 21 |     Saves the given image to the image templates folder and returns the generated name.58
 22 |     """
 23 | 
 24 |     unique_filename = \
 25 |         str(datetime.datetime.now().date()) + '_' + \
 26 |         str(datetime.datetime.now().time()).replace(':', '.') + \
 27 |         ".png"
 28 | 
 29 |     templates_directory = get_image_template_directory()
 30 |     full_filename = os.path.join(templates_directory, unique_filename)
 31 | 
 32 |     if not os.path.exists(templates_directory):
 33 |         os.mkdir(templates_directory)
 34 | 
 35 |     image.write_file(full_filename)
 36 | 
 37 |     return unique_filename
 38 | 
 39 | 
 40 | def handle_image_click_builder(result):
 41 |     """
 42 |     Result handler for the image click command builder.
 43 |     """
 44 |     if result is None:
 45 |         return
 46 | 
 47 |     filename = save_image_template(result["image"])
 48 |     index = result["index"]
 49 | 
 50 |     offset_bit = ""
 51 |     if result["offset"]:
 52 |         offset_bit = ", ".join([""] + list(map(lambda x: str(int(x)), result["offset"])))
 53 | 
 54 |     command = "\n".join([
 55 |         "",
 56 |         ":",
 57 |         "    user.mouse_helper_position_save()",
 58 |         f'    user.mouse_helper_move_image_relative("{filename}", {index}{offset_bit})',
 59 |         "    sleep(0.05)",
 60 |         "    mouse_click(0)",
 61 |         "    sleep(0.05)",
 62 |         "    user.mouse_helper_position_restore()",
 63 |     ])
 64 |     actions.clip.set_text(command)
 65 |     actions.app.notify("Copied new command to clipboard")
 66 | 
 67 | 
 68 | def handle_multi_image_builder(result):
 69 |     """
 70 |     Result handler for the multi image marker command builder.
 71 |     """
 72 |     if result is None:
 73 |         return
 74 | 
 75 |     filename = save_image_template(result["image"])
 76 | 
 77 |     offset_bit = ""
 78 |     if result["offset"]:
 79 |         offset_bit = ", ".join([""] + list(map(lambda x: str(int(x)), result["offset"])))
 80 | 
 81 |     command = "\n".join([
 82 |         "",
 83 |         ":",
 84 |         f'    matches = user.mouse_helper_find_template_relative("{filename}"{offset_bit})',
 85 |         "    user.marker_ui_show(matches)",
 86 |     ])
 87 |     actions.clip.set_text(command)
 88 |     actions.app.notify("Copied new command to clipboard")
 89 | 
 90 | 
 91 | def handle_blob_detect_builder(result):
 92 |     """
 93 |     Result handler for the blob detect command builder.
 94 |     """
 95 |     if result is None:
 96 |         return
 97 | 
 98 |     active_rectangle = active_rectangle_before_overlay
 99 |     def calculate_offset(position, minimum, width):
100 |         # Split each axis into two to determine which side of the screen
101 |         # the coordinate is offset from
102 |         maximum = minimum + width
103 |         split = (minimum + maximum) / 2
104 |         if position > split:
105 |             return str(position - maximum)
106 |         else:
107 |             return str(position - minimum)
108 | 
109 |     offsets = " ".join([
110 |         calculate_offset(result.x, active_rectangle.x, active_rectangle.width),
111 |         calculate_offset(result.y, active_rectangle.y, active_rectangle.height),
112 |         calculate_offset(result.x + result.width, active_rectangle.x, active_rectangle.width),
113 |         calculate_offset(result.y + result.height, active_rectangle.y, active_rectangle.height),
114 |     ])
115 | 
116 |     command = "\n".join([
117 |         "",
118 |         ":",
119 |         f'    bounding_rectangle = user.mouse_helper_calculate_relative_rect("{offsets}", "active_window")',
120 |         f'    user.mouse_helper_blob_picker(bounding_rectangle)',
121 |     ])
122 |     actions.clip.set_text(command)
123 |     actions.app.notify("Copied new command to clipboard")
124 | 
125 | 
126 | command_wizards = [
127 |     (
128 |         "Click a single image on the screen",
129 |         ImageSelectorOverlay,
130 |         handle_image_click_builder,
131 |         (
132 |             "Select a region of the screen as an image to find in your voice command "
133 |             "then press enter to confirm your selection. Press escape to cancel.\n\n"
134 |             "After selecting and before enter, optionally right click to define an "
135 |             "offset from the selected region. This will be clicked instead of the "
136 |             "center of the region."
137 |         )
138 |     ),
139 |     (
140 |         "Show markers on all image matches on screen",
141 |         ImageSelectorOverlay,
142 |         handle_multi_image_builder,
143 |         (
144 |             "Select a region of the screen as an image to find in your voice command "
145 |             "then press enter to confirm your selection. Press escape to cancel.\n\n"
146 |             "After selecting and before enter, optionally right click to define an "
147 |             "offset from the selected region. This will be clicked instead of the "
148 |             "center of the region."
149 |         )
150 |     ),
151 |     (
152 |         "Find items in a box to click",
153 |         BlobBoxOverlay,
154 |         handle_blob_detect_builder,
155 |         (
156 |             "Select a region of the active window to use in your voice command then press "
157 |             "enter to confirm your selection. Press escape to cancel.\n\n"
158 |             "When drawing a box wider than tall, the first row of pixels will be "
159 |             "considered background color. When the box is taller than wide the first "
160 |             "column of pixels will be considered background color.\n\n"
161 |             "The background color will be used to detect clickable regions in the area "
162 |             "you selected. "
163 |         )
164 |     )
165 | ]
166 | 
167 | existing_overlay = None
168 | # On OSX the overlay window itself shows up as the ui.active_window() after the overlay is nominally
169 | # closed. So we save what the active window rect was just before the overlay is shown.
170 | active_rectangle_before_overlay = None
171 | 
172 | 
173 | def open_overlay(index):
174 |     global active_rectangle_before_overlay
175 | 
176 |     active_rectangle_before_overlay = ui.active_window().rect
177 | 
178 |     builder_picker_toggle(False)
179 | 
180 |     if existing_overlay:
181 |         existing_overlay.destroy()
182 | 
183 |     _, overlay, handler, help = command_wizards[index]
184 |     def _open():
185 |         global existing_overlay
186 |         existing_overlay = overlay(handler, text=help)
187 |     # Do this after a delay to give the imgui a chance to get removed from
188 |     # the screen so it won't show up in our screenshot.
189 |     cron.after("50ms", _open)
190 | 
191 | 
192 | @imgui.open(y=0)
193 | def builder_picker(gui: imgui.GUI):
194 |     t = gui.text("Choose the command type that you would like to build:")
195 |     gui.spacer()
196 | 
197 |     for i, (text, _, _, _) in enumerate(command_wizards):
198 |         gui.text(text)
199 |         if gui.button(f"Choose {i+1}"):
200 |             open_overlay(i)
201 |         gui.spacer()
202 | 
203 |     if gui.button("Command wizard hide"):
204 |         builder_picker_toggle(False)
205 | 
206 | 
207 | def builder_picker_toggle(visible: bool):
208 |     if visible:
209 |         builder_picker.show()
210 |         ctx.tags = ["user.command_wizard_showing"]
211 |     else:
212 |         builder_picker.hide()
213 |         ctx.tags = []
214 | 
215 | 
216 | @mod.action_class
217 | class CommandWizardActions:
218 |     """
219 |     Actions related to the command builder wizard.
220 |     """
221 | 
222 |     def command_wizard_show():
223 |         """
224 |         Brings up the command wizard UI
225 |         """
226 | 
227 |         builder_picker_toggle(True)
228 | 
229 |     def command_wizard_hide():
230 |         """
231 |         Closes the command wizard UI
232 |         """
233 | 
234 |         builder_picker_toggle(False)
235 | 
236 |     def command_wizard_choose_option(option: int):
237 |         """
238 |         Chooses one of the command wizards
239 |         """
240 | 
241 |         open_overlay(option - 1)
242 | 


--------------------------------------------------------------------------------
/mouse_helper.py:
--------------------------------------------------------------------------------
  1 | """
  2 | Useful actions related to moving the mouse
  3 | """
  4 | 
  5 | import os
  6 | import math
  7 | import subprocess
  8 | from typing import Union, Optional, List
  9 | 
 10 | from talon import actions, ui, clip, screen, settings, Module
 11 | from talon.types import Rect as TalonRect
 12 | from talon.experimental import locate
 13 | 
 14 | from .blob_detector import calculate_blob_rects
 15 | from .marker_ui import MarkerUi
 16 | 
 17 | 
 18 | mod = Module()
 19 | setting_template_directory = mod.setting(
 20 |     "mouse_helper_template_directory",
 21 |     type=str,
 22 |     desc=(
 23 |         "The folder that templated images are saved to."
 24 |         " Defaults to image_templates in your user folder"
 25 |     ),
 26 |     default=None
 27 | )
 28 | 
 29 | 
 30 | def get_image_template_directory():
 31 |     """
 32 |     Gets the full path to the directory where template images are stored.
 33 |     """
 34 | 
 35 |     maybe_value = settings.get("user.mouse_helper_template_directory")
 36 |     if maybe_value:
 37 |         return maybe_value
 38 |     else:
 39 |         return os.path.join(
 40 |             os.path.dirname(os.path.realpath(__file__)),
 41 |             "../image_templates"
 42 |         )
 43 | 
 44 | 
 45 | def find_active_window_rect() -> TalonRect:
 46 |     return ui.active_window().rect
 47 | 
 48 | 
 49 | def screencap_to_image(rect: TalonRect) -> 'talon.skia.image.Image':
 50 |     """
 51 |     Captures the given rectangle off the screen
 52 |     """
 53 | 
 54 |     return screen.capture(rect.x, rect.y, rect.width, rect.height, retina=False)
 55 | 
 56 | 
 57 | def calculate_relative(modifier: str, start: float, end: float) -> float:
 58 |     """
 59 |     Helper method for settings. Lets you specify numbers relative to a
 60 |     range. For example:
 61 | 
 62 |         calculate_relative("-10.0", 0, 100) == 90
 63 |         calculate_relative("10", 0, 100) == 10
 64 |         calculate_relative("-0", 0, 100) == 100
 65 | 
 66 |     Note that positions and offset are floats.
 67 |     """
 68 |     if modifier.startswith("-"):
 69 |         modifier_ = float(modifier[1:])
 70 |         rel_end = True
 71 |     elif modifier == ".":
 72 |         # In the middle
 73 |         return (end + start) // 2
 74 |     else:
 75 |         modifier_ = float(modifier)
 76 |         rel_end = False
 77 | 
 78 |     if rel_end:
 79 |         return end - modifier_
 80 |     else:
 81 |         return start + modifier_
 82 | 
 83 | 
 84 | saved_mouse_pos = None
 85 | 
 86 | 
 87 | @mod.action_class
 88 | class MouseActions:
 89 |     def mouse_helper_position_save():
 90 |         """
 91 |         Saves the mouse position to a global variable
 92 |         """
 93 | 
 94 |         global saved_mouse_pos
 95 | 
 96 |         saved_mouse_pos = (actions.mouse_x(), actions.mouse_y())
 97 | 
 98 |     def mouse_helper_position_restore():
 99 |         """
100 |         Restores a saved mouse position
101 |         """
102 | 
103 |         if saved_mouse_pos is None:
104 |             return
105 | 
106 |         actions.mouse_move(
107 |             saved_mouse_pos[0],
108 |             saved_mouse_pos[1]
109 |         )
110 | 
111 |     def mouse_helper_move_active_window_relative(xpos: str, ypos: str):
112 |         """
113 |         Positions the mouse relative to the active window
114 |         """
115 | 
116 |         rect = find_active_window_rect()
117 | 
118 |         actions.mouse_move(
119 |             calculate_relative(xpos, 0, rect.width) + rect.x,
120 |             calculate_relative(ypos, 0, rect.height) + rect.y,
121 |         )
122 | 
123 |     def mouse_helper_move_relative(xdelta: float, ydelta: float):
124 |         """
125 |         Moves the mouse relative to its current position
126 |         """
127 | 
128 |         new_xpos = actions.mouse_x() + xdelta
129 |         new_ypos = actions.mouse_y() + ydelta
130 |         actions.mouse_move(new_xpos, new_ypos)
131 | 
132 |     def mouse_helper_calculate_relative_rect(relative_rect_offsets: str, region:str ="active_screen") -> TalonRect:
133 |         """
134 |         Calculates a talon rectangle relative to the entire screen based on the given region
135 |         of interest and a set of offsets. Examples:
136 | 
137 |             "0 0 -0 -0", "active_screen": Would indicate the entire active screen.
138 |             "10 20 30 40", "active_window": Would indicate the region between pixels (10, 20) and (30, 40)
139 |               on the currently focussed window.
140 |             "10 20 -30 40", "active_window": Would indicate the region between pixels (10, 20) and
141 |               the pixel 30 units from the right hand side of the window and 40 units from the top.
142 |         """
143 | 
144 |         if region == "active_screen":
145 |             active_window = ui.active_window()
146 |             if active_window.id == -1:
147 |                 base_rect = ui.main_screen().rect
148 |             else:
149 |                 base_rect = active_window.screen.rect
150 |         elif region == "active_window":
151 |             base_rect = find_active_window_rect()
152 |         else:
153 |             assert "Unhandled region"
154 | 
155 |         mods = relative_rect_offsets.split(" ")
156 |         _calc_pos = calculate_relative
157 |         x = _calc_pos(mods[0], base_rect.x, base_rect.x + base_rect.width)
158 |         y = _calc_pos(mods[1], base_rect.y, base_rect.y + base_rect.height)
159 |         rect = TalonRect(
160 |             x,
161 |             y,
162 |             _calc_pos(mods[2], base_rect.x, base_rect.x + base_rect.width) - x,
163 |             _calc_pos(mods[3], base_rect.y, base_rect.y + base_rect.height) - y,
164 |         )
165 | 
166 |         return rect
167 | 
168 |     def mouse_helper_find_template_relative(
169 |         template_path: str,
170 |         xoffset: float=0,
171 |         yoffset: float=0,
172 |         region: Optional[TalonRect]=None
173 |     ) -> List[TalonRect]:
174 |         """
175 |         Finds all matches for the given image template within the given region.
176 | 
177 |         :param template_path: Filename of the image to find. Can be an absolute path or
178 |             if no '/' or '\\' character is specified, it is relative to the image
179 |             templates directory.
180 |         :param xoffset: Amount to shift in the x direction relative to the
181 |             center of the template.
182 |         :param yoffset: Amount to shift in the y direction relative to the
183 |             center of the template.
184 |         :param region: The region to search for the template in. Either a screen relative
185 |             TalonRect (see mouse_helper_calculate_relative_rect) or None to just use the
186 |             active screen.
187 |         """
188 | 
189 |         if region is None:
190 |             rect = actions.user.mouse_helper_calculate_relative_rect(
191 |                 "0 0 -0 -0",
192 |                 "active_screen"
193 |             )
194 |         else:
195 |             rect = region
196 | 
197 |         if os.pathsep in template_path:
198 |             # Absolute path specified
199 |             template_file = template_path
200 |         else:
201 |             # Filename in image templates directory specified
202 |             template_file = os.path.join(get_image_template_directory(), template_path)
203 | 
204 |         matches = [
205 |             TalonRect(
206 |                 match.x + xoffset,
207 |                 match.y + yoffset,
208 |                 match.width,
209 |                 match.height
210 |             )
211 | 
212 |             for match in locate.locate(
213 |                 template_file,
214 |                 rect=rect,
215 |                 threshold=0.9
216 |             )
217 |         ]
218 | 
219 |         return sorted(
220 |             matches,
221 |             key=lambda m: (m.x, m.y)
222 |         )
223 | 
224 |     def mouse_helper_move_image_relative(
225 |         template_path: str,
226 |         disambiguator: Union[int, str]=0,
227 |         xoffset: float=0,
228 |         yoffset: float=0,
229 |         region: Optional[TalonRect]=None
230 |     ):
231 |         """
232 |         Moves the mouse relative to the template image given in template_path.
233 | 
234 |         :param template_path: Filename of the image to find. Can be an absolute path or
235 |             if no '/' or '\\' character is specified, it is relative to the image
236 |             templates directory.
237 |         :param disambiguator: If there are multiple matches, use this to indicate
238 |             which one you want to match. Matches are ordered left to right top to
239 |             bottom. If disambiguator is an integer then it's just an index into that list.
240 |             If it's the string "mouse" then it's the next match in the region to the right
241 |             and down from the mouse after shifting back the offset amount and up and left
242 |             half the size and width of the template. If it is "mouse_cycle" then if there
243 |             are no further matches it will attempt to start from the top of the screen again.
244 |             This is useful for iterating through rows in a table for example.
245 |         :param xoffset: Amount to shift in the x direction relative to the
246 |             center of the template.
247 |         :param yoffset: Amount to shift in the y direction relative to the
248 |             center of the template.
249 |         :param region: The region to search for the template in. Either a screen relative
250 |             TalonRect (see mouse_helper_calculate_relative_rect) or None to just use the
251 |             active screen.
252 |         """
253 | 
254 |         if region is None:
255 |             rect = actions.user.mouse_helper_calculate_relative_rect(
256 |                 "0 0 -0 -0",
257 |                 "active_screen"
258 |             )
259 |         else:
260 |             rect = region
261 | 
262 |         sorted_matches = actions.user.mouse_helper_find_template_relative(
263 |             template_path,
264 |             xoffset,
265 |             yoffset,
266 |             rect
267 |         )
268 | 
269 |         if len(sorted_matches) == 0:
270 |             # Throw an exception to cancel any following commands in the .talon file
271 |             raise RuntimeError("No matches")
272 | 
273 |         if disambiguator in ("mouse", "mouse_cycle"):
274 |             # math.ceil is needed here to ensure we only look at pixels after the current template match if we're
275 |             # cycling between matches. math.floor would pick up the current one again.
276 |             xnorm = math.ceil(actions.mouse_x() - sorted_matches[0].width / 2)
277 |             ynorm = math.ceil(actions.mouse_y() - sorted_matches[0].height / 2)
278 |             filtered_matches = [
279 |                 match
280 |                 for match in sorted_matches
281 |                 if (match.y == ynorm and match.x > xnorm) or match.y > ynorm
282 |             ]
283 | 
284 |             if len(filtered_matches) > 0:
285 |                 match_rect = filtered_matches[0]
286 |             elif disambiguator == "mouse_cycle":
287 |                 match_rect = sorted_matches[0]
288 |             else:
289 |                 return
290 |         else:
291 |             if len(sorted_matches) <= disambiguator:
292 |                 return
293 | 
294 |             match_rect = sorted_matches[disambiguator]
295 | 
296 |         actions.mouse_move(
297 |             math.ceil(match_rect.x + (match_rect.width / 2)),
298 |             math.ceil(match_rect.y + (match_rect.height / 2)),
299 |         )
300 | 
301 |     def mouse_helper_blob_picker(bounding_rectangle: TalonRect, min_gap_size: int=5):
302 |         """
303 |         Attempts to find clickable elements within the given bounding rectangle, then
304 |         draws a labelled overlay allowing you to click or move the mouse to them.
305 | 
306 |         See mouse_helper_calculate_relative_rect for how to get the bounding rectangle.
307 |         """
308 | 
309 |         image = screencap_to_image(bounding_rectangle)
310 |         rects = calculate_blob_rects(image, bounding_rectangle, min_gap_size=min_gap_size)
311 | 
312 |         if len(rects) == 0:
313 |             return
314 | 
315 |         actions.user.marker_ui_show(rects)
316 | 


--------------------------------------------------------------------------------
/overlays.py:
--------------------------------------------------------------------------------
  1 | import abc
  2 | import numpy as np
  3 | import threading
  4 | import datetime
  5 | from typing import Optional
  6 | 
  7 | from talon import Module, app, actions, ui, imgui, canvas, screen, cron
  8 | 
  9 | from talon.skia import image, rrect, paint
 10 | from talon.types import Rect as TalonRect
 11 | from talon.experimental import locate
 12 | 
 13 | from .ui_widgets import layout_text, render_text
 14 | from .marker_ui import MarkerUi
 15 | from .blob_detector import calculate_blob_rects
 16 | 
 17 | 
 18 | mod = Module()
 19 | 
 20 | 
 21 | def find_active_window_rect() -> TalonRect:
 22 |     return ui.active_window().rect
 23 | 
 24 | 
 25 | def screencap_to_image(rect: TalonRect) -> 'talon.skia.image.Image':
 26 |     """
 27 |     Captures the given rectangle off the screen
 28 |     """
 29 | 
 30 |     return screen.capture(rect.x, rect.y, rect.width, rect.height, retina=False)
 31 | 
 32 | 
 33 | class ScreenshotOverlay(abc.ABC):
 34 |     """
 35 |     Abstract base class for overlay windows operating on a static screenshot.
 36 |     """
 37 | 
 38 |     # See _calculate_rect_handler
 39 |     CALCULATE_RECT_CANVAS_COLOR = "010203ff"
 40 |     CALCULATE_RECT_CANVAS_COLOR_NUMERIC = [1, 2, 3, 255]
 41 | 
 42 |     def __init__(self, result_handler, text=None, screen_idx=None):
 43 |         self.result_handler = result_handler
 44 | 
 45 |         if screen_idx is not None:
 46 |             self.screen_rect = ui.screens()[screen_idx].rect
 47 |         else:
 48 |             active_window = ui.active_window()
 49 |             if active_window.id == -1:
 50 |                 self.screen_rect = ui.main_screen().rect
 51 |             else:
 52 |                 self.screen_rect = active_window.screen.rect
 53 | 
 54 |         self.can = canvas.Canvas.from_rect(self.screen_rect)
 55 |         # Need panel = True for keyboard events to be captured on Linux
 56 |         self.can.panel = True
 57 |         # See _calculate_rect_handler for where this is used to refine
 58 |         # canvas size.
 59 |         self.calculate_rect_state = "initial"
 60 | 
 61 |         # We want an unfocus event to destroy the overlay normally, but
 62 |         # we get some initial illegitimate unfocus events, so ignore those.
 63 |         self.unfocus_destroy_enabled = False
 64 |         def _set_unfocus_destroy_enabled():
 65 |             if not self.can.focused:
 66 |                 self.destroy()
 67 |             else:
 68 |                 self.unfocus_destroy_enabled = True
 69 |         cron.after("1000ms", _set_unfocus_destroy_enabled)
 70 | 
 71 |         self.image = screencap_to_image(self.screen_rect)
 72 |         self.text = text
 73 |         self.text_position = "bottom"
 74 |         self.text_rect = None
 75 |         self.flash_text = None
 76 | 
 77 |         self.can.register("draw", self._draw_wrapper)
 78 |         self.can.blocks_mouse = True
 79 |         self.can.register("mouse", self._mouse_event)
 80 |         self.can.register("key", self._key_event)
 81 |         self.can.register("focus", self._focus_event)
 82 |         self.can.focused = True
 83 |         self.can.freeze()
 84 | 
 85 |     def destroy(self):
 86 |         self.can.close()
 87 | 
 88 |     def _calculate_rect_handler(self):
 89 |         """
 90 |         Working out how big to make the canvas isn't straightforward.
 91 |         Window manager's may move us (OSX, Linux), and may also have
 92 |         floating panels that are positioned above our window (due to
 93 |         panel = True).
 94 | 
 95 |         So what we do is take a screenshot after filling our attempt
 96 |         at a full screen canvas with a particular color. We then
 97 |         search the image to find that color, and this tells us what
 98 |         parts aren't covered by window manager decorations. We can
 99 |         then resize and move the canvas to fit within that area.
100 | 
101 |         An alternative would be to change the whole UI to implement
102 |         a scrollable window that shows the whole screenshot.
103 |         """
104 | 
105 |         if app.platform == "windows":
106 |             # Windows screen capture doesn't seem to include the overlay,
107 |             # and doesn't need this trickery anyway, so just go to the regular
108 |             # draw handler without changing anything.
109 |             self.calculate_rect_state = "normal"
110 |             self.can.freeze()
111 |             return
112 | 
113 |         talon_img = screencap_to_image(self.screen_rect)
114 |         img = np.array(
115 |             talon_img
116 |         )
117 |         # Array of True/False indicating whether the pixel is our
118 |         # special color
119 |         masked_array = np.all(
120 |             img == self.CALCULATE_RECT_CANVAS_COLOR_NUMERIC,
121 |             axis=2
122 |         )
123 | 
124 |         def _calculate_bounds(mask, axis):
125 |             background_items = np.any(
126 |                 mask,
127 |                 axis=axis
128 |             )
129 |             bg_start = None
130 |             bg_end = None
131 |             for idx, is_background in enumerate(background_items):
132 |                 if is_background:
133 |                     bg_start = idx if bg_start is None else min(idx, bg_start)
134 |                     bg_end = idx if bg_end is None else max(idx, bg_end)
135 |             return bg_start, bg_end + 1
136 | 
137 |         row_start_idx, row_end_idx = _calculate_bounds(masked_array, 1)
138 |         col_start_idx, col_end_idx = _calculate_bounds(
139 |             masked_array[row_start_idx:row_end_idx],
140 |             0
141 |         )
142 | 
143 |         self.can.rect = TalonRect(
144 |             col_start_idx,
145 |             row_start_idx,
146 |             col_end_idx - col_start_idx,
147 |             row_end_idx - row_start_idx,
148 |         )
149 | 
150 |         # Clean up, our work is done.
151 |         self.calculate_rect_state = "normal"
152 |         self.can.freeze()
153 | 
154 |     def _get_keyboard_commands(self):
155 |         return [
156 |             ("escape", "Close overlay"),
157 |             ("enter", "Confirm selection and close overlay"),
158 |         ]
159 | 
160 |     def _calculate_result(self):
161 |         raise NotImplementedError
162 | 
163 |     def _draw_wrapper(self, canvas):
164 |         # This will delegate to _draw for normal drawing, so override that.
165 |         # See _calculate_rect_handler for what's happening here
166 |         if self.calculate_rect_state in ("initial", "drawn"):
167 |             canvas.paint = paint.Paint()
168 |             canvas.paint.color = self.CALCULATE_RECT_CANVAS_COLOR
169 |             canvas.draw_rect(canvas.rect)
170 |             if self.calculate_rect_state == "initial":
171 |                 # Give the canvas a bit of time to redraw
172 |                 cron.after("200ms", self._calculate_rect_handler)
173 |             self.calculate_rect_state = "drawn"
174 |         elif self.calculate_rect_state == "normal":
175 |             self._draw(canvas)
176 |         else:
177 |             assert False, "Unhandled state: " + self.calculate_rect_state
178 | 
179 |     def _draw(self, canvas):
180 |         # This is the normal draw routine
181 |         canvas.draw_image(self.image, 0, 0)
182 |         canvas.paint = paint.Paint()
183 |         canvas.paint.color = "000000aa"
184 |         canvas.draw_rect(canvas.rect)
185 | 
186 |         self._draw_widgets(canvas)
187 | 
188 |         self._draw_text(canvas)
189 | 
190 |         self._draw_flash(canvas)
191 | 
192 |     def _draw_widgets(self, canvas):
193 |         pass
194 | 
195 |     def _draw_text(self, canvas):
196 |         all_text = self.text or ""
197 |         all_text += "\n\nKeyboard shortcuts (or use equivalent voice command):\n"
198 |         all_text += "\n".join([
199 |             f" - {key}: {description}"
200 |             for key, description in self._get_keyboard_commands()
201 |         ])
202 | 
203 |         canvas.paint = paint.Paint()
204 |         canvas.paint.antialias = True
205 |         canvas.paint.color = "ffffffff"
206 |         ((width, height), formatted_text) = layout_text(all_text, canvas.paint, 600)
207 | 
208 |         xpos = canvas.rect.x + (canvas.width - width - 20) / 2
209 |         ypos = 10 + canvas.rect.y
210 |         if self.text_position == "bottom":
211 |             ypos = canvas.rect.y + canvas.height - height - 20 - 10
212 |         self.text_rect = TalonRect(xpos, ypos, width + 20, height + 20)
213 | 
214 |         canvas.paint.color = "000000ff"
215 |         canvas.paint.style = canvas.paint.Style.FILL
216 |         thing = rrect.RoundRect.from_rect(
217 |             self.text_rect,
218 |             x=10,
219 |             y=10,
220 |             radii=(10, 10)
221 |         )
222 |         canvas.draw_rrect(thing)
223 | 
224 |         render_text(canvas, formatted_text, xpos + 10, ypos + 20)
225 | 
226 |     def _draw_flash(self, canvas):
227 |         if self.flash_text is None:
228 |             return
229 | 
230 |         canvas.paint = paint.Paint()
231 |         canvas.paint.antialias = True
232 |         canvas.paint.color = "ffffffff"
233 |         ((width, height), formatted_text) = layout_text(self.flash_text, canvas.paint, 300)
234 | 
235 |         xpos = canvas.rect.x + (canvas.width - width - 20) / 2
236 |         ypos = canvas.rect.y + (canvas.height - height - 20) / 2
237 |         text_rect = TalonRect(xpos, ypos, width + 20, height + 20)
238 | 
239 |         canvas.paint.color = "660000ff"
240 |         canvas.paint.style = canvas.paint.Style.FILL
241 |         thing = rrect.RoundRect.from_rect(
242 |             text_rect,
243 |             x=10,
244 |             y=10,
245 |             radii=(10, 10)
246 |         )
247 |         canvas.draw_rrect(thing)
248 | 
249 |         render_text(canvas, formatted_text, xpos + 10, ypos + 20)
250 | 
251 |     def _show_flash(self, text):
252 |         self.flash_text = text
253 |         def clear_flash():
254 |             self.flash_text = None
255 |             self.can.freeze()
256 |         cron.after("2s", clear_flash)
257 | 
258 |     def _focus_event(self, focussed):
259 |         if not focussed and self.unfocus_destroy_enabled:
260 |             self.destroy()
261 |             self.result_handler(None)
262 | 
263 |     def _key_event(self, evt):
264 |         if evt.down:
265 |             return
266 | 
267 |         # Lowercase events are for Talon 0.3, upper for 0.4+ (Rust)
268 |         if evt.key in ("Escape", "esc"):
269 |             self.destroy()
270 |             self.result_handler(None)
271 | 
272 |         if evt.key in ("Return", "return", "Enter"):
273 |             self.destroy()
274 |             self.result_handler(self._calculate_result())
275 | 
276 |     def _mouse_event(self, evt):
277 |         if self.text_rect:
278 |             if self.text_rect.contains(evt.gpos.x, evt.gpos.y):
279 |                 self.text_position = "top" if self.text_position == "bottom" else "bottom"
280 |                 self.can.freeze()
281 | 
282 | 
283 | class BoxSelectorOverlay(ScreenshotOverlay):
284 |     """
285 |     Show an overlay allowing the user to select a region of the screen.
286 |     """
287 | 
288 |     def __init__(self, *args, **kwargs):
289 |         super().__init__(*args, **kwargs)
290 | 
291 |         self.hl_region = None
292 |         self.is_selecting = False
293 |         self.settled_countdown_timer = None
294 | 
295 |     def _calculate_result(self):
296 |         return self.hl_region
297 | 
298 |     def _get_keyboard_commands(self):
299 |         commands = super()._get_keyboard_commands()
300 |         commands += [
301 |             ("up/down/left/right", "Nudge selection in indicated direction"),
302 |             ("shift + up/down/left/right", "Nudge selection a larger amount"),
303 |             ("ctrl + up/down/left/right", "Shrink/grow selection"),
304 |             ("shift + ctrl + up/down/left/right", "Shrink/grow selection a larger amount"),
305 |         ]
306 |         return commands
307 | 
308 |     def _selection_settled(self, finished_selection):
309 |         """
310 |         Called when we can assume that the user has finished selecting a region, or when
311 |         they've just started drawing a new one. If you want to draw any other markers
312 |         after the user has finished selecting this is the place to set and unset
313 |         a flag for _draw_widgets.
314 |         """
315 | 
316 |     def _get_region(self):
317 |         """
318 |         Gets the selected region, normalising any negative widths
319 |         """
320 | 
321 |         if self.hl_region is None:
322 |             return None
323 | 
324 |         if self.hl_region.width < 0:
325 |             x = self.hl_region.x + self.hl_region.width
326 |             width = self.hl_region.width * -1
327 |         else:
328 |             x = self.hl_region.x
329 |             width = self.hl_region.width
330 | 
331 |         if self.hl_region.height < 0:
332 |             y = self.hl_region.y + self.hl_region.height
333 |             height = self.hl_region.height * -1
334 |         else:
335 |             y = self.hl_region.y
336 |             height = self.hl_region.height
337 | 
338 |         return TalonRect(
339 |             int(x),
340 |             int(y),
341 |             int(width),
342 |             int(height)
343 |         )
344 | 
345 |     def _draw_widgets(self, canvas):
346 |         super()._draw_widgets(canvas)
347 | 
348 |         if not self.hl_region:
349 |             return
350 |         canvas.save()
351 |         canvas.clip_rect(self.hl_region, canvas.ClipOp.INTERSECT)
352 |         canvas.draw_image(self.image, 0, 0)
353 |         canvas.restore()
354 |         canvas.paint = paint.Paint()
355 |         canvas.paint.style = canvas.paint.Style.STROKE
356 |         canvas.paint.color = 'ffffffff'
357 |         if self.hl_region.width == 0 or self.hl_region.height == 0:
358 |             # Deal with the zero thickness cases that happen when using voice commands
359 |             canvas.draw_line(
360 |                 self.hl_region.x,
361 |                 self.hl_region.y,
362 |                 self.hl_region.x + self.hl_region.width,
363 |                 self.hl_region.y + self.hl_region.height
364 |             )
365 |         else:
366 |             canvas.draw_rect(self.hl_region)
367 | 
368 |     def _get_region_centre(self):
369 |         if self.hl_region:
370 |             return (
371 |                 self.hl_region.x + self.hl_region.width / 2,
372 |                 self.hl_region.y + self.hl_region.height / 2
373 |             )
374 |         else:
375 |             return None
376 | 
377 |     def _get_cropped_image(self) -> Optional['talon.Image']:
378 |         if self.hl_region is None:
379 |             return None
380 | 
381 |         # I think make_subset does this more cleanly, but I don't know what the Talon API is
382 |         region = self._get_region()
383 |         xpos = int(region.x)
384 |         ypos = int(region.y)
385 |         height = int(region.height)
386 |         width = int(region.width)
387 |         arr = np.array(self.image)[
388 |             ypos:(ypos + height),
389 |             xpos:(xpos + width),
390 |         ]
391 |         if len(arr) == 0:
392 |             return None
393 | 
394 |         return image.Image.from_array(arr)
395 | 
396 |     def _mouse_event(self, evt):
397 |         super()._mouse_event(evt)
398 | 
399 |         if evt.event == "mousedown" and evt.button == 0:
400 |             self.hl_region = TalonRect(evt.gpos.x, evt.gpos.y, 0, 0)
401 |             self.is_selecting = True
402 |             self._selection_settled(False)
403 |             self.can.freeze()
404 |         elif evt.event == "mousemove" and self.is_selecting and self.hl_region:
405 |             self.hl_region = TalonRect(
406 |                 self.hl_region.x,
407 |                 self.hl_region.y,
408 |                 evt.gpos.x - self.hl_region.x,
409 |                 evt.gpos.y - self.hl_region.y
410 |             )
411 |             self.can.freeze()
412 |         elif evt.event == "mouseup" and evt.button == 0:
413 |             self.is_selecting = False
414 |             self._selection_settled(True)
415 |             self.can.freeze()
416 | 
417 |     def _key_event(self, evt):
418 |         super()._key_event(evt)
419 | 
420 |         if evt.down or self.hl_region is None:
421 |             return
422 | 
423 |         keymap = [
424 |             # Needed between Talon 0.4.0-185 and 0.4.0-335. Can be removed after 2024-12-01 (once -335 changes have reached everybody).
425 |             ("ArrowLeft", ["x", "width", "-"]),
426 |             ("ArrowRight", ["x", "width", "+"]),
427 |             ("ArrowUp", ["y", "height", "-"]),
428 |             ("ArrowDown", ["y", "height", "+"]),
429 | 
430 |             # Talon 0.4 public
431 |             ("Left", ["x", "width", "-"]),
432 |             ("Right", ["x", "width", "+"]),
433 |             ("Up", ["y", "height", "-"]),
434 |             ("Down", ["y", "height", "+"]),
435 | 
436 |             # Talon 0.3 public
437 |             ("left", ["x", "width", "-"]),
438 |             ("right", ["x", "width", "+"]),
439 |             ("up", ["y", "height", "-"]),
440 |             ("down", ["y", "height", "+"]),
441 |         ]
442 | 
443 |         for key, args in keymap:
444 |             if key != evt.key:
445 |                 continue
446 | 
447 |             position, scale, direction = args
448 |             magnitude = 25 if 'shift' in evt.mods else 1
449 |             magnitude = -1 * magnitude if direction == "-" else magnitude
450 | 
451 |             if 'ctrl' in evt.mods:
452 |                 curr = getattr(self.hl_region, scale)
453 |                 new_val = curr + magnitude
454 |                 new_val = new_val if new_val >= 0 else 1
455 |                 setattr(self.hl_region, scale, new_val)
456 |             else:
457 |                 curr = getattr(self.hl_region, position)
458 |                 new_val = curr + magnitude
459 |                 new_val = new_val if new_val >= 0 else 0
460 |                 setattr(self.hl_region, position, new_val)
461 | 
462 |             # Ensure the region is still within the bounds of the canvas
463 |             for (position, scale) in (("x", "width"), ("y", "height")):
464 |                 min_pos = getattr(self.can.rect, position)
465 |                 max_pos = min_pos + getattr(self.can.rect, scale)
466 |                 curr_extent = getattr(self.hl_region, position) + getattr(self.hl_region, scale)
467 | 
468 |                 if getattr(self.hl_region, position) < min_pos:
469 |                     setattr(self.hl_region, position, min_pos)
470 |                 if getattr(self.hl_region, position) > max_pos:
471 |                     setattr(self.hl_region, position, max_pos - 1)
472 | 
473 |                 if curr_extent > max_pos:
474 |                     setattr(
475 |                         self.hl_region,
476 |                         scale,
477 |                         max_pos - getattr(self.hl_region, position)
478 |                     )
479 | 
480 |             # Start the selection settled countdown timer
481 |             self._selection_settled(False)
482 |             self._reset_settled_countdown("2s")
483 | 
484 |             self.can.freeze()
485 | 
486 |     def _reset_settled_countdown(self, countdown):
487 |         def _inner():
488 |             self._selection_settled(True)
489 |             self.settled_countdown_timer = None
490 | 
491 |         if self.settled_countdown_timer:
492 |             cron.cancel(self.settled_countdown_timer)
493 |         self.settled_countdown_timer = cron.after(countdown, _inner)
494 | 
495 | 
496 | class ImageSelectorOverlay(BoxSelectorOverlay):
497 |     """
498 |     Allows the user to select a region on the screen to use as an image for the locator API.
499 |     """
500 |     def __init__(self, *args, **kwargs):
501 |         super().__init__(*args, **kwargs)
502 |         self.offset_coord = None
503 |         self.result_rects = []
504 | 
505 |     def _calculate_result(self):
506 |         if self.hl_region:
507 |             if self.offset_coord is not None:
508 |                 cx, cy = self._get_region_centre()
509 |                 offset = (
510 |                     self.offset_coord.x - cx,
511 |                     self.offset_coord.y - cy
512 |                 )
513 |             else:
514 |                 offset = None
515 | 
516 |             index = 0
517 |             for i, rect in enumerate(self.result_rects):
518 |                 if rect == self._get_region():
519 |                     index = i
520 | 
521 |             return {
522 |                 "image": self._get_cropped_image(),
523 |                 "offset": offset,
524 |                 "index": index
525 |             }
526 |         else:
527 |             return None
528 | 
529 |     def _selection_settled(self, finished_selection):
530 |         if finished_selection == False:
531 |             self.result_rects = []
532 |             return
533 | 
534 |         th = threading.Thread(target=self._find_matches, daemon=True)
535 |         th.start()
536 |         th.join(timeout=1)
537 |         timed_out = th.is_alive()
538 | 
539 |         if timed_out or len(self.result_rects) > 20:
540 |             self._show_flash(
541 |                 "Too many matches, not showing any of them"
542 |             )
543 |             self.result_rects = []
544 | 
545 |         self.can.freeze()
546 | 
547 |     def _draw_widgets(self, canvas):
548 |         super()._draw_widgets(canvas)
549 |         if not self.hl_region:
550 |             return
551 | 
552 |         # Draw other matching regions
553 |         if len(self.result_rects) > 0:
554 |             self._draw_matches(canvas)
555 | 
556 |         # Draw the offset marker
557 |         canvas.paint = paint.Paint()
558 |         canvas.paint.style = canvas.paint.Style.FILL
559 |         canvas.paint.color = "ff00ffff"
560 |         canvas.paint.stroke_width = 1
561 |         if self.offset_coord is None:
562 |             args = [
563 |                 self.hl_region.x + self.hl_region.width / 2,
564 |                 self.hl_region.y + self.hl_region.height / 2
565 |             ]
566 |         else:
567 |             args = [
568 |                 self.offset_coord.x,
569 |                 self.offset_coord.y
570 |             ]
571 |             canvas.paint.antialias = True
572 |             canvas.draw_line(
573 |                 *self._get_region_centre(),
574 |                 *args
575 |             )
576 | 
577 |         canvas.draw_circle(
578 |             *args,
579 |             2
580 |         )
581 | 
582 |     def _find_matches(self):
583 |         cropped_img = self._get_cropped_image()
584 |         if cropped_img is None:
585 |             self.result_rects = []
586 |             return
587 | 
588 |         self.result_rects = locate.locate_in_image(
589 |             self.image,
590 |             cropped_img,
591 |             threshold=0.9
592 |         )
593 |         self.result_rects = [
594 |             TalonRect(
595 |                 rect.x,
596 |                 rect.y,
597 |                 rect.width,
598 |                 rect.height
599 |             )
600 |             for rect in self.result_rects
601 |         ]
602 | 
603 |     def _draw_matches(self, canvas):
604 |         canvas.paint = paint.Paint()
605 |         canvas.paint.color = "ff0000aa"
606 |         canvas.paint.style = canvas.paint.Style.STROKE
607 |         canvas.paint.stroke_width = 1
608 |         for rect in self.result_rects:
609 |             if rect == self._get_region():
610 |                 continue
611 |             canvas.save()
612 |             canvas.clip_rect(rect, canvas.ClipOp.INTERSECT)
613 |             canvas.draw_image(self.image, 0, 0)
614 |             canvas.restore()
615 |             canvas.draw_rect(rect)
616 | 
617 |     def _mouse_event(self, evt):
618 |         super()._mouse_event(evt)
619 | 
620 |         if evt.event == "mousedown" and evt.button == 0:
621 |             # Reset the coord when a new box is started
622 |             self.offset_coord = None
623 | 
624 |         if evt.event == "mouseup" and evt.button == 1:
625 |             self.offset_coord = evt.gpos
626 |             self.can.freeze()
627 | 
628 | 
629 | class BlobBoxOverlay(BoxSelectorOverlay):
630 |     """
631 |     And overlay that helps the user build a blob box by displaying the matched blobs
632 |     live as they define boxes.
633 |     """
634 | 
635 |     def _selection_settled(self, finished_selection):
636 |         if finished_selection == False:
637 |             self.markers = []
638 |             return
639 | 
640 |         maybe_image = self._get_cropped_image()
641 |         if maybe_image is None:
642 |             return
643 | 
644 |         img = np.array(maybe_image)
645 |         region = self._get_region()
646 |         rects = calculate_blob_rects(img, region)
647 | 
648 |         self.markers = [
649 |             MarkerUi.Marker(
650 |                 rect,
651 |                 label
652 |             )
653 |             for rect, label in zip(rects, "abcdefghijklmnopqrstuvwxyz0123456789"*3)
654 |         ]
655 |         self.can.freeze()
656 | 
657 |     def _draw_widgets(self, canvas):
658 |         super()._draw_widgets(canvas)
659 | 
660 |         if not self.hl_region:
661 |             return
662 | 
663 |         if self.markers:
664 |             MarkerUi.draw_markers(canvas, self.markers)
665 | 


--------------------------------------------------------------------------------