Click-by-Voice Extension's Popup

Iframe srcdoc button test

The iframe below uses srcdoc to define its contents.

at end 8 | let hint_descriptor = command_text; 9 | let operation = ""; 10 | // Allow :'s inside 1 level of balanced {}'s to not count as the before operation separator: 11 | const match = command_text.match(/^((?:[^:\{]|\{[^\}]*\})*):(.*)$/); 12 | if (match) { 13 | hint_descriptor = match[1]; 14 | operation = match[2]; 15 | } 16 | 17 | // Send hint number and operation to content_script.js for current tab: 18 | chrome.tabs.query({active: true, currentWindow: true}, function(tabs) { 19 | chrome.tabs.sendMessage(tabs[0].id, 20 | {hint_descriptor: hint_descriptor, 21 | operation: operation}); 22 | if (close_window) { 23 | // Closing the pop up window ends its JavaScript execution 24 | // so need to do it only after all done here. 25 | window.close(); 26 | } 27 | }); 28 | } 29 | -------------------------------------------------------------------------------- /doc/config.md: -------------------------------------------------------------------------------- 1 | # Experimental config 2 | 3 | On the Click-by-Voice options page as of version 0.20 there is a form 4 | for a Click-by-voice config. This config allows specifying switch 5 | defaults on a per-website basis. 6 | 7 | Better documentation to be written, but here's an example config in the 8 | meanwhile: 9 | ``` 10 | # default case: 11 | when .* 12 | h 13 | 14 | when https://www.reddit.com 15 | ^{.rank, .arrow, div.thing} 16 | !{.tagline} 17 | 18 | when github.com 19 | # don't hint line numbers 20 | !{table.js-file-line-container} 21 | 22 | when quip.com 23 | # disable CbV because hints confuse quip 24 | - 25 | ``` 26 | 27 | Very roughly, there are a series of stanzas each of which contains a 28 | regular expression that is matched against the current URL. The 29 | contents of the stanzas whose regexes match are concatenated together in 30 | order to make the default switches. 31 | 32 | For the example config, `https://quip.com/product` produces switch 33 | defaults of `h-` and `https://www.reddit.com/?count=25&after=t3_b7rjf7` 34 | produces `h^{.rank, .arrow, div.thing}!{.tagline}`. These defaults apply 35 | before any switches of the current show hints command. Thus, if the 36 | current command is `:+ic` then for the reddit page, the complete set of 37 | switches would be `h^{.rank, .arrow, div.thing}!{.tagline}+{1}ic` 38 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2016-2018 by Mark Lillibridge. 2 | 3 | I am providing code in this repository to you under an open source 4 | license. Because this is my personal repository, the license you 5 | receive to my code is from me and not from my employer (Facebook). 6 | 7 | 8 | 9 | [MIT license] 10 | 11 | Permission is hereby granted, free of charge, to any person 12 | obtaining a copy of this software and associated documentation files 13 | (the "Software"), to deal in the Software without restriction, including 14 | without limitation the rights to use, copy, modify, merge, publish, 15 | distribute, sublicense, and/or sell copies of the Software, and to 16 | permit persons to whom the Software is furnished to do so, subject to 17 | the following conditions: 18 | 19 | The above copyright notice and this permission notice shall be 20 | included in all copies or substantial portions of the Software. 21 | 22 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 23 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 24 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 25 | IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 26 | CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 27 | TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 28 | SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 29 | -------------------------------------------------------------------------------- /src/background_clipboard_offscreen.js: -------------------------------------------------------------------------------- 1 | // 2 | // JavaScript to implement background_clipboard_offscreen.html page. 3 | // 4 | // That is an offscreen document used to implement clipboard access 5 | // for the service worker; see background_clipboard.js for how it is used. 6 | // 7 | 8 | 9 | // 10 | // Dispatch according to background_clipboard function desired 11 | // 12 | 13 | chrome.runtime.onMessage.addListener(function(message, sender, sendResponse) { 14 | if (message.target !== 'background_clipboard_offscreen') { 15 | return; 16 | } 17 | 18 | switch (message.type) { 19 | case 'getClipboard': 20 | sendResponse({value: getClipboard()}); 21 | break; 22 | case 'putClipboard': 23 | putClipboard(message.value); 24 | break; 25 | default: 26 | console.error(`Unexpected message type received: '${message.type}'.`); 27 | } 28 | }); 29 | 30 | 31 | // 32 | // The functions themselves 33 | // 34 | 35 | function getClipboard() { 36 | const pasteTarget = document.querySelector('#text'); 37 | pasteTarget.contentEditable = true; 38 | pasteTarget.value = ""; 39 | pasteTarget.focus(); 40 | document.execCommand("paste"); 41 | return pasteTarget.value; 42 | }; 43 | 44 | function putClipboard(text) { 45 | const copyTarget = document.querySelector('#text'); 46 | copyTarget.value = text; 47 | copyTarget.select(); 48 | document.execCommand("copy"); 49 | }; 50 | -------------------------------------------------------------------------------- /src/options.js: -------------------------------------------------------------------------------- 1 | // 2 | // JavaScript to implement options.html page, which is the pop-up for setting extension options. 3 | // 4 | 5 | import * as option_storage from './option_storage.js'; 6 | 7 | 8 | // Restores shown options using the preferences stored in chrome.storage.sync. 9 | async function restore_options() { 10 | const saved_options = await option_storage.get_saved_options(); 11 | document.getElementById('command').value = saved_options.startingCommand; 12 | document.getElementById('config').value = saved_options.config; 13 | } 14 | 15 | // Saves options to chrome.storage.sync. 16 | async function save_options() { 17 | const command = document.getElementById('command').value; 18 | const config = document.getElementById('config').value; 19 | await option_storage.put_saved_options({ 20 | startingCommand: command, 21 | config: config, 22 | }); 23 | 24 | // Change only per-session config, not per-session starting command: 25 | const current_starting_command = (await option_storage.get_per_session_options()).startingCommand; 26 | await option_storage.put_per_session_options({ 27 | startingCommand: current_starting_command, 28 | config: config, 29 | }); 30 | 31 | // Update status to let user know options were saved. 32 | const status = document.getElementById('status'); 33 | status.textContent = 'Click-by-Voice options successfully saved.'; 34 | setTimeout(function() { 35 | status.textContent = ''; 36 | }, 750); 37 | } 38 | 39 | 40 | document.addEventListener('DOMContentLoaded', restore_options); 41 | document.getElementById('save').addEventListener('click', save_options); 42 | -------------------------------------------------------------------------------- /src/batching_updates.js: -------------------------------------------------------------------------------- 1 | /// 2 | /// Framework for alternating sensing and mutating the DOM to avoid 3 | /// extra forced layouts. 4 | /// 5 | /// Provides Batcher 6 | 7 | "use strict"; 8 | 9 | let Batcher = null; 10 | 11 | (function() { 12 | 13 | let sensing_work = []; 14 | let mutating_work = []; 15 | 16 | // Use these functions to submit work for this cycle. 17 | function sensing(thunk) { 18 | sensing_work.push(thunk); 19 | } 20 | function mutating(thunk) { 21 | mutating_work.push(thunk); 22 | } 23 | 24 | // Run all work for the current cycle, continuing until no more 25 | // work submitted for the current cycle. 26 | function do_work() { 27 | let result = ""; 28 | while (sensing_work.length + mutating_work.length > 0) { 29 | let work = sensing_work; sensing_work = []; 30 | result += "; " + run_sensing_stage(work); 31 | work = mutating_work; mutating_work = []; 32 | result += "; " + run_mutating_stage(work); 33 | } 34 | return result.substring(2); 35 | } 36 | 37 | 38 | // These are separate helping functions so they can be 39 | // distinguished in stack traces. 40 | function run_sensing_stage(work) { 41 | const start = performance.now(); 42 | work.map(function (thunk) { 43 | thunk(); 44 | }); 45 | return Util.time(start); 46 | } 47 | function run_mutating_stage(work) { 48 | const start = performance.now(); 49 | work.map(function (thunk) { 50 | thunk(); 51 | }); 52 | return Util.time(start); 53 | } 54 | 55 | 56 | Batcher = { 57 | sensing: sensing, 58 | mutating: mutating, 59 | do_work: do_work 60 | }; 61 | })(); 62 | -------------------------------------------------------------------------------- /src/background_clipboard.js: -------------------------------------------------------------------------------- 1 | // 2 | // Routines for accessing the clipboard from the background service worker. 3 | // 4 | 5 | 6 | // 7 | // As of September 2023, manifest version 3 requires a workaround to 8 | // achieve this using an offscreen document. 9 | // 10 | 11 | let creating; // A global promise to avoid concurrency issues 12 | async function setupOffscreenDocument(path) { 13 | // Check all windows controlled by the service worker to see if one 14 | // of them is the offscreen document with the given path. 15 | const offscreenUrl = chrome.runtime.getURL(path); 16 | const existingContexts = await chrome.runtime.getContexts({ 17 | contextTypes: ['OFFSCREEN_DOCUMENT'], 18 | documentUrls: [offscreenUrl] 19 | }); 20 | 21 | if (existingContexts.length > 0) { 22 | return; 23 | } 24 | 25 | // create offscreen document 26 | if (creating) { 27 | await creating; 28 | } else { 29 | creating = chrome.offscreen.createDocument({ 30 | url: path, 31 | reasons: ['CLIPBOARD'], 32 | justification: 'Reading and writing text from/to the clipboard', 33 | }); 34 | await creating; 35 | creating = null; 36 | } 37 | } 38 | 39 | async function createOffscreenDocument() { 40 | await setupOffscreenDocument('background_clipboard_offscreen.html'); 41 | } 42 | 43 | 44 | // 45 | // The actual routines using the offscreen document. 46 | // 47 | 48 | export async function getClipboard() { 49 | await createOffscreenDocument(); 50 | let response = await chrome.runtime.sendMessage({ 51 | type: 'getClipboard', 52 | target: 'background_clipboard_offscreen' 53 | }); 54 | return response.value; 55 | }; 56 | 57 | export async function putClipboard(text) { 58 | await createOffscreenDocument(); 59 | await chrome.runtime.sendMessage({ 60 | type: 'putClipboard', 61 | target: 'background_clipboard_offscreen', 62 | value: text 63 | }); 64 | }; 65 | -------------------------------------------------------------------------------- /doc/activation_commands.md: -------------------------------------------------------------------------------- 1 | # Hint activation commands 2 | 3 | Click by Voice hint activation commands consist of a 4 | *hint_specification* optionally followed by a colon then an *operation*. 5 | For example, `42` specifies activating the element with hint number 42 6 | using the default operation and `34:k` specifies activating hint 34 7 | using operation `k`. An empty operation (e.g., `153:`) is equivalent to 8 | specifying the default operation. 9 | 10 | 11 | ## Available operations 12 | 13 | The following operations are currently available: 14 | 15 | * `f` focuses the element (doesn't work on all elements, does work on iframes) 16 | * `c` clicks the element 17 | * `t` opens links and iframes in a new tab, changing focus to that tab 18 | * `b` opens links and iframes in a new tab, but does not change focus to 19 | that tab 20 | * `w` opens links and iframes in a new window, changing focus to that window 21 | * `k` copies link and iframe locations to the clipboard 22 | * `h` simulates hovering the mouse over the element; repeat to unhover 23 | * `s` copies the text contents of the element to the clipboard 24 | 25 | The default operation either clicks or focuses the given element using 26 | heuristics to decide which makes more sense. If CbV guesses wrong, you 27 | can explicity specify `c` or `f` to force clicking or focusing 28 | respectively. 29 | 30 | Note that `t`, `b`, `w`, and `k` work only on links and iframes that 31 | explicitly give a target address (currently `` and 32 | ` 129 | 130 |

Plain paragraph: click sets caret/selection.

131 | 132 | 133 | 134 |

135 |

SVG interactive map using <path> regions

136 | 137 |

Click inside left or right shape — each navigates separately.

138 | 139 | 156 | 157 |

This uses SVG's native hit-testing on individual <path> shapes.

158 |

159 | 160 |

161 |

Miscellaneous

162 | This anchor has no href and is thus not applicable or focusable 163 |

164 | 165 |

166 |

Focus only?: HTML inside <object>

167 | 168 |

175 |

176 | 177 |

178 |

Elements made focusable via `contenteditable`

179 | 180 |

181 | Normally, a plain <div> or <span> is not focusable. 182 | Adding contenteditable makes it an editable region 183 | that can receive focus and caret. 184 |

185 | 186 |

189 | This is a <div> with contenteditable="true".
190 | Click here and type to test focus and caret. 191 |

192 | 193 |

194 | You can also make inline elements editable: 195 |

196 | 197 |

198 | Normal text before – 199 | 202 | editable <span> (click and type) 203 | 204 | – normal text after. 205 |

206 |

207 | 208 |

209 |

Elements made focusable via `tabindex`

210 | 211 |

212 | The tabindex attribute controls whether an element is focusable, 213 | and where it appears in the tab order. 214 |

215 | 216 |

tabindex="0" → focusable, participates in normal tab order
tabindex=">0" → focusable, custom tab order (discouraged)
tabindex="-1" → focusable by script only (no tab stop)

221 | 222 |

Try tabbing through the controls below and watch the order.

223 | 224 | 225 |

226 | 227 |

228 | 229 | 230 |

233 | <div tabindex="0"> — focusable, in normal tab order 234 |

235 | 236 | 237 |

240 | <div tabindex="2"> — focusable, with positive tabindex (comes earlier in tab order than some elements) 241 |

242 | 243 | 244 |

247 | <div tabindex="1"> — focusable, custom tab order (will usually get focus before tabindex="2" and 0) 248 |

249 | 250 | 251 |

255 | <div tabindex="-1"> — focusable via script only; you cannot tab to this. 256 |

257 | 258 |

259 | 262 |

263 |

264 | 265 | 266 | 267 | -------------------------------------------------------------------------------- /src/hints.js: -------------------------------------------------------------------------------- 1 | /// 2 | /// Overall control code for labeling elements with hint tags 3 | /// 4 | /// Provides Hints 5 | /// 6 | 7 | "use strict"; 8 | 9 | var Hints = null; 10 | 11 | (function() { 12 | 13 | let config_ = ""; 14 | let hinting_on_ = true; 15 | let options_ = new Map(); 16 | 17 | 18 | // 19 | // Main exported actions: 20 | // 21 | 22 | function set_config(config) { 23 | config_= config; 24 | } 25 | 26 | function add_hints(parameters) { 27 | set_hinting_parameters(parameters); 28 | if (option_value('+', 1) > 0) { 29 | hinting_on_ = true; 30 | place_hints(false); 31 | } else { 32 | Util.vlog(0, "not adding hints: " + options_to_string()); 33 | remove_hints(); 34 | } 35 | } 36 | 37 | function refresh_hints() { 38 | if (hinting_on_) 39 | place_hints(true); 40 | } 41 | 42 | function remove_hints() { 43 | HintManager.discard_hints(); 44 | remove_hints_from(document) 45 | hinting_on_ = false; 46 | } 47 | function remove_hints_from(from) { 48 | $("[CBV_hint_element]", from).remove(); 49 | const $frame = $("iframe, frame", from); 50 | if ($frame.length != 0) { 51 | remove_hints_from($frame.contents()); 52 | } 53 | } 54 | 55 | // 56 | // Parameters for hinting: 57 | // 58 | 59 | function reset_option(option_name) { 60 | options_.delete(option_name); 61 | } 62 | function set_option(option_name, args) { 63 | // The main mode switches are exclusive: 64 | if (/^[ioh]$/.test(option_name)) { 65 | reset_option("i"); 66 | reset_option("o"); 67 | reset_option("h"); 68 | } 69 | // +/- are special cases: 70 | if (option_name == '-') { 71 | options_.set('+', [0]); 72 | return; 73 | } else if (option_name == '+') { 74 | if (args.length > 0) { 75 | options_.set('+', args); 76 | } else { 77 | options_.set('+', [option_value('+',0) + 1]); 78 | } 79 | return; 80 | } 81 | // syntax for long option names & reseting options: 82 | if (option_name == 'X') { 83 | if (args.length > 0) { 84 | option_name = args[0]; 85 | args = args.slice(1); 86 | if (/-$/.test(option_name)) { 87 | // X{-} 88 | reset_option(option_name.substring(0, option_name.length-1)); 89 | } else { 90 | // X{} 91 | set_option(option_name, args); 92 | } 93 | } 94 | return; 95 | } 96 | options_.set(option_name, args); 97 | } 98 | 99 | function option(option_name) { 100 | return options_.has(option_name); 101 | } 102 | function option_value(option_name, default_value) { 103 | if (options_.has(option_name) && options_.get(option_name).length>0) { 104 | return options_.get(option_name)[0]; 105 | } else { 106 | return default_value; 107 | } 108 | } 109 | 110 | function options_to_string() { 111 | let result = ""; 112 | let flags = ""; 113 | options_.forEach(function(value, key) { 114 | if (value.length==0 && key.length==1) { 115 | flags += key; 116 | } else { 117 | result += ' ' + key + value.map(function (v) { return '{' + v + '}';}).join(''); 118 | } 119 | }); 120 | return flags + result; 121 | } 122 | 123 | function parse_option(text) { 124 | let m; 125 | if (m = text.match(/^\s+(.*)/)) { 126 | text = m[1]; 127 | } 128 | if (m = text.match(/^([^{])\{([^{}]*)\}\{([^{}]*)\}(.*)/)) { 129 | return [m[1], [m[2],m[3]], m[4]]; 130 | } 131 | if (m = text.match(/^([^{])\{([^{}]*)\}(.*)/)) { 132 | return [m[1], [m[2]], m[3]]; 133 | } 134 | return [text[0], [], text.substring(1)]; 135 | } 136 | function set_hinting_parameters(value) { 137 | options_ = new Map(); 138 | let text = get_effective_hints(value, window.location.href); 139 | while (text != "") { 140 | // console.log(text); 141 | const r = parse_option(text); 142 | const name = r[0]; 143 | const args = r[1]; 144 | text = r[2]; 145 | // console.log([name, args, text]); 146 | set_option(name, args); 147 | } 148 | } 149 | 150 | function with_high_contrast(callback) { 151 | const saved = options_; 152 | options_= new Map(options_); 153 | set_option('c', []); 154 | callback(); 155 | options_ = saved; 156 | } 157 | 158 | 159 | 160 | // 161 | // 162 | // 163 | 164 | function get_effective_hints(user_hints, url) { 165 | // Config stanzas are separated by one or more blank lines: 166 | const without_trailing_whitespace = config_.replace(/[ \t]+$/gm, ""); 167 | const stanzas = without_trailing_whitespace.split(/\n\n+/); 168 | 169 | let config_hints = ""; 170 | for (let stanza of stanzas) { 171 | // Comments are # at beginning of line (possibly indented) to end of line: 172 | // (CSS selectors can contain #'s) 173 | stanza = stanza.replace(/^[ \t]*#.*$/gm, ""); 174 | // Remove any blank lines inside the stanza that result from removing comments: 175 | stanza = stanza.replace(/^\n/gm, ""); 176 | 177 | let match; 178 | if (match = stanza.match(/^when +(.+?) *\n((?:.|\n)*)/)) { 179 | const regex = match[1]; 180 | // Remove obviously unnecessary whitespace at beginning and end of lines, newlines: 181 | // (CSS selectors can contain spaces; we do not allow them to span lines) 182 | const options = match[2].replace(/^\s+/gm,'').replace(/\s+$/gm, '').replace(/\n/gm,''); 183 | // console.log(`${regex} => ${options}`); 184 | if (new RegExp(regex).test(url)) { 185 | // console.log("match!"); 186 | config_hints += options 187 | } 188 | } else if (stanza != '') { 189 | console.error("Bad click-by-voice config stanza:\n", stanza); 190 | } 191 | } 192 | 193 | const default_hints = "h"; 194 | return default_hints + config_hints + user_hints; 195 | } 196 | 197 | 198 | // 199 | // 200 | // 201 | 202 | function place_hints(refreshing) { 203 | if (!refreshing) { 204 | Util.vlog(0, "adding hints: " + options_to_string()); 205 | } 206 | 207 | const starting_hint_count = HintManager.get_hint_number_stats().hints_made; 208 | const start = performance.now(); 209 | 210 | // DomWalk.each_displaying( 211 | // function (element, styles) {}, 212 | // function (element, styles) {}, 213 | // ""); 214 | // console.log(" just DomWalk time: " + (performance.now()-start) + " ms"); 215 | // start = performance.now(); 216 | 217 | // FindHint.each_hintable(function(element) {}); 218 | // console.log(" just FindHint.each_hintable time: " + (performance.now()-start) + " ms"); 219 | // start = performance.now(); 220 | 221 | FindHint.each_hintable(function($element) { 222 | if (HintManager.is_hinted_element($element[0])) 223 | return; 224 | AddHint.add_hint($element); 225 | }); 226 | const work_start = performance.now(); 227 | Batcher.sensing( () => { HintManager.adjust_hints(); } ); 228 | const result = Batcher.do_work(); 229 | 230 | const stats = HintManager.get_hint_number_stats(); 231 | const hints_made = stats.hints_made - starting_hint_count; 232 | if (Hints.option("timing") || hints_made > 0) { 233 | const max_hint_number = stats.max_hint_number_used; 234 | const hints_in_use = stats.hints_in_use; 235 | Util.vlog(1, `+${hints_made} -> ${hints_in_use} hints` + 236 | ` (number high water ${max_hint_number})` + 237 | ` in ${Util.time(start)}: walk: ${Util.time(start, work_start)};` + 238 | ` add/adjust: ${result}`); 239 | 240 | // for (let i = 1; i < 10; i++) { 241 | // Batcher.sensing( () => { HintManager.adjust_hints(); } ); 242 | // console.log("again: " +Batcher.do_work()); 243 | // } 244 | } 245 | } 246 | 247 | 248 | 249 | Hints = { 250 | set_config: set_config, 251 | 252 | add_hints: add_hints, 253 | refresh_hints: refresh_hints, 254 | remove_hints: remove_hints, 255 | 256 | option: option, 257 | option_value: option_value, 258 | with_high_contrast: with_high_contrast, 259 | }; 260 | })(); 261 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Click by Voice 2 | 3 | This Chrome browser extension provides support for activating links and 4 | other HTML elements using voice commands. It displays small numbers 5 | next to each activatable element called *hints* and provides mechanisms 6 | to activate these elements using the hint numbers. This allows creating 7 | voice commands (via other software) that lets users activate links by 8 | saying their hint numbers. 9 | 10 | Like many Chrome browser extensions, this extension can also be used 11 | with the Microsoft Edge browser. 12 | 13 | 14 | ## Using Click by Voice manually 15 | 16 | Click by Voice provides two keyboard shortcuts suitable for manual use; 17 | they are bound by default to `{ctrl+shift+space}` (*pop up command 18 | dialog box*) and `{ctrl+shift+,}` (*blur*). You can rebind these as 19 | desired like with any Chrome extension by following the link 20 | chrome://extensions/shortcuts in your Chrome browser. 21 | 22 | The blur shortcut removes keyboard focus from an element, returning it 23 | to the overall webpage. This can be useful, for example, when you want 24 | to page the website up and down but an input element like a text field 25 | has focus. 26 | 27 | ### Activating hints 28 | 29 | The pop-up command shortcut pops up a small dialog box in the upper 30 | right asking for the hint number that should be activated. At its 31 | simplest, typing the number displayed next to an element then pressing 32 | enter will dismiss the dialog box then click or focus that element as 33 | appropriate. (Click by Voice uses heuristics to attempt to determine 34 | whether an element should be clicked or focused if you don't specify 35 | which to do.) 36 | 37 | To specify that something different be done with the element, add a 38 | colon then an operation code. For example, `153:t` opens the link with 39 | hint number 153 in a new tab. Many different operations on hinted 40 | elements are available, including copying the destination URL for a link 41 | and copying the text of an element; see the 42 | [list of available operations](./doc/activation_commands.md) for more. 43 | 44 | Instead of providing a hint number, you can provide a CSS selector that 45 | specifies which element you wish to activate. For example, 46 | `${button.go}:c` clicks the first element that is both a button and of 47 | class `go`. This feature is useful for programmatically activating 48 | elements. 49 | 50 | You can dismiss the command dialog box without activating anything by 51 | typing `{escape}`. 52 | 53 | ### Displaying hints 54 | 55 | You can change how hints are displayed for the current tab by using a 56 | _show hints_ command. The simplest such commands are: 57 | 58 | * `:+` shows standard hints 59 | * `:++` is similar but displays more hints, attempting to hint every 60 | element that might be clickable or focusable, however unlikely that 61 | might be 62 | * `:-` shows no hints 63 | 64 | To use these commands, just enter them into the hint number popup 65 | instead of a hint number. 66 | 67 | Click by Voice normally remembers the last such command you have given 68 | (in any tab) and automatically uses it when a new page is loaded or the 69 | current tab is reloaded. If you want to only temporarily change how 70 | hints are displayed for a tab, add `once` after the colon; for example, 71 | `:once-` turns off hints for the current tab until it is refreshed and 72 | does not affect future loads of other tabs. 73 | 74 | The hinting system is highly flexible, with these commands taking many 75 | optional switches. For details, including how to change startup and 76 | per-webpage defaults, see 77 | [displaying hints in detail](./doc/displaying_hints.md). 78 | 79 | Hint numbers are not shown when printing and should not show up when you 80 | copy from a hinted webpage. 81 | 82 | 83 | ## Using Click by Voice with voice commands 84 | 85 | **This extension by itself provides no voice functionality**; 86 | procurement of the needed voice commands is the user's responsibility. 87 | One recommended means of doing this is to use Vocola 88 | (http://vocola.net/) to create the needed voice commands. 89 | 90 | Writing voice commands to use Click by Voice should be straightforward. 91 | As an example, here are some Vocola 2 commands that provide access to 92 | much of the Click by Voice functionality: 93 | 94 | CbV(command) := Clipboard.Set($command!!! Clipboard.Get("")) {ctrl+shift+.}; 95 | 96 | blur me = "{ctrl+shift+,}"; 97 | 98 | := (once); 99 | := (inline=i | overlay=o | hybrid=h | contrasting=c); 100 | show [] hints [] = CbV(:$2+$1); 101 | show more [] hints [] = CbV(:$2++$1); 102 | hide hints [] = CbV(:$1-); 103 | 104 | 0..9 [0..9 [0..9 [0..9]]] = CbV($2$3$4$5:$1); 105 | 106 | := ( pick = "" # guess whether to click or focus 107 | | go pick = f 108 | | click pick = c 109 | | push pick = b # stay but open new tab w/ link or iframe 110 | | tab pick = t 111 | | window pick = w 112 | | hover pick = h 113 | | link pick = k # copy link destination address 114 | | copy pick = s 115 | ); 116 | 117 | These commands take advantage of another Click by Voice keyboard 118 | shortcut, `{ctrl+shift+.}` by default, which makes Click by Voice accept 119 | a command from the clipboard rather than via the pop-up dialog box. For 120 | more on how this shortcut works, see 121 | [on making voice commands](./doc/making_voice_commands.md). 122 | 123 | 124 | ## Known issues (7/12/2024) 125 | 126 | ### Selection of elements to hint 127 | 128 | * Elements inside of cross-origin iframes are missed 129 | * same-origin [i]frames should work fine now 130 | * iframes themselves are now hinted and can be focused or opened in a 131 | new tab or window 132 | * Elements added after a page is first loaded can take a while to get 133 | hinted 134 | * to keep performance reasonable, Click by Voice only automatically 135 | refreshes hints every three seconds 136 | * CbV will automatically refresh a page's hints shortly after you 137 | activate a hint to quickly handle cases where activating a hint 138 | reveals new elements (e.g., a drop-down menu) 139 | * Normal hint level (`:+`) does not find elements that are only clickable 140 | because of event listeners 141 | * hopefully, `:++` should find most of these. 142 | * Some invisible elements are still hinted 143 | * Some elements come into existence only after a mouse hovers over 144 | another element; although hinted once they appear, the hover action 145 | may be necessary to make them appear. 146 | 147 | ### Hint activation 148 | 149 | * Some elements can be difficult to select even using CSS selectors 150 | * e.g., multiple elements that differ only by their contained text 151 | * Some hint activations do not work properly due to insufficient 152 | fidelity of the synthetically generated events 153 | * e.g., the generated mouse events do not include coordinates and 154 | hover does not simulate moving the mouse over all the parent 155 | elements to the target element 156 | * Chrome clipboard bugs: 157 | * using !!! without following text (e.g., wanted an empty clipboard afterwards) leaves the command in the clipboard because there is no way to empty the clipboard in Chrome extensions 158 | * using !!! with thousands of lines can hang the browser for quite a long time 159 | 160 | ### Hint display 161 | 162 | * The default hinting mode, hybrid, disrupts the flow of some webpages 163 | and its hints can be clipped 164 | * switching to the overlay mode (`:+o`) should not disturb the flow at 165 | all at the cost of making text hard to read 166 | * Sometime hints are too hard to read due to inadequate contrast between 167 | foreground and background colors 168 | * Adding the high contrast hints switch (e.g, `:+c`) should make the 169 | hints stand out more and be easier to read at the cost of making 170 | them more distracting 171 | * Webpages changing an element after the initial page load can make that 172 | element's hint disappear 173 | * usually refreshing hints will make the hint reappear 174 | * When a webpage removes then re-creates an element, it will get a new 175 | hint; when done repeatedly, this causes the element's hint number to 176 | keep increasing 177 | 178 | ### Other issues 179 | 180 | * Click by Voice, like any Chrome extension, is unable to run on 181 | `chrome://` URLs like the settings and extensions pages or in built-in 182 | dialog boxes like the "add bookmarks" dialog box 183 | * ditto `https://chrome.google.com` URLs (e.g., the developer dashboard) 184 | * Some applications actually read out data from the browser webpage 185 | representation (DOM) and can become confused by the hints 186 | * this unfortunately appears to include Dragon's Chrome extension 187 | * Dragon may think the name of a link includes the hint number at the end 188 | * a simple workaround is to either include the hint number or only 189 | use a prefix of the link name; e.g., say `click submit` or `click 190 | submit form twelve` for a link named `Submit Form` with hint 191 | number 12. 192 | * Also affected are many Google docs/mail sites 193 | * e.g., Gmail 194 | * When used with Microsoft edge, the extension options dialog box is 195 | too small and appears with scrollbars 196 | 197 | 198 | ## News 199 | 200 | * 11/2025: New major version 0.32 released 201 | * now reuses hint numbers of no longer existing/connected elements 202 | * refreshes hints more often while limiting CPU 203 | * no longer adds attribute CBV_hint_element to hinted_elements by default 204 | * this should reduce interference with some applications 205 | * only logs to console in response to user actions or if verbosity level has been raised 206 | * 6/2024: New major version 0.30 released 207 | * switched to using manifest version 3 208 | * 11/2021: New version 0.23.4 released 209 | * nested same-origin [i]frames now work properly courtesy of Quinn Tucker 210 | * the hints no longer appear as text in the DOM 211 | * this means they should no longer be copied when you cut-and-paste a region containing them 212 | * may also reduce interference with applications that inspect their DOM 213 | * 12/2020: New major version 0.22 released 214 | * major performance improvements; perceived lag of using CBV should be 215 | greatly reduced as well as CPU usage 216 | * 8/2020: New major version 0.21 released 217 | * Same-origin iframes are now supported 218 | * as of 0.21.4, click-by-voice periodically checks and updates overlay 219 | positioning if needed; this includes removing the overlay if the 220 | underlying element is no longer connected/present 221 | * 4/2019: New major version 0.20 released 222 | * no official changes, but new experimental config feature available 223 | * 4/2018: New major version 0.19 released. 224 | * default mode is now hybrid (was inline; use `:+i` to get previous behavior) 225 | * different basic modes and switching between them is no longer experimental 226 | * documentation now covers various features introduced recently 227 | * sending commands by clipboard, nonpersistent show hints commands 228 | 229 | 230 | ## Other 231 | 232 | Please address questions and issues to this 234 | KnowBrainer thread. 235 | -------------------------------------------------------------------------------- /src/HintManager.js: -------------------------------------------------------------------------------- 1 | /// 2 | /// TBD 3 | /// 4 | /// Provides HintManager 5 | 6 | "use strict"; 7 | 8 | let HintManager = null; 9 | 10 | (function() { 11 | 12 | // HintNumberGenerator: responsible for doling out hint numbers 13 | // that are not currently used. 14 | // 15 | // To allow reuse, release hint_numbers when they are no longer in use. 16 | // We attempt to use the smallest hint numbers possible. 17 | class HintNumberGenerator { 18 | #next_hint_number = 0; 19 | #hints_made = 0; 20 | #hints_retired = 0; 21 | #max_hint_number_used = -1; 22 | 23 | #retired_hint_numbers = []; 24 | #is_retired_hint_numbers_sorted = true; 25 | 26 | generate() { 27 | this.#hints_made++; 28 | if (!Hints.option("avoiding_reuse") && this.#retired_hint_numbers.length > 0) { 29 | if (!this.#is_retired_hint_numbers_sorted) { 30 | // Sort on demand so we sort only once per batch of numbers generated. 31 | this.#retired_hint_numbers.sort(); 32 | this.#is_retired_hint_numbers_sorted = true; 33 | } 34 | const number = this.#retired_hint_numbers.shift(); 35 | Util.vlog(2, `reusing hint number ${number}`); 36 | return number; 37 | } 38 | 39 | this.#max_hint_number_used = this.#next_hint_number++; 40 | return this.#max_hint_number_used; 41 | } 42 | 43 | release(hint_number) { 44 | this.#hints_retired++; 45 | this.#retired_hint_numbers.push(hint_number); 46 | this.#is_retired_hint_numbers_sorted = false; 47 | } 48 | 49 | get stats() { 50 | return { 51 | hints_made: this.#hints_made, 52 | max_hint_number_used: this.#max_hint_number_used, 53 | hints_in_use: this.#hints_made - this.#hints_retired 54 | }; 55 | } 56 | } 57 | 58 | 59 | // forward: function _remove_hint(hint_number, hinted_element) 60 | var _remove_hint; 61 | 62 | // Hint: 63 | // 64 | // 65 | class Hint { 66 | #hint_number; 67 | #hinted_element; 68 | #hint_tag; 69 | 70 | displacement; 71 | show_at_end; 72 | 73 | constructor(hint_number, hinted_element) { 74 | this.#hint_number = hint_number; 75 | this.#hinted_element = new WeakRef(hinted_element); 76 | } 77 | 78 | initialize(hint_tag) { 79 | this.#hint_tag = hint_tag; 80 | } 81 | 82 | 83 | get hint_number() { 84 | return this.#hint_number; 85 | } 86 | 87 | // Note that this verbose logs to the console as a side effect. 88 | get hinted_element() { 89 | const element = this.#hinted_element.deref(); 90 | if (!element) { 91 | Util.vlog(0, `The element that had hint ${this.#hint_number} longer exists`); 92 | return undefined; 93 | } 94 | if (!element.isConnected) { 95 | Util.vlog(0, `The element with hint ${this.#hint_number} is no longer connected`); 96 | return undefined; 97 | } 98 | return element; 99 | } 100 | 101 | dump() { 102 | console.log(`Hint information for hint number ${this.#hint_number}:`); 103 | console.log(this); 104 | const hinted_element = this.#hinted_element.deref(); 105 | if (hinted_element) { 106 | console.log(hinted_element); 107 | } else { 108 | console.log("hinted element has been garbage collected"); 109 | } 110 | console.log(this.#hint_tag); 111 | } 112 | 113 | 114 | // precondition: we are in a sensing step 115 | adjust() { 116 | const hinted_element = this.#hinted_element.deref(); 117 | if (!hinted_element || !hinted_element.isConnected) { 118 | Util.vlog(2, `The element with hint ${this.#hint_number} is no longer connected`); 119 | this.#remove(); 120 | return; 121 | } 122 | 123 | if (!this.displacement) { 124 | return; 125 | } 126 | 127 | // It's an overlay hint... 128 | 129 | const hint_number = this.#hint_number; 130 | const $element = $(hinted_element); 131 | const $inner = $(this.#hint_tag).children().first(); 132 | const show_at_end = this.show_at_end; 133 | const displacement = this.displacement; 134 | 135 | if (!$inner[0].isConnected) { 136 | if (Hints.option("keep_hints")) { 137 | // some webpages seem to temporarily disconnect then reconnect hints 138 | return; 139 | } 140 | Batcher.mutating(() => { 141 | Util.vlog(2, `lost hint for ${this.#hint_number}; removing...`); 142 | // TODO: automatically reconnect at bottom of body? <<<>>> 143 | // do we need to preserve $outer as well then? 144 | this.#remove(); 145 | }); 146 | return; 147 | } 148 | 149 | const target_box = $element[0].getBoundingClientRect(); 150 | const inner_box = $inner[0] .getBoundingClientRect(); 151 | 152 | 153 | // Figure out whether the element and/or hint tag are hidden 154 | 155 | let element_hidden = false; 156 | 157 | // Below detects display: none. 158 | if (target_box.top == 0 && target_box.left == 0) { 159 | element_hidden = "display: none"; 160 | } 161 | const inner_hidden = (inner_box .top == 0 && inner_box .left == 0); 162 | 163 | // Check for other hiding via CSS. 164 | // 165 | // Don't need to worry about offscreen (the tag moves 166 | // offscreen as well); under something is handled in next section. 167 | if (!element_hidden) { 168 | if (Util.is_under_low_opacity(hinted_element) && Hints.option("hide_opacity")) { 169 | element_hidden = "low opacity"; 170 | } else if (Hints.option("hide_visibility") && Util.css($(hinted_element), "visibility") === "hidden") { 171 | element_hidden = "visibility: hidden"; 172 | } 173 | } 174 | 175 | if (Hints.option("hide_seen") && !element_hidden) { 176 | // transparent padding can pass through clicks 177 | let test_y = target_box.top + Util.css_pixels($element,"padding-top") + 1; 178 | test_y = (target_box.top + target_box.bottom)/2; 179 | let test_x; 180 | if (show_at_end) { 181 | test_x = target_box.right - Util.css_pixels($element,"padding-right") - 1; 182 | } else { 183 | test_x = target_box.left + Util.css_pixels($element,"padding-left") + 1; 184 | } 185 | 186 | // TODO: deal with iframes 187 | // TODO: consider better test point 188 | const topmost_element = document.elementFromPoint(test_x, test_y); 189 | // topmost_element is null if the test point is 190 | // offscreen, which we don't count as hidden 191 | if (topmost_element) { 192 | if (!$element[0].contains(topmost_element)) { 193 | if (true || !topmost_element.contains($element[0])) { 194 | // console.log(`hint ${this.#hint_number} not visible: ${test_x},${test_y}`); 195 | // console.log(topmost_element); 196 | // console.log(topmost_element.getBoundingClientRect()); 197 | // console.log($element[0]); 198 | // console.log($element[0].getBoundingClientRect()); 199 | // if (topmost_element.contains($element[0])) { 200 | // console.log("fell through?") 201 | // } 202 | const elements = document.elementsFromPoint(test_x, test_y) 203 | elements.forEach((element, index) => { 204 | if (index <= 2) { 205 | // console.log(element); 206 | } 207 | }); 208 | // console.log(`${target_box.right - target_box.left}x${target_box.bottom - target_box.top}`); 209 | element_hidden = "not seen"; 210 | } 211 | } 212 | } 213 | } 214 | 215 | 216 | if (Hints.option("reverse-hiding")) { 217 | element_hidden = !element_hidden; 218 | } 219 | 220 | if (element_hidden) { 221 | if (inner_hidden) { 222 | return; 223 | } 224 | Batcher.mutating(() => { 225 | Util.vlog(3, `hiding hint for hidden element ${hint_number};` + 226 | ` due to ${element_hidden}`); 227 | $inner.attr("CBV_hidden", "true"); 228 | }); 229 | return; 230 | } 231 | 232 | let target_top = target_box.top; 233 | let target_left = target_box.left; 234 | if (show_at_end) { 235 | target_left += target_box.width - inner_box.width; 236 | } 237 | target_top -= displacement.up; 238 | target_left += displacement.right; 239 | if (inner_hidden) { 240 | // TODO: what if hidden attribute already removed? 241 | const style = $inner[0].style; 242 | if (style == undefined) return; // XML case... 243 | let inner_top = parseFloat(style.top); 244 | let inner_left = parseFloat(style.left); 245 | Batcher.mutating(() => { 246 | Util.vlog(3, `unhiding hint for unhidden element ${hint_number}`); 247 | $inner.removeAttr("CBV_hidden"); 248 | $inner[0].style.top = `${inner_top + target_top - inner_box.top}px`; 249 | $inner[0].style.left = `${inner_left + target_left - inner_box.left}px`; 250 | }); 251 | return; 252 | } 253 | 254 | if (Math.abs(inner_box.left - target_left) > 0.5 || 255 | Math.abs(inner_box.top - target_top) > 0.5) { 256 | const style = $inner[0].style; 257 | if (style == undefined) return; // XML case... 258 | let inner_top = parseFloat(style.top); 259 | let inner_left = parseFloat(style.left); 260 | Batcher.mutating(() => { 261 | Util.vlog(4, `(re)positioning overlay for ${hint_number}`); 262 | Util.vlog(4, ` ${inner_box.top} x ${inner_box.left}` + 263 | ` -> ${target_top} x ${target_left}`); 264 | 265 | $inner[0].style.top = `${inner_top + target_top - inner_box.top}px`; 266 | $inner[0].style.left = `${inner_left + target_left - inner_box.left}px`; 267 | }); 268 | } 269 | } 270 | 271 | #remove() { 272 | Batcher.mutating(() => { 273 | $(this.#hint_tag).remove(); 274 | const hinted_element = this.#hinted_element.deref(); 275 | _remove_hint(this.#hint_number, hinted_element); 276 | }); 277 | } 278 | } 279 | 280 | 281 | 282 | // 283 | // Keeping track of hints 284 | // 285 | 286 | let hint_number_generator = new HintNumberGenerator(); 287 | let hint_number_to_hint = new Map(); 288 | let hinted_elements = new WeakSet(); 289 | 290 | 291 | function make_hint(hinted_element) { 292 | const hint_number = hint_number_generator.generate(); 293 | let hint = new Hint(hint_number, hinted_element); 294 | 295 | hint_number_to_hint.set(hint_number, hint); 296 | hinted_elements.add(hinted_element); 297 | 298 | if (Hints.option("mark_hinted")) { 299 | // Optionally for debugging mark the hinted element in the DOM. 300 | $(hinted_element).attr("CBV_hint_number", hint_number); 301 | } 302 | 303 | return hint; 304 | } 305 | 306 | // precondition: call this during a mutating step 307 | var _remove_hint = function _remove_hint(hint_number, hinted_element) { 308 | Util.vlog(2, `removing ${hint_number}:`); 309 | Util.vlog(2, hinted_element); 310 | hinted_elements.delete(hinted_element); 311 | hint_number_to_hint.delete(hint_number); 312 | if (Hints.option("mark_hinted")) { 313 | $(`[CBV_hint_number='${hint_number}']`).removeAttr("CBV_hint_number"); 314 | } 315 | hint_number_generator.release(hint_number); 316 | } 317 | 318 | 319 | function locate_hint(hint_number) { 320 | return hint_number_to_hint.get(Number(hint_number)); 321 | } 322 | 323 | function is_hinted_element(element) { 324 | return hinted_elements.has(element); 325 | } 326 | 327 | function discard_hints() { 328 | hint_number_generator = new HintNumberGenerator(); 329 | hint_number_to_hint.clear(); 330 | hinted_elements = new WeakSet(); 331 | _remove_hint_numbers_from(document); 332 | } 333 | 334 | function _remove_hint_numbers_from(from) { 335 | $("[CBV_hint_number]", from).removeAttr("CBV_hint_number"); 336 | const $frame = $("iframe, frame", from); 337 | if ($frame.length != 0) { 338 | _remove_hint_numbers_from($frame.contents()); 339 | } 340 | } 341 | 342 | 343 | 344 | // 345 | // Inspecting hint numbers 346 | // 347 | 348 | function get_hint_number_stats() { 349 | return hint_number_generator.stats; 350 | } 351 | 352 | 353 | // 354 | // Operations on hints 355 | // 356 | 357 | // precondition: currently in a sensing step 358 | function adjust_hints() { 359 | for (const [hint_number, hint] of hint_number_to_hint) { 360 | hint.adjust(); 361 | } 362 | } 363 | 364 | 365 | HintManager = { 366 | get_hint_number_stats: get_hint_number_stats, 367 | 368 | make_hint: make_hint, 369 | 370 | locate_hint: locate_hint, 371 | is_hinted_element: is_hinted_element, 372 | discard_hints: discard_hints, 373 | 374 | adjust_hints: adjust_hints 375 | }; 376 | })(); 377 | -------------------------------------------------------------------------------- /src/add_hint.js: -------------------------------------------------------------------------------- 1 | /// 2 | /// Labeling an element with a hint tag 3 | /// 4 | /// Provides AddHint 5 | 6 | let AddHint = null; 7 | 8 | "use strict"; 9 | 10 | (function() { 11 | 12 | 13 | // 14 | // Generic manipulations of DOM elements 15 | // 16 | 17 | // add CSS declaration ': !important' to element's 18 | // inline styles; 19 | // has no effect on XML elements, which ignore inline styles 20 | function set_important($element, item, value) { 21 | try { 22 | // jquery .css(-,-) does not handle !important correctly: 23 | $element[0].style.setProperty(item, value, "important"); 24 | //$element[0].style.setProperty(item, value); 25 | } catch (e) {} // XML elements throw an exception 26 | } 27 | 28 | 29 | // insert element before/after target or (if put_inside), at the 30 | // beginning of target's contents or at the end of target's contents 31 | function insert_element($target, $element, put_before, put_inside) { 32 | if (put_inside) { 33 | if (put_before) 34 | $target.prepend($element); 35 | else 36 | $target.append($element); 37 | } else { 38 | if (put_before) 39 | $target.before($element); 40 | else 41 | $target.after($element); 42 | } 43 | } 44 | 45 | 46 | 47 | // 48 | // Building hint tags 49 | // 50 | 51 | function $build_base_element() { 52 | const $element = $(""); 53 | // mark our inserted elements so we can distinguish them: 54 | $element.attr("CBV_hint_element", "true"); 55 | return $element; 56 | } 57 | 58 | function compute_z_index($element) { 59 | // beat hinted element's z-index by at least one; 60 | // if we are not in a different stacking context, this should 61 | // put us on top of it. 62 | let zindex = Util.css($element, "z-index", 0); 63 | if (Hints.option("zindex")) { 64 | const min_zindex = Hints.option_value("zindex"); 65 | if (zindex < min_zindex || zindex == "auto") 66 | zindex = min_zindex; 67 | } 68 | return zindex; 69 | } 70 | 71 | function $build_hint(hint_number, use_overlay, zindex) { 72 | const $outer = $build_base_element(); 73 | $outer.attr("CBV_hint_tag", hint_number); 74 | 75 | if (use_overlay) { 76 | $outer.attr("CBV_outer_overlay", "true"); 77 | 78 | const $inner = $build_base_element(); 79 | $outer.append($inner); 80 | 81 | $inner.attr("CBV_inner_overlay2", "true"); 82 | $inner.attr("CBV_hint_tag", hint_number); 83 | if (Hints.option("c")) 84 | $inner.attr("CBV_high_contrast", "true"); 85 | 86 | // IMPORTANT: need to have top, left set so offset(-[,-]) 87 | // works correctly on this element: 88 | set_important($inner, "top", "0"); 89 | set_important($inner, "left", "0"); 90 | 91 | if (zindex > 0) 92 | set_important($inner, "z-index", zindex+1); 93 | 94 | } else { 95 | $outer.attr("CBV_outer_inline", "true"); 96 | if (Hints.option("c")) 97 | $outer.attr("CBV_high_contrast", "true"); 98 | } 99 | 100 | return $outer; 101 | } 102 | 103 | 104 | 105 | // 106 | // Analysis routines 107 | // 108 | 109 | // Can we legally put a span element inside of element and have it be 110 | // visible? Does not take CSS properties into account. 111 | function can_put_span_inside($element) { 112 | // unconditionally _empty elements_ that cannot have any child 113 | // nodes (text or nested elements): 114 | if ($element.is("area, base, br, col, command, embed, hr, img, input, keygen, link, meta, param, source, track, wbr")) 115 | return false; 116 | 117 | if ($element.is("select, option, textarea")) 118 | return false; 119 | 120 | if ($element.is("iframe, frame")) 121 | // [i]frame contents are displayed only if browser doesn't support iframe's 122 | return false; 123 | 124 | // only actual webpage elements are fair game: 125 | if (CBV_inserted_element($element)) 126 | return false; 127 | 128 | if ($element.is("div, span, a, button, li, th, td")) 129 | return true; 130 | 131 | // above absolutely correct; below is heuristic: 132 | try { 133 | if ($element.contents().length > 0) 134 | return true; 135 | } catch (e) {} 136 | return false; 137 | } 138 | 139 | 140 | // is it okay to put a span before this element? 141 | function span_before_okay($element) { 142 | // don't put spans before tr elements (else messes up table 143 | // formatting as treats span as first column): 144 | if ($element.is("tr")) 145 | return false; 146 | 147 | return true; 148 | } 149 | 150 | 151 | 152 | // 153 | // Adding overlay hints 154 | // 155 | 156 | function compute_displacement($element) { 157 | const displacement = parseInt(Hints.option_value('E', '0')); 158 | const displacement_right = parseInt(Hints.option_value('displaceX', displacement)); 159 | const displacement_up = parseInt(Hints.option_value('displaceY', displacement)); 160 | let extra_displacement_right = 0; 161 | if (Hints.option("?") && $element.is("input")) { 162 | const padding = Util.css_pixels($element,"padding-right"); 163 | // too large padding mean something's probably being 164 | // positioned there absolutely so don't put overlay there 165 | if (padding > 10) 166 | extra_displacement_right = -padding + 5; 167 | } 168 | 169 | let use_displacement = false; 170 | if ($element.is('a, code, b, i, strong, em, abbr, input[type="checkbox"], input[type="radio"]') 171 | && $element.children().length == 0) { 172 | use_displacement = true; 173 | } 174 | if (Hints.option('f')) { 175 | use_displacement = true; 176 | } 177 | if (Hints.option('alwaysDisplace')) { 178 | use_displacement = true; 179 | } 180 | 181 | if (use_displacement) { 182 | return {up: displacement_up, right: displacement_right+extra_displacement_right}; 183 | } else { 184 | return {up: 0, right: extra_displacement_right}; 185 | } 186 | } 187 | 188 | 189 | function add_overlay_hint($element, hint) { 190 | let show_at_end = !Hints.option("s"); 191 | 192 | // needs to be before we insert the hint tag <<<>>> 193 | const displacement = compute_displacement($element); 194 | 195 | // 196 | // compute where to put overlay 197 | // 198 | let $container = $element; 199 | let inside = false; 200 | let after = true; 201 | 202 | if (Hints.option("exclude")) { 203 | while ($container.is(Hints.option_value("exclude"))) { 204 | $container = $container.parent(); 205 | } 206 | } 207 | 208 | if (Hints.option("f")) { 209 | $container = $("body"); 210 | inside = false; 211 | after = true; 212 | } else if ($container.is("table, tr, td, th, colgroup, tbody, thead, tfoot")) { 213 | // temporary kludge for Gmail: <<<>>> 214 | while ($container.is("table, tr, td, th, colgroup, tbody, thead, tfoot")) 215 | $container = $container.parent(); 216 | inside = false; 217 | after = false; 218 | } else { 219 | // 220 | // We prefer to put overlays inside the element so they share the 221 | // element's fate. If we cannot legally do that, we prefer before 222 | // the element because after the element has caused the inserted 223 | // span to wrap to the next line box, adding space. 224 | // 225 | if (can_put_span_inside($container)) { 226 | inside = true; 227 | after = false; 228 | } else { 229 | inside = false; 230 | after = !span_before_okay($container); 231 | } 232 | } 233 | 234 | const zindex = compute_z_index($element); 235 | 236 | Batcher.mutating(() => { 237 | // console.log("added hint " + hint.hint_number); 238 | // console.log($element[0]); 239 | 240 | const $hint_tag = $build_hint(hint.hint_number, true, zindex); 241 | const $inner = $hint_tag.children().first(); 242 | insert_element($container, $hint_tag, !after, inside); 243 | hint.initialize($hint_tag[0]); 244 | 245 | // move overlay into place at end after all inline hints have been 246 | // inserted so their insertion doesn't mess up the overlay's position: 247 | hint.displacement = displacement; 248 | hint.show_at_end = show_at_end; 249 | Batcher.sensing(()=>{ hint.adjust(); }); 250 | }); 251 | } 252 | 253 | 254 | 255 | // 256 | // 257 | // 258 | 259 | 260 | function $visual_contents($element) { 261 | if ($element.is("iframe, frame")) 262 | return []; 263 | 264 | const indent = Util.css($element, "text-indent"); 265 | if (indent && /^-999/.test(indent)) 266 | return []; 267 | const font_size = Util.css($element, "font-size"); 268 | if (font_size && /^0[^0-9]/.test(font_size)) 269 | return []; 270 | 271 | return $element.contents().filter(function () { 272 | if (this.nodeType === Node.COMMENT_NODE) 273 | return false; 274 | 275 | // ignore nodes intended solely for screen readers and the like 276 | if (this.nodeType === Node.ELEMENT_NODE) { 277 | if (Util.css($(this),"display") == "none") 278 | return false; 279 | if (Util.css($(this),"visibility") == "hidden") 280 | return false; 281 | // if ($(this).width()==0 && $(this).height()==0) 282 | if (Util.css($(this),"position") == "absolute" 283 | || Util.css($(this),"position") == "fixed") 284 | return false; 285 | } 286 | 287 | return true; 288 | }); 289 | } 290 | 291 | 292 | function get_text_overflow_ellipisis_clip($element) { 293 | for (;;) { 294 | if (Util.css($element, "text-overflow", "clip") != "clip") { 295 | let clip = {right: $element[0].getBoundingClientRect().right}; 296 | 297 | clip.right -= Util.css_pixels($element,"border-right-width") 298 | - Util.css_pixels($element,"padding-right"); 299 | // We ignore various values like "fit-content" here. 300 | const slop = Util.css_pixels($element,"max-width",-1,-1) - $element.width(); 301 | if (slop>0) 302 | clip.right += slop; 303 | if (slop>0) 304 | clip.top = -slop; 305 | // if (slop>0) 306 | // console.log(clip); 307 | // console.log($element[0]); 308 | // console.log($element.css("max-width")); 309 | // console.log(slop); 310 | 311 | return clip; 312 | } 313 | if (Util.css($element, "display") != "inline") 314 | return null; 315 | $element = $element.parent(); 316 | } 317 | } 318 | 319 | 320 | function ellipsis_clipping_possible($element) { 321 | const clip = get_text_overflow_ellipisis_clip($element); 322 | if (!clip) 323 | return false; 324 | 325 | // hardwire 40 pixels as the maximum hint tag length for now: <<<>>> 326 | if ($element[0].getBoundingClientRect().right + 40 < clip.right) 327 | return false; 328 | 329 | return true; 330 | } 331 | 332 | 333 | // 334 | // Adding inline hints 335 | // 336 | 337 | // returns false iff unable to safely add hint 338 | function add_inline_hint_inside($element, hint) { 339 | if (Hints.option("exclude") && $element.is(Hints.option_value("exclude"))) { 340 | return false; 341 | } 342 | 343 | let $current = $element; 344 | for (;;) { 345 | if (!can_put_span_inside($current)) 346 | return false; 347 | 348 | const $inside = $visual_contents($current); 349 | if ($inside.length == 0) 350 | return false; 351 | const $last_inside = $inside.last(); 352 | 353 | if ($last_inside[0].nodeType == Node.ELEMENT_NODE 354 | && $last_inside.is("div, span, i, b, strong, em, code, font, abbr")) { 355 | $current = $last_inside; 356 | continue; 357 | } 358 | 359 | if ($last_inside[0].nodeType != Node.TEXT_NODE) 360 | return false; 361 | if ($last_inside.text() == " ") // &nsbp 362 | // Google docs uses &nsbp; to take up space for visual items 363 | return false; 364 | if (Util.css($current, "display") == "flex") 365 | return false; 366 | 367 | let put_before = false; 368 | if (!Hints.option(".") && ellipsis_clipping_possible($current)) { 369 | if (!Hints.option(">")) 370 | put_before = true; 371 | else 372 | return false; 373 | } 374 | 375 | Batcher.mutating(() => { 376 | const $hint_tag = $build_hint(hint.hint_number, false, 0); 377 | insert_element($current, $hint_tag, put_before, true); 378 | hint.initialize($hint_tag[0]); 379 | }); 380 | return true; 381 | } 382 | } 383 | 384 | 385 | // this is often unsafe; prefer add_inline_hint_inside 386 | function add_inline_hint_outside($element, hint) { 387 | Batcher.mutating(() => { 388 | const $hint_tag = $build_hint(hint.hint_number, false, 0); 389 | insert_element($element, $hint_tag, false, false); 390 | hint.initialize($hint_tag[0]); 391 | }); 392 | } 393 | 394 | 395 | 396 | function add_hint($element) { 397 | const hint = HintManager.make_hint($element[0]); 398 | Batcher.sensing(() => { 399 | if (Hints.option("o")) { 400 | add_overlay_hint($element, hint); 401 | return; 402 | } 403 | 404 | if (Hints.option("h")) { 405 | if (!add_inline_hint_inside($element, hint)) { 406 | // if ($element.is("input[type=checkbox], input[type=radio]")) { 407 | // add_inline_hint_outside($element, hint); 408 | // return null; 409 | // } 410 | return add_overlay_hint($element, hint); 411 | } 412 | return; 413 | } 414 | 415 | // current fallback is inline 416 | if (Hints.option("i") || true) { 417 | if (!add_inline_hint_inside($element, hint)) 418 | add_inline_hint_outside($element, hint); 419 | return; 420 | } 421 | }); 422 | } 423 | 424 | 425 | AddHint = { 426 | add_hint: add_hint 427 | }; 428 | })(); 429 | -------------------------------------------------------------------------------- /src/activate.js: -------------------------------------------------------------------------------- 1 | /// 2 | /// Activating a hint by hint descriptor (usually a number) 3 | /// 4 | /// Provides Activate 5 | 6 | "use strict"; 7 | 8 | var Activate = null; 9 | 10 | (function() { 11 | 12 | // 13 | // Working with points 14 | // 15 | 16 | // return position in viewpoint to click on $element 17 | function point_to_click($element) { 18 | // If $element takes up multiple boxes, pretend it just is the first box 19 | const rectangles = $element[0].getClientRects(); 20 | const rectangle = rectangles[0]; 21 | 22 | let x = (rectangle.left + rectangle.right) /2; 23 | let y = (rectangle.top + rectangle.bottom)/2; 24 | 25 | // If inside iframes, accumulate offsets up to the top window 26 | let win = $element[0].ownerDocument.defaultView; 27 | while (win && win.frameElement) { 28 | const fr = win.frameElement.getBoundingClientRect(); 29 | x += fr.left; 30 | y += fr.top; 31 | win = win.parent; 32 | } 33 | 34 | return {x: x, y: y}; 35 | } 36 | 37 | // return position in viewpoint of top right point of $element 38 | function top_right_point($element) { 39 | // If $element takes up multiple boxes, pretend it just is the first box 40 | const rectangles = $element[0].getClientRects(); 41 | const rectangle = rectangles[0]; 42 | 43 | let x = rectangle.right; 44 | let y = rectangle.top; 45 | 46 | // If inside iframes, accumulate offsets up to the top window 47 | let win = $element[0].ownerDocument.defaultView; 48 | while (win && win.frameElement) { 49 | const fr = win.frameElement.getBoundingClientRect(); 50 | x += fr.left; 51 | y += fr.top; 52 | win = win.parent; 53 | } 54 | 55 | return {x: x, y: y}; 56 | } 57 | 58 | // Convert viewpoint point to physical(*) pixel offset relative to 59 | // inner bottom-left corner of browser Windows client area. 60 | // 61 | // * - at least physical as far as the PC sees; the monitor might 62 | // do some stretching after the fact. 63 | // 64 | // This is accurate to within +/- 1 after rounding so long as 65 | // there isn't any UI stuff like a downloads bar at the bottom or 66 | // left of the browser. This also may be inaccurate if the 67 | // browser window crosses monitors with different DPIs. 68 | function viewportToBottomLeftPhysicalOffset(clientX, clientY) { 69 | const stretch = window.devicePixelRatio; // includes both in-browser zoom and monitor stretch 70 | const screenX = clientX * stretch; 71 | const screenY = (window.innerHeight - clientY) * stretch; 72 | return {x: screenX, y: screenY}; 73 | } 74 | 75 | // Convert viewpoint point to screen coordinates relative to window 76 | // and place in clipboard 77 | function output_bottom_left_physical_offset(point) { 78 | const physicalPixelOffset = viewportToBottomLeftPhysicalOffset(point.x, point.y); 79 | const answer = physicalPixelOffset.x + "," + physicalPixelOffset.y; 80 | 81 | Util.vlog(1, "********************************************************************************"); 82 | Util.vlog(1, "input viewport point: " + point.x + " , " + point.y); 83 | Util.vlog(1, "window.devicePixelRatio: " + window.devicePixelRatio); 84 | Util.vlog(1, "bottom left physical offset: " +answer); 85 | 86 | act("copy_to_clipboard", {text: answer}); 87 | } 88 | 89 | 90 | // 91 | // 92 | // 93 | 94 | 95 | 96 | 97 | // Apply heuristics to determine if an element should be clicked or 98 | // focused. 99 | function wants_click($element) { 100 | if ($element.is("button")) { 101 | return true; 102 | } else if ($element.is("a")) { 103 | return true; 104 | } else if ($element.is(":input")) { 105 | if ($element.attr("type") == "submit") 106 | return true; 107 | if ($element.attr("type") == "checkbox") 108 | return true; 109 | if ($element.attr("type") == "radio") 110 | return true; 111 | if ($element.attr("type") == "button") 112 | return true; 113 | } 114 | if ($element.attr("onclick")) { 115 | return true; 116 | } 117 | const role = $element.attr("role"); 118 | switch (role) { 119 | case "button": 120 | case "link": 121 | return true; 122 | break; 123 | } 124 | 125 | if (Util.css($element, "cursor") === "pointer") 126 | return true; 127 | 128 | return false; 129 | } 130 | 131 | function dispatch_mouse_events($element, event_names) { 132 | event_names.forEach(function(event_name) { 133 | const event = document.createEvent('MouseEvents'); 134 | event.initMouseEvent(event_name, true, true, window, 1, 0, 0, 0, 0, 135 | false, false, false, false, 0, null); 136 | $element[0].dispatchEvent(event); 137 | }); 138 | } 139 | 140 | function area($element) { 141 | try { 142 | return $element.height() * $element.width(); 143 | } catch (e) { 144 | return -1; 145 | } 146 | } 147 | 148 | function href($element) { 149 | if ($element.is("iframe, frame")) 150 | return $element[0].src; 151 | if ($element.attr("href")) 152 | return $element[0].href; 153 | return undefined; 154 | } 155 | 156 | 157 | 158 | let last_hover = null; 159 | 160 | function silently_activate($element, hint_if_known, operation) { 161 | // It is impossible to measure this from inside the browser so 162 | // we are just assuming it's 1.0, which means that the 163 | // physical-move-the-mouse commands will only work on monitors 164 | // with no DPI scaling. 165 | switch (operation) { 166 | // Focusing: 167 | case "f": 168 | // this also works for [i]frames 169 | $element[0].focus(); 170 | break; 171 | 172 | // Clicking: 173 | case "c": 174 | // quora.com needs the mouseover event for clicking 'comments': 175 | dispatch_mouse_events($element, ['mouseover', 'mousedown']); 176 | $element[0].focus(); 177 | // we are not simulating leaving the mouse hovering over the element here <<<>>> 178 | dispatch_mouse_events($element, ['mouseup', 'click', 'mouseout']); 179 | break; 180 | 181 | // Following or copying explicit links: 182 | case "t": 183 | if (href($element)) 184 | // change focus to new tab 185 | act("create_tab", {URL: href($element), active: true}); 186 | break; 187 | case "b": 188 | if (href($element)) 189 | // do not change focus to new tab 190 | act("create_tab", {URL: href($element), active: false}); 191 | break; 192 | case "w": 193 | if (href($element)) 194 | act("create_window", {URL: href($element)}); 195 | break; 196 | case "k": 197 | if (href($element)) 198 | act("copy_to_clipboard", {text: href($element)}); 199 | break; 200 | 201 | // Hovering: 202 | case "h": 203 | if (last_hover) { 204 | dispatch_mouse_events(last_hover, ['mouseout', 'mouseleave']); 205 | } 206 | // hover same element means unhover 207 | if (last_hover==null || last_hover[0] !== $element[0]) { 208 | dispatch_mouse_events($element, ['mouseover', 'mouseenter']); 209 | last_hover = $element; 210 | } else 211 | last_hover = null; 212 | break; 213 | 214 | // Copying element text: 215 | case "s": { 216 | const clone = $element.clone(); 217 | clone.find("[CBV_hint_element]").remove(); 218 | Util.vlog(1, clone[0]); 219 | const text = clone[0].textContent; 220 | Util.vlog(1, '"' + text + '"'); 221 | act("copy_to_clipboard", {text: text}); 222 | break; 223 | } 224 | 225 | 226 | // Debug information: 227 | case "D": { 228 | console.log(""); 229 | if (hint_if_known) { 230 | hint_if_known.dump(); 231 | console.log(""); 232 | } 233 | console.log("Element information:"); 234 | console.log($element[0].getBoundingClientRect()); 235 | console.log($element[0].getClientRects()); 236 | console.log($element[0]); 237 | console.log(`display: ${Util.css($element, "display")}; ` + 238 | `visibility: ${Util.css($element, "visibility")}; ` + 239 | `is_under_low_opacity: ${Util.is_under_low_opacity($element[0])}`); 240 | console.log(`cursor: ${Util.css($element, "cursor")}`); 241 | break; 242 | } 243 | 244 | // Moving the physical mouse: 245 | case "Xnew": 246 | output_bottom_left_physical_offset(point_to_click($element)); 247 | break; 248 | case "XnewTL": 249 | output_bottom_left_physical_offset(top_right_point($element)); 250 | break; 251 | 252 | 253 | // experimental: 254 | case "R": 255 | dispatch_mouse_events($element, ['mouseover', 'contextmenu']); 256 | break; 257 | 258 | case ">": 259 | dispatch_mouse_events($element, ['mouseover', 'mousedown', 'mouseout']); 260 | break; 261 | case "<": 262 | dispatch_mouse_events($element, ['mouseover', 'mouseup', 'click', 'mouseout']); 263 | break; 264 | 265 | case "K": 266 | $element[0].remove(); 267 | break; 268 | case "V": 269 | $element.css("visibility", "hidden"); 270 | break; 271 | case "ZAP": 272 | $element.value = "fill"; 273 | break; 274 | 275 | 276 | 277 | 278 | // old versions for comparison purposes; depreciated 279 | case "C": 280 | $element[0].click(); 281 | break; 282 | case "CC": 283 | dispatch_mouse_events($element, ['mouseover', 'mousedown', 'mouseup', 284 | 'click']); 285 | break; 286 | case "DC": 287 | if ($element.children().length>0) 288 | $element = $element.children().first(); 289 | $element[0].click(); 290 | break; 291 | 292 | case "TT": 293 | $element.attr("tabindex", "0"); 294 | $element.siblings().attr("tabindex", "-1"); 295 | break; 296 | 297 | case "FF": 298 | $element[0].focusin(); 299 | $element[0].focus(); 300 | break; 301 | 302 | case "INSPECT": 303 | $('body').click(function (event) { 304 | console.log(event.originalEvent); 305 | console.log("window.devicePixelRatio: " + window.devicePixelRatio); 306 | console.log("Y: " + (event.screenY - event.clientY)); 307 | console.log("X: " + (event.screenX - event.clientX)); 308 | console.log("WY: " + (event.screenY - window.screenY - event.clientY)); 309 | console.log("WX: " + (event.screenX - window.screenX - event.clientX)); 310 | console.log("X ratio: " + (event.clientX / event.screenX)); 311 | console.log("outer height: " + window.outerHeight); 312 | console.log("inner height: " + window.innerHeight); 313 | 314 | // Viewport coordinates (CSS px) 315 | console.log("Viewport (CSS px): x=" + event.clientX + ", y=" + event.clientY); 316 | 317 | // Same point in device pixels (after page zoom / DPR) 318 | const zoom = window.devicePixelRatio; 319 | console.log("Viewport (device px): x=" + Math.round(event.clientX * zoom) 320 | + ", y=" + Math.round(event.clientY * zoom)); 321 | 322 | // Optional: normalized within the viewport [0..1] 323 | console.log("Viewport [0..1]: x=" + (event.clientX / window.innerWidth).toFixed(4) + 324 | ", y=" + (event.clientY / window.innerHeight).toFixed(4)); 325 | 326 | output_viewport_point({x:0, y:0}, 1, false); 327 | 328 | console.log("measured Delta: " + (event.screenY - window.screenY - event.clientY*zoom)); 329 | 330 | const elements = document.elementsFromPoint(event.clientX, event.clientY); 331 | console.log(elements); 332 | elements.forEach((element, index) => { 333 | if (index <= 10) { 334 | console.log(element); 335 | } 336 | }); 337 | }); 338 | break; 339 | 340 | 341 | default: 342 | Util.vlog(0, "unknown activate operation: " + operation); 343 | } 344 | } 345 | 346 | 347 | function activate($element, hint_if_known, operation) { 348 | if (operation=="c" && $element.is("div, span")) { 349 | const $parent = $element; 350 | let max_area = 0; 351 | $parent.children().each(function(index) { 352 | if (//!disabled_or_hidden($(this)) && // <<<>>> 353 | area($(this)) > max_area) { 354 | max_area = area($(this)); 355 | $element = $(this); 356 | } 357 | }); 358 | Util.vlog(1, $parent[1] + " -> " + $element[0]); 359 | } 360 | 361 | 362 | $element.addClass("CBV_highlight_class"); 363 | 364 | setTimeout(function() { 365 | setTimeout(function() { 366 | $element.removeClass("CBV_highlight_class"); 367 | // sometimes elements get cloned so do this globally also... 368 | // TODO: do we need to make this work inside of [i]frames also? <<<>>> 369 | $(".CBV_highlight_class").removeClass("CBV_highlight_class"); 370 | }, 500); 371 | 372 | silently_activate($element, hint_if_known, operation); 373 | }, 250); 374 | } 375 | 376 | // Locate an element described by a hint descriptor in the page, or in a nested [i]frame. 377 | // Returns object with optional fields $element, hint_if_known. 378 | function find_hint_descriptor(hint_descriptor, ...contents) { 379 | const match = hint_descriptor.match(/^\$\{(.*)\}("(.*)")?$/); 380 | if (match) { 381 | // ${CSS selector} or ${CSS selector}"text" 382 | let $element = $(match[1], ...contents); 383 | if (match[3]) { 384 | const target = match[3].toLowerCase(); 385 | $element = $element.filter(function(index, e) { 386 | return e.textContent.toLowerCase().includes(target); 387 | }); 388 | } 389 | $element = $element.first(); 390 | if ($element.length > 0) { 391 | return { $element: $element }; 392 | } 393 | } else { 394 | // hint_number 395 | const hint = HintManager.locate_hint(hint_descriptor); 396 | if (!hint) { 397 | Util.vlog(0, `The hint ${hint_descriptor} is not currently in use`); 398 | return {}; 399 | } 400 | const element = hint.hinted_element; 401 | if (!element) { 402 | return {}; 403 | } 404 | return { $element: $(element), hint_if_known: hint }; 405 | } 406 | 407 | // If the hint_descriptor was not found, search recursively in any [i]frames. 408 | const $frames = $("iframe, frame", ...contents); 409 | if ($frames.length > 0) { 410 | return find_hint_descriptor(hint_descriptor, $frames.contents()); 411 | } 412 | 413 | return {}; 414 | } 415 | 416 | function goto_hint_descriptor(hint_descriptor, operation) { 417 | const lookup = find_hint_descriptor(hint_descriptor); 418 | const $element = lookup.$element; 419 | if (!$element) { 420 | Util.vlog(0, "goto_hint_descriptor: unable to find hint descriptor: " + hint_descriptor); 421 | return; 422 | } 423 | 424 | if (operation == "") { 425 | if (wants_click($element)) 426 | operation = "c"; 427 | else 428 | operation = "f"; 429 | Util.vlog(1, $element[0]); 430 | Util.vlog(1, "defaulting to: " + operation); 431 | } 432 | 433 | activate($element, lookup.hint_if_known, operation); 434 | } 435 | 436 | 437 | Activate = { 438 | goto_hint_descriptor: goto_hint_descriptor 439 | }; 440 | })(); 441 | --------------------------------------------------------------------------------