├── .gitignore ├── Cargo.toml ├── LICENSE ├── README.md ├── bevy_guessture ├── Cargo.toml ├── LICENSE ├── README.md ├── assets │ └── data.gestures ├── examples │ └── train.rs └── src │ └── lib.rs └── guessture ├── .gitignore ├── Cargo.toml ├── LICENSE ├── README.md └── src └── lib.rs /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | Cargo.lock 3 | 4 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [workspace] 2 | resolver = "2" 3 | members = [ 4 | "guessture", 5 | "bevy_guessture", 6 | ] -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Josh Matthews 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # guessture & bevy_guessture 2 | 3 | Implementation of [the $1 unistroke recognizer](http://depts.washington.edu/acelab/proj/dollar/index.html) algorithm in Rust, 4 | with a Bevy integration. See the [guessture](guessture/README.md) and [bevy_guessture](bevy_guessture/README.md) crate readmes for more information. 5 | -------------------------------------------------------------------------------- /bevy_guessture/Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "bevy_guessture" 3 | version = "0.1.0" 4 | edition = "2021" 5 | authors = ["Josh Matthews "] 6 | license = "MIT" 7 | description = "Bevy plugin wrapping the `guessture` crate's gesture recognition API." 8 | repository = "https://github.com/jdm/bevy_guessture" 9 | 10 | [[example]] 11 | name = "train" 12 | 13 | [lib] 14 | name = "bevy_guessture" 15 | 16 | [dependencies] 17 | bevy = { version = "0.13", default-features = false, features = ["bevy_asset"] } 18 | bevy_common_assets = { version = "0.10.0", features = ["json"] } 19 | guessture = { path = "../guessture", version = "0.1" } 20 | serde = "1.0" 21 | serde_json = "1" 22 | 23 | [dev-dependencies] 24 | bevy = { version = "0.13", default-features = false, features = [ 25 | "bevy_asset", 26 | "bevy_winit", 27 | "bevy_core_pipeline", 28 | "bevy_sprite", 29 | "bevy_text", 30 | "bevy_ui", 31 | "default_font", 32 | ] } 33 | -------------------------------------------------------------------------------- /bevy_guessture/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Josh Matthews 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /bevy_guessture/README.md: -------------------------------------------------------------------------------- 1 | # bevy_guessture 2 | 3 | This library integrates the `guessture` library into the Bevy ecosystem. Its responsibilities include: 4 | * recording mouse position data in response to app-initiated events 5 | * providing mouse path data for a completed recording window to the app 6 | * storing app-accessible gesture templates 7 | * exposing gesture template serialization and asset loading mechanisms 8 | 9 | Bevy apps using `bevy_guessture` are responsible for setting up gesture templates, 10 | triggering recording windows, and initiating gesture matching with the recorded mouse path data. 11 | There is an example app that demonstrates visual integration of gesture recognition, as well as 12 | serializing gesture information as a loadable asset. 13 | 14 | To get started, install the `GuessturePlugin` in your app and prepare a set of guesture templates: 15 | ```rs 16 | App::new() 17 | .add_plugins(GuessturePlugin::default()); 18 | ``` 19 | Then prepare a set of gesture templates: 20 | ```rs 21 | fn setup(server: Res) { 22 | let _handle: Handle = server.load("data.gestures"); 23 | } 24 | ``` 25 | 26 | To start recording a potential gesture, send the appropriate event: 27 | ```rs 28 | fn start_record(mut record_events: EventWriter) { 29 | record_events.send(GestureRecord::Start); 30 | } 31 | ``` 32 | 33 | After later sending a `GestureRecord::Stop` event, wait for a `RecordedPath` event with the complete recording: 34 | ```rs 35 | fn recorded_path( 36 | mut events: EventReader, 37 | mut state: ResMut, 38 | ) { 39 | for event in events.read() { 40 | let matched_template = find_matching_template_with_defaults( 41 | &state.templates, 42 | &event.path, 43 | ); 44 | match matched_template { 45 | Ok((template, score)) => 46 | println!("matched {} with score {}", template.name, score), 47 | Err(err) => 48 | println!("failed to match: {:?}", err), 49 | } 50 | } 51 | } 52 | ``` 53 | 54 | ## Bevy compatibility 55 | 56 | | bevy_guessture | Bevy | 57 | |---------------:|------| 58 | | main | 0.13 | 59 | | 0.1 | 0.13 | 60 | -------------------------------------------------------------------------------- /bevy_guessture/assets/data.gestures: -------------------------------------------------------------------------------- 1 | {"templates":[{"name":"0","path":[[-98.79413,-0.000091552734],[-97.281006,12.29599],[-95.208984,24.504974],[-91.43219,36.42108],[-87.0802,48.17856],[-81.95868,59.699585],[-76.44177,71.01721],[-67.53558,80.59073],[-58.11743,89.760864],[-48.275208,98.5965],[-37.691956,106.742676],[-26.550903,114.26602],[-14.670227,120.85382],[-1.7426147,125.56259],[12.173096,126.308075],[25.757263,123.13446],[39.143066,119.34259],[51.131958,112.98987],[62.19171,105.4234],[72.42297,96.93646],[81.83569,87.76013],[91.028564,78.39865],[99.33606,68.418365],[107.30255,58.253845],[113.50165,47.149994],[119.70068,36.046143],[124.74219,24.557343],[128.44391,12.622986],[130.87909,0.4716797],[132.539,-11.8124695],[131.50696,-24.110474],[129.2738,-36.2872],[125.51819,-48.208527],[120.31433,-59.666046],[114.479004,-70.9214],[106.71979,-80.917175],[96.138306,-89.06505],[85.125854,-96.7081],[73.27911,-103.37367],[61.307617,-109.82721],[48.281555,-114.48872],[34.947205,-118.32414],[21.314453,-121.35971],[7.45459,-123.06679],[-6.590149,-123.691925],[-20.434692,-121.74342],[-34.10028,-118.91788],[-47.401245,-114.9731],[-59.041504,-108.03174],[-67.43463,-98.249466],[-75.26227,-87.97241],[-82.66388,-77.48711],[-88.44818,-66.211395],[-94.22131,-54.93132],[-99.9491,-43.63333],[-104.93805,-32.084473],[-109.44165,-20.365692],[-113.924194,-8.645477],[-116.74933,3.4729614],[-117.46094,15.803162],[-114.42218,27.739868],[-110.111694,39.515015],[-103.40735,50.37799],[-99.01715,61.856537]]},{"name":"1","path":[[-134.38162,0.00012207031],[-124.39432,-3.897766],[-114.43863,-7.963318],[-104.49423,-12.089264],[-94.55023,-16.217224],[-84.6084,-20.35608],[-74.666504,-24.494873],[-64.72464,-28.633698],[-54.782745,-32.77249],[-44.85556,-36.9859],[-34.928864,-41.20172],[-25.002136,-45.417603],[-15.075439,-49.633514],[-5.151245,-53.861572],[4.706543,-58.41046],[14.56427,-62.95929],[24.421997,-67.50818],[34.293884,-71.98996],[44.194397,-76.33557],[54.09491,-80.68121],[63.995422,-85.026794],[73.91852,-89.258545],[83.87317,-93.331024],[93.74066,-97.819244],[103.54419,-102.61203],[113.46057,-106.848785],[115.61841,-95.36627],[112.98236,-80.78241],[108.92401,-66.87897],[104.741455,-53.06122],[99.99054,-39.635254],[95.239685,-26.209259],[90.0105,-13.190979],[84.36426,-0.5281372],[78.71814,12.134674],[73.15448,24.875183],[67.61609,37.639465],[61.959167,50.28412],[55.76764,62.389526],[49.223694,74.08984],[43.032227,86.19525],[36.76477,98.21246],[30.10913,109.779175],[23.627808,121.55292],[17.151245,133.33252],[9.867493,143.15118],[1.3928833,134.51825],[-6.64859,125.020935],[-14.215698,114.747986],[-21.507904,104.03357],[-28.80008,93.319214],[-36.064148,82.565186],[-43.12863,71.5293],[-50.193085,60.49341],[-57.257538,49.45746],[-64.32153,38.42102],[-71.38547,27.3844],[-78.44931,16.347717],[-85.581085,5.4062805],[-92.83069,-5.36969],[-100.10034,-16.116669],[-107.39398,-26.82898],[-114.59079,-37.679535],[-120.54141,-48.93094]]}]} -------------------------------------------------------------------------------- /bevy_guessture/examples/train.rs: -------------------------------------------------------------------------------- 1 | use bevy::prelude::*; 2 | use bevy::sprite::MaterialMesh2dBundle; 3 | use bevy_guessture::{GuessturePlugin, GestureRecord, GestureState, RecordedPath, GestureTemplates}; 4 | use guessture::{Template, find_matching_template_with_defaults}; 5 | use std::fs::File; 6 | use std::io::Write; 7 | use std::path::Path; 8 | 9 | #[derive(Copy, Clone)] 10 | enum RecordType { 11 | Template, 12 | Attempt, 13 | } 14 | 15 | #[derive(Default, Resource)] 16 | struct RecordState { 17 | state: Option, 18 | } 19 | 20 | fn main() { 21 | App::new() 22 | .insert_resource(Msaa::Off) 23 | .insert_resource(ClearColor(Color::rgb(0.0, 0.0, 0.0))) 24 | .init_resource::() 25 | .add_event::() 26 | .add_event::() 27 | .add_systems(Update, ( 28 | recorded_path, 29 | keyboard_input, 30 | create_visible_path, 31 | fade_visible_path, 32 | update_text, 33 | )) 34 | .add_systems(Startup, setup) 35 | .add_plugins(DefaultPlugins.set(WindowPlugin { 36 | primary_window: Some(Window { 37 | title: "Gesture trainer".to_string(), 38 | canvas: Some("#bevy".to_owned()), 39 | prevent_default_event_handling: false, 40 | ..default() 41 | }), 42 | ..default() 43 | })) 44 | .add_plugins(GuessturePlugin::default()) 45 | .run(); 46 | } 47 | 48 | fn setup( 49 | mut commands: Commands, 50 | ) { 51 | commands.spawn((Camera2dBundle::default(), MainCamera)); 52 | 53 | commands.spawn(( 54 | TextBundle::from_section( 55 | "Space: record a template\nShift: attempt a gesture\nEnter: save all templates\nO: load templates", 56 | TextStyle { 57 | font_size: 20.0, 58 | color: Color::WHITE, 59 | ..default() 60 | } 61 | ).with_style( 62 | Style { 63 | position_type: PositionType::Absolute, 64 | top: Val::Px(5.0), 65 | left: Val::Px(15.0), 66 | ..default() 67 | }, 68 | ), 69 | )); 70 | } 71 | 72 | #[derive(Event)] 73 | struct VisiblePathEvent { 74 | path: Vec<(f32, f32)>, 75 | color: Color, 76 | } 77 | 78 | #[derive(Component)] 79 | struct PathComponent; 80 | 81 | #[derive(Component)] 82 | struct MainCamera; 83 | 84 | const RADIUS: f32 = 50.; 85 | 86 | fn create_visible_path( 87 | mut events: EventReader, 88 | mut meshes: ResMut>, 89 | mut materials: ResMut>, 90 | mut commands: Commands, 91 | q_camera: Query<(&Camera, &GlobalTransform), With>, 92 | ) { 93 | for ev in events.read() { 94 | let (camera, camera_transform) = q_camera.single(); 95 | for point in &ev.path { 96 | let Some(remapped) = camera.viewport_to_world_2d( 97 | camera_transform, 98 | Vec2::new(point.0, point.1), 99 | ) else { 100 | continue 101 | }; 102 | 103 | commands.spawn(( 104 | PathComponent, 105 | MaterialMesh2dBundle { 106 | mesh: meshes.add(Circle::new(RADIUS)).into(), 107 | material: materials.add(ColorMaterial::from(ev.color)), 108 | transform: Transform::from_translation( 109 | Vec3::new(remapped.x, remapped.y, 0.), 110 | ), 111 | ..default() 112 | }, 113 | )); 114 | } 115 | } 116 | } 117 | 118 | fn fade_visible_path( 119 | mut path: Query<(Entity, &mut Transform), With>, 120 | mut commands: Commands, 121 | ) { 122 | for (entity, mut transform) in &mut path { 123 | transform.scale.x *= 0.9; 124 | transform.scale.y *= 0.9; 125 | if transform.scale.x < 0.01 { 126 | commands.entity(entity).despawn(); 127 | } 128 | } 129 | } 130 | 131 | fn recorded_path( 132 | mut events: EventReader, 133 | mut state: ResMut, 134 | mut path_events: EventWriter, 135 | mut record_state: ResMut, 136 | ) { 137 | for event in events.read() { 138 | match record_state.state.as_ref().unwrap() { 139 | RecordType::Attempt => { 140 | let matched_template = find_matching_template_with_defaults( 141 | &state.templates, 142 | &event.path, 143 | ); 144 | match matched_template { 145 | Ok((template, score)) if score >= 0.8 => { 146 | println!("matched {} with score {}", template.name, score); 147 | path_events.send(VisiblePathEvent { 148 | color: Color::GREEN, 149 | path: event.path.points(), 150 | }); 151 | } 152 | Ok((template, score)) => { 153 | println!("matched {} but with score {}", template.name, score); 154 | } 155 | Err(err) => println!("failed to match: {:?}", err), 156 | } 157 | } 158 | 159 | RecordType::Template => { 160 | let Ok(template) = Template::new( 161 | state.templates.len().to_string(), 162 | &event.path, 163 | ) else { 164 | continue; 165 | }; 166 | println!("done recording template {}", template.name); 167 | state.templates.push(template); 168 | path_events.send(VisiblePathEvent { 169 | color: Color::BLUE, 170 | path: event.path.points(), 171 | }); 172 | } 173 | } 174 | record_state.state = None; 175 | } 176 | } 177 | 178 | fn save_templates(state: &GestureState) -> Result<(), ()> { 179 | let serialized = state.serialize_templates()?; 180 | let path = Path::new("data.gestures"); 181 | let mut f = File::create(path).map_err(|_| ())?; 182 | f.write(serialized.as_bytes()).map_err(|_| ())?; 183 | Ok(()) 184 | } 185 | 186 | fn keyboard_input( 187 | keys: Res>, 188 | mut record_events: EventWriter, 189 | mut ui_events: EventWriter, 190 | state: Res, 191 | server: Res, 192 | mut record_state: ResMut, 193 | ) { 194 | if keys.just_pressed(KeyCode::ShiftLeft) { 195 | record_state.state = Some(RecordType::Attempt); 196 | record_events.send(GestureRecord::Start); 197 | ui_events.send(TextEvent::Show("Recording".to_owned())); 198 | } 199 | if keys.just_released(KeyCode::ShiftLeft) { 200 | record_events.send(GestureRecord::Stop); 201 | ui_events.send(TextEvent::Hide); 202 | } 203 | 204 | if keys.just_pressed(KeyCode::Space) { 205 | record_state.state = Some(RecordType::Template); 206 | record_events.send(GestureRecord::Start); 207 | ui_events.send(TextEvent::Show("Recording template".to_owned())); 208 | } 209 | if keys.just_released(KeyCode::Space) { 210 | record_events.send(GestureRecord::Stop); 211 | ui_events.send(TextEvent::Hide); 212 | } 213 | 214 | if keys.just_released(KeyCode::Enter) { 215 | match save_templates(&state) { 216 | Ok(()) => { 217 | ui_events.send(TextEvent::Show("Saved templates".to_owned())); 218 | } 219 | Err(()) => { 220 | ui_events.send(TextEvent::Show("Error saving templates".to_owned())); 221 | } 222 | } 223 | } 224 | 225 | if keys.just_released(KeyCode::KeyO) { 226 | let _handle: Handle = server.load("data.gestures"); 227 | ui_events.send(TextEvent::Show("Loading templates".to_owned())); 228 | } 229 | } 230 | 231 | #[derive(Event)] 232 | enum TextEvent { 233 | Show(String), 234 | Hide, 235 | } 236 | 237 | #[derive(Component)] 238 | struct UiText; 239 | 240 | fn update_text( 241 | mut events: EventReader, 242 | query: Query>, 243 | mut commands: Commands, 244 | ) { 245 | for event in events.read() { 246 | for entity in &query { 247 | commands.entity(entity).despawn(); 248 | } 249 | 250 | match event { 251 | TextEvent::Show(ref text) => { 252 | commands.spawn(( 253 | UiText, 254 | TextBundle::from_section( 255 | text.clone(), 256 | TextStyle { 257 | font_size: 50.0, 258 | color: Color::GOLD, 259 | ..default() 260 | } 261 | ).with_style( 262 | Style { 263 | position_type: PositionType::Absolute, 264 | bottom: Val::Px(5.0), 265 | left: Val::Px(15.0), 266 | ..default() 267 | }, 268 | ), 269 | )); 270 | } 271 | 272 | TextEvent::Hide => () 273 | } 274 | } 275 | } 276 | -------------------------------------------------------------------------------- /bevy_guessture/src/lib.rs: -------------------------------------------------------------------------------- 1 | use bevy::prelude::*; 2 | use bevy::reflect::TypePath; 3 | use bevy_common_assets::json::JsonAssetPlugin; 4 | pub use guessture::*; 5 | use std::mem; 6 | 7 | /// Plugin object to automatically integrate gesture recognition into your Bevy app. 8 | #[derive(Default)] 9 | pub struct GuessturePlugin; 10 | 11 | impl Plugin for GuessturePlugin { 12 | fn build(&self, app: &mut App) { 13 | app 14 | .add_plugins( 15 | JsonAssetPlugin::::new(&["gestures"]) 16 | ) 17 | .add_systems(Update, ( 18 | change_recording_state, 19 | update_templates, 20 | record_mouse 21 | .run_if(|state: Res| state.current_recording.is_some()) 22 | )) 23 | .add_event::() 24 | .add_event::() 25 | .init_resource::(); 26 | } 27 | } 28 | 29 | /// A resource containing all gesture templates that will be considered. 30 | /// Updating the `templates` member will affect all future match attempts. 31 | #[derive(Default, Resource)] 32 | pub struct GestureState { 33 | pub templates: Vec