├── .gitignore ├── Cargo.toml ├── LICENSE ├── README.md └── src ├── algorithm.rs ├── debug.rs ├── job.rs ├── lib.rs ├── lip_sync.rs └── model.rs /.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | .idea/ 3 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "real-time-lip-sync-gd" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html 7 | 8 | [lib] 9 | crate-type = ["cdylib"] 10 | 11 | [dependencies] 12 | godot = { git = "https://github.com/godot-rust/gdext.git" } 13 | lazy_static = "1.4.0" 14 | rand = "0.8.4" 15 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | License shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | Licensor shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | Legal Entity shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | control means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | You (or Your) shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | Source form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | Object form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | Work shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | Derivative Works shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | Contribution shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, submitted 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as Not a Contribution. 61 | 62 | Contributor shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a NOTICE text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an AS IS BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets [] 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same printed page as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2021 Timothy Yuen 190 | 191 | Licensed under the Apache License, Version 2.0 (the License); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an AS IS BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Real-time Lip Sync GD 2 | A Rust port of [uLipSync](https://github.com/hecomi/uLipSync) connected to Godot via [godot-rust](https://github.com/godot-rust/godot-rust). In theory, should work with any game engine with a C api. 3 | 4 | ## Porting notes 5 | - uLipSync 6 | - Runtime 7 | - Core 8 | - [x] Algorithm.cs 9 | - [x] Common.cs 10 | - [x] LipSyncJob.cs 11 | - [ ] ~~MicUtil.cs~~ 12 | - Won't do, should be done from GDScript 13 | - [x] Profile.cs 14 | - [ ] uLipSync.cs 15 | - [ ] uLipSyncAudioSource.cs 16 | - I don't think I need this? 17 | - [ ] uLipSyncBlendShape.cs 18 | - [ ] ~~uLipSyncMicrophone.cs~~ 19 | - Won't do, should be done from GDScript 20 | -------------------------------------------------------------------------------- /src/algorithm.rs: -------------------------------------------------------------------------------- 1 | use std::boxed::Box; 2 | 3 | use crate::model::{DataPoint, INV_LOG10, PI2}; 4 | 5 | pub fn rms(data: &[f32]) -> f32 { 6 | let mut rms: f32 = 0.0; 7 | 8 | for i in data.iter() { 9 | rms += i.powi(2); 10 | } 11 | 12 | rms = (rms / data.len() as f32).sqrt(); 13 | rms = 20.0 * (rms.ln() * *INV_LOG10); 14 | 15 | rms 16 | } 17 | 18 | pub fn normalize(data: &mut [f32]) { 19 | let mut v_max: f32 = 0.0; 20 | let mut v_min: f32 = 0.0; 21 | 22 | for i in data.iter() { 23 | v_max = v_max.max(*i); 24 | v_min = v_min.min(*i); 25 | } 26 | 27 | let diff = v_max - v_min; 28 | let d: f32 = if diff != 0.0 { 1.0 / diff } else { 1.0 }; 29 | for i in data.iter_mut() { 30 | *i = (*i - v_min) * d; 31 | } 32 | } 33 | 34 | pub fn smoothing(data: &mut [f32], before: &[f32]) { 35 | let n = data.len(); 36 | for i in 0..n { 37 | data[i] = (data[i] + before[i]) * 0.5; 38 | } 39 | } 40 | 41 | pub fn hamming(data: &mut [f32]) { 42 | let n = data.len(); 43 | for i in data.iter_mut() { 44 | let h = 0.54 - 0.46 * (*PI2 * *i / (n as f32 - 1.0)); 45 | *i = *i * h; 46 | } 47 | data[0] = 0.0; 48 | data[n - 1] = 0.0; 49 | } 50 | 51 | pub fn rfft(data: &mut [f32], reverse: bool, positive: bool) { 52 | let n = data.len(); 53 | let mut cmp_vec = vec![]; 54 | for i in data.iter() { 55 | let dp = DataPoint(*i, 0.0); 56 | cmp_vec.push(Box::new(dp)); 57 | } 58 | fft(cmp_vec.as_mut_slice(), reverse); 59 | if positive { 60 | for i in 0..n { 61 | data[i] = cmp_vec[i].0.abs(); 62 | } 63 | } else { 64 | for i in 0..n { 65 | data[i] = cmp_vec[i].0; 66 | } 67 | } 68 | if reverse { 69 | let inv_n: f32 = 1.0 / n as f32; 70 | for i in data { 71 | *i *= inv_n; 72 | } 73 | } 74 | } 75 | 76 | pub fn fft(data: &mut [Box], reverse: bool) { 77 | let n = data.len(); 78 | if n == 1 { 79 | return; 80 | } 81 | 82 | let mut b = vec![]; 83 | let mut c = vec![]; 84 | for i in 0..n { 85 | if i % 2 == 0 { 86 | b.push(data[i].clone()); 87 | } else if i % 2 == 1 { 88 | c.push(data[i].clone()); 89 | } 90 | } 91 | fft(b.as_mut_slice(), reverse); 92 | fft(c.as_mut_slice(), reverse); 93 | let circle = if reverse { -*PI2 } else { *PI2 }; 94 | for i in 0..n { 95 | // TODO this doesn't feel correct 96 | *data[i] = *b[i % (n / 2)].clone() 97 | + *c[i % (n / 2)].clone() * (DataPoint(0.0, circle * i as f32 / n as f32)).exp() 98 | } 99 | } 100 | 101 | pub fn lifter(data: &mut [f32], level: i32) { 102 | let i_min = level; 103 | let i_max = data.len() as i32 - 1 - level; 104 | for i in 0..data.len() as i32 { 105 | if i > i_min && i <= i_max { 106 | data[i as usize] = 0.0; 107 | } 108 | } 109 | } 110 | 111 | pub fn filter(data: &mut [f32], lowcut: i32, highcut: i32) { 112 | let mut minimum = data[0]; 113 | for i in data.iter() { 114 | minimum = minimum.min(*i); 115 | } 116 | 117 | if minimum == 0.0 { 118 | minimum = 0.000001 119 | } 120 | 121 | for i in data { 122 | if *i <= lowcut as f32 || *i >= highcut as f32 { 123 | *i = minimum; 124 | } 125 | } 126 | } 127 | 128 | pub fn lerp(a: f32, b: f32, f: f32) -> f32 { 129 | // l = a + f * (b - a) 130 | a + f * (b - a) 131 | } 132 | 133 | pub fn inverse_lerp(a: f32, b: f32, l: f32) -> f32 { 134 | // l = a + f * (b - a) 135 | // l - a = f * (b - a) 136 | // (l - a)/(b - a) = f 137 | (l - a) / (b - a) 138 | } 139 | -------------------------------------------------------------------------------- /src/debug.rs: -------------------------------------------------------------------------------- 1 | use godot::prelude::*; 2 | 3 | pub fn print_max(sample_vec: Vec) { 4 | let mut max: f32 = 0.0; 5 | for i in sample_vec { 6 | max = max.max(i.abs()); 7 | } 8 | godot_print!("{}", max); 9 | } 10 | 11 | pub fn print_min(sample_vec: Vec) { 12 | let mut min: f32 = 0.0; 13 | for i in sample_vec { 14 | min = min.min(i.abs()); 15 | } 16 | godot_print!("{}", min); 17 | } 18 | -------------------------------------------------------------------------------- /src/job.rs: -------------------------------------------------------------------------------- 1 | use crate::{algorithm::*, model::*}; 2 | use godot::prelude::*; 3 | use rand::Rng; 4 | use std::{ 5 | collections::{HashMap, VecDeque}, 6 | sync::mpsc, 7 | thread, 8 | }; 9 | 10 | struct Job { 11 | before_sample_array: Vec, 12 | // TODO pretty sure these are just ring buffers 13 | peaks3_log: VecDeque>, 14 | peaks4_log: VecDeque>, 15 | vowel_log: VecDeque, 16 | estimate_log: VecDeque, 17 | } 18 | 19 | impl Job { 20 | pub fn new() -> Self { 21 | Job { 22 | before_sample_array: vec![], 23 | peaks3_log: VecDeque::new(), 24 | peaks4_log: VecDeque::new(), 25 | vowel_log: VecDeque::from(vec![-1, -1, -1]), 26 | estimate_log: VecDeque::from(vec![-1, -1, -1]), 27 | } 28 | } 29 | 30 | pub fn execute(&mut self, stream: &Array) -> Option { 31 | // let mut data = Job::read_16_bit_samples(stream); 32 | let mut data = vec![]; 33 | for i in stream.iter_shared() { 34 | data.push(i); 35 | } 36 | 37 | if data.len() < FFT_SAMPLES { 38 | godot_print!("Audio data size is too small, skipped!"); 39 | return None; 40 | } 41 | 42 | let rms = rms(data.as_slice()); 43 | 44 | data = data[..FFT_SAMPLES as usize].to_vec(); 45 | hamming(data.as_mut_slice()); 46 | rfft(data.as_mut_slice(), false, true); 47 | data = data[..((FFT_SAMPLES as f32 * 0.5) as usize) + 1].to_vec(); 48 | if self.before_sample_array.len() > 0 { 49 | smoothing(data.as_mut_slice(), self.before_sample_array.as_slice()); 50 | } 51 | self.before_sample_array = data.clone(); 52 | filter(data.as_mut_slice(), 10, 95); 53 | for i in data.iter_mut() { 54 | *i = i.powi(2).ln() * *INV_LOG10; 55 | } 56 | normalize(data.as_mut_slice()); 57 | rfft(data.as_mut_slice(), true, false); 58 | lifter(data.as_mut_slice(), 26); 59 | rfft(data.as_mut_slice(), false, false); 60 | data = data[..((FFT_SAMPLES as f32 * 0.25) as usize) + 1].to_vec(); 61 | normalize(data.as_mut_slice()); 62 | for i in data.iter_mut() { 63 | *i = i.powi(2); 64 | } 65 | normalize(data.as_mut_slice()); 66 | let nrm_rms = DYNAMIC_RANGE.min((rms + DYNAMIC_RANGE).max(0.0)); 67 | for i in data.iter_mut() { 68 | *i = *i * nrm_rms * *INV_DYNAMIC_RANGE; 69 | } 70 | let amount = inverse_lerp(-DYNAMIC_RANGE, 0.0, rms).clamp(0.0, 1.0); 71 | let current_vowel = self.get_vowel(data.as_slice(), amount); 72 | self.push_estimate(current_vowel.estimate); 73 | self.push_vowel(current_vowel.vowel); 74 | 75 | Some(current_vowel) 76 | } 77 | 78 | // TODO this is returning values that are not in range -1..1 79 | fn read_16_bit_samples(stream: &Array) -> Vec { 80 | let mut res = vec![]; 81 | let mut i = 0; 82 | while i < stream.len() { 83 | let b0 = stream.get(i); 84 | let b1 = stream.get(i + 1); 85 | let mut u = b0 as u16 | ((b1 as u16) << 8); 86 | u = (u + 32768) & 0xffff; 87 | let s = (u - 32768) as f32 / 32768.0; 88 | res.push(s); 89 | i += 2; 90 | } 91 | res 92 | } 93 | 94 | fn get_peaks(&self, data: &[f32], threshold: f32) -> Vec { 95 | let n = data.len() - 1; 96 | let mut i = 1; 97 | let mut out = vec![]; 98 | let mut div = 1.0; 99 | while i < n { 100 | if data[i] > threshold && data[i] > data[i - 1] && data[i] > data[i + 1] { 101 | if out.len() > 0 { 102 | out.push(DataPoint(i as f32, data[i] * div)); 103 | } else { 104 | out.push(DataPoint(i as f32, 1.0)); 105 | div = 1.0 / data[i]; 106 | } 107 | } 108 | i += 1; 109 | } 110 | out 111 | } 112 | fn get_peaks_average(&mut self, size: usize) -> Vec { 113 | let mut out = vec![]; 114 | let mut i = 1; 115 | let mut j: usize; 116 | let mut div = 1.0; 117 | match size { 118 | 3 => { 119 | out = self.peaks3_log[0].clone(); 120 | while i < self.peaks3_log.len() { 121 | j = 0; 122 | while j < out.len() { 123 | out[j].0 += self.peaks3_log[i][j].0; 124 | out[j].1 += self.peaks3_log[i][j].1; 125 | j += 1; 126 | } 127 | i += 1; 128 | } 129 | div = 1.0 / self.peaks3_log.len() as f32; 130 | } 131 | 4 => { 132 | out = self.peaks4_log[0].clone(); 133 | while i < self.peaks4_log.len() { 134 | j = 0; 135 | while j < out.len() { 136 | out[j].0 += self.peaks4_log[i][j].0; 137 | out[j].1 += self.peaks4_log[i][j].1; 138 | j += 1; 139 | } 140 | i += 1; 141 | } 142 | div = 1.0 / self.peaks4_log.len() as f32; 143 | } 144 | _ => {} 145 | } 146 | 147 | for k in out.iter_mut() { 148 | *k *= div; 149 | } 150 | 151 | out 152 | } 153 | 154 | fn get_distance_from_db(&self, data: &[DataPoint]) -> Vec { 155 | let mut out = vec![]; 156 | 157 | let mut dist: f32; 158 | 159 | let peak_est: &HashMap = match data.len() { 160 | 3 => &DEFAULT_ESTIMATES["peak3"], 161 | 4 => &DEFAULT_ESTIMATES["peak4"], 162 | _ => { 163 | return out; 164 | } 165 | }; 166 | 167 | for i in VOWELS { 168 | dist = 0.0; 169 | for j in 0..data.len() { 170 | let est = &peak_est[i][j]; 171 | dist += (est.0 - data[j].0).abs() * *INV_255 + (est.1 - data[j].1); 172 | } 173 | out.push(dist); 174 | } 175 | 176 | out 177 | } 178 | 179 | fn push_peaks(&mut self, data: &[DataPoint]) { 180 | match data.len() { 181 | 3 => { 182 | if self.peaks3_log.len() < 3 { 183 | self.peaks3_log.push_back(data.to_owned()); 184 | } else { 185 | self.peaks3_log.push_front(data.to_owned()); 186 | self.peaks3_log.pop_back(); 187 | } 188 | } 189 | 4 => { 190 | if self.peaks4_log.len() < 3 { 191 | self.peaks4_log.push_back(data.to_owned()); 192 | } else { 193 | self.peaks4_log.push_front(data.to_owned()); 194 | self.peaks4_log.pop_back(); 195 | } 196 | } 197 | _ => godot_print!("push_peaks encountered invalid data"), 198 | } 199 | } 200 | 201 | fn estimate_vowel(&mut self, data: &[f32]) -> i32 { 202 | let peaks = self.get_peaks(data, 0.1); 203 | if peaks.len() != 3 && peaks.len() != 4 { 204 | return -1; 205 | } 206 | 207 | self.push_peaks(peaks.as_slice()); 208 | 209 | let peaks_ave = self.get_peaks_average(peaks.len()); 210 | let distance_vowel = self.get_distance_from_db(peaks_ave.as_slice()); 211 | 212 | let mut i = 1; 213 | let mut min_distance = distance_vowel[0]; 214 | let mut min_idx = 0; 215 | while i < VOWELS.len() { 216 | let dist = distance_vowel[i]; 217 | if dist < min_distance { 218 | min_distance = dist; 219 | min_idx = i as i32; 220 | } 221 | i += 1; 222 | } 223 | 224 | min_idx 225 | } 226 | 227 | fn get_vowel(&mut self, data: &[f32], amount: f32) -> VowelEstimate { 228 | let current = self.estimate_vowel(data); 229 | 230 | let f_vowel = self.vowel_log[0]; 231 | 232 | if self.vowel_log[0] != -1 { 233 | if amount < 0.5 { 234 | return VowelEstimate::new(current, f_vowel, amount); 235 | } 236 | } 237 | 238 | if self.vowel_log.len() > 2 { 239 | if current == self.estimate_log[0] { 240 | if current != -1 { 241 | return VowelEstimate::new(current, current, amount); 242 | } 243 | } else { 244 | if f_vowel != -1 { 245 | return VowelEstimate::new(current, f_vowel, amount); 246 | } 247 | } 248 | } 249 | 250 | return VowelEstimate::new(current, rand::thread_rng().gen_range(0..5), amount); 251 | } 252 | 253 | fn push_vowel(&mut self, vowel: i32) { 254 | if self.vowel_log.len() < 3 { 255 | self.vowel_log.push_back(vowel); 256 | } else { 257 | self.vowel_log.push_front(vowel); 258 | self.vowel_log.pop_back(); 259 | } 260 | } 261 | 262 | fn push_estimate(&mut self, vowel: i32) { 263 | if self.estimate_log.len() < 3 { 264 | self.estimate_log.push_back(vowel); 265 | } else { 266 | self.estimate_log.push_front(vowel); 267 | self.estimate_log.pop_back(); 268 | } 269 | } 270 | } 271 | 272 | pub enum JobMessage { 273 | InputData(Array), 274 | OutputData(VowelEstimate), 275 | Shutdown, 276 | } 277 | 278 | unsafe impl Send for JobMessage {} 279 | 280 | pub fn create_job() -> Option<( 281 | thread::JoinHandle<()>, 282 | mpsc::Sender, 283 | mpsc::Receiver, 284 | )> { 285 | let (s1, r2) = mpsc::channel(); 286 | let (s2, r1) = mpsc::channel(); 287 | 288 | let mut job = Job::new(); 289 | 290 | let builder = thread::Builder::new(); 291 | match builder.spawn(move || loop { 292 | let new_data: Array; 293 | if let Ok(msg) = r1.recv() { 294 | match msg { 295 | JobMessage::InputData(d) => new_data = d, 296 | JobMessage::Shutdown => break, 297 | _ => { 298 | godot_print!("Error when matching job data"); 299 | break; 300 | } 301 | } 302 | } else { 303 | godot_print!("Error when receiving job data"); 304 | break; 305 | } 306 | 307 | if let Some(vowel) = job.execute(&new_data) { 308 | match s1.send(JobMessage::OutputData(vowel)) { 309 | Ok(_) => {} 310 | Err(e) => { 311 | godot_print!("Error when sending output from job: {:?}", e); 312 | break; 313 | } 314 | } 315 | } 316 | }) { 317 | Ok(v) => return Some((v, s2, r2)), 318 | Err(_) => None, 319 | } 320 | } 321 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | use godot::prelude::*; 2 | 3 | mod lip_sync; 4 | 5 | mod algorithm; 6 | mod debug; 7 | mod job; 8 | mod model; 9 | 10 | struct LipSyncLib; 11 | 12 | #[gdextension] 13 | unsafe impl ExtensionLibrary for LipSyncLib {} 14 | -------------------------------------------------------------------------------- /src/lip_sync.rs: -------------------------------------------------------------------------------- 1 | use godot::prelude::*; 2 | use lazy_static::lazy_static; 3 | use rand::{rngs::ThreadRng, Rng}; 4 | use std::{ 5 | collections::{HashMap, VecDeque}, 6 | ops::{Add, Div, Index, Mul, MulAssign}, 7 | sync::mpsc, 8 | sync::{Arc, Mutex}, 9 | thread, 10 | }; 11 | 12 | use crate::{job, job::JobMessage}; 13 | 14 | const LIP_SYNC_UPDATED: &str = "updated"; 15 | const LIP_SYNC_PANICKED: &str = "panicked"; 16 | 17 | #[derive(GodotClass)] 18 | #[class(base = Node)] 19 | pub struct LipSyncRs { 20 | join_handle: Option>, 21 | sender: mpsc::Sender, 22 | receiver: mpsc::Receiver, 23 | #[base] 24 | base: Base, 25 | } 26 | 27 | unsafe impl Sync for LipSyncRs {} 28 | 29 | unsafe impl Send for LipSyncRs {} 30 | 31 | #[godot_api] 32 | impl LipSyncRs { 33 | #[signal] 34 | fn updated(); 35 | 36 | #[signal] 37 | fn panicked(); 38 | 39 | #[func] 40 | pub fn update(&mut self, stream: Array) { 41 | self.sender 42 | .send(JobMessage::InputData(stream)) 43 | .expect("Unable to send stream to thread"); 44 | } 45 | 46 | #[func] 47 | pub fn poll(&mut self) { 48 | match self.receiver.try_recv() { 49 | Ok(v) => match v { 50 | JobMessage::OutputData(od) => { 51 | // godot_print!("Emitted signal: {:?}", LIP_SYNC_UPDATED); 52 | 53 | self.base.emit_signal( 54 | LIP_SYNC_UPDATED.into(), 55 | &[Variant::from(Dictionary::from(od))], 56 | ); 57 | } 58 | _ => { 59 | // Unexpected data 60 | self.sender.send(JobMessage::Shutdown).expect("When shutting down thread because of invalid message, encoutered error. Shutting down anyways."); 61 | } 62 | }, 63 | Err(e) => { 64 | if e == mpsc::TryRecvError::Disconnected { 65 | // godot_print!("Emitted signal: {:?}", LIP_SYNC_PANICKED); 66 | 67 | self.base 68 | .emit_signal(LIP_SYNC_PANICKED.into(), &[Variant::from(format!("{}", e))]); 69 | } 70 | } 71 | } 72 | } 73 | 74 | #[func] 75 | pub fn shutdown(&mut self) { 76 | self.sender.send(JobMessage::Shutdown).expect("When shutting down thread because of invalid message, encountered error. Shutting down anyways."); 77 | self.join_handle 78 | .take() 79 | .expect("Unable to take join_handle") 80 | .join() 81 | .expect("Unable to join thread"); 82 | } 83 | } 84 | 85 | #[godot_api] 86 | impl INode for LipSyncRs { 87 | fn init(base: Base) -> Self { 88 | let (jh, s, r) = job::create_job().expect("Unable to create job thread"); 89 | 90 | LipSyncRs { 91 | join_handle: Some(jh), 92 | sender: s, 93 | receiver: r, 94 | base, 95 | } 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /src/model.rs: -------------------------------------------------------------------------------- 1 | use godot::prelude::*; 2 | use lazy_static::lazy_static; 3 | use std::{ 4 | collections::{HashMap, VecDeque}, 5 | ops::{Add, Div, Index, Mul, MulAssign}, 6 | }; 7 | 8 | pub const FFT_SAMPLES: usize = 1024; 9 | // pub const UPDATE_FRAME: usize = 5; 10 | pub const DYNAMIC_RANGE: f32 = 100.0; 11 | 12 | pub const VOWELS: [&str; 5] = ["A", "E", "I", "O", "U"]; 13 | 14 | lazy_static! { 15 | pub static ref DEFAULT_ESTIMATES: HashMap> = HashMap::from([ 16 | ( 17 | "peak3".to_owned(), 18 | HashMap::from([ 19 | ( 20 | "A".to_owned(), 21 | Phoneme(vec![ 22 | DataPoint(18.0, 1.0), 23 | DataPoint(41.0, 0.9), 24 | DataPoint(85.0, 0.75), 25 | ]), 26 | ), 27 | ( 28 | "E".to_owned(), 29 | Phoneme(vec![ 30 | DataPoint(21.0, 1.0), 31 | DataPoint(60.0, 0.75), 32 | DataPoint(84.0, 0.65), 33 | ]), 34 | ), 35 | ( 36 | "I".to_owned(), 37 | Phoneme(vec![ 38 | DataPoint(21.0, 1.0), 39 | DataPoint(42.0, 1.1), 40 | DataPoint(84.0, 1.0), 41 | ]), 42 | ), 43 | ( 44 | "O".to_owned(), 45 | Phoneme(vec![ 46 | DataPoint(20.0, 1.0), 47 | DataPoint(63.0, 0.9), 48 | DataPoint(85.0, 0.8), 49 | ]), 50 | ), 51 | ( 52 | "U".to_owned(), 53 | Phoneme(vec![ 54 | DataPoint(19.0, 1.0), 55 | DataPoint(47.0, 0.65), 56 | DataPoint(84.0, 0.7), 57 | ]), 58 | ), 59 | ]), 60 | ), 61 | ( 62 | "peak4".to_owned(), 63 | HashMap::from([ 64 | ( 65 | "A".to_owned(), 66 | Phoneme(vec![ 67 | DataPoint(18.0, 1.0), 68 | DataPoint(41.0, 0.9), 69 | DataPoint(68.0, 0.7), 70 | DataPoint(85.0, 0.55), 71 | ]), 72 | ), 73 | ( 74 | "E".to_owned(), 75 | Phoneme(vec![ 76 | DataPoint(22.0, 1.0), 77 | DataPoint(43.0, 0.9), 78 | DataPoint(66.0, 0.7), 79 | DataPoint(84.0, 0.65) 80 | ]) 81 | ), 82 | ( 83 | "I".to_owned(), 84 | Phoneme(vec![ 85 | DataPoint(21.0, 1.0), 86 | DataPoint(42.0, 1.1), 87 | DataPoint(60.0, 1.0), 88 | DataPoint(84.0, 1.1) 89 | ]) 90 | ), 91 | ( 92 | "O".to_owned(), 93 | Phoneme(vec![ 94 | DataPoint(20.0, 1.0), 95 | DataPoint(39.0, 0.9), 96 | DataPoint(63.0, 0.75), 97 | DataPoint(85.0, 0.8) 98 | ]) 99 | ), 100 | ( 101 | "U".to_owned(), 102 | Phoneme(vec![ 103 | DataPoint(20.0, 1.0), 104 | DataPoint(39.0, 0.7), 105 | DataPoint(65.0, 0.6), 106 | DataPoint(84.0, 0.75) 107 | ]) 108 | ) 109 | ]), 110 | ), 111 | ]); 112 | pub static ref PI2: f32 = 2.0 * std::f32::consts::PI; 113 | pub static ref INV_255: f32 = 1.0 / 255.0; 114 | pub static ref INV_32767: f32 = 1.0 / 32767.0; 115 | pub static ref INV_LOG10: f32 = 1.0 / (10.0 as f32).ln(); 116 | pub static ref INV_DYNAMIC_RANGE: f32 = 1.0 / DYNAMIC_RANGE; 117 | } 118 | 119 | #[derive(Debug, PartialEq, Clone)] 120 | pub struct DataPoint(pub f32, pub f32); 121 | 122 | impl DataPoint { 123 | pub fn exp(self) -> DataPoint { 124 | let e = self.0.exp(); 125 | 126 | DataPoint(e * self.1.cos(), e * self.1.sin()) 127 | } 128 | 129 | pub fn zero() -> DataPoint { 130 | DataPoint(0.0, 0.0) 131 | } 132 | } 133 | 134 | impl Add for DataPoint { 135 | type Output = DataPoint; 136 | fn add(self, other: DataPoint) -> DataPoint { 137 | DataPoint(self.0 + other.0, self.1 + other.1) 138 | } 139 | } 140 | 141 | impl Mul for DataPoint { 142 | type Output = DataPoint; 143 | fn mul(self, other: DataPoint) -> DataPoint { 144 | let r = self.0 * other.0 - self.1 * other.1; 145 | let i = self.0 * other.0 + self.1 * other.1; 146 | 147 | DataPoint(r, i) 148 | } 149 | } 150 | 151 | impl MulAssign for DataPoint { 152 | fn mul_assign(&mut self, other: f32) { 153 | self.0 *= other; 154 | self.1 *= other; 155 | } 156 | } 157 | 158 | impl Div for DataPoint { 159 | type Output = DataPoint; 160 | fn div(self, other: DataPoint) -> DataPoint { 161 | let r = self.0 * other.0 + self.1 * other.1; 162 | let i = self.1 * other.0 - self.1 * other.1; 163 | let d = other.0 * other.0 + other.1 * other.1; 164 | 165 | DataPoint(r / d, i / d) 166 | } 167 | } 168 | 169 | #[derive(Debug, PartialEq)] 170 | pub struct Phoneme(Vec); 171 | 172 | impl Index for Phoneme { 173 | type Output = DataPoint; 174 | fn index(&self, idx: usize) -> &DataPoint { 175 | &self.0[idx] 176 | } 177 | } 178 | 179 | #[derive(Debug)] 180 | pub struct VowelEstimate { 181 | pub estimate: i32, 182 | pub vowel: i32, 183 | pub amount: f32, 184 | } 185 | 186 | impl VowelEstimate { 187 | pub fn new(estimate: i32, vowel: i32, amount: f32) -> Self { 188 | VowelEstimate { 189 | estimate, 190 | vowel, 191 | amount, 192 | } 193 | } 194 | } 195 | 196 | impl From for Dictionary { 197 | fn from(ve: VowelEstimate) -> Self { 198 | let mut dict = Dictionary::new(); 199 | 200 | dict.insert("estimate", ve.estimate); 201 | dict.insert("vowel", ve.vowel); 202 | dict.insert("amount", ve.amount); 203 | 204 | dict 205 | } 206 | } 207 | --------------------------------------------------------------------------------