├── .gitignore ├── README.md ├── ei-um-uh-ah-detector-3-center_curated_fillers_02-arduino-1.0.4.zip └── mind-the-uuh-arduino.ino /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | .DS_Store 3 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Mind the “Uuh” – for Arduino Nano 33 BLE Sense 2 | 3 | By [Benedikt Groß](https://benedikt-gross.de), [Maik Groß](https://twitter.com/thatsmaik), [Thibault Durand](http://thibault-durand.fr/) 4 | 5 | 6 | youtube-preview 7 | 8 | 9 | Mind the “Uuh” is an experimental training device helping everyone to become a better public speaker. The cute little companion is constantly listening to the sound of your voice, aiming to make you aware of “uuh” fill words. These fillers are easy to avoid, but you have to start noticing them. Now every time you give a presentation and you say “uuh” – you will be aware :) 10 | 11 | The prototype of Mind the “Uuh” was carefully designed as simple as possible: There is a bell, a volume knob which controls how hard the bell is hit or to turn it silent, a counter for your “uuh” stats and a reset button. The product design is deliberately making references to classic alarm clocks to convey the nature of Mind the “Uuh” intuitively. 12 | 13 | For the detection of the “uuh”s the device runs a custom trained machine learning model, trained on 1500 samples of various durations from 300ms to 1 sec. This proof of concept model will notice distinctive “uuh” fillers but ignore very short utterances. All speech data is processed directly on device, nothing is sent to the cloud. 14 | 15 | ## 💻 Hardware pre-requisite (part list) 16 | 17 | Part list: 18 | 19 | - Microcontroller: Arduino Nano 33 BLE Sense 20 | - Bell: we are using a standard bike bell for handle bars 21 | - Servo: Tower Pro SG51R 22 | - Display: standart seven segment display (sourced via 74HC595 shift register / drain via TPIC6B595) 23 | - Poti: 10 kΩ 24 | - Stripboard: 70x90 mm 25 | - Housing: 3d printed PLA ([print your own device](https://www.thingiverse.com/thing:4910005)) 26 | 27 | Schaltplan 28 | 29 | 30 | ## 📦 Software pre-requisite 31 | 32 | 1. Add the Edge Impulse library through the Arduino IDE via: `Sketch` > `Include Library` > `Add .ZIP Library` ... see .zip file `ei-um-uh-ah-detector-...something.zip` in this repo. 33 | 2. Upload the `mind-the-uuh-arduino.ino` onto your Arduino. 34 | 3. Done :) 35 | 36 | 37 | ## 💪 Training your own model 38 | 39 | The "uuh" model was completely trained with [Edge Impulse Studio](https://studio.edgeimpulse.com/) on a free account. If you want to train your own hotword, we recommend following carefully the super great [Responding to your voice](https://docs.edgeimpulse.com/docs/responding-to-your-voice) video tutorial. You will have to prepare ca. 1500 samples of 1 sec duration as mono .wav files of 16000 hz for the training. 40 | 41 | 42 | ## Acknowledgment 43 | 44 | - Training of the model: [Edge Impulse Studio](https://studio.edgeimpulse.com/) 45 | - Understanding how to train the model: Edge Impulse Tutorial [Responding to your voice](https://docs.edgeimpulse.com/docs/responding-to-your-voice) 46 | - Web App: based on Edge Impulse example [Demo-Shower-Timer]( https://github.com/edgeimpulse/demo-shower-timer) 47 | -------------------------------------------------------------------------------- /ei-um-uh-ah-detector-3-center_curated_fillers_02-arduino-1.0.4.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/b-g/mind-the-uuh-arduino/eed512acac34d8805cf502329711ac0599b290e8/ei-um-uh-ah-detector-3-center_curated_fillers_02-arduino-1.0.4.zip -------------------------------------------------------------------------------- /mind-the-uuh-arduino.ino: -------------------------------------------------------------------------------- 1 | int uhm_count = 0; 2 | 3 | // Button 4 | const int BUTTON_PIN = 12; 5 | int currentState; 6 | int lastState = HIGH; 7 | bool pressed = false; 8 | 9 | // Poti 10 | int potPin = 6; 11 | int potiVal = 0; 12 | 13 | // Seven Segment Display 14 | #include "SevSegShift.h" 15 | #define SHIFT_PIN_SHCP 10 16 | #define SHIFT_PIN_STCP 9 17 | #define SHIFT_PIN_DS 6 18 | 19 | /* Instantiate a seven segment controller object with: 20 | - segment pins controlled via 1 shift register and 21 | - digit pins connected to the Arduino directly 22 | */ 23 | SevSegShift sevseg(SHIFT_PIN_DS, SHIFT_PIN_SHCP, SHIFT_PIN_STCP, 1, true); 24 | 25 | // Servo 26 | #include 27 | bool run_servo = false; 28 | 29 | class BellServo 30 | { 31 | Servo servo; // the servo 32 | int pos; // current servo position 33 | int increment; // increment to move for each interval 34 | unsigned long updateInterval; // interval between updates 35 | unsigned long lastUpdate; // last update of position 36 | bool cycle_ended = false; 37 | unsigned long cycleInterval = 1000; 38 | unsigned long lastCycleUpdate = 0; 39 | 40 | public: 41 | BellServo(unsigned long interval) 42 | { 43 | updateInterval = interval; 44 | increment = 1; //20 45 | } 46 | 47 | void Attach(int pin) 48 | { 49 | servo.attach(pin); 50 | } 51 | 52 | void Detach() 53 | { 54 | servo.detach(); 55 | } 56 | 57 | void Update() 58 | { 59 | if (run_servo && potiVal > 600) { 60 | 61 | if ((millis() - lastUpdate) > updateInterval) { 62 | lastUpdate = millis(); 63 | servo.write(80); 64 | } else { 65 | servo.write(0); 66 | } 67 | 68 | if ((millis() - lastCycleUpdate) > cycleInterval) { 69 | lastCycleUpdate = millis(); 70 | run_servo = false; 71 | } 72 | 73 | } else { 74 | lastCycleUpdate = millis(); 75 | } 76 | } 77 | }; 78 | 79 | BellServo bellservo(1000); 80 | 81 | //-------------------------- 82 | // LEDs 83 | #define PIN_LED (13u) 84 | #define LED_BUILTIN PIN_LED 85 | #define LEDR (22u) 86 | #define LEDG (23u) 87 | #define LEDB (24u) 88 | #define LED_PWR (25u) 89 | 90 | // If your target is limited in memory remove this macro to save 10K RAM 91 | #define EIDSP_QUANTIZE_FILTERBANK 0 92 | 93 | /** 94 | * Define the number of slices per model window. E.g. a model window of 1000 ms 95 | * with slices per model window set to 4. Results in a slice size of 250 ms. 96 | * For more info: https://docs.edgeimpulse.com/docs/continuous-audio-sampling 97 | */ 98 | #define EI_CLASSIFIER_SLICES_PER_MODEL_WINDOW 3 99 | 100 | /* Includes ---------------------------------------------------------------- */ 101 | #include 102 | #include 103 | 104 | /** Audio buffers, pointers and selectors */ 105 | typedef struct { 106 | signed short *buffers[2]; 107 | unsigned char buf_select; 108 | unsigned char buf_ready; 109 | unsigned int buf_count; 110 | unsigned int n_samples; 111 | } inference_t; 112 | 113 | static inference_t inference; 114 | static bool record_ready = false; 115 | static signed short *sampleBuffer; 116 | static bool debug_nn = false; // Set this to true to see e.g. features generated from the raw signal 117 | static int print_results = -(EI_CLASSIFIER_SLICES_PER_MODEL_WINDOW); 118 | 119 | /** 120 | * @brief Arduino setup function 121 | */ 122 | void setup() 123 | { 124 | // put your setup code here, to run once: 125 | Serial.begin(115200); 126 | 127 | //---------MAIK---------------- 128 | setupSevsegDisplay(); 129 | setupServo(); 130 | setupButton(); 131 | //----------------------------- 132 | 133 | Serial.println("Edge Impulse Inferencing Demo"); 134 | 135 | // summary of inferencing settings (from model_metadata.h) 136 | ei_printf("Inferencing settings:\n"); 137 | ei_printf("\tInterval: %.2f ms.\n", (float)EI_CLASSIFIER_INTERVAL_MS); 138 | ei_printf("\tFrame size: %d\n", EI_CLASSIFIER_DSP_INPUT_FRAME_SIZE); 139 | ei_printf("\tSample length: %d ms.\n", EI_CLASSIFIER_RAW_SAMPLE_COUNT / 16); 140 | ei_printf("\tNo. of classes: %d\n", sizeof(ei_classifier_inferencing_categories) / 141 | sizeof(ei_classifier_inferencing_categories[0])); 142 | 143 | run_classifier_init(); 144 | if (microphone_inference_start(EI_CLASSIFIER_SLICE_SIZE) == false) { 145 | ei_printf("ERR: Failed to setup audio sampling\r\n"); 146 | return; 147 | } 148 | } 149 | 150 | /** 151 | * @brief Arduino main function. Runs the inferencing loop. 152 | */ 153 | void loop() 154 | { 155 | //-------------MAIK------------- 156 | displayLoop(); 157 | servoLoop(); 158 | buttonLoop(); 159 | 160 | potiVal = analogRead(potPin); 161 | //------------------------------ 162 | 163 | 164 | bool m = microphone_inference_record(); 165 | if (!m) { 166 | ei_printf("ERR: Failed to record audio...\n"); 167 | return; 168 | } 169 | 170 | signal_t signal; 171 | signal.total_length = EI_CLASSIFIER_SLICE_SIZE; 172 | signal.get_data = µphone_audio_signal_get_data; 173 | ei_impulse_result_t result = {0}; 174 | 175 | EI_IMPULSE_ERROR r = run_classifier_continuous(&signal, &result, debug_nn); 176 | if (r != EI_IMPULSE_OK) { 177 | ei_printf("ERR: Failed to run classifier (%d)\n", r); 178 | return; 179 | } 180 | 181 | if (++print_results >= (EI_CLASSIFIER_SLICES_PER_MODEL_WINDOW)) { 182 | 183 | // flash LED if filler is detected 184 | float filler_classification = result.classification[0].value; 185 | if (filler_classification > 0.6) { 186 | Serial.println("FILLER DETECTED"); 187 | run_servo = true; 188 | digitalWrite(LEDR, LOW); 189 | digitalWrite(LEDG, LOW); 190 | digitalWrite(LEDB, LOW); 191 | } else { 192 | digitalWrite(LEDR, HIGH); 193 | digitalWrite(LEDG, HIGH); 194 | digitalWrite(LEDB, HIGH); 195 | } 196 | 197 | // print the predictions 198 | ei_printf("Predictions "); 199 | ei_printf("( DSP: %d ms., Classification: %d ms., Anomaly: %d ms. )", 200 | result.timing.dsp, result.timing.classification, result.timing.anomaly); 201 | ei_printf(": \n"); 202 | for (size_t ix = 0; ix < EI_CLASSIFIER_LABEL_COUNT; ix++) { 203 | ei_printf(" %s: %.5f\n", result.classification[ix].label, 204 | result.classification[ix].value); 205 | } 206 | 207 | #if EI_CLASSIFIER_HAS_ANOMALY == 1 208 | ei_printf(" anomaly score: %.3f\n", result.anomaly); 209 | #endif 210 | 211 | print_results = 0; 212 | } 213 | } 214 | 215 | /** 216 | * @brief Printf function uses vsnprintf and output using Arduino Serial 217 | * 218 | * @param[in] format Variable argument list 219 | */ 220 | void ei_printf(const char *format, ...) { 221 | static char print_buf[1024] = { 0 }; 222 | 223 | va_list args; 224 | va_start(args, format); 225 | int r = vsnprintf(print_buf, sizeof(print_buf), format, args); 226 | va_end(args); 227 | 228 | if (r > 0) { 229 | Serial.write(print_buf); 230 | } 231 | } 232 | 233 | /** 234 | * @brief PDM buffer full callback 235 | * Get data and call audio thread callback 236 | */ 237 | static void pdm_data_ready_inference_callback(void) 238 | { 239 | int bytesAvailable = PDM.available(); 240 | 241 | // read into the sample buffer 242 | int bytesRead = PDM.read((char *)&sampleBuffer[0], bytesAvailable); 243 | 244 | if (record_ready == true) { 245 | for (int i = 0; i> 1; i++) { 246 | inference.buffers[inference.buf_select][inference.buf_count++] = sampleBuffer[i]; 247 | 248 | if (inference.buf_count >= inference.n_samples) { 249 | inference.buf_select ^= 1; 250 | inference.buf_count = 0; 251 | inference.buf_ready = 1; 252 | } 253 | } 254 | } 255 | } 256 | 257 | /** 258 | * @brief Init inferencing struct and setup/start PDM 259 | * 260 | * @param[in] n_samples The n samples 261 | * 262 | * @return { description_of_the_return_value } 263 | */ 264 | static bool microphone_inference_start(uint32_t n_samples) 265 | { 266 | inference.buffers[0] = (signed short *)malloc(n_samples * sizeof(signed short)); 267 | 268 | if (inference.buffers[0] == NULL) { 269 | return false; 270 | } 271 | 272 | inference.buffers[1] = (signed short *)malloc(n_samples * sizeof(signed short)); 273 | 274 | if (inference.buffers[0] == NULL) { 275 | free(inference.buffers[0]); 276 | return false; 277 | } 278 | 279 | sampleBuffer = (signed short *)malloc((n_samples >> 1) * sizeof(signed short)); 280 | 281 | if (sampleBuffer == NULL) { 282 | free(inference.buffers[0]); 283 | free(inference.buffers[1]); 284 | return false; 285 | } 286 | 287 | inference.buf_select = 0; 288 | inference.buf_count = 0; 289 | inference.n_samples = n_samples; 290 | inference.buf_ready = 0; 291 | 292 | // configure the data receive callback 293 | PDM.onReceive(&pdm_data_ready_inference_callback); 294 | 295 | // optionally set the gain, defaults to 20 296 | PDM.setGain(80); 297 | 298 | PDM.setBufferSize((n_samples >> 1) * sizeof(int16_t)); 299 | 300 | // initialize PDM with: 301 | // - one channel (mono mode) 302 | // - a 16 kHz sample rate 303 | if (!PDM.begin(1, EI_CLASSIFIER_FREQUENCY)) { 304 | ei_printf("Failed to start PDM!"); 305 | } 306 | 307 | record_ready = true; 308 | 309 | return true; 310 | } 311 | 312 | /** 313 | * @brief Wait on new data 314 | * 315 | * @return True when finished 316 | */ 317 | static bool microphone_inference_record(void) 318 | { 319 | bool ret = true; 320 | 321 | if (inference.buf_ready == 1) { 322 | ei_printf( 323 | "Error sample buffer overrun. Decrease the number of slices per model window " 324 | "(EI_CLASSIFIER_SLICES_PER_MODEL_WINDOW)\n"); 325 | ret = false; 326 | } 327 | 328 | while (inference.buf_ready == 0) { 329 | delay(1); 330 | } 331 | 332 | inference.buf_ready = 0; 333 | 334 | return ret; 335 | } 336 | 337 | /** 338 | * Get raw audio signal data 339 | */ 340 | static int microphone_audio_signal_get_data(size_t offset, size_t length, float *out_ptr) 341 | { 342 | numpy::int16_to_float(&inference.buffers[inference.buf_select ^ 1][offset], out_ptr, length); 343 | 344 | return 0; 345 | } 346 | 347 | /** 348 | * @brief Stop PDM and release buffers 349 | */ 350 | static void microphone_inference_end(void) 351 | { 352 | PDM.end(); 353 | free(inference.buffers[0]); 354 | free(inference.buffers[1]); 355 | free(sampleBuffer); 356 | } 357 | 358 | #if !defined(EI_CLASSIFIER_SENSOR) || EI_CLASSIFIER_SENSOR != EI_CLASSIFIER_SENSOR_MICROPHONE 359 | #error "Invalid model for current sensor." 360 | #endif 361 | 362 | //---------MAIK--------------// 363 | //---------------------------// 364 | //---SEVEN SEGMENT DISPLAY---// 365 | 366 | void setupSevsegDisplay() { 367 | 368 | byte numDigits = 4; 369 | byte digitPins[] = {2, 3, 4, 5}; //{2, 3, 4, 5}; // These are the PINS of the ** Arduino ** 370 | byte segmentPins[] = {1, 2, 3, 4, 5, 6, 7}; // these are the PINs of the ** Shift register ** 371 | bool resistorsOnSegments = false; // 'false' means resistors are on digit pins 372 | byte hardwareConfig = COMMON_CATHODE; // See README.md for options 373 | bool updateWithDelays = false; // Default 'false' is Recommended 374 | bool leadingZeros = false; // Use 'true' if you'd like to keep the leading zeros 375 | bool disableDecPoint = true; // Use 'true' if your decimal point doesn't exist or isn't connected 376 | 377 | sevseg.begin(hardwareConfig, numDigits, digitPins, segmentPins, resistorsOnSegments, updateWithDelays, leadingZeros, disableDecPoint); 378 | sevseg.setBrightness(90); 379 | } 380 | 381 | void displayLoop() { 382 | sevseg.setNumber(uhm_count, 0); 383 | sevseg.refreshDisplay(); // Must run repeatedly 384 | } 385 | 386 | //---------------------------// 387 | //------SERVO MOTOR----------// 388 | 389 | void setupServo() { 390 | bellservo.Attach(11); // attaches the servo on pin 9 to the servo object 391 | } 392 | 393 | void servoLoop() { 394 | bellservo.Update(); 395 | } 396 | 397 | //---------------------------// 398 | //------BUTTON---------------// 399 | 400 | void setupButton() { 401 | pinMode(BUTTON_PIN, INPUT_PULLUP); 402 | } 403 | 404 | void buttonLoop() { 405 | currentState = digitalRead(BUTTON_PIN); 406 | 407 | if (currentState == LOW && pressed == false) { 408 | Serial.println("Button pressed!"); 409 | uhm_count++; 410 | run_servo = true; 411 | pressed = true; 412 | } 413 | else if (currentState == HIGH && pressed == true) { 414 | pressed = false; 415 | } 416 | } 417 | --------------------------------------------------------------------------------