├── voicecontrol.css ├── snowboy ├── _snowboydetect.so ├── resources │ ├── ding.wav │ ├── dong.wav │ └── common.res ├── snowboydecoder.pyc ├── snowboydetect.pyc ├── _snowboydetect-osx.so ├── kws.py ├── kws-multiple.py ├── snowboydetect.py └── snowboydecoder.py ├── voicecontrol.js ├── node_helper.js └── README.md /voicecontrol.css: -------------------------------------------------------------------------------- 1 | .voicecontrol .top { 2 | position: relative; 3 | top: 5px; 4 | } -------------------------------------------------------------------------------- /snowboy/_snowboydetect.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexyak/voicecontrol/HEAD/snowboy/_snowboydetect.so -------------------------------------------------------------------------------- /snowboy/resources/ding.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexyak/voicecontrol/HEAD/snowboy/resources/ding.wav -------------------------------------------------------------------------------- /snowboy/resources/dong.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexyak/voicecontrol/HEAD/snowboy/resources/dong.wav -------------------------------------------------------------------------------- /snowboy/snowboydecoder.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexyak/voicecontrol/HEAD/snowboy/snowboydecoder.pyc -------------------------------------------------------------------------------- /snowboy/snowboydetect.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexyak/voicecontrol/HEAD/snowboy/snowboydetect.pyc -------------------------------------------------------------------------------- /snowboy/resources/common.res: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexyak/voicecontrol/HEAD/snowboy/resources/common.res -------------------------------------------------------------------------------- /snowboy/_snowboydetect-osx.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/alexyak/voicecontrol/HEAD/snowboy/_snowboydetect-osx.so -------------------------------------------------------------------------------- /snowboy/kws.py: -------------------------------------------------------------------------------- 1 | import snowboydecoder 2 | import sys 3 | import signal 4 | 5 | interrupted = False 6 | 7 | 8 | def signal_handler(signal, frame): 9 | global interrupted 10 | interrupted = True 11 | 12 | 13 | def interrupt_callback(): 14 | global interrupted 15 | return interrupted 16 | 17 | def hotword_detected_callback(): 18 | print("!Hotword Detected") 19 | #snowboydecoder.play_audio_file(snowboydecoder.DETECT_DING) 20 | 21 | if len(sys.argv) < 2: 22 | print("Error: need to specify model name and sensitivity") 23 | print("Usage: python demo.py your.model 0.5") 24 | sys.exit(-1) 25 | 26 | model = sys.argv[1] 27 | detectionSensitivity = round(float(sys.argv[2]), 2) 28 | 29 | # capture SIGINT signal, e.g., Ctrl+C 30 | signal.signal(signal.SIGINT, signal_handler) 31 | 32 | detector = snowboydecoder.HotwordDetector(model, sensitivity=detectionSensitivity) 33 | print('Listening... Press Ctrl+C to exit') 34 | 35 | # main loop 36 | detector.start(detected_callback=hotword_detected_callback, 37 | interrupt_check=interrupt_callback, 38 | sleep_time=0.03) 39 | 40 | detector.terminate() 41 | -------------------------------------------------------------------------------- /snowboy/kws-multiple.py: -------------------------------------------------------------------------------- 1 | import snowboydecoder 2 | import sys 3 | import signal 4 | 5 | # Demo code for listening two hotwords at the same time 6 | 7 | interrupted = False 8 | 9 | 10 | def signal_handler(signal, frame): 11 | global interrupted 12 | interrupted = True 13 | 14 | 15 | def interrupt_callback(): 16 | global interrupted 17 | return interrupted 18 | 19 | # if len(sys.argv) != 3: 20 | # print("Error: need to specify 2 model names") 21 | # print("Usage: python demo.py 1st.model 2nd.model") 22 | # sys.exit(-1) 23 | 24 | models = sys.argv[1:] 25 | 26 | def hotword_detected_callback(): 27 | print("!Hotword Detected") 28 | 29 | 30 | # capture SIGINT signal, e.g., Ctrl+C 31 | signal.signal(signal.SIGINT, signal_handler) 32 | 33 | sensitivity = [0.5]*len(models) 34 | detector = snowboydecoder.HotwordDetector(models, sensitivity=sensitivity) 35 | # callbacks = [print("DETECTED:1"), 36 | # print("DETECTED:2")] 37 | print('Listening... Press Ctrl+C to exit') 38 | 39 | # main loop 40 | # make sure you have the same numbers of callbacks and models 41 | detector.start(detected_callback=hotword_detected_callback, 42 | interrupt_check=interrupt_callback, 43 | sleep_time=0.03) 44 | 45 | detector.terminate() 46 | -------------------------------------------------------------------------------- /voicecontrol.js: -------------------------------------------------------------------------------- 1 | 2 | 'use strict'; 3 | /* Magic Mirror 4 | * Module: voicecontrol 5 | * 6 | * By Alex Yakhnin 7 | * MIT Licensed. 8 | */ 9 | 10 | Module.register("voicecontrol", { 11 | 12 | // Default module config. 13 | 14 | defaults: { 15 | models: [ 16 | { 17 | keyword: "Show Camera", 18 | description: "Say 'Show Camera' to display camera", 19 | file: "showCamera.pmdl", 20 | message: "SHOW_CAMERA" 21 | }, 22 | { 23 | keyword: "Hide Camera", 24 | description: "Say 'Hide Camera' to hide camera", 25 | file: "hideCamera.pmdl", 26 | message: "HIDE_CAMERA" 27 | }, 28 | { 29 | keyword: "Selfie", 30 | description: "Say 'Selfie' when camera is visible", 31 | file: "selfie.pmdl", 32 | message: "SELFIE" 33 | }, 34 | ] 35 | }, 36 | 37 | start: function() { 38 | 39 | this.sendSocketNotification("CONNECT", this.config); 40 | 41 | }, 42 | 43 | getStyles: function() { 44 | return ['voicecontrol.css']; 45 | }, 46 | 47 | socketNotificationReceived: function(notification, payload){ 48 | if (notification === "KEYWORD_SPOTTED"){ 49 | //Broadcast the message 50 | this.sendNotification(payload.message, {type: "notification"}); 51 | } 52 | }, 53 | 54 | getDom: function() { 55 | var wrapper = document.createElement("div"); 56 | var header = document.createElement("header"); 57 | header.innerHTML = "Voice Commands"; 58 | wrapper.appendChild(header); 59 | var models = this.config.models; 60 | 61 | models.forEach(function(model) { 62 | var command = document.createElement("div"); 63 | command.innerHTML = model.description; 64 | command.className = "small dimmed top"; 65 | wrapper.appendChild(command); 66 | }, this); 67 | 68 | return wrapper; 69 | } 70 | 71 | }); -------------------------------------------------------------------------------- /node_helper.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | /* Magic Mirror 4 | * Module: voicecontrol 5 | * 6 | * By Alex Yaknin 7 | * MIT Licensed. 8 | */ 9 | 10 | const NodeHelper = require('node_helper'); 11 | const spawn = require('child_process').spawn; 12 | 13 | module.exports = NodeHelper.create({ 14 | start: function () { 15 | this.started = false; 16 | 17 | }, 18 | 19 | socketNotificationReceived: function(notification, payload) { 20 | if (notification === "CONNECT") { 21 | this.startRecognition(payload); 22 | return; 23 | } 24 | }, 25 | 26 | startRecognition : function(config) { 27 | 28 | var models = config.models; 29 | 30 | var kwsSensitivity = 0.5; 31 | this.started = true; 32 | var self = this; 33 | // Initilize the keyword spotter 34 | var params = ['./modules/voicecontrol/snowboy/kws-multiple.py']; //, modelFile1, modelFile2]; 35 | 36 | 37 | models.forEach(function(model) { 38 | params.push(model.file); 39 | }, this); 40 | 41 | //var kwsProcess = spawn('python', ['./speech-osx/kws-multiple.py', modelFile1, modelFile2], { detached: false }); 42 | var kwsProcess = spawn('python', params, { detached: false }); 43 | // Handel messages from python script 44 | kwsProcess.stderr.on('data', function (data) { 45 | var message = data.toString(); 46 | if (message.startsWith('INFO')) { 47 | var items = message.split(':'); 48 | var index = parseInt(items[2].split(' ')[1]); 49 | var model = models[index - 1]; 50 | self.sendSocketNotification("KEYWORD_SPOTTED", model); 51 | 52 | } else { 53 | console.error(message); 54 | } 55 | }) 56 | kwsProcess.stdout.on('data', function (data) { 57 | console.log(data.toString()); 58 | }) 59 | } 60 | 61 | }); -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Module: Voice Control 2 | The `voicecontrol` module allows to use voice control in the MagicMirror. 3 | This module based on the https://snowboy.kitt.ai/. The snowboy.kitt has a few dependencies which you can install running this command line on your raspberry pi: 4 | 5 | ````javascript 6 | sudo apt-get install python-pyaudio python3-pyaudio sox 7 | ```` 8 | 9 | 10 | ## Using the module 11 | 12 | In order to use this module you should create a trained model for each command/keyword at https://snowboy.kitt.ai/. 13 | Download the model and copy it into the root of the MagicMirror directory. Besides recognizing the voice commands the module could display a list of commands on the mirror. 14 | 15 | 16 | To use this module, add it to the modules array in the `config/config.js` file with the following settings: 17 | ````javascript 18 | modules: [ 19 | { 20 | module: 'voicecontrol', 21 | position: 'bottom_left', 22 | config: { 23 | models: [ 24 | { 25 | keyword: "playMusic", // keyword 26 | description: "Say 'Play Music' to start playing", 27 | file: "playMusic.pmdl", // trained model file name 28 | message: "PLAY_MUSIC" // notification message that's broadcast in the MagicMirror app 29 | }, 30 | { 31 | keyword: "stopMusic", 32 | description: "Say 'Stop Music' to stop playing", 33 | file: "stopMusic.pmdl", 34 | message: "STOP_MUSIC" 35 | }, 36 | ] 37 | } 38 | } 39 | ] 40 | ```` 41 | 42 | When a command is detected a notification message is send with sendNotification to every other module. You will need to subscribe for a specific type of message in your module: 43 | 44 | ````javascript 45 | 46 | notificationReceived: function(notification, payload, sender) { 47 | if (notification === "PLAY_MUSIC"){ 48 | this.media.play(); 49 | } 50 | 51 | if (notification === "STOP_MUSIC"){ 52 | this.media.pause(); 53 | } 54 | }, 55 | 56 | ```` 57 | -------------------------------------------------------------------------------- /snowboy/snowboydetect.py: -------------------------------------------------------------------------------- 1 | # This file was automatically generated by SWIG (http://www.swig.org). 2 | # Version 3.0.7 3 | # 4 | # Do not make changes to this file unless you know what you are doing--modify 5 | # the SWIG interface file instead. 6 | 7 | 8 | 9 | 10 | 11 | from sys import version_info 12 | if version_info >= (2, 6, 0): 13 | def swig_import_helper(): 14 | from os.path import dirname 15 | import imp 16 | fp = None 17 | try: 18 | fp, pathname, description = imp.find_module('_snowboydetect', [dirname(__file__)]) 19 | except ImportError: 20 | import _snowboydetect 21 | return _snowboydetect 22 | if fp is not None: 23 | try: 24 | _mod = imp.load_module('_snowboydetect', fp, pathname, description) 25 | finally: 26 | fp.close() 27 | return _mod 28 | _snowboydetect = swig_import_helper() 29 | del swig_import_helper 30 | else: 31 | import _snowboydetect 32 | del version_info 33 | try: 34 | _swig_property = property 35 | except NameError: 36 | pass # Python < 2.2 doesn't have 'property'. 37 | 38 | 39 | def _swig_setattr_nondynamic(self, class_type, name, value, static=1): 40 | if (name == "thisown"): 41 | return self.this.own(value) 42 | if (name == "this"): 43 | if type(value).__name__ == 'SwigPyObject': 44 | self.__dict__[name] = value 45 | return 46 | method = class_type.__swig_setmethods__.get(name, None) 47 | if method: 48 | return method(self, value) 49 | if (not static): 50 | if _newclass: 51 | object.__setattr__(self, name, value) 52 | else: 53 | self.__dict__[name] = value 54 | else: 55 | raise AttributeError("You cannot add attributes to %s" % self) 56 | 57 | 58 | def _swig_setattr(self, class_type, name, value): 59 | return _swig_setattr_nondynamic(self, class_type, name, value, 0) 60 | 61 | 62 | def _swig_getattr_nondynamic(self, class_type, name, static=1): 63 | if (name == "thisown"): 64 | return self.this.own() 65 | method = class_type.__swig_getmethods__.get(name, None) 66 | if method: 67 | return method(self) 68 | if (not static): 69 | return object.__getattr__(self, name) 70 | else: 71 | raise AttributeError(name) 72 | 73 | def _swig_getattr(self, class_type, name): 74 | return _swig_getattr_nondynamic(self, class_type, name, 0) 75 | 76 | 77 | def _swig_repr(self): 78 | try: 79 | strthis = "proxy of " + self.this.__repr__() 80 | except: 81 | strthis = "" 82 | return "<%s.%s; %s >" % (self.__class__.__module__, self.__class__.__name__, strthis,) 83 | 84 | try: 85 | _object = object 86 | _newclass = 1 87 | except AttributeError: 88 | class _object: 89 | pass 90 | _newclass = 0 91 | 92 | 93 | class SnowboyDetect(_object): 94 | __swig_setmethods__ = {} 95 | __setattr__ = lambda self, name, value: _swig_setattr(self, SnowboyDetect, name, value) 96 | __swig_getmethods__ = {} 97 | __getattr__ = lambda self, name: _swig_getattr(self, SnowboyDetect, name) 98 | __repr__ = _swig_repr 99 | 100 | def __init__(self, resource_filename, model_str): 101 | this = _snowboydetect.new_SnowboyDetect(resource_filename, model_str) 102 | try: 103 | self.this.append(this) 104 | except: 105 | self.this = this 106 | 107 | def Reset(self): 108 | return _snowboydetect.SnowboyDetect_Reset(self) 109 | 110 | def RunDetection(self, data): 111 | return _snowboydetect.SnowboyDetect_RunDetection(self, data) 112 | 113 | def SetSensitivity(self, sensitivity_str): 114 | return _snowboydetect.SnowboyDetect_SetSensitivity(self, sensitivity_str) 115 | 116 | def GetSensitivity(self): 117 | return _snowboydetect.SnowboyDetect_GetSensitivity(self) 118 | 119 | def SetAudioGain(self, audio_gain): 120 | return _snowboydetect.SnowboyDetect_SetAudioGain(self, audio_gain) 121 | 122 | def UpdateModel(self): 123 | return _snowboydetect.SnowboyDetect_UpdateModel(self) 124 | 125 | def NumHotwords(self): 126 | return _snowboydetect.SnowboyDetect_NumHotwords(self) 127 | 128 | def SampleRate(self): 129 | return _snowboydetect.SnowboyDetect_SampleRate(self) 130 | 131 | def NumChannels(self): 132 | return _snowboydetect.SnowboyDetect_NumChannels(self) 133 | 134 | def BitsPerSample(self): 135 | return _snowboydetect.SnowboyDetect_BitsPerSample(self) 136 | __swig_destroy__ = _snowboydetect.delete_SnowboyDetect 137 | __del__ = lambda self: None 138 | SnowboyDetect_swigregister = _snowboydetect.SnowboyDetect_swigregister 139 | SnowboyDetect_swigregister(SnowboyDetect) 140 | 141 | # This file is compatible with both classic and new-style classes. 142 | 143 | 144 | -------------------------------------------------------------------------------- /snowboy/snowboydecoder.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import collections 4 | import pyaudio 5 | import snowboydetect 6 | import time 7 | import wave 8 | import os 9 | import logging 10 | 11 | logging.basicConfig() 12 | logger = logging.getLogger("snowboy") 13 | logger.setLevel(logging.INFO) 14 | TOP_DIR = os.path.dirname(os.path.abspath(__file__)) 15 | 16 | RESOURCE_FILE = os.path.join(TOP_DIR, "resources/common.res") 17 | DETECT_DING = os.path.join(TOP_DIR, "resources/ding.wav") 18 | DETECT_DONG = os.path.join(TOP_DIR, "resources/dong.wav") 19 | 20 | 21 | class RingBuffer(object): 22 | """Ring buffer to hold audio from PortAudio""" 23 | def __init__(self, size = 4096): 24 | self._buf = collections.deque(maxlen=size) 25 | 26 | def extend(self, data): 27 | """Adds data to the end of buffer""" 28 | self._buf.extend(data) 29 | 30 | def get(self): 31 | """Retrieves data from the beginning of buffer and clears it""" 32 | tmp = ''.join(self._buf) 33 | self._buf.clear() 34 | return tmp 35 | 36 | 37 | def play_audio_file(fname=DETECT_DING): 38 | """Simple callback function to play a wave file. By default it plays 39 | a Ding sound. 40 | 41 | :param str fname: wave file name 42 | :return: None 43 | """ 44 | ding_wav = wave.open(fname, 'rb') 45 | ding_data = ding_wav.readframes(ding_wav.getnframes()) 46 | audio = pyaudio.PyAudio() 47 | stream_out = audio.open( 48 | format=audio.get_format_from_width(ding_wav.getsampwidth()), 49 | channels=ding_wav.getnchannels(), 50 | rate=ding_wav.getframerate(), input=False, output=True) 51 | stream_out.start_stream() 52 | stream_out.write(ding_data) 53 | time.sleep(0.2) 54 | stream_out.stop_stream() 55 | stream_out.close() 56 | audio.terminate() 57 | 58 | 59 | class HotwordDetector(object): 60 | """ 61 | Snowboy decoder to detect whether a keyword specified by `decoder_model` 62 | exists in a microphone input stream. 63 | 64 | :param decoder_model: decoder model file path, a string or a list of strings 65 | :param resource: resource file path. 66 | :param sensitivity: decoder sensitivity, a float of a list of floats. 67 | The bigger the value, the more senstive the 68 | decoder. If an empty list is provided, then the 69 | default sensitivity in the model will be used. 70 | :param audio_gain: multiply input volume by this factor. 71 | """ 72 | def __init__(self, decoder_model, 73 | resource=RESOURCE_FILE, 74 | sensitivity=[], 75 | audio_gain=1): 76 | 77 | def audio_callback(in_data, frame_count, time_info, status): 78 | self.ring_buffer.extend(in_data) 79 | play_data = chr(0) * len(in_data) 80 | return play_data, pyaudio.paContinue 81 | 82 | tm = type(decoder_model) 83 | ts = type(sensitivity) 84 | if tm is not list: 85 | decoder_model = [decoder_model] 86 | if ts is not list: 87 | sensitivity = [sensitivity] 88 | model_str = ",".join(decoder_model) 89 | 90 | self.detector = snowboydetect.SnowboyDetect( 91 | resource_filename=resource, model_str=model_str) 92 | self.detector.SetAudioGain(audio_gain) 93 | self.num_hotwords = self.detector.NumHotwords() 94 | 95 | if len(decoder_model) > 1 and len(sensitivity) == 1: 96 | sensitivity = sensitivity*self.num_hotwords 97 | if len(sensitivity) != 0: 98 | assert self.num_hotwords == len(sensitivity), \ 99 | "number of hotwords in decoder_model (%d) and sensitivity " \ 100 | "(%d) does not match" % (self.num_hotwords, len(sensitivity)) 101 | sensitivity_str = ",".join([str(t) for t in sensitivity]) 102 | if len(sensitivity) != 0: 103 | self.detector.SetSensitivity(sensitivity_str); 104 | 105 | self.ring_buffer = RingBuffer( 106 | self.detector.NumChannels() * self.detector.SampleRate() * 5) 107 | self.audio = pyaudio.PyAudio() 108 | self.stream_in = self.audio.open( 109 | input=True, output=False, 110 | format=self.audio.get_format_from_width( 111 | self.detector.BitsPerSample() / 8), 112 | channels=self.detector.NumChannels(), 113 | rate=self.detector.SampleRate(), 114 | frames_per_buffer=2048, 115 | stream_callback=audio_callback) 116 | 117 | 118 | def start(self, detected_callback=play_audio_file, 119 | interrupt_check=lambda: False, 120 | sleep_time=0.03): 121 | """ 122 | Start the voice detector. For every `sleep_time` second it checks the 123 | audio buffer for triggering keywords. If detected, then call 124 | corresponding function in `detected_callback`, which can be a single 125 | function (single model) or a list of callback functions (multiple 126 | models). Every loop it also calls `interrupt_check` -- if it returns 127 | True, then breaks from the loop and return. 128 | 129 | :param detected_callback: a function or list of functions. The number of 130 | items must match the number of models in 131 | `decoder_model`. 132 | :param interrupt_check: a function that returns True if the main loop 133 | needs to stop. 134 | :param float sleep_time: how much time in second every loop waits. 135 | :return: None 136 | """ 137 | if interrupt_check(): 138 | logger.debug("detect voice return") 139 | return 140 | 141 | tc = type(detected_callback) 142 | if tc is not list: 143 | detected_callback = [detected_callback] 144 | if len(detected_callback) == 1 and self.num_hotwords > 1: 145 | detected_callback *= self.num_hotwords 146 | 147 | assert self.num_hotwords == len(detected_callback), \ 148 | "Error: hotwords in your models (%d) do not match the number of " \ 149 | "callbacks (%d)" % (self.num_hotwords, len(detected_callback)) 150 | 151 | logger.debug("detecting...") 152 | 153 | while True: 154 | if interrupt_check(): 155 | logger.debug("detect voice break") 156 | break 157 | data = self.ring_buffer.get() 158 | if len(data) == 0: 159 | time.sleep(sleep_time) 160 | continue 161 | 162 | ans = self.detector.RunDetection(data) 163 | if ans == -1: 164 | logger.warning("Error initializing streams or reading audio data") 165 | elif ans > 0: 166 | message = "Keyword " + str(ans) + " detected at time: " 167 | message += time.strftime("%Y-%m-%d %H:%M:%S", 168 | time.localtime(time.time())) 169 | logger.info(message) 170 | callback = detected_callback[ans-1] 171 | if callback is not None: 172 | callback() 173 | 174 | logger.debug("finished.") 175 | 176 | def terminate(self): 177 | """ 178 | Terminate audio stream. Users cannot call start() again to detect. 179 | :return: None 180 | """ 181 | self.stream_in.stop_stream() 182 | self.stream_in.close() 183 | self.audio.terminate() 184 | --------------------------------------------------------------------------------