├── .gitattributes ├── .gitignore ├── README.md ├── deploy ├── dashboard │ ├── Dockerfile │ ├── assets │ │ └── custom_style.css │ ├── bell_dash.py │ ├── gunicorn.config.py │ └── requirements.txt ├── data │ └── original_data │ │ ├── ambient_data.npy │ │ └── bell_data.npy ├── docker-compose.yaml ├── model_server │ ├── Dockerfile │ ├── asoundrc │ ├── config.py │ ├── main.py │ └── requirements.txt └── retraining │ ├── Dockerfile │ ├── analyze_training.ipynb │ ├── api.py │ ├── config.py │ ├── requirements.txt │ └── retraining_pipeline.py ├── experiments ├── asyncio_generators.py └── plot_input.py ├── notebook ├── confusion_matrix_pretty_print.py └── train.ipynb ├── raw_data ├── ambient.wav ├── bel.wav └── bel2.wav └── requirements.txt /.gitattributes: -------------------------------------------------------------------------------- 1 | *.npy filter=lfs diff=lfs merge=lfs -text 2 | raw_data/*.wav filter=lfs diff=lfs merge=lfs -text 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | creds.py 2 | __pycache__ 3 | .vscode 4 | split_data 5 | experiments 6 | .venv 7 | *.p 8 | *.wav 9 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # doorbell_detector 2 | An over engineered doorbell detector based on machine learning 3 | 4 | ## Install 5 | Install docker: 6 | ```bash 7 | curl -fsSL https://get.docker.com -o get-docker.sh 8 | sudo get-docker.sh 9 | ``` 10 | 11 | Install git, git lfs and docker compose 12 | `sudo apt install git git-lfs docker-compose` 13 | 14 | 15 | This repository is structured as follows: 16 | 17 | raw_data: folder to keep track of original, uncut recordings for reference. It's always a good idea to keep the original data around. 18 | split_data: folder to put the manually extracted bell sounds in and the automatically extracted non-bell and non-silence parts. 19 | bell: bell sounds of varying lengths 20 | noise: ambient noise sounds of varying lengths 21 | 22 | 23 | Quickstart on your own data: 24 | Extract the bell sounds and put each audio file under `split_data/bell` 25 | Throw the ambient sounds file in `raw_data` as `ambient.wav`, it will be split as part of the notebook 26 | 27 | Run the notebook to generate a model file 28 | 29 | Add a `creds.py` file under `deploy` and add an `app_token` and `client_token` variable of your pushover api keys 30 | Copy the deploy folder (also containing the trained model file now) to the raspberry pi 31 | 32 | You can use `screen` to run the python script until it is deployed with docker in the next version 33 | 34 | 35 | TODO: add retraining code for corrected windows 36 | 37 | https://github.com/veirs/sounddevice/blob/master/Dockerfile 38 | 39 | TODO: 40 | - [*] Get model server running 41 | - [ ] Get dashboard running 42 | - [ ] Get retraining pipeline running 43 | - [*] Update model server features 44 | - [ ] Implement model hotswap in model server 45 | -------------------------------------------------------------------------------- /deploy/dashboard/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM balenalib/raspberry-pi-python:3.7.4 2 | 3 | # Copy requirements first. Docker caches every step so this way we don't have to reinstall every pip package every time our code changes 4 | COPY requirements.txt / 5 | RUN pip3 install --extra-index-url=https://www.piwheels.org/simple --no-cache-dir -r requirements.txt 6 | 7 | COPY . /app 8 | WORKDIR /app 9 | 10 | ENTRYPOINT [ "gunicorn", "-c", "gunicorn.config.py", "bell_dash:SERVER" ] -------------------------------------------------------------------------------- /deploy/dashboard/assets/custom_style.css: -------------------------------------------------------------------------------- 1 | .custom-control-label::after, 2 | .custom-control-label::before { 3 | display: none; 4 | } 5 | 6 | .custom-control { 7 | padding-left: 0; 8 | } 9 | 10 | .custom-control-inline { 11 | margin-right: 0; 12 | } 13 | 14 | .date-group-items input { 15 | visibility: hidden; 16 | } 17 | 18 | .date-group-labels { 19 | border: 1px solid #ccc; 20 | border-radius: 0px; 21 | font: bold 11px Roboto, sans-serif; 22 | padding: 6px 7px; 23 | background-image: -webkit-linear-gradient(top, #fefefe, #f3f3f3); 24 | background-image: -moz-linear-gradient(top, #fefefe, #f3f3f3); 25 | cursor: pointer; 26 | } 27 | 28 | .date-group-labels.date-group-labels-checked { 29 | background-image: -webkit-linear-gradient(top, #eee, #e3e3e3); 30 | background-image: -moz-linear-gradient(top, #eee, #e3e3e3); 31 | box-shadow: inset 0px 1px 2px rgba(0, 0, 0, 0.2); 32 | } 33 | 34 | .custom-radio:first-child .date-group-labels { 35 | border-radius: 3px 0 0 3px; 36 | } 37 | 38 | .custom-radio:last-child .date-group-labels { 39 | border-radius: 0 3px 3px 0; 40 | } 41 | 42 | .custom-radio:not(:last-child) .date-group-labels { 43 | border-right: none; 44 | } 45 | 46 | .date-group-labels:hover { 47 | box-shadow: inset 0px 1px 2px rgba(0, 0, 0, 0.2); 48 | } 49 | 50 | .card { 51 | margin: 15px; 52 | padding: 15px; 53 | } -------------------------------------------------------------------------------- /deploy/dashboard/bell_dash.py: -------------------------------------------------------------------------------- 1 | import os 2 | import shutil 3 | import socket 4 | import logging 5 | import requests 6 | 7 | import dash 8 | import rq_dashboard 9 | import dash_core_components as dcc 10 | import dash_html_components as html 11 | import dash_bootstrap_components as dbc 12 | from dash.exceptions import PreventUpdate 13 | from dash.dependencies import Input, Output, State, ALL 14 | 15 | # Create the dash app object 16 | APP = dash.Dash(__name__, external_stylesheets=[dbc.themes.BOOTSTRAP]) 17 | SERVER = APP.server 18 | 19 | CLIP_FOLDER = '/app/assets/unlabeled_data' 20 | BELL_FOLDER = '/app/labeled_data/bell' 21 | AMBIENT_FOLDER = '/app/labeled_data/ambient' 22 | 23 | 24 | def generate_clips(): 25 | clips = [] 26 | for i, clip in enumerate(sorted(os.listdir(CLIP_FOLDER))): 27 | clips.append( 28 | dbc.Col([ 29 | dbc.Card([ 30 | dbc.Row([ 31 | dbc.Col(html.Audio(src=f'/assets/unlabeled_data/{clip}', controls=True)) 32 | ]), 33 | dbc.Row([ 34 | dbc.Col(html.P(clip)), 35 | dbc.Col(dbc.RadioItems( 36 | id={ 37 | 'type': 'filter-dropdown', 38 | 'index': i 39 | }, 40 | options=[ 41 | {"label": "Correct", "value": True}, 42 | {"label": "Wrong", "value": False}, 43 | ], 44 | labelClassName="date-group-labels", 45 | labelCheckedClassName="date-group-labels-checked", 46 | className="date-group-items", 47 | inline=True, 48 | ), align='center') 49 | ]) 50 | ], color='light') 51 | ], width=12) 52 | ) 53 | if not clips: 54 | clips = [dbc.Row(dbc.Col(html.P('Nothing to review!')))] 55 | return clips 56 | 57 | 58 | 59 | # Create the HTML layout 60 | # Use html for 1:1 html tags 61 | # Use dcc for more interactive stuff like button and sliders 62 | APP.layout = dbc.Container([ 63 | dcc.Location(id='url'), 64 | dcc.Interval(id='poll-timer', interval=5000), 65 | dcc.Store(id='job_id'), 66 | dbc.Row([ 67 | dbc.Col(html.H1('Review doorbells'), width={"size": 4, "offset": 4}) 68 | ]), 69 | dbc.Row([ 70 | dbc.Col([ 71 | dbc.Row(generate_clips(), id='clips'), 72 | dbc.Row([ 73 | dbc.Col(dbc.Button('Trigger Retraining', id='training_btn', color="primary", className="mr-1")), 74 | dbc.Col(dbc.Button('Save labels', id='save_btn', color="primary", className="mr-1")), 75 | ]), 76 | dbc.Row([ 77 | dbc.Col([ 78 | html.P(id='test') 79 | ]) 80 | ]), 81 | dbc.Row([ 82 | dbc.Alert( 83 | "Training Successfully started!", 84 | id="alert-good", 85 | is_open=False, 86 | duration=4000, 87 | ), 88 | dbc.Alert( 89 | "Something went wrong in retraining!", 90 | id="alert-bad", 91 | is_open=False, 92 | duration=4000, 93 | ), 94 | ]) 95 | ], width={"size": 6, "offset": 3}, sm={"size": 12, "offset": 0}), 96 | ]) 97 | ], style={'text-align': 'center'}) 98 | 99 | # Callbacks! 100 | # These are the real magic of plotly dash 101 | # Use any attribute within the layout as input 102 | # and ouput to any other attribute based on the 103 | # return value of a function 104 | @APP.callback( 105 | Output('clips', 'children'), 106 | [Input('save_btn', 'n_clicks')], 107 | [State({'type': 'filter-dropdown', 'index': ALL}, 'value')] 108 | ) 109 | def update_output_div(_, values): 110 | files = sorted(os.listdir(CLIP_FOLDER)) 111 | if files: 112 | for i, value in enumerate(values): 113 | filename = files[i] 114 | if value is None: 115 | continue 116 | elif value: 117 | shutil.move(os.path.join(CLIP_FOLDER, filename), 118 | os.path.join(BELL_FOLDER, filename)) 119 | else: 120 | shutil.move(os.path.join(CLIP_FOLDER, filename), 121 | os.path.join(AMBIENT_FOLDER, filename)) 122 | print(f'File {files[i]} is {value}') 123 | return generate_clips() 124 | 125 | 126 | @APP.callback( 127 | [Output('alert-good', 'is_open'), 128 | Output('alert-bad', 'is_open'), 129 | Output('job_id', 'data')], 130 | [Input('training_btn', 'n_clicks')] 131 | ) 132 | def trigger_training(n_clicks): 133 | if n_clicks: 134 | response = requests.post('http://retraining_api:8080/start_retraining') 135 | if response.status_code == 201: 136 | return True, False, response.json()['job_id'] 137 | else: 138 | return False, True, None 139 | else: 140 | raise PreventUpdate 141 | 142 | 143 | @APP.callback( 144 | [Output('training_btn', 'children'), 145 | Output('training_btn', 'disabled')], 146 | [Input('poll-timer', 'n_intervals')], 147 | State('job_id', 'data') 148 | ) 149 | def status_update(n_clicks, job_id): 150 | if n_clicks and job_id: 151 | response = requests.get(f'http://retraining_api:8080/status_retraining/{job_id}') 152 | if response.status_code == 200: 153 | meta = response.json() 154 | if meta['progress'] != 'Done!': 155 | return [dbc.Spinner(size="sm"), ' ' + meta['progress']], True 156 | else: 157 | return 'Trigger Retraining', False 158 | else: 159 | raise PreventUpdate 160 | else: 161 | raise PreventUpdate 162 | 163 | if __name__ == '__main__': 164 | APP.run_server(host='0.0.0.0', debug=False) 165 | -------------------------------------------------------------------------------- /deploy/dashboard/gunicorn.config.py: -------------------------------------------------------------------------------- 1 | # pylint: skip-file 2 | """Gunicorn configuration.""" 3 | bind = ':{}'.format(8000) 4 | 5 | workers = 1 6 | timeout = 90 7 | #worker_class = 'gevent' 8 | 9 | accesslog = '-' 10 | -------------------------------------------------------------------------------- /deploy/dashboard/requirements.txt: -------------------------------------------------------------------------------- 1 | dash==1.20.0 2 | dash-bootstrap-components==0.12.0 3 | gunicorn==20.1.0 4 | rq-dashboard==0.6.1 5 | rq==1.8.0 6 | requests==2.25.1 7 | -------------------------------------------------------------------------------- /deploy/data/original_data/ambient_data.npy: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:f5f934499f44e060b035f4093c4260853941ebc916a5aabae68312320f73b6a7 3 | size 118673228 4 | -------------------------------------------------------------------------------- /deploy/data/original_data/bell_data.npy: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:fda0232d1e447328606142fed07a26633e729a50cb1673b43b93ee9415e11955 3 | size 2072828 4 | -------------------------------------------------------------------------------- /deploy/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: "2.3" 2 | 3 | services: 4 | model_server: 5 | build: 6 | context: ./model_server 7 | dockerfile: Dockerfile 8 | devices: 9 | - "/dev/snd:/dev/snd" 10 | volumes: 11 | - "original_data:/app/original_data" 12 | - "models:/app/models" 13 | - "unlabeled_data:/app/unlabeled_data" 14 | 15 | dashboard: 16 | build: 17 | context: ./dashboard 18 | dockerfile: Dockerfile 19 | devices: 20 | - "/dev/snd:/dev/snd" 21 | volumes: 22 | - "labeled_data:/app/labeled_data" 23 | - "unlabeled_data:/app/assets/unlabeled_data" 24 | environment: 25 | - RQ_DASHBOARD_REDIS_URL=redis://redis:6379 26 | ports: 27 | - '8000:8000' 28 | 29 | retraining_api: 30 | build: 31 | context: ./retraining 32 | dockerfile: Dockerfile 33 | depends_on: 34 | - redis 35 | volumes: 36 | - "labeled_data:/app/labeled_data" 37 | - "unlabeled_data:/app/original_data" 38 | ports: 39 | - "8080:8080" 40 | 41 | retraining_worker: 42 | build: 43 | context: ./retraining 44 | dockerfile: Dockerfile 45 | depends_on: 46 | - redis 47 | volumes: 48 | - "labeled_data:/app/labeled_data" 49 | - "original_data:/app/original_data" 50 | - "models:/app/models" 51 | entrypoint: rq worker --url=redis://redis:6379 52 | 53 | redis: 54 | image: redis 55 | 56 | volumes: 57 | original_data: 58 | driver: local 59 | driver_opts: 60 | type: 'none' 61 | o: 'bind' 62 | device: '/home/pi/doorbell_detector/deploy/data/original_data' 63 | unlabeled_data: 64 | driver: local 65 | driver_opts: 66 | type: 'none' 67 | o: 'bind' 68 | device: '/home/pi/doorbell_detector/deploy/data/unlabeled_data/' 69 | labeled_data: 70 | driver: local 71 | driver_opts: 72 | type: 'none' 73 | o: 'bind' 74 | device: '/home/pi/doorbell_detector/deploy/data/labeled_data/' 75 | models: 76 | driver: local 77 | driver_opts: 78 | type: 'none' 79 | o: 'bind' 80 | device: '/home/pi/doorbell_detector/deploy/models' 81 | -------------------------------------------------------------------------------- /deploy/model_server/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM balenalib/raspberry-pi-python:3.7.4 2 | 3 | # Install needed packages to install llvmlite (needed by librosa) 4 | RUN apt-get update && apt-get install -y llvm-9 llvm-9-dev make g++ 5 | 6 | # Copy requirements first. Docker caches every step so this way we don't have to reinstall every pip package every time our code changes 7 | COPY requirements.txt / 8 | RUN LLVM_CONFIG=/usr/bin/llvm-config-9 pip3 install --extra-index-url=https://www.piwheels.org/simple -r requirements.txt 9 | 10 | RUN apt-get install -y libatlas-base-dev libsndfile1-dev libportaudio2 libasound-dev 11 | 12 | COPY . /app 13 | WORKDIR /app 14 | 15 | COPY asoundrc /root/.asoundrc 16 | 17 | ENTRYPOINT [ "python3", "main.py" ] 18 | # ENTRYPOINT [ "python3", "-m", "sounddevice" ] 19 | # ENTRYPOINT ["tail", "-F", "/dev/null"] -------------------------------------------------------------------------------- /deploy/model_server/asoundrc: -------------------------------------------------------------------------------- 1 | pcm.!default { 2 | type asym 3 | playback.pcm { 4 | type plug 5 | slave.pcm "hw:0,0" 6 | } 7 | capture.pcm { 8 | type plug 9 | slave.pcm "hw:1,0" 10 | } 11 | } -------------------------------------------------------------------------------- /deploy/model_server/config.py: -------------------------------------------------------------------------------- 1 | # MODEL = '/app/models/latest.p' 2 | MODEL = '/home/victor/Projects/doorbell_detector/models/latest.p' 3 | SR = 22050 4 | # SAVE_LOCATION = '/app/unlabeled_data' 5 | SAVE_LOCATION = '/home/victor/Projects/doorbell_detector/deploy/data/unlabeled_data' 6 | -------------------------------------------------------------------------------- /deploy/model_server/main.py: -------------------------------------------------------------------------------- 1 | import hashlib 2 | import logging 3 | import os 4 | import pickle 5 | import sys 6 | from datetime import datetime, timedelta 7 | from queue import Queue 8 | from threading import Thread 9 | from time import sleep, time 10 | 11 | import librosa 12 | import numpy as np 13 | import requests 14 | import sounddevice as sd 15 | import soundfile as sf 16 | from pushover import Client, init 17 | 18 | from config import MODEL, SAVE_LOCATION, SR 19 | from creds import app_token, client_token 20 | 21 | # sd.default.device = 1 22 | logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(thread)d - %(message)s') 23 | 24 | logger = logging.getLogger(__name__) 25 | 26 | class PushWorker(Thread): 27 | 28 | def __init__(self, notif_queue): 29 | Thread.__init__(self) 30 | self.notif_queue = notif_queue 31 | 32 | init(app_token) 33 | self.members = [Client(client_token)] # Mine 34 | 35 | self.last_pinged = time() 36 | 37 | def run(self): 38 | logging.info('Staring Push Thread!') 39 | while True: 40 | datetimestr = self.notif_queue.get() 41 | if time() - self.last_pinged > 5: 42 | for client in self.members: 43 | client.send_message(f'') 44 | self.last_pinged = time() 45 | self.notif_queue.task_done() 46 | 47 | 48 | class SaveWorker(Thread): 49 | 50 | def __init__(self, save_queue, sr, location): 51 | Thread.__init__(self) 52 | self.save_queue = save_queue 53 | self.location = location 54 | self.sr = sr 55 | 56 | def run(self): 57 | logging.info('Staring Save Thread!') 58 | while True: 59 | recording, datetimestr = self.save_queue.get() 60 | sf.write(os.path.join(self.location, f'{datetimestr}.wav'), 61 | recording, self.sr, subtype='PCM_24') 62 | self.save_queue.task_done() 63 | 64 | 65 | class RecordingWorker(Thread): 66 | 67 | def __init__(self, bell_queue, seconds, sr): 68 | Thread.__init__(self) 69 | self.bell_queue = bell_queue 70 | self.seconds = seconds 71 | self.sr = sr 72 | self.stream = sd.InputStream(channels=1, samplerate=self.sr, callback=self.audio_callback) 73 | self.block_queue = Queue() 74 | self.window_data = np.zeros((int(self.seconds * self.sr), 1)) 75 | 76 | 77 | def audio_callback(self, indata, frames, time, status): 78 | """This is called (from a separate thread) for each audio block.""" 79 | if status: 80 | logging.warning(status) 81 | # Fancy indexing with mapping creates a (necessary!) copy: 82 | self.block_queue.put(indata.copy()) 83 | 84 | 85 | def run(self): 86 | logging.info('Staring Recording Thread!') 87 | prev_time = time() 88 | with self.stream: 89 | while True: 90 | # logging.info(f'Elapsed Time Between recordings: {time() - prev_time}') 91 | # Get the work from the bell_queue and expand the tuple 92 | # recording = sd.rec(int(self.seconds * self.sr), samplerate=self.sr, channels=1) 93 | while not self.block_queue.empty(): 94 | #logging.info(f'Block Q length: {self.block_queue.qsize()}') 95 | data = self.block_queue.get() 96 | shift = len(data) 97 | self.window_data = np.roll(self.window_data, -shift, axis=0) 98 | self.window_data[-shift:, :] = data 99 | self.block_queue.task_done() 100 | datetimestr = datetime.now().strftime("%Y-%m-%d_%H:%M:%S") 101 | logging.debug(f'Putting current window in queue! {datetimestr}') 102 | self.bell_queue.put((self.sr, self.window_data.reshape(self.window_data.shape[0]), datetimestr)) 103 | prev_time = time() 104 | sleep(self.seconds / 2) 105 | 106 | 107 | class DetectionWorker(Thread): 108 | 109 | def __init__(self, bell_queue, notif_queue, save_queue): 110 | Thread.__init__(self) 111 | self.bell_queue = bell_queue 112 | self.notif_queue = notif_queue 113 | self.save_queue = save_queue 114 | # Load the Sklearn model. 115 | self.model = pickle.load(open(MODEL, 'rb')) 116 | self.model_hash = hashlib.md5(open(MODEL,'rb').read()).hexdigest() 117 | logging.info(f'Loaded model with hash: {self.model_hash}') 118 | self.last_updated = datetime.now() 119 | self.model_check_interval = 1 # minutes 120 | 121 | 122 | def process_recording(self, signal, sr): 123 | 124 | if sr == 0: 125 | # audio file IO problem 126 | return -1, -1, -1 127 | X = signal.T 128 | 129 | mfccs = np.mean(librosa.feature.mfcc(y=librosa.util.normalize(X), sr=sr, n_mfcc=13).T, axis=0) 130 | ext_features = np.expand_dims(mfccs, axis=0) 131 | 132 | logging.debug(f'{ext_features.shape}, {np.mean(ext_features)}, {np.std(ext_features)}') 133 | 134 | # classification 135 | pred = self.model.predict(ext_features) 136 | 137 | # logging.info(f'Pred: {pred}') 138 | 139 | return pred[0] 140 | 141 | def update_model_if_new(self): 142 | new_model_hash = hashlib.md5(open(MODEL, 'rb').read()).hexdigest() 143 | if new_model_hash != self.model_hash: 144 | logging.info('New model version detected! Updating model now.') 145 | self.model = pickle.load(open(MODEL, 'rb')) 146 | self.model_hash = new_model_hash 147 | logging.info(f'Loaded model with hash: {self.model_hash}') 148 | 149 | 150 | def run(self): 151 | logging.info('Staring Detection Thread!') 152 | loop_count = 0 153 | while True: 154 | logging.debug(f'Detection Q length: {self.bell_queue.qsize()}') 155 | sr, recording, recording_ts = self.bell_queue.get() 156 | start_detection = time() 157 | try: 158 | logging.debug(f'processing recording @ {recording_ts}') 159 | class_out = self.process_recording(recording, sr) 160 | logging.debug(f'{class_out}') 161 | datetimestr = datetime.now().strftime("%Y-%m-%d_%H:%M:%S") 162 | if class_out == 1: 163 | logging.info('Bell detected!! Sending push notification to queue.') 164 | self.notif_queue.put((datetimestr)) 165 | self.save_queue.put((recording, datetimestr)) 166 | except Exception as e: 167 | logging.error(e) 168 | finally: 169 | self.bell_queue.task_done() 170 | if datetime.now() - self.last_updated > timedelta(minutes=self.model_check_interval): 171 | self.update_model_if_new() 172 | # logging.info(f'Elapsed Detection Time: {time() - start_detection}') 173 | # logging.info(f'Queue size: {self.bell_queue.qsize()}') 174 | 175 | 176 | def main(): 177 | # Create a bell_queue to communicate with the worker threads 178 | bell_queue = Queue() 179 | # Create a notif_queue to send push notifications 180 | notif_queue = Queue() 181 | # Create a save_queue to save sounds for later reuse in training 182 | save_queue = Queue() 183 | 184 | # Start processing thread first 185 | det_worker = DetectionWorker(bell_queue, notif_queue, save_queue) 186 | # Setting daemon to True will let the main thread exit even though the workers are blocking 187 | # det_worker.daemon = False 188 | det_worker.start() 189 | 190 | # Start recording 191 | rec_worker = RecordingWorker(bell_queue, 0.5, SR) 192 | # Setting daemon to True will let the main thread exit even though the workers are blocking 193 | # rec_worker.daemon = False 194 | rec_worker.start() 195 | 196 | # Start the push notification listener 197 | notif_worker = PushWorker(notif_queue) 198 | # Setting daemon to True will let the main thread exit even though the workers are blocking 199 | # notif_worker.daemon = False 200 | notif_worker.start() 201 | 202 | # Start the saving worker which will save all bell instances 203 | save_worker = SaveWorker(save_queue, SR, location=SAVE_LOCATION) 204 | # Setting daemon to True will let the main thread exit even though the workers are blocking 205 | # save_worker.daemon = False 206 | save_worker.start() 207 | 208 | # Causes the main thread to wait for the bell_queue to finish processing all the tasks 209 | det_worker.join() 210 | rec_worker.join() 211 | notif_worker.join() 212 | save_worker.join() 213 | 214 | if __name__ == '__main__': 215 | main() 216 | -------------------------------------------------------------------------------- /deploy/model_server/requirements.txt: -------------------------------------------------------------------------------- 1 | pyAudioAnalysis==0.3.5 2 | sounddevice==0.4.0 3 | python-pushover==0.4 4 | SoundFile==0.10.3.post1 5 | # matplotlib==3.1.2 6 | simplejson==3.16.0 7 | # hmmlearn==0.2.2 8 | eyeD3==0.8.12 9 | pydub==0.23.1 10 | # scikit_learn==0.21.3 11 | tqdm==4.44.1 12 | plotly==4.1.1 13 | dash==1.11.0 14 | dash-bootstrap-components==0.10.6 15 | librosa==0.8.0 16 | -------------------------------------------------------------------------------- /deploy/retraining/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM balenalib/raspberry-pi-python:3.7.4 2 | 3 | # Install needed packages to install llvmlite (needed by librosa) 4 | RUN apt-get update && apt-get install -y llvm-9 llvm-9-dev make g++ 5 | 6 | # Copy requirements first. Docker caches every step so this way we don't have to reinstall every pip package every time our code changes 7 | COPY requirements.txt / 8 | RUN LLVM_CONFIG=/usr/bin/llvm-config-9 pip3 install --extra-index-url=https://www.piwheels.org/simple -r requirements.txt 9 | 10 | RUN apt-get install -y libatlas-base-dev libsndfile1-dev 11 | 12 | COPY . /app 13 | WORKDIR /app 14 | 15 | ENV CONTAINER=True 16 | 17 | CMD ["uvicorn", "api:app", "--host", "0.0.0.0", "--port", "8080"] -------------------------------------------------------------------------------- /deploy/retraining/analyze_training.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 2, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "from retraining_pipeline import *\n", 10 | "import soundfile" 11 | ] 12 | }, 13 | { 14 | "cell_type": "code", 15 | "execution_count": 3, 16 | "metadata": {}, 17 | "outputs": [], 18 | "source": [ 19 | "data_loader = NumpyDataLoader()\n", 20 | "feature_calculator = MFCCs()\n", 21 | "pre_processor = TrainTestSplit()\n", 22 | "trainer = SklearnTrainer()\n", 23 | "model_runner = SVCRunner()\n", 24 | "model_loader = ModelLoader()\n", 25 | "model_comparison = BasicComparison()\n", 26 | "model_saver = ModelSaver()" 27 | ] 28 | }, 29 | { 30 | "cell_type": "code", 31 | "execution_count": 4, 32 | "metadata": {}, 33 | "outputs": [ 34 | { 35 | "name": "stdout", 36 | "output_type": "stream", 37 | "text": [ 38 | "/home/victor/Projects/doorbell_detector\n" 39 | ] 40 | } 41 | ], 42 | "source": [ 43 | "%cd /home/victor/Projects/doorbell_detector\n", 44 | "data, labels = data_loader.run()" 45 | ] 46 | }, 47 | { 48 | "cell_type": "code", 49 | "execution_count": 5, 50 | "metadata": {}, 51 | "outputs": [ 52 | { 53 | "name": "stdout", 54 | "output_type": "stream", 55 | "text": [ 56 | "(3329, 11025)\n", 57 | "(3329,)\n", 58 | "[1 1 1 ... 0 0 0]\n" 59 | ] 60 | } 61 | ], 62 | "source": [ 63 | "print(data.shape)\n", 64 | "print(labels.shape)\n", 65 | "print(labels)" 66 | ] 67 | }, 68 | { 69 | "cell_type": "code", 70 | "execution_count": 6, 71 | "metadata": {}, 72 | "outputs": [], 73 | "source": [ 74 | "import soundfile as sf\n", 75 | "from playsound import playsound\n", 76 | "\n", 77 | "for i in range(10):\n", 78 | " sf.write('temp.wav', data[i], 22050, 'PCM_24')\n", 79 | " playsound(\"temp.wav\")" 80 | ] 81 | }, 82 | { 83 | "cell_type": "code", 84 | "execution_count": 7, 85 | "metadata": {}, 86 | "outputs": [ 87 | { 88 | "name": "stderr", 89 | "output_type": "stream", 90 | "text": [ 91 | "100%|██████████| 3329/3329 [00:17<00:00, 185.89it/s]\n" 92 | ] 93 | } 94 | ], 95 | "source": [ 96 | "features = feature_calculator.run(data)" 97 | ] 98 | }, 99 | { 100 | "cell_type": "code", 101 | "execution_count": 35, 102 | "metadata": {}, 103 | "outputs": [], 104 | "source": [ 105 | "from sklearn.model_selection import train_test_split\n", 106 | "\n", 107 | "X_train, X_test, y_train, y_test, data_train, data_test \\\n", 108 | " = train_test_split(features, labels, data, test_size=0.2,\n", 109 | " random_state=42)" 110 | ] 111 | }, 112 | { 113 | "cell_type": "code", 114 | "execution_count": 36, 115 | "metadata": {}, 116 | "outputs": [ 117 | { 118 | "data": { 119 | "text/plain": [ 120 | "(2663, 13)" 121 | ] 122 | }, 123 | "execution_count": 36, 124 | "metadata": {}, 125 | "output_type": "execute_result" 126 | } 127 | ], 128 | "source": [ 129 | "X_train.shape" 130 | ] 131 | }, 132 | { 133 | "cell_type": "code", 134 | "execution_count": 37, 135 | "metadata": {}, 136 | "outputs": [], 137 | "source": [ 138 | "old_model = model_loader.run()" 139 | ] 140 | }, 141 | { 142 | "cell_type": "code", 143 | "execution_count": 38, 144 | "metadata": {}, 145 | "outputs": [], 146 | "source": [ 147 | "old_test_pred = old_model.predict(X_test)\n", 148 | "wrong = np.where(old_test_pred != y_test)" 149 | ] 150 | }, 151 | { 152 | "cell_type": "code", 153 | "execution_count": 44, 154 | "metadata": {}, 155 | "outputs": [], 156 | "source": [ 157 | "sf.write('temp.wav', data_test[wrong[0][0]], 22050, 'PCM_24')\n", 158 | "playsound(\"temp.wav\")" 159 | ] 160 | }, 161 | { 162 | "cell_type": "code", 163 | "execution_count": 46, 164 | "metadata": {}, 165 | "outputs": [], 166 | "source": [ 167 | "old_train_pred = old_model.predict(X_train)\n", 168 | "wrong = np.where(old_train_pred != y_train)" 169 | ] 170 | }, 171 | { 172 | "cell_type": "code", 173 | "execution_count": 47, 174 | "metadata": {}, 175 | "outputs": [ 176 | { 177 | "data": { 178 | "text/plain": [ 179 | "(array([], dtype=int64),)" 180 | ] 181 | }, 182 | "execution_count": 47, 183 | "metadata": {}, 184 | "output_type": "execute_result" 185 | } 186 | ], 187 | "source": [ 188 | "wrong" 189 | ] 190 | }, 191 | { 192 | "cell_type": "code", 193 | "execution_count": 21, 194 | "metadata": {}, 195 | "outputs": [ 196 | { 197 | "name": "stderr", 198 | "output_type": "stream", 199 | "text": [ 200 | "100%|██████████| 1/1 [00:00<00:00, 83.87it/s]\n" 201 | ] 202 | }, 203 | { 204 | "data": { 205 | "text/plain": [ 206 | "(1, 13)" 207 | ] 208 | }, 209 | "execution_count": 21, 210 | "metadata": {}, 211 | "output_type": "execute_result" 212 | } 213 | ], 214 | "source": [ 215 | "data_val = data_loader.load_wav('deploy/data/unlabeled_data')\n", 216 | "X_val = feature_calculator.run(data_val)\n", 217 | "X_val.shape" 218 | ] 219 | } 220 | ], 221 | "metadata": { 222 | "interpreter": { 223 | "hash": "e7370f93d1d0cde622a1f8e1c04877d8463912d04d973331ad4851f04de6915a" 224 | }, 225 | "kernelspec": { 226 | "display_name": "Python 3.9.7 64-bit", 227 | "language": "python", 228 | "name": "python3" 229 | }, 230 | "language_info": { 231 | "codemirror_mode": { 232 | "name": "ipython", 233 | "version": 3 234 | }, 235 | "file_extension": ".py", 236 | "mimetype": "text/x-python", 237 | "name": "python", 238 | "nbconvert_exporter": "python", 239 | "pygments_lexer": "ipython3", 240 | "version": "3.9.7" 241 | }, 242 | "orig_nbformat": 4 243 | }, 244 | "nbformat": 4, 245 | "nbformat_minor": 2 246 | } 247 | -------------------------------------------------------------------------------- /deploy/retraining/api.py: -------------------------------------------------------------------------------- 1 | from fastapi import FastAPI 2 | from redis import Redis 3 | from rq import Queue 4 | import rq 5 | 6 | from retraining_pipeline import run_pipeline 7 | 8 | app = FastAPI() 9 | redis_conn = Redis(host='redis', port=6379) 10 | q = Queue(connection=redis_conn) 11 | 12 | 13 | @app.post("/start_retraining", status_code=201) 14 | def start_retraining(): 15 | # Retraining can take a loooong time 16 | retraining_job = q.enqueue(run_pipeline, job_timeout=3600) 17 | return {'job_id': retraining_job.get_id()} 18 | 19 | 20 | @app.get("/status_retraining/{job_id}", status_code=200) 21 | def status_retraining(job_id): 22 | # Retraining can take a loooong time 23 | retraining_job = rq.job.Job.fetch(job_id, connection=redis_conn) 24 | job_info = { 25 | 'job_id': retraining_job.get_id(), 26 | 'state': retraining_job.get_status(), 27 | 'progress': retraining_job.meta.get('progress'), 28 | 'result': retraining_job.result 29 | } 30 | return job_info -------------------------------------------------------------------------------- /deploy/retraining/config.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | if os.environ.get('CONTAINER'): 4 | ORIGINAL_DATA_PATH='/app/original_data' 5 | LABELED_DATA_PATH='/app/labeled_data' 6 | MODEL_PATH='/app/models' 7 | else: 8 | ORIGINAL_DATA_PATH='deploy/data/original_data' 9 | LABELED_DATA_PATH='deploy/data/labeled_data' 10 | MODEL_PATH='models' -------------------------------------------------------------------------------- /deploy/retraining/requirements.txt: -------------------------------------------------------------------------------- 1 | scikit_learn==0.24.2 2 | tqdm==4.44.1 3 | librosa==0.8.0 4 | fastapi==0.63.0 5 | uvicorn==0.13.4 6 | rq==1.8.0 7 | -------------------------------------------------------------------------------- /deploy/retraining/retraining_pipeline.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | import logging 3 | import pickle 4 | import os 5 | 6 | import librosa 7 | import numpy as np 8 | from tqdm import tqdm 9 | from sklearn.model_selection import train_test_split 10 | from sklearn.svm import SVC 11 | from sklearn.model_selection import GridSearchCV 12 | from sklearn.model_selection import train_test_split 13 | from sklearn.metrics import roc_auc_score, confusion_matrix 14 | from rq import get_current_job 15 | 16 | from config import ORIGINAL_DATA_PATH, LABELED_DATA_PATH, MODEL_PATH 17 | 18 | 19 | class Pipeline: 20 | def __init__(self, rq_job_obj): 21 | self.job = rq_job_obj 22 | self.data_loader = NumpyDataLoader() 23 | self.feature_calculator = MFCCs() 24 | self.pre_processor = TrainTestSplit() 25 | self.trainer = SklearnTrainer() 26 | self.model_runner = SVCRunner() 27 | self.model_loader = ModelLoader() 28 | self.model_comparison = BasicComparison() 29 | self.model_saver = ModelSaver() 30 | 31 | def log_progress(self, message): 32 | logging.info(message) 33 | print(message) 34 | if self.job: 35 | self.job.meta['progress'] = message 36 | self.job.save_meta() 37 | 38 | def run(self): 39 | self.log_progress('Loading data') 40 | 41 | data, labels = self.data_loader.run() 42 | self.log_progress('Calculating features') 43 | 44 | features = self.feature_calculator.run(data) 45 | self.log_progress('Preprocessing') 46 | 47 | X_train, X_test, y_train, y_test = self.pre_processor.run(features, labels) 48 | self.log_progress('Training new model') 49 | 50 | new_model = self.trainer.run(X_train, X_test, y_train, y_test) 51 | self.log_progress('Loading old model') 52 | 53 | old_model = self.model_loader.run() 54 | 55 | if not old_model: 56 | is_better = True 57 | else: 58 | self.log_progress('Benchmarking both models') 59 | new_predictions = self.model_runner.run(X_test, new_model) 60 | old_predictions = self.model_runner.run(X_test, old_model) 61 | 62 | # is_better = self.model_comparison.run(y_test, old_predictions, new_predictions) 63 | is_better = True 64 | 65 | self.log_progress('Saving new model') 66 | 67 | model_name = self.model_saver.run(new_model) 68 | 69 | self.log_progress(f'New model is better: {is_better}') 70 | 71 | if is_better: 72 | self.log_progress('Symlinking new model') 73 | 74 | old_cwd = os.getcwd() 75 | os.chdir(MODEL_PATH) 76 | latest_path = 'latest.p' 77 | if os.path.lexists(latest_path): 78 | os.unlink(latest_path) 79 | os.symlink(model_name, latest_path) 80 | os.chdir(old_cwd) 81 | 82 | self.log_progress('Done!') 83 | 84 | 85 | class NumpyDataLoader: 86 | def __init__(self, original_data_path=ORIGINAL_DATA_PATH, labeled_data_path=LABELED_DATA_PATH): 87 | self.original_data_path = original_data_path 88 | self.labeled_data_path = labeled_data_path 89 | 90 | def run(self): 91 | ambient_path = os.path.join(self.original_data_path, 'ambient_data.npy') 92 | bell_path = os.path.join(self.original_data_path, 'bell_data.npy') 93 | ambient_data = np.load(open(ambient_path, 'rb')) 94 | bell_data = np.load(open(bell_path, 'rb')) 95 | 96 | ambient_labeled = self.load_wav(os.path.join(self.labeled_data_path, 'ambient')) 97 | bell_labeled = self.load_wav(os.path.join(self.labeled_data_path, 'bell')) 98 | 99 | features = np.vstack([bell_data, bell_labeled, ambient_data, ambient_labeled]) 100 | labels = np.ravel(np.vstack([1]*(len(bell_data) + len(bell_labeled)) + [0]*(len(ambient_data) + len(ambient_labeled)))) 101 | 102 | return features, labels 103 | 104 | def load_wav(self, folder): 105 | clips = [] 106 | for clip_name in os.listdir(folder): 107 | # TODO: implement check on SR 108 | x , _ = librosa.load(os.path.join(folder, clip_name)) 109 | clips.append(x) 110 | return np.array(clips) 111 | 112 | 113 | class MFCCs: 114 | def __init__(self, sr=22050): 115 | self.sr = sr 116 | 117 | def run(self, data): 118 | mfccs = np.array([np.mean(librosa.feature.mfcc(y=librosa.util.normalize(entry), sr=self.sr, n_mfcc=13).T, axis=0) 119 | for entry in tqdm(data)]) 120 | return mfccs 121 | 122 | 123 | class TrainTestSplit: 124 | def __init__(self, test_split=0.2, random_state=42): 125 | self.test_split = test_split 126 | self.random_state = random_state 127 | 128 | def run(self, features, labels): 129 | X_train, X_test, y_train, y_test = train_test_split(features, labels, 130 | test_size=self.test_split, 131 | random_state=self.random_state) 132 | return X_train, X_test, y_train, y_test 133 | 134 | 135 | class SklearnTrainer: 136 | def __init__(self): 137 | self.param_grid = [ 138 | {'kernel': ['rbf'], 139 | 'gamma': [1e-3, 1e-4], 140 | 'C': [1, 10, 100, 1000]}, 141 | # {'kernel': ['linear'], 142 | # 'C': [1, 10, 100, 1000]} 143 | ] 144 | 145 | def _algorithm_pipeline(self, X_train_data, X_test_data, y_train_data, y_test_data, 146 | model, param_grid, cv=5, scoring_fit='accuracy'): 147 | gs = GridSearchCV( 148 | estimator=model, 149 | param_grid=param_grid, 150 | cv=cv, 151 | n_jobs=1, 152 | scoring=scoring_fit, 153 | verbose=3 154 | ) 155 | fitted_model = gs.fit(X_train_data, y_train_data) 156 | 157 | return fitted_model 158 | 159 | def run(self, X_train, X_test, y_train, y_test): 160 | model = SVC() 161 | model = self._algorithm_pipeline(X_train, X_test, y_train, y_test, model, 162 | self.param_grid, cv=5, scoring_fit='accuracy') 163 | 164 | print(model.best_score_) 165 | print(model.best_params_) 166 | return model 167 | 168 | class ModelLoader: 169 | def __init__(self, path=MODEL_PATH): 170 | self.path = path 171 | 172 | def run(self): 173 | try: 174 | model = pickle.load(open(os.path.join(self.path, 'latest.p'), 'rb')) 175 | except FileNotFoundError: 176 | model = False 177 | 178 | return model 179 | 180 | class SVCRunner: 181 | def __init__(self): 182 | pass 183 | 184 | def run(self, test_data, model, do_probabilities=False): 185 | if do_probabilities: 186 | pred = model.predict_proba(test_data) 187 | else: 188 | pred = model.predict(test_data) 189 | 190 | return pred 191 | 192 | 193 | class BasicComparison: 194 | def __init__(self): 195 | pass 196 | 197 | def run(self, y_true, old_predictions, new_predictions): 198 | old_auc = roc_auc_score(y_true, old_predictions) 199 | new_auc = roc_auc_score(y_true, new_predictions) 200 | 201 | old_cm = confusion_matrix(y_true, old_predictions) 202 | new_cm = confusion_matrix(y_true, new_predictions) 203 | 204 | print(f'Old AUC: {old_auc}') 205 | print('Old CM') 206 | print(old_cm) 207 | print(f'New AUC: {new_auc}') 208 | print('New CM') 209 | print(new_cm) 210 | 211 | return new_auc > old_auc 212 | 213 | 214 | class ModelSaver: 215 | def __init__(self, dt_format="%Y-%m-%d_%H:%M:%S", model_path=MODEL_PATH): 216 | self.dt_format = dt_format 217 | self.model_path = model_path 218 | 219 | def run(self, fitted_model): 220 | datetimestr = datetime.now().strftime(self.dt_format) 221 | model_name = f'{datetimestr}.p' 222 | pickle.dump(fitted_model, open(os.path.join(self.model_path, model_name), 'wb')) 223 | 224 | return model_name 225 | 226 | 227 | def run_pipeline(): 228 | self_job = get_current_job() 229 | pipeline = Pipeline(self_job) 230 | pipeline.run() 231 | 232 | 233 | if __name__ == '__main__': 234 | run_pipeline() 235 | -------------------------------------------------------------------------------- /experiments/asyncio_generators.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """Creating an asyncio generator for blocks of audio data. 3 | 4 | This example shows how a generator can be used to analyze audio input blocks. 5 | In addition, it shows how a generator can be created that yields not only input 6 | blocks but also output blocks where audio data can be written to. 7 | 8 | You need Python 3.7 or newer to run this. 9 | 10 | """ 11 | import asyncio 12 | import queue 13 | import sys 14 | 15 | import numpy as np 16 | import sounddevice as sd 17 | 18 | 19 | async def inputstream_generator(channels=1, **kwargs): 20 | """Generator that yields blocks of input data as NumPy arrays.""" 21 | q_in = asyncio.Queue() 22 | loop = asyncio.get_event_loop() 23 | 24 | def callback(indata, frame_count, time_info, status): 25 | loop.call_soon_threadsafe(q_in.put_nowait, (indata.copy(), status)) 26 | 27 | stream = sd.InputStream(callback=callback, channels=channels, **kwargs) 28 | with stream: 29 | while True: 30 | indata, status = await q_in.get() 31 | yield indata, status 32 | 33 | 34 | async def stream_generator(blocksize, *, channels=1, dtype='float32', 35 | pre_fill_blocks=10, **kwargs): 36 | """Generator that yields blocks of input/output data as NumPy arrays. 37 | 38 | The output blocks are uninitialized and have to be filled with 39 | appropriate audio signals. 40 | 41 | """ 42 | assert blocksize != 0 43 | q_in = asyncio.Queue() 44 | q_out = queue.Queue() 45 | loop = asyncio.get_event_loop() 46 | 47 | def callback(indata, outdata, frame_count, time_info, status): 48 | loop.call_soon_threadsafe(q_in.put_nowait, (indata.copy(), status)) 49 | outdata[:] = q_out.get_nowait() 50 | 51 | # pre-fill output queue 52 | for _ in range(pre_fill_blocks): 53 | q_out.put(np.zeros((blocksize, channels), dtype=dtype)) 54 | 55 | stream = sd.Stream(blocksize=blocksize, callback=callback, dtype=dtype, 56 | channels=channels, **kwargs) 57 | with stream: 58 | while True: 59 | indata, status = await q_in.get() 60 | outdata = np.empty((blocksize, channels), dtype=dtype) 61 | yield indata, outdata, status 62 | q_out.put_nowait(outdata) 63 | 64 | 65 | async def print_input_infos(**kwargs): 66 | """Show minimum and maximum value of each incoming audio block.""" 67 | async for indata, status in inputstream_generator(**kwargs): 68 | if status: 69 | print(status) 70 | print('min:', indata.min(), '\t', 'max:', indata.max()) 71 | 72 | 73 | async def wire_coro(**kwargs): 74 | """Create a connection between audio inputs and outputs. 75 | 76 | Asynchronously iterates over a stream generator and for each block 77 | simply copies the input data into the output block. 78 | 79 | """ 80 | async for indata, outdata, status in stream_generator(**kwargs): 81 | if status: 82 | print(status) 83 | outdata[:] = indata 84 | 85 | 86 | async def main(**kwargs): 87 | print('Some informations about the input signal:') 88 | try: 89 | await asyncio.wait_for(print_input_infos(), timeout=2) 90 | except asyncio.TimeoutError: 91 | pass 92 | print('\nEnough of that, activating wire ...\n') 93 | audio_task = asyncio.create_task(wire_coro(**kwargs)) 94 | for i in range(10, 0, -1): 95 | print(i) 96 | await asyncio.sleep(1) 97 | audio_task.cancel() 98 | try: 99 | await audio_task 100 | except asyncio.CancelledError: 101 | print('\nwire was cancelled') 102 | 103 | 104 | if __name__ == "__main__": 105 | try: 106 | asyncio.run(main(blocksize=1024)) 107 | except KeyboardInterrupt: 108 | sys.exit('\nInterrupted by user') -------------------------------------------------------------------------------- /experiments/plot_input.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """Plot the live microphone signal(s) with matplotlib. 3 | 4 | Matplotlib and NumPy have to be installed. 5 | 6 | """ 7 | from datetime import datetime 8 | import argparse 9 | import queue 10 | import sys 11 | 12 | from matplotlib.animation import FuncAnimation 13 | import matplotlib.pyplot as plt 14 | import numpy as np 15 | import sounddevice as sd 16 | import soundfile as sf 17 | import os 18 | 19 | 20 | def int_or_str(text): 21 | """Helper function for argument parsing.""" 22 | try: 23 | return int(text) 24 | except ValueError: 25 | return text 26 | 27 | 28 | parser = argparse.ArgumentParser(add_help=False) 29 | parser.add_argument( 30 | '-l', '--list-devices', action='store_true', 31 | help='show list of audio devices and exit') 32 | args, remaining = parser.parse_known_args() 33 | if args.list_devices: 34 | print(sd.query_devices()) 35 | parser.exit(0) 36 | parser = argparse.ArgumentParser( 37 | description=__doc__, 38 | formatter_class=argparse.RawDescriptionHelpFormatter, 39 | parents=[parser]) 40 | parser.add_argument( 41 | 'channels', type=int, default=[1], nargs='*', metavar='CHANNEL', 42 | help='input channels to plot (default: the first)') 43 | parser.add_argument( 44 | '-d', '--device', type=int_or_str, 45 | help='input device (numeric ID or substring)') 46 | parser.add_argument( 47 | '-w', '--window', type=float, default=500, metavar='DURATION', 48 | help='visible time slot (default: %(default)s ms)') 49 | parser.add_argument( 50 | '-i', '--interval', type=float, default=250, 51 | help='minimum time between plot updates (default: %(default)s ms)') 52 | parser.add_argument( 53 | '-b', '--blocksize', type=int, help='block size (in samples)') 54 | parser.add_argument( 55 | '-r', '--samplerate', type=float, default=44100, help='sampling rate of audio device') 56 | parser.add_argument( 57 | '-n', '--downsample', type=int, default=1, metavar='N', 58 | help='display every Nth sample (default: %(default)s)') 59 | args = parser.parse_args(remaining) 60 | if any(c < 1 for c in args.channels): 61 | parser.error('argument CHANNEL: must be >= 1') 62 | mapping = [c - 1 for c in args.channels] # Channel numbers start with 1 63 | q = queue.Queue() 64 | last_time = datetime.now().time() 65 | 66 | def audio_callback(indata, frames, time, status): 67 | """This is called (from a separate thread) for each audio block.""" 68 | if status: 69 | print(status, file=sys.stderr) 70 | # Fancy indexing with mapping creates a (necessary!) copy: 71 | q.put(indata[::args.downsample, mapping]) 72 | 73 | 74 | def update_plot(frame): 75 | """This is called by matplotlib for each plot update. 76 | 77 | Typically, audio callbacks happen more frequently than plot updates, 78 | therefore the queue tends to contain multiple blocks of audio data. 79 | 80 | """ 81 | global plotdata 82 | while True: 83 | try: 84 | data = q.get_nowait() 85 | except queue.Empty: 86 | break 87 | shift = len(data) 88 | plotdata = np.roll(plotdata, -shift, axis=0) 89 | plotdata[-shift:, :] = data 90 | for column, line in enumerate(lines): 91 | line.set_ydata(plotdata[:, column]) 92 | print(datetime.now().time()) 93 | print(len(plotdata)) 94 | datetimestr = datetime.now().strftime("%Y-%m-%d_%H:%M:%S") 95 | sf.write(os.path.join('/home/victor/Projects/doorbell_detector/experiments', 96 | f'{datetimestr}.wav'), plotdata, 44100, subtype='PCM_24') 97 | return lines 98 | 99 | 100 | try: 101 | if args.samplerate is None: 102 | device_info = sd.query_devices(args.device, 'input') 103 | print(device_infoq) 104 | args.samplerate = device_info['default_samplerate'] 105 | 106 | length = int(args.window * args.samplerate / (1000 * args.downsample)) 107 | plotdata = np.zeros((length, len(args.channels))) 108 | 109 | fig, ax = plt.subplots() 110 | lines = ax.plot(plotdata) 111 | if len(args.channels) > 1: 112 | ax.legend(['channel {}'.format(c) for c in args.channels], 113 | loc='lower left', ncol=len(args.channels)) 114 | ax.axis((0, len(plotdata), -1, 1)) 115 | ax.set_yticks([0]) 116 | ax.yaxis.grid(True) 117 | ax.tick_params(bottom=False, top=False, labelbottom=False, 118 | right=False, left=False, labelleft=False) 119 | fig.tight_layout(pad=0) 120 | 121 | print(f'device: {args.device}') 122 | 123 | stream = sd.InputStream( 124 | device=args.device, channels=max(args.channels), 125 | samplerate=args.samplerate, callback=audio_callback) 126 | ani = FuncAnimation(fig, update_plot, interval=args.interval, blit=True) 127 | with stream: 128 | plt.show() 129 | except Exception as e: 130 | parser.exit(type(e).__name__ + ': ' + str(e)) 131 | -------------------------------------------------------------------------------- /notebook/confusion_matrix_pretty_print.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | plot a pretty confusion matrix with seaborn 4 | Created on Mon Jun 25 14:17:37 2018 5 | @author: Wagner Cipriano - wagnerbhbr - gmail - CEFETMG / MMC 6 | REFerences: 7 | https://www.mathworks.com/help/nnet/ref/plotconfusion.html 8 | https://stackoverflow.com/questions/28200786/how-to-plot-scikit-learn-classification-report 9 | https://stackoverflow.com/questions/5821125/how-to-plot-confusion-matrix-with-string-axis-rather-than-integer-in-python 10 | https://www.programcreek.com/python/example/96197/seaborn.heatmap 11 | https://stackoverflow.com/questions/19233771/sklearn-plot-confusion-matrix-with-labels/31720054 12 | http://scikit-learn.org/stable/auto_examples/model_selection/plot_confusion_matrix.html#sphx-glr-auto-examples-model-selection-plot-confusion-matrix-py 13 | """ 14 | 15 | #imports 16 | from pandas import DataFrame 17 | import numpy as np 18 | import matplotlib.pyplot as plt 19 | import matplotlib.font_manager as fm 20 | from matplotlib.collections import QuadMesh 21 | import seaborn as sn 22 | 23 | 24 | def get_new_fig(fn, figsize=[9,9]): 25 | """ Init graphics """ 26 | fig1 = plt.figure(fn, figsize) 27 | ax1 = fig1.gca() #Get Current Axis 28 | ax1.cla() # clear existing plot 29 | return fig1, ax1 30 | # 31 | 32 | def configcell_text_and_colors(array_df, lin, col, oText, facecolors, posi, fz, fmt, show_null_values=0): 33 | """ 34 | config cell text and colors 35 | and return text elements to add and to dell 36 | @TODO: use fmt 37 | """ 38 | text_add = []; text_del = []; 39 | cell_val = array_df[lin][col] 40 | tot_all = array_df[-1][-1] 41 | per = (float(cell_val) / tot_all) * 100 42 | curr_column = array_df[:,col] 43 | ccl = len(curr_column) 44 | 45 | #last line and/or last column 46 | if(col == (ccl - 1)) or (lin == (ccl - 1)): 47 | #tots and percents 48 | if(cell_val != 0): 49 | if(col == ccl - 1) and (lin == ccl - 1): 50 | tot_rig = 0 51 | for i in range(array_df.shape[0] - 1): 52 | tot_rig += array_df[i][i] 53 | per_ok = (float(tot_rig) / cell_val) * 100 54 | elif(col == ccl - 1): 55 | tot_rig = array_df[lin][lin] 56 | per_ok = (float(tot_rig) / cell_val) * 100 57 | elif(lin == ccl - 1): 58 | tot_rig = array_df[col][col] 59 | per_ok = (float(tot_rig) / cell_val) * 100 60 | per_err = 100 - per_ok 61 | else: 62 | per_ok = per_err = 0 63 | 64 | per_ok_s = ['%.2f%%'%(per_ok), '100%'] [per_ok == 100] 65 | 66 | #text to DEL 67 | text_del.append(oText) 68 | 69 | #text to ADD 70 | font_prop = fm.FontProperties(weight='bold', size=fz) 71 | text_kwargs = dict(color='w', ha="center", va="center", gid='sum', fontproperties=font_prop) 72 | lis_txt = ['%d'%(cell_val), per_ok_s, '%.2f%%'%(per_err)] 73 | lis_kwa = [text_kwargs] 74 | dic = text_kwargs.copy(); dic['color'] = 'g'; lis_kwa.append(dic); 75 | dic = text_kwargs.copy(); dic['color'] = 'r'; lis_kwa.append(dic); 76 | lis_pos = [(oText._x, oText._y-0.3), (oText._x, oText._y), (oText._x, oText._y+0.3)] 77 | for i in range(len(lis_txt)): 78 | newText = dict(x=lis_pos[i][0], y=lis_pos[i][1], text=lis_txt[i], kw=lis_kwa[i]) 79 | #print 'lin: %s, col: %s, newText: %s' %(lin, col, newText) 80 | text_add.append(newText) 81 | #print '\n' 82 | 83 | #set background color for sum cells (last line and last column) 84 | carr = [0.27, 0.30, 0.27, 1.0] 85 | if(col == ccl - 1) and (lin == ccl - 1): 86 | carr = [0.17, 0.20, 0.17, 1.0] 87 | facecolors[posi] = carr 88 | 89 | else: 90 | if(per > 0): 91 | txt = '%s\n%.2f%%' %(cell_val, per) 92 | else: 93 | if(show_null_values == 0): 94 | txt = '' 95 | elif(show_null_values == 1): 96 | txt = '0' 97 | else: 98 | txt = '0\n0.0%' 99 | oText.set_text(txt) 100 | 101 | #main diagonal 102 | if(col == lin): 103 | #set color of the textin the diagonal to white 104 | oText.set_color('w') 105 | # set background color in the diagonal to blue 106 | facecolors[posi] = [0.35, 0.8, 0.55, 1.0] 107 | else: 108 | oText.set_color('r') 109 | 110 | return text_add, text_del 111 | # 112 | 113 | def insert_totals(df_cm): 114 | """ insert total column and line (the last ones) """ 115 | sum_col = [] 116 | for c in df_cm.columns: 117 | sum_col.append( df_cm[c].sum() ) 118 | sum_lin = [] 119 | for item_line in df_cm.iterrows(): 120 | sum_lin.append( item_line[1].sum() ) 121 | df_cm['sum_lin'] = sum_lin 122 | sum_col.append(np.sum(sum_lin)) 123 | df_cm.loc['sum_col'] = sum_col 124 | #print ('\ndf_cm:\n', df_cm, '\n\b\n') 125 | # 126 | 127 | def pretty_plot_confusion_matrix(df_cm, annot=True, cmap="Oranges", fmt='.2f', fz=11, 128 | lw=0.5, cbar=False, figsize=[8,8], show_null_values=0, pred_val_axis='y'): 129 | """ 130 | print conf matrix with default layout (like matlab) 131 | params: 132 | df_cm dataframe (pandas) without totals 133 | annot print text in each cell 134 | cmap Oranges,Oranges_r,YlGnBu,Blues,RdBu, ... see: 135 | fz fontsize 136 | lw linewidth 137 | pred_val_axis where to show the prediction values (x or y axis) 138 | 'col' or 'x': show predicted values in columns (x axis) instead lines 139 | 'lin' or 'y': show predicted values in lines (y axis) 140 | """ 141 | if(pred_val_axis in ('col', 'x')): 142 | xlbl = 'Predicted' 143 | ylbl = 'Actual' 144 | else: 145 | xlbl = 'Actual' 146 | ylbl = 'Predicted' 147 | df_cm = df_cm.T 148 | 149 | # create "Total" column 150 | insert_totals(df_cm) 151 | 152 | #this is for print allways in the same window 153 | fig, ax1 = get_new_fig('Conf matrix default', figsize) 154 | 155 | #thanks for seaborn 156 | ax = sn.heatmap(df_cm, annot=annot, annot_kws={"size": fz}, linewidths=lw, ax=ax1, 157 | cbar=cbar, cmap=cmap, linecolor='w', fmt=fmt) 158 | 159 | #set ticklabels rotation 160 | ax.set_xticklabels(ax.get_xticklabels(), rotation = 45, fontsize = 10) 161 | ax.set_yticklabels(ax.get_yticklabels(), rotation = 25, fontsize = 10) 162 | 163 | # Turn off all the ticks 164 | for t in ax.xaxis.get_major_ticks(): 165 | t.tick1On = False 166 | t.tick2On = False 167 | for t in ax.yaxis.get_major_ticks(): 168 | t.tick1On = False 169 | t.tick2On = False 170 | 171 | #face colors list 172 | quadmesh = ax.findobj(QuadMesh)[0] 173 | facecolors = quadmesh.get_facecolors() 174 | 175 | #iter in text elements 176 | array_df = np.array( df_cm.to_records(index=False).tolist() ) 177 | text_add = []; text_del = []; 178 | posi = -1 #from left to right, bottom to top. 179 | for t in ax.collections[0].axes.texts: #ax.texts: 180 | pos = np.array( t.get_position()) - [0.5,0.5] 181 | lin = int(pos[1]); col = int(pos[0]); 182 | posi += 1 183 | #print ('>>> pos: %s, posi: %s, val: %s, txt: %s' %(pos, posi, array_df[lin][col], t.get_text())) 184 | 185 | #set text 186 | txt_res = configcell_text_and_colors(array_df, lin, col, t, facecolors, posi, fz, fmt, show_null_values) 187 | 188 | text_add.extend(txt_res[0]) 189 | text_del.extend(txt_res[1]) 190 | 191 | #remove the old ones 192 | for item in text_del: 193 | item.remove() 194 | #append the new ones 195 | for item in text_add: 196 | ax.text(item['x'], item['y'], item['text'], **item['kw']) 197 | 198 | #titles and legends 199 | ax.set_title('Confusion matrix') 200 | ax.set_xlabel(xlbl) 201 | ax.set_ylabel(ylbl) 202 | plt.tight_layout() #set layout slim 203 | plt.show() 204 | # 205 | 206 | def plot_confusion_matrix_from_data(y_test, predictions, columns=None, annot=True, cmap="Oranges", 207 | fmt='.2f', fz=11, lw=0.5, cbar=False, figsize=[8,8], show_null_values=0, pred_val_axis='lin'): 208 | """ 209 | plot confusion matrix function with y_test (actual values) and predictions (predic), 210 | whitout a confusion matrix yet 211 | """ 212 | from sklearn.metrics import confusion_matrix 213 | from pandas import DataFrame 214 | 215 | #data 216 | if(not columns): 217 | #labels axis integer: 218 | ##columns = range(1, len(np.unique(y_test))+1) 219 | #labels axis string: 220 | from string import ascii_uppercase 221 | columns = ['class %s' %(i) for i in list(ascii_uppercase)[0:len(np.unique(y_test))]] 222 | 223 | confm = confusion_matrix(y_test, predictions) 224 | cmap = 'Oranges'; 225 | fz = 11; 226 | figsize=[9,9]; 227 | show_null_values = 2 228 | df_cm = DataFrame(confm, index=columns, columns=columns) 229 | pretty_plot_confusion_matrix(df_cm, fz=fz, cmap=cmap, figsize=figsize, show_null_values=show_null_values, pred_val_axis=pred_val_axis) 230 | # 231 | 232 | 233 | 234 | # 235 | #TEST functions 236 | # 237 | def _test_cm(): 238 | #test function with confusion matrix done 239 | array = np.array( [[13, 0, 1, 0, 2, 0], 240 | [ 0, 50, 2, 0, 10, 0], 241 | [ 0, 13, 16, 0, 0, 3], 242 | [ 0, 0, 0, 13, 1, 0], 243 | [ 0, 40, 0, 1, 15, 0], 244 | [ 0, 0, 0, 0, 0, 20]]) 245 | #get pandas dataframe 246 | df_cm = DataFrame(array, index=range(1,7), columns=range(1,7)) 247 | #colormap: see this and choose your more dear 248 | cmap = 'PuRd' 249 | pretty_plot_confusion_matrix(df_cm, cmap=cmap) 250 | # 251 | 252 | def _test_data_class(): 253 | """ test function with y_test (actual values) and predictions (predic) """ 254 | #data 255 | y_test = np.array([1,2,3,4,5, 1,2,3,4,5, 1,2,3,4,5, 1,2,3,4,5, 1,2,3,4,5, 1,2,3,4,5, 1,2,3,4,5, 1,2,3,4,5, 1,2,3,4,5, 1,2,3,4,5, 1,2,3,4,5, 1,2,3,4,5, 1,2,3,4,5, 1,2,3,4,5, 1,2,3,4,5, 1,2,3,4,5, 1,2,3,4,5, 1,2,3,4,5, 1,2,3,4,5, 1,2,3,4,5, 1,2,3,4,5, 1,2,3,4,5]) 256 | predic = np.array([1,2,4,3,5, 1,2,4,3,5, 1,2,3,4,4, 1,4,3,4,5, 1,2,4,4,5, 1,2,4,4,5, 1,2,4,4,5, 1,2,4,4,5, 1,2,3,3,5, 1,2,3,3,5, 1,2,3,4,4, 1,2,3,4,1, 1,2,3,4,1, 1,2,3,4,1, 1,2,4,4,5, 1,2,4,4,5, 1,2,4,4,5, 1,2,4,4,5, 1,2,3,4,5, 1,2,3,4,5, 1,2,3,4,5, 1,2,3,4,5]) 257 | """ 258 | Examples to validate output (confusion matrix plot) 259 | actual: 5 and prediction 1 >> 3 260 | actual: 2 and prediction 4 >> 1 261 | actual: 3 and prediction 4 >> 10 262 | """ 263 | columns = [] 264 | annot = True; 265 | cmap = 'Oranges'; 266 | fmt = '.2f' 267 | lw = 0.5 268 | cbar = False 269 | show_null_values = 2 270 | pred_val_axis = 'y' 271 | #size:: 272 | fz = 12; 273 | figsize = [9,9]; 274 | if(len(y_test) > 10): 275 | fz=9; figsize=[14,14]; 276 | plot_confusion_matrix_from_data(y_test, predic, columns, 277 | annot, cmap, fmt, fz, lw, cbar, figsize, show_null_values, pred_val_axis) 278 | # 279 | 280 | 281 | # 282 | #MAIN function 283 | # 284 | if(__name__ == '__main__'): 285 | print('__main__') 286 | print('_test_cm: test function with confusion matrix done\nand pause') 287 | _test_cm() 288 | plt.pause(5) 289 | print('_test_data_class: test function with y_test (actual values) and predictions (predic)') 290 | _test_data_class() 291 | 292 | -------------------------------------------------------------------------------- /raw_data/ambient.wav: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:67f7c514f441978add6f9a49d4b46cacececcfd72d9c51c31232a3759b726daa 3 | size 635040044 4 | -------------------------------------------------------------------------------- /raw_data/bel.wav: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:46c6866bc55c3802a3e3026cf2552fcd5b70f8c47e63ec8249b96747f5f7fdd1 3 | size 4489484 4 | -------------------------------------------------------------------------------- /raw_data/bel2.wav: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:d28b0b581cc95ea69bd3b4cb4c3b515d9e18afccf46f0b23fb580358abadaf03 3 | size 5305404 4 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | appdirs==1.4.4 2 | astroid==2.5.6 3 | asyncio==3.4.3 4 | audioread==2.1.9 5 | backcall==0.2.0 6 | Brotli==1.0.9 7 | certifi==2020.12.5 8 | cffi==1.14.5 9 | chardet==4.0.0 10 | click==7.1.2 11 | coverage==5.5 12 | cycler==0.10.0 13 | dash==1.11.0 14 | dash-bootstrap-components==0.10.6 15 | dash-core-components==1.9.1 16 | dash-html-components==1.0.3 17 | dash-renderer==1.4.0 18 | dash-table==4.6.2 19 | decorator==4.4.2 20 | deprecation==2.1.0 21 | eyeD3==0.8.12 22 | filetype==1.0.7 23 | Flask==1.1.2 24 | Flask-Compress==1.9.0 25 | future==0.18.2 26 | hmmlearn==0.2.5 27 | idna==2.10 28 | ipykernel==5.5.0 29 | ipython==7.16.1 30 | ipython-genutils==0.2.0 31 | isort==5.8.0 32 | itsdangerous==1.1.0 33 | jedi==0.18.0 34 | Jinja2==2.11.3 35 | joblib==1.0.1 36 | jupyter-client==6.1.12 37 | jupyter-core==4.7.1 38 | kiwisolver==1.3.1 39 | lazy-object-proxy==1.6.0 40 | librosa==0.8.0 41 | llvmlite==0.36.0 42 | MarkupSafe==1.1.1 43 | matplotlib==3.3.4 44 | mccabe==0.6.1 45 | numba==0.53.0 46 | numpy==1.19.5 47 | packaging==20.9 48 | pandas==1.1.5 49 | parso==0.8.1 50 | pexpect==4.8.0 51 | pickleshare==0.7.5 52 | Pillow==8.1.2 53 | plotly==4.1.1 54 | pooch==1.3.0 55 | prompt-toolkit==3.0.17 56 | ptyprocess==0.7.0 57 | pyAudioAnalysis==0.3.5 58 | pycodestyle==2.7.0 59 | pycparser==2.20 60 | pydub==0.23.1 61 | Pygments==2.8.1 62 | pylint==2.8.1 63 | pyparsing==2.4.7 64 | python-dateutil==2.8.1 65 | python-magic==0.4.22 66 | python-pushover==0.4 67 | pytz==2021.1 68 | pyzmq==22.0.3 69 | requests==2.25.1 70 | resampy==0.2.2 71 | retrying==1.3.3 72 | scikit-learn==0.24.1 73 | scipy==1.5.4 74 | seaborn==0.11.1 75 | simplejson==3.16.0 76 | six==1.15.0 77 | sounddevice==0.4.0 78 | SoundFile==0.10.3.post1 79 | threadpoolctl==2.1.0 80 | toml==0.10.2 81 | tornado==6.1 82 | tqdm==4.44.1 83 | traitlets==4.3.3 84 | typed-ast==1.4.3 85 | urllib3==1.26.4 86 | wcwidth==0.2.5 87 | Werkzeug==1.0.1 88 | wrapt==1.12.1 89 | --------------------------------------------------------------------------------