├── README.md
├── RNNoise-License
├── Scripts
    ├── DataPreparation.sh
    └── WaveConcat.sh
├── TRAINING-README
├── c-audio-experiments
    └── main.c
├── evaluation
    ├── CDenoiseClassicMethods.m
    ├── CDownsample_WAV_Classic.m
    ├── CDownsample_WAV_Noisy.m
    ├── CEVALUATION_script_Classic.m
    ├── Downsample_WAV.m
    ├── EVALUATION_script.m
    ├── EvalPlotFun.m
    ├── EvalStatFun.m
    ├── README.txt
    ├── spectrograms.m
    └── stoi.m
├── evaluation_batch.sh
├── featureExtractor
    ├── .deps
    │   ├── .dirstamp
    │   └── rnnoise_demo.Po
    ├── .dirstamp
    ├── .libs
    │   ├── lt-rnnoise_demo
    │   └── rnnoise_demo
    ├── denoise.c
    ├── denoise.h
    ├── denoise.o
    ├── feature_extractor
    └── feature_extractor.c
├── log_testset.csv
├── src
    ├── denoise.c
    ├── denoise.h
    └── featureExtraction.py
└── training
    ├── bin2hdf5.py
    ├── evaluation.py
    ├── evaluation_batch.py
    └── rnn_train.py


/README.md:
--------------------------------------------------------------------------------
 1 | # RNNoise-Ex
 2 | RNNoise-Ex is an attempt to extend the RNNoise hybrid speech enhancement system based on RNN and spectral features.
 3 | 
 4 | ## Network Architecture
 5 | <img src="https://user-images.githubusercontent.com/5578714/202531544-d739ad56-75e3-4ac0-a480-95e64e9b0905.png" width="600" />
 6 | 
 7 | ## Usage
 8 | See [TRAINING-README](TRAINING-README) for instructions on how to train and run the model.
 9 | 
10 | ## Paper
11 | You can find the [RNNoise-Ex paper on arXiv](https://arxiv.org/abs/2105.11813).
12 | Citation:
13 | ```
14 | @misc{rnnoise-ex,
15 |   doi = {10.48550/ARXIV.2105.11813},
16 |   url = {https://arxiv.org/abs/2105.11813},
17 |   author = {Doumanidis, Constantine C. and Anagnostou, Christina and Arvaniti, Evangelia-Sofia and Papadopoulou, Anthi},
18 |   title = {RNNoise-Ex: Hybrid Speech Enhancement System based on RNN and Spectral Features},
19 |   publisher = {arXiv},
20 |   year = {2021},
21 |   copyright = {arXiv.org perpetual, non-exclusive license}
22 | }
23 | ```
24 | 


--------------------------------------------------------------------------------
/RNNoise-License:
--------------------------------------------------------------------------------
 1 | Copyright (c) 2017, Mozilla
 2 | Copyright (c) 2007-2017, Jean-Marc Valin
 3 | Copyright (c) 2005-2017, Xiph.Org Foundation
 4 | Copyright (c) 2003-2004, Mark Borgerding
 5 | 
 6 | This is the original license of RNNoise. Files modified from the 
 7 | original RNNoise project are licensed under the following terms.
 8 | Other files included in the rnnoise-ex project are licensed under
 9 | different terms.
10 | 
11 | Redistribution and use in source and binary forms, with or without
12 | modification, are permitted provided that the following conditions
13 | are met:
14 | 
15 | - Redistributions of source code must retain the above copyright
16 | notice, this list of conditions and the following disclaimer.
17 | 
18 | - Redistributions in binary form must reproduce the above copyright
19 | notice, this list of conditions and the following disclaimer in the
20 | documentation and/or other materials provided with the distribution.
21 | 
22 | - Neither the name of the Xiph.Org Foundation nor the names of its
23 | contributors may be used to endorse or promote products derived from
24 | this software without specific prior written permission.
25 | 
26 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
27 | ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
28 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
29 | A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION
30 | OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
31 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
32 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
33 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
34 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
35 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
36 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37 | 


--------------------------------------------------------------------------------
/Scripts/DataPreparation.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | # Bash script to prepare noisy audio data for analysis
 3 | 
 4 | echo "[$(date +%T)]: Starting..."
 5 | 
 6 | while IFS=, read -r file scene db; 
 7 | do 
 8 |     echo "[$(date +%T)]: Processing $file";
 9 |     # Create Directories
10 |     mkdir -p $scene/"$db DB"/noisy_wav/
11 |     mkdir -p $scene/"$db DB"/noisy_raw/
12 |     mkdir -p $scene/"$db DB"/clean_raw/
13 |     mkdir -p $scene/"$db DB"/clean_wav/
14 |     # Categorize based on log file
15 |     cp noisy_testset_wav/"$file.wav" $scene/"$db DB"/noisy_wav/"$file.wav"
16 |     # Covert wave to raw
17 |     sox $scene/"$db DB"/noisy_wav/"$file.wav" $scene/"$db DB"/noisy_raw/"$file.raw"
18 |     # Use RNNoise to clean the audio
19 |     rnnoise/rnnoise_demo $scene/"$db DB"/noisy_raw/"$file.raw" $scene/"$db DB"/clean_raw/"$file.raw"
20 |     # Convert clean raw sample to wave for analysis
21 |     sox -t raw -r 48000 -b 16 -e signed-integer -c 1 $scene/"$db DB"/clean_raw/"$file.raw" $scene/"$db DB"/clean_wav/"$file.wav"
22 | done < log_testset.csv


--------------------------------------------------------------------------------
/Scripts/WaveConcat.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | # Bash script to prepare two big audio files for training the RNNoise 
 3 | 
 4 | echo "[$(date +%T)]: Starting..."
 5 | 
 6 | # Concatenate all speech wave files into one
 7 | sox $(ls speech) speech_concatenated.wav
 8 | 
 9 | # Concatenate all noise wave files into one
10 | sox $(ls noise) noise_concatenated.wav


--------------------------------------------------------------------------------
/TRAINING-README:
--------------------------------------------------------------------------------
 1 | # Training
 2 | 
 3 | (1) Compile Valin's feature extraction application as modified by us: 
 4 |     cd src ; ./compile.sh
 5 | 
 6 | (2) Concatenate speech and noise tracks and convert them to raw format
 7 |     cd ../speech; sox $(ls *.wav) ../trainingSpeech.wav; sox trainingSpeech.wav trainingSpeech.raw
 8 |     cd ../noise; sox $(ls *.wav) trainingNoise.wav; sox trainingNoise.wav trainingNoise.raw
 9 | 
10 | (3) Run the feature extraction application that also dumps the raw noisy samples in ./src/noisySpeechSamples.raw
11 |     It is advised to use 500000 or more for count
12 |     cd ../src;
13 |     ./denoise_training ../speech/trainingSpeech.raw ../noise/trainingNoise.raw count > trainingRNNoiseFeatures.f32
14 | 
15 | (4) Extract extended features by running featureExtraction.py. Features will be saved as trainingExtendedFeatures.h5
16 |     cd ../src; python ./featureExtraction.py training noisySpeechSamples.raw trainingExtendedFeatures.h5 count
17 | 
18 | (5) Join feature sets using bin2hdf5
19 |     cd ../training; ./bin2hdf5.py ../src/trainingRNNoiseFeatures.f32 ../src/trainingExtendedFeatures.h5 trainingFeatures.h5 training
20 | 
21 | (6) Train RNNoise
22 |     ./rnn_train.py trainingFeatures.h5 trainedModel.hdf5
23 | 
24 | # Evaluation
25 | 
26 | (1) Compile the RNNoise feature extractor
27 |     cd ../featureExtraction
28 |     gcc -Wall -o feature_extractor -lm feature_extractor.c denoise.c ../src/kiss_fft.c ../src/rnn_data.c ../src/pitch.c ../src/rnn.c ../src/celt_lpc.c
29 | 
30 | (2) Convert desired audio track to raw
31 |     sox testAudio.wav testAudio.raw
32 | 
33 | (3) Extract RNNoise features using the feature extractor
34 |     ./feature_extractor testAudio.raw testAudioFeatures.f32
35 | 
36 | (4) Extract extended features using featureExtraction.py
37 |     python ../training/featureExtraction.py testing testAudio.wav testAudioExtendedFeatures.h5
38 | 
39 | (5) Join feature sets using bin2hdf5
40 |     cd ../training; ./bin2hdf5.py ../featureExtraction/testAudioFeatures.f32 ../featureExtraction/testAudioExtendedFeatures.h5 testAudioFeatures.h5 testing
41 | 
42 | (6) Run RNNoise using evaluation.py
43 |     python ./evaluation.py trainedModel.hdf5 testAudioFeatures.h5 ../featureExtraction/testAudio.wav testAudioClean.wav
44 | 
45 | (6a) If you want to batch evaluate a folder with wav files use
46 |     python ./evaluation_batch.py noisyWavsDirectory/ trainedModel.hdf5
47 | 


--------------------------------------------------------------------------------
/c-audio-experiments/main.c:
--------------------------------------------------------------------------------
 1 | #define FRAME_SIZE 236300
 2 | #include <stdlib.h>
 3 | #include <string.h>
 4 | #include <stdio.h>
 5 | 
 6 | int main(){
 7 | 
 8 |     // Import wave file converted to raw to a 16bit array
 9 |     FILE *fin, *foutShort, *foutFloat;
10 |     fin = fopen("sample.raw", "r");
11 |     foutShort = fopen("processedShort.raw", "w");
12 |     foutFloat = fopen("processedFloat.raw", "w");
13 |     short shortRaw[FRAME_SIZE];
14 |     float floatRaw[FRAME_SIZE];
15 |     fread(shortRaw, sizeof(short), FRAME_SIZE, fin);
16 | 
17 |     // Copy to 32 bit array
18 |     for(int i=0; i < FRAME_SIZE; i++){
19 |         floatRaw[i] = shortRaw[i];
20 |     }
21 |     
22 |     // Dump both to raw files
23 |     fwrite(shortRaw, sizeof(short), FRAME_SIZE, foutShort);
24 |     fwrite(floatRaw, sizeof(float), FRAME_SIZE, foutFloat);
25 | 
26 |     return 0;
27 | }
28 | 
29 | /*
30 |     Pre-processing:
31 |     1. Converted to raw: sox sample.wav sample.raw
32 |     2. Opened raw with audacity 16bit signed mono 48khz little endian
33 |     
34 |     3. Compile: gcc main.c -o experiment
35 |     4. Processing: ./experiment
36 | 
37 |     Post Processing:
38 |     5. Opening processedShort.raw with previously mentioned settings in audacity yields desired results
39 |     6. Opening processedFloat.raw with previously mentioned settings in audacity yields desired results
40 |     but has some trailing garbage after the full length of the useful file has been played. To trim them
41 |     out just use: 
42 |     
43 |     sox -t raw -r 48000 -b 16 -e signed-integer -c 1 processedFloat.raw processedFloat.wav
44 | 
45 |     to convert the file to wav and then to trim it use:
46 | 
47 |     sox processedFloat.wav processedFloatTrimmed.wav trim 0 4.92
48 |     
49 |     (keeps file from 0s to 4.92s)
50 | 
51 | */


--------------------------------------------------------------------------------
/evaluation/CDenoiseClassicMethods.m:
--------------------------------------------------------------------------------
 1 | clear all;
 2 | close all;
 3 | 
 4 | technic = "wiener-as";
 5 | 
 6 | %% Store filename of Enhanced Signals
 7 | folders = {'bus','cafe','living','office','psquare'};
 8 | folders2 = {'2.5 DB','7.5 DB','12.5 DB','17.5 DB'};
 9 | 
10 | k = 0; %number of 20 contditions
11 | data_Noisy = cell(length(folders)*length(folders2),1); %filenames of enhanced
12 | data_Noisy_16 = cell(length(folders)*length(folders2),1); %filenames of enhanced
13 | descriptor = cell(length(folders)*length(folders2),1); %descriptors of conditions
14 | for i=1:length(folders) %folders of noise types
15 |     for j = 1:length(folders2) %folders of dB
16 |         k = k+1;
17 |         names = "RNNoise2\" + folders{i} + "\ " + folders2{j} +"\noisy_wav"; %folder of enhanced signals
18 |         names = dir(names + "\p*"); %initial storage of files
19 |         data_cell = regexpi({names.name},'p\w*_down16.wav','match'); %get only wanted filenames
20 |         data_cell = vertcat(data_cell{:}); %discard empty records
21 |         data_Noisy_16(k) = {data_cell}; %store them in a cell array
22 |         data_cell = regexpi({names.name},'p\d{3}_\d{3}.wav','match'); %get only wanted filenames
23 |         data_cell = vertcat(data_cell{:}); %discard empty records
24 |         data_Noisy(k) = {data_cell}; %store them in a cell array
25 |         descriptor(k) = {folders{i} + "_" + folders2{j}}; %create a descriptor for the conditions
26 |     end
27 | end
28 | 
29 | %% Denoise
30 | for i=1:size(data_Noisy,1)%for every condition (20 in total)
31 |     files = data_Noisy{i}; %filenames for enhanced
32 |     files_16 = data_Noisy_16{i};
33 |     for j=1:length(data_Noisy{i}) %for every file in the folder
34 |         %get full path to .wav of clean and enhanced signals 
35 |         desc = strsplit(descriptor{i},'_'); %split descriptor (type of noise & SNR)
36 |         name_Noisy = "RNNoise2\"+ desc{1} + "\ " + desc{2} + "\noisy_wav\" + files{j}; %full path to enhanced file
37 |         name_Denoised = "RNNoise2\clean_ClassicMeth\" + technic + "\" + desc{1} + "\" + desc{2} + "\noisy_wav\" + files{j};
38 |         %16 KHz
39 |         name_Noisy_16 = "RNNoise2\"+ desc{1} + "\ " + desc{2} + "\noisy_wav\" + files_16{j}; %full path to enhanced file
40 |         name_Denoised_16 = "RNNoise2\clean_ClassicMeth\" + technic + "\"+ desc{1} + "\" + desc{2} + "\noisy_wav\" + files_16{j};
41 | %         logmmse(convertStringsToChars(name_Noisy),convertStringsToChars(name_Denoised));
42 | %         logmmse(convertStringsToChars(name_Noisy_16),convertStringsToChars(name_Denoised_16));
43 |         wiener_as(convertStringsToChars(name_Noisy),convertStringsToChars(name_Denoised));
44 |         wiener_as(convertStringsToChars(name_Noisy_16),convertStringsToChars(name_Denoised_16));
45 |     end
46 | end


--------------------------------------------------------------------------------
/evaluation/CDownsample_WAV_Classic.m:
--------------------------------------------------------------------------------
 1 | clear all;
 2 | 
 3 | %for the downsampling of denoised 48KHz signals
 4 | technic = "wiener-as";
 5 | %% FOR ALL FOLDERS EXCEPT "clean"
 6 | folders = {'bus','cafe','living','office','psquare'};
 7 | folders2 = {'2.5 DB','7.5 DB','12.5 DB','17.5 DB'};
 8 | Fresample = 16000; %final sampling frequency
 9 | 
10 | for k=1:length(folders) %for every noise type folder
11 |     for j=1:length(folders2) %for every SNR
12 |         sAudioFolder="RNNoise2\clean_ClassicMeth\" + technic + "\" + folders{k} + "\" + folders2{j} +"\noisy_wav"; %path of files
13 |         eFiles=dir(sAudioFolder+"\*.wav"); %get all .wav files
14 |         for i=1:length(eFiles) %for every file
15 |             if(~isempty(regexpi(eFiles(i).name,'p\d{3}_\d{3}.wav','match')))
16 |                 sAudioFile=fullfile(sAudioFolder,eFiles(i).name); %full path to file
17 |                 [y,Fs] = audioread(sAudioFile); %read file
18 |                 y_resamp = resample(y,Fresample,Fs); %resample at Fresample frequency
19 |                 sAudioFileOut=fullfile(sAudioFolder,[strrep(eFiles(i).name,'.wav','') '_down16_AfterDen.wav']); %create new filename
20 |                 audiowrite(convertStringsToChars(sAudioFileOut),y_resamp,Fresample); %store the downsampled signal
21 |             end
22 |         end
23 |     end
24 | end


--------------------------------------------------------------------------------
/evaluation/CDownsample_WAV_Noisy.m:
--------------------------------------------------------------------------------
 1 | clear all;
 2 | 
 3 | %cretes downsampled to 16KHz noisy wav files
 4 | %% FOR ALL FOLDERS EXCEPT "clean"
 5 | folders = {'bus','cafe','living','office','psquare'};
 6 | folders2 = {'2.5 DB','7.5 DB','12.5 DB','17.5 DB'};
 7 | Fresample = 16000; %final sampling frequency
 8 | 
 9 | for k=1:length(folders) %for every noise type folder
10 |     for j=1:length(folders2) %for every SNR
11 |         sAudioFolder="RNNoise2\" + folders{k} + "\ " + folders2{j} +"\noisy_wav"; %path of files
12 |         eFiles=dir(sAudioFolder+"\*.wav"); %get all .wav files
13 |         for i=1:length(eFiles) %for every file
14 |             sAudioFile=fullfile(sAudioFolder,eFiles(i).name); %full path to file
15 |             [y,Fs] = audioread(sAudioFile); %read file
16 |             y_resamp = resample(y,Fresample,Fs); %resample at Fresample frequency
17 |             sAudioFileOut=fullfile(sAudioFolder,[strrep(eFiles(i).name,'.wav','') '_down16.wav']); %create new filename
18 |             audiowrite(convertStringsToChars(sAudioFileOut),y_resamp,Fresample); %store the downsampled signal
19 |         end
20 |     end
21 | end


--------------------------------------------------------------------------------
/evaluation/CEVALUATION_script_Classic.m:
--------------------------------------------------------------------------------
  1 | clear all;
  2 | close all;
  3 | 
  4 | technic = "wiener-as";
  5 | DOWNSAMPLED_BEFORE = 0;
  6 | %% Store filename of Enhanced Signals
  7 | folders = {'bus','cafe','living','office','psquare'};
  8 | folders2 = {'2.5 DB','7.5 DB','12.5 DB','17.5 DB'};
  9 | 
 10 | k = 0; %number of 20 contditions
 11 | data_Enh = cell(length(folders)*length(folders2),1); %filenames of enhanced
 12 | data_Enh_16 = cell(length(folders)*length(folders2),1); %filenames of enhanced 16 KHz
 13 | data_Enh_16_After = cell(length(folders)*length(folders2),1); %filenames of enhanced 16 KHz
 14 | descriptor = cell(length(folders)*length(folders2),1); %descriptors of conditions
 15 | for i=1:length(folders) %folders of noise types
 16 |     for j = 1:length(folders2) %folders of dB
 17 |         k = k+1;
 18 |         names = "RNNoise2\clean_ClassicMeth\" + technic +"\"+ folders{i} + "\" + folders2{j} +"\noisy_wav"; %folder of enhanced signals
 19 |         names = dir(names + "\p*"); %initial storage of files
 20 |         data_cell = regexpi({names.name},'p\w*_down16.wav','match'); %get only wanted filenames
 21 |         data_cell = vertcat(data_cell{:}); %discard empty records
 22 |         data_Enh_16(k) = {data_cell}; %store them in a cell array
 23 |         data_cell = regexpi({names.name},'p\d{3}_\d{3}.wav','match'); %get only wanted filenames
 24 |         data_cell = vertcat(data_cell{:}); %discard empty records
 25 |         data_Enh(k) = {data_cell}; %store them in a cell array
 26 |         data_cell = regexpi({names.name},'p\w*_down16_AfterDen.wav','match'); %get only wanted filenames
 27 |         data_cell = vertcat(data_cell{:}); %discard empty records
 28 |         data_Enh_16_After(k) = {data_cell}; %store them in a cell array
 29 |         descriptor(k) = {folders{i} + "_" + folders2{j}}; %create a descriptor for the conditions
 30 |     end
 31 | end
 32 | 
 33 | %% Compute Metrics
 34 | %find number of files in every folder
 35 | a = [];
 36 | for i=1:length(data_Enh)
 37 |     a(i) = length(data_Enh{i});
 38 | end
 39 | 
 40 | %initialization of metrics' arrays
 41 | Csig = zeros(max(a),length(data_Enh));
 42 | Cbak = zeros(max(a),length(data_Enh));
 43 | Covl = zeros(max(a),length(data_Enh));
 44 | PESQ = zeros(max(a),length(data_Enh),2);
 45 | STOI = zeros(max(a),length(data_Enh));
 46 | LLR = zeros(max(a),length(data_Enh));
 47 | fwSNRseg = zeros(max(a),length(data_Enh));
 48 | Snr_mean = zeros(max(a),length(data_Enh));
 49 | SegSNR_mean = zeros(max(a),length(data_Enh));
 50 | Pers = zeros(max(a),length(data_Enh));
 51 | 
 52 | %computation of metrics
 53 | for i=1:size(data_Enh,1)%for every condition (20 in total)
 54 |     files = data_Enh{i}; %filenames for enhanced
 55 |     files_16 = data_Enh_16{i};
 56 |     files_16_After = data_Enh_16_After{i};
 57 |     for j=1:length(data_Enh{i}) %for every file in the folder
 58 |         %get full path to .wav of clean and enhanced signals 
 59 |         desc = strsplit(descriptor{i},'_'); %split descriptor (type of noise & SNR)
 60 | %         name_Enh = "RNNoise2\"+ desc{1} + "\ " + desc{2} + "\clean_wav\" + files{j}; %full path to enhanced file
 61 |         name_Enh = "RNNoise2\clean_ClassicMeth\" + technic + "\"+ desc{1} + "\" + desc{2} + "\noisy_wav\" + files{j}; %full path to enhanced file
 62 |         name_Clean = "RNNoise2\clean\" + files{j}; %full path to clear file
 63 |         %16 KHz
 64 | %         name_Enh_16 = "RNNoise2\"+ desc{1} + "\ " + desc{2} + "\clean_wav\" + files_16{j}; %full path to enhanced file
 65 |         name_Enh_16 = "RNNoise2\clean_ClassicMeth\" + technic + "\"+ desc{1} + "\" + desc{2} + "\noisy_wav\" + files_16{j}; %full path to enhanced file
 66 |         name_Clean_16 = "RNNoise2\clean\" + files_16{j}; %full path to clear file
 67 |         %16 KHz After Denoising
 68 |         name_Enh_16_After = "RNNoise2\clean_ClassicMeth\" + technic+ "\" +  desc{1} + "\" + desc{2} + "\noisy_wav\" + files_16_After{j}; %full path to enhanced file
 69 |         
 70 |         if(DOWNSAMPLED_BEFORE == 1) %16KHz from the start
 71 |             %Csig/Cbak/Covl
 72 |             [Csig(j,i),Cbak(j,i),Covl(j,i)]=composite(convertStringsToChars(name_Clean_16),convertStringsToChars(name_Enh_16));
 73 |             %PESQ
 74 |             PESQ(j,i,:) = pesq(convertStringsToChars(name_Clean_16),convertStringsToChars(name_Enh_16));
 75 |             %Log-likelihood ratio (LLR)
 76 |             LLR(j,i) = comp_llr(convertStringsToChars(name_Clean_16),convertStringsToChars(name_Enh_16));
 77 |             %fwSNRseg
 78 |             fwSNRseg(j,i) = comp_fwseg(convertStringsToChars(name_Clean_16),convertStringsToChars(name_Enh_16));
 79 |             %SNR
 80 |             [Snr_mean(j,i), SegSNR_mean(j,i)]= comp_snr(convertStringsToChars(name_Clean_16),convertStringsToChars(name_Enh_16));
 81 |             %STOI
 82 |             clean_audio = audioread(name_Clean_16); %get clean audio
 83 |             [enh_audio,fs] = audioread(name_Enh_16); %get enhanced audio
 84 |             samples = min(length(clean_audio),length(enh_audio)); %find minimum number of samples between clean and enhanced
 85 |             STOI(j,i) = stoi(clean_audio(1:samples),enh_audio(1:samples),fs);
 86 |             %Pearson
 87 |             Pers(j,i) = corr(clean_audio(1:samples),enh_audio(1:samples));
 88 |         else
 89 |             %Csig/Cbak/Covl
 90 |             [Csig(j,i),Cbak(j,i),Covl(j,i)]=composite(convertStringsToChars(name_Clean_16),convertStringsToChars(name_Enh_16_After));
 91 |             %PESQ
 92 |             PESQ(j,i,:) = pesq(convertStringsToChars(name_Clean_16),convertStringsToChars(name_Enh_16_After));
 93 |             %Log-likelihood ratio (LLR)
 94 |             LLR(j,i) = comp_llr(convertStringsToChars(name_Clean),convertStringsToChars(name_Enh));
 95 |             %fwSNRseg
 96 |             fwSNRseg(j,i) = comp_fwseg(convertStringsToChars(name_Clean),convertStringsToChars(name_Enh));
 97 |             %SNR
 98 |             [Snr_mean(j,i), SegSNR_mean(j,i)]= comp_snr(convertStringsToChars(name_Clean),convertStringsToChars(name_Enh));
 99 |             %STOI
100 |             clean_audio = audioread(name_Clean); %get clean audio
101 |             [enh_audio,fs] = audioread(name_Enh); %get enhanced audio
102 |             samples = min(length(clean_audio),length(enh_audio)); %find minimum number of samples between clean and enhanced
103 |             STOI(j,i) = stoi(clean_audio(1:samples),enh_audio(1:samples),fs);
104 |             %Pearson
105 |             Pers(j,i) = corr(clean_audio(1:samples),enh_audio(1:samples));
106 |         end
107 | 
108 | 
109 |     end
110 | end
111 | 
112 | % %% Statistics
113 | % %initializaton
114 | % Csig_mean = zeros(size(Csig,2),1);
115 | % Cbak_mean = zeros(size(Csig,2),1);
116 | % Covl_mean = zeros(size(Csig,2),1);
117 | % STOI_mean = zeros(size(Csig,2),1);
118 | % PESQ_mean = zeros(size(Csig,2),2);
119 | % LLR_mean = zeros(size(Csig,2),1);
120 | % fwSNRseg_mean = zeros(size(Csig,2),1);
121 | % SNR_mean = zeros(size(Csig,2),1); %SNR_mean
122 | % SNRseg_mean = zeros(size(Csig,2),1); %segSNR_mean
123 | % Corr_mean = zeros(size(Csig,2),1); %Pearson correlation
124 | % 
125 | % 
126 | % for i=1:size(Csig,2) %for every condition (noise type & SNR)
127 | %     %for all metrics remove 0s (missing records) elements and then
128 | %     %compute the mean for every condition
129 | %     Csig_mean(i) = mean(Csig(Csig(:,i) ~= 0,i),1);
130 | %     Cbak_mean(i) = mean(Cbak(Cbak(:,i) ~= 0,i),1);
131 | %     Covl_mean(i) = mean(Covl(Covl(:,i) ~= 0,i),1);
132 | %     PESQ_mean(i,1) = mean(PESQ(PESQ(:,i,1)~= 0,i,1),1); %2 for narrow- and wideband
133 | %     PESQ_mean(i,2) = mean(PESQ(PESQ(:,i,2)~= 0,i,2),1);
134 | %     STOI_mean(i) = mean(STOI(STOI(:,i) ~= 0,i),1);
135 | %     LLR_mean(i) = mean(LLR(LLR(:,i) ~= 0,i),1);
136 | %     fwSNRseg_mean(i) = mean(fwSNRseg(fwSNRseg(:,i) ~= 0,i),1);
137 | %     SNR_mean(i) = mean(Snr_mean(Snr_mean(:,i) ~= 0,i),1);
138 | %     SNRseg_mean(i) = mean(SegSNR_mean(SegSNR_mean(:,i) ~= 0,i),1);
139 | %     Corr_mean(i) = mean(Pers(Pers(:,i) ~= 0,i),1);
140 | % end
141 | 
142 | ALL_Metrics=[Csig Cbak Covl PESQ(:,:,1) PESQ(:,:,2) STOI LLR fwSNRseg Snr_mean SegSNR_mean Pers];
143 | numberOfMetrics=11;
144 | 
145 | AllResults1_MEAN=zeros(1,11);
146 | AllResults2_MEAN=zeros(20,11);
147 | AllResults3_MEAN=zeros(4,11);
148 | AllResults4_MEAN=zeros(5,11);
149 | AllResults1_STD=zeros(1,11);
150 | AllResults2_STD=zeros(20,11);
151 | AllResults3_STD=zeros(4,11);
152 | AllResults4_STD=zeros(5,11);
153 | 
154 | for i=1:numberOfMetrics
155 |     [AllResults1_MEAN(i),AllResults2_MEAN(:,i),AllResults3_MEAN(:,i),AllResults4_MEAN(:,i),AllResults1_STD(i),AllResults2_STD(:,i),AllResults3_STD(:,i),AllResults4_STD(:,i),]=EvalPlotFun(ALL_Metrics(:,(i-1)*20+(1:20)),i);
156 | end
157 | 
158 | FigList = findobj(allchild(0), 'flat', 'Type', 'figure');
159 | %FigList = flip(FigList);
160 | for iFig = 1:1:length(FigList)
161 | FigHandle = FigList(iFig);
162 | FigHandle.WindowState = 'maximized';
163 | saveas(FigHandle,fullfile("resultsClassicWiener-AS48KHz",num2str(length(FigList)-iFig+1)+".png"));
164 | end


--------------------------------------------------------------------------------
/evaluation/Downsample_WAV.m:
--------------------------------------------------------------------------------
 1 | clear all;
 2 | 
 3 | %% FOR ALL FOLDERS EXCEPT "clean"
 4 | folders = {'bus','cafe','living','office','psquare'};
 5 | folders2 = {'2.5 DB','7.5 DB','12.5 DB','17.5 DB'};
 6 | Fresample = 16000; %final sampling frequency
 7 | 
 8 | for k=1:length(folders) %for every noise type folder
 9 |     for j=1:length(folders2) %for every SNR
10 |         sAudioFolder="RNNoise2\" + folders{k} + "\ " + folders2{j} +"\clean_wav"; %path of files
11 |         eFiles=dir(sAudioFolder+"\*.wav"); %get all .wav files
12 |         for i=1:length(eFiles) %for every file
13 |             sAudioFile=fullfile(sAudioFolder,eFiles(i).name); %full path to file
14 |             [y,Fs] = audioread(sAudioFile); %read file
15 |             y_resamp = resample(y,Fresample,Fs); %resample at Fresample frequency
16 |             sAudioFileOut=fullfile(sAudioFolder,[strrep(eFiles(i).name,'.wav','') '_down16.wav']); %create new filename
17 |             audiowrite(convertStringsToChars(sAudioFileOut),y_resamp,Fresample); %store the downsampled signal
18 |         end
19 |     end
20 | end
21 | 
22 | % %% FOR FOLDER "clean"
23 | % sAudioFolder="RNNoise2\clean"; %path to files
24 | % eFiles=dir(sAudioFolder+"\*.wav"); %get all .wav files of folder
25 | % for i=1:length(eFiles) %for every file
26 | %     sAudioFile=fullfile(sAudioFolder,eFiles(i).name); %full path to file
27 | %     [y,Fs] = audioread(sAudioFile); %read file
28 | %     y_resamp = resample(y,Fresample,Fs); %resample at Fresample frequency
29 | %     sAudioFileOut=fullfile(sAudioFolder,[strrep(eFiles(i).name,'.wav','') '_down16.wav']); %create new filename
30 | %     audiowrite(convertStringsToChars(sAudioFileOut),y_resamp,Fresample); %store the downsampled signal
31 | % end


--------------------------------------------------------------------------------
/evaluation/EVALUATION_script.m:
--------------------------------------------------------------------------------
  1 | clear all;
  2 | close all;
  3 | 
  4 | %% Store filename of Enhanced Signals
  5 | folders = {'bus','cafe','living','office','psquare'};
  6 | folders2 = {'2.5 DB','7.5 DB','12.5 DB','17.5 DB'};
  7 | 
  8 | k = 0; %number of 20 contditions
  9 | data_Enh = cell(length(folders)*length(folders2),1); %filenames of enhanced
 10 | data_Enh_16 = cell(length(folders)*length(folders2),1); %filenames of enhanced 16 KHz
 11 | descriptor = cell(length(folders)*length(folders2),1); %descriptors of conditions
 12 | for i=1:length(folders) %folders of noise types
 13 |     for j = 1:length(folders2) %folders of dB
 14 |         k = k+1;
 15 |         names = "RNNoise2\" + folders{i} + "\ " + folders2{j} +"\clean_wav"; %folder of enhanced signals
 16 |         names = dir(names + "\p*"); %initial storage of files
 17 |         data_cell = regexpi({names.name},'p\w*_down16.wav','match'); %get only wanted filenames
 18 |         data_cell = vertcat(data_cell{:}); %discard empty records
 19 |         data_Enh_16(k) = {data_cell}; %store them in a cell array
 20 |         data_cell = regexpi({names.name},'p\d{3}_\d{3}.wav','match'); %get only wanted filenames
 21 |         data_cell = vertcat(data_cell{:}); %discard empty records
 22 |         data_Enh(k) = {data_cell}; %store them in a cell array
 23 |         descriptor(k) = {folders{i} + "_" + folders2{j}}; %create a descriptor for the conditions
 24 |     end
 25 | end
 26 | 
 27 | %% Compute Metrics
 28 | %find number of files in every folder
 29 | a = [];
 30 | for i=1:length(data_Enh)
 31 |     a(i) = length(data_Enh{i});
 32 | end
 33 | 
 34 | %initialization of metrics' arrays
 35 | Csig = zeros(max(a),length(data_Enh));
 36 | Cbak = zeros(max(a),length(data_Enh));
 37 | Covl = zeros(max(a),length(data_Enh));
 38 | PESQ = zeros(max(a),length(data_Enh),2);
 39 | STOI = zeros(max(a),length(data_Enh));
 40 | LLR = zeros(max(a),length(data_Enh));
 41 | fwSNRseg = zeros(max(a),length(data_Enh));
 42 | Snr_mean = zeros(max(a),length(data_Enh));
 43 | SegSNR_mean = zeros(max(a),length(data_Enh));
 44 | Pers = zeros(max(a),length(data_Enh));
 45 | 
 46 | %computation of metrics
 47 | for i=1:size(data_Enh,1)%for every condition (20 in total)
 48 |     files = data_Enh{i}; %filenames for enhanced
 49 |     files_16 = data_Enh_16{i};
 50 |     for j=1:length(data_Enh{i}) %for every file in the folder
 51 |         %get full path to .wav of clean and enhanced signals 
 52 |         desc = strsplit(descriptor{i},'_'); %split descriptor (type of noise & SNR)
 53 |         name_Enh = "RNNoise2\"+ desc{1} + "\ " + desc{2} + "\clean_wav\" + files{j}; %full path to enhanced file
 54 |         name_Clean = "RNNoise2\clean\" + files{j}; %full path to clear file
 55 |         %16 KHz
 56 |         name_Enh_16 = "RNNoise2\"+ desc{1} + "\ " + desc{2} + "\clean_wav\" + files_16{j}; %full path to enhanced file
 57 |         name_Clean_16 = "RNNoise2\clean\" + files_16{j}; %full path to clear file
 58 |         
 59 |         %Csig/Cbak/Covl
 60 |         [Csig(j,i),Cbak(j,i),Covl(j,i)]=composite(convertStringsToChars(name_Clean_16),convertStringsToChars(name_Enh_16));
 61 |         
 62 |         %PESQ
 63 |         PESQ(j,i,:) = pesq(convertStringsToChars(name_Clean_16),convertStringsToChars(name_Enh_16));
 64 |           
 65 |         %Log-likelihood ratio (LLR)
 66 |         LLR(j,i) = comp_llr(convertStringsToChars(name_Clean),convertStringsToChars(name_Enh));
 67 |         
 68 |         %fwSNRseg
 69 |         fwSNRseg(j,i) = comp_fwseg(convertStringsToChars(name_Clean),convertStringsToChars(name_Enh));
 70 |         
 71 |         %SNR
 72 |         [Snr_mean(j,i), SegSNR_mean(j,i)]= comp_snr(convertStringsToChars(name_Clean),convertStringsToChars(name_Enh));
 73 |         
 74 |         %STOI
 75 |         clean_audio = audioread(name_Clean); %get clean audio
 76 |         [enh_audio,fs] = audioread(name_Enh); %get enhanced audio
 77 |         samples = min(length(clean_audio),length(enh_audio)); %find minimum number of samples between clean and enhanced
 78 |         STOI(j,i) = stoi(clean_audio(1:samples),enh_audio(1:samples),fs);
 79 |         
 80 |         %Pearson
 81 |         Pers(j,i) = corr(clean_audio(1:samples),enh_audio(1:samples));
 82 | 
 83 |     end
 84 | end
 85 | 
 86 | % %% Statistics
 87 | % %initializaton
 88 | % Csig_mean = zeros(size(Csig,2),1);
 89 | % Cbak_mean = zeros(size(Csig,2),1);
 90 | % Covl_mean = zeros(size(Csig,2),1);
 91 | % STOI_mean = zeros(size(Csig,2),1);
 92 | % PESQ_mean = zeros(size(Csig,2),2);
 93 | % LLR_mean = zeros(size(Csig,2),1);
 94 | % fwSNRseg_mean = zeros(size(Csig,2),1);
 95 | % SNR_mean = zeros(size(Csig,2),1); %SNR_mean
 96 | % SNRseg_mean = zeros(size(Csig,2),1); %segSNR_mean
 97 | % Corr_mean = zeros(size(Csig,2),1); %Pearson correlation
 98 | % 
 99 | % 
100 | % for i=1:size(Csig,2) %for every condition (noise type & SNR)
101 | %     %for all metrics remove 0s (missing records) elements and then
102 | %     %compute the mean for every condition
103 | %     Csig_mean(i) = mean(Csig(Csig(:,i) ~= 0,i),1);
104 | %     Cbak_mean(i) = mean(Cbak(Cbak(:,i) ~= 0,i),1);
105 | %     Covl_mean(i) = mean(Covl(Covl(:,i) ~= 0,i),1);
106 | %     PESQ_mean(i,1) = mean(PESQ(PESQ(:,i,1)~= 0,i,1),1); %2 for narrow- and wideband
107 | %     PESQ_mean(i,2) = mean(PESQ(PESQ(:,i,2)~= 0,i,2),1);
108 | %     STOI_mean(i) = mean(STOI(STOI(:,i) ~= 0,i),1);
109 | %     LLR_mean(i) = mean(LLR(LLR(:,i) ~= 0,i),1);
110 | %     fwSNRseg_mean(i) = mean(fwSNRseg(fwSNRseg(:,i) ~= 0,i),1);
111 | %     SNR_mean(i) = mean(Snr_mean(Snr_mean(:,i) ~= 0,i),1);
112 | %     SNRseg_mean(i) = mean(SegSNR_mean(SegSNR_mean(:,i) ~= 0,i),1);
113 | %     Corr_mean(i) = mean(Pers(Pers(:,i) ~= 0,i),1);
114 | % end
115 | 
116 | ALL_Metrics=[Csig Cbak Covl PESQ(:,:,1) PESQ(:,:,2) STOI LLR fwSNRseg Snr_mean SegSNR_mean Pers];
117 | numberOfMetrics=11;
118 | 
119 | AllResults1_MEAN=zeros(1,11);
120 | AllResults2_MEAN=zeros(20,11);
121 | AllResults3_MEAN=zeros(4,11);
122 | AllResults4_MEAN=zeros(5,11);
123 | AllResults1_STD=zeros(1,11);
124 | AllResults2_STD=zeros(20,11);
125 | AllResults3_STD=zeros(4,11);
126 | AllResults4_STD=zeros(5,11);
127 | 
128 | for i=1:numberOfMetrics
129 |     [AllResults1_MEAN(i),AllResults2_MEAN(:,i),AllResults3_MEAN(:,i),AllResults4_MEAN(:,i),AllResults1_STD(i),AllResults2_STD(:,i),AllResults3_STD(:,i),AllResults4_STD(:,i),]=EvalPlotFun(ALL_Metrics(:,(i-1)*20+(1:20)),i);
130 | end
131 | 
132 | FigList = findobj(allchild(0), 'flat', 'Type', 'figure');
133 | %FigList = flip(FigList);
134 | for iFig = 1:1:length(FigList)
135 | FigHandle = FigList(iFig);
136 | FigHandle.WindowState = 'maximized';
137 | saveas(FigHandle,fullfile("resultsRNNoise",num2str(iFig)+".png"));
138 | end
139 | % FigList = findobj(allchild(0), 'flat', 'Type', 'figure');
140 | % %FigList = flip(FigList);
141 | % for iFig = 1:1:length(FigList)
142 | % FigHandle = FigList(iFig);
143 | % FigHandle.WindowState = 'maximized';
144 | % saveas(FigHandle,fullfile("resultsRNNoiseModified_500.000",num2str(length(FigList)-iFig)+".png"));
145 | % end


--------------------------------------------------------------------------------
/evaluation/EvalPlotFun.m:
--------------------------------------------------------------------------------
  1 | function [out1, out2, out3, out4, out5, out6, out7, out8]=EvalPlotFun(Vmatrix,metricType)
  2 | %% Preamble
  3 | metricNames={"Csig","Cbak","Covl","narrow-band PESQ","wide-band PESQ","STOI","LLR","fwSNRseg","SNR","SegSNR","Pearson's Coefficient"}
  4 | 
  5 | [out1, out2, out3, out4]=EvalStatFun(Vmatrix,metricType,1);
  6 | [out5, out6, out7, out8]=EvalStatFun(Vmatrix,metricType,2);
  7 | 
  8 | 
  9 | %% 1st Figure
 10 | databar=[out2(1),out2(2),out2(3),out2(4);out2(5),out2(6),out2(7),out2(8);out2(9),out2(10),out2(11),out2(12);out2(13),out2(14),out2(15),out2(16);out2(17),out2(18),out2(19),out2(20)];
 11 | stdbar=[out6(1),out6(2),out6(3),out6(4);out6(5),out6(6),out6(7),out6(8);out6(9),out6(10),out6(11),out6(12);out6(13),out6(14),out6(15),out6(16);out6(17),out6(18),out6(19),out6(20)];
 12 | 
 13 | h1=figure 
 14 | hb=bar(databar);
 15 | 
 16 | hold on
 17 | [ngroups,nbars]=size(databar);
 18 | groupwidth = min(0.8, nbars/(nbars + 1.5));
 19 | % Set the position of each error bar in the centre of the main bar
 20 | % Based on barweb.m by Bolu Ajiboye from MATLAB File Exchange
 21 | for i = 1:nbars
 22 |     % Calculate center of each bar
 23 |     x = (1:ngroups) - groupwidth/2 + (2*i-1) * groupwidth / (2*nbars);
 24 |     errorbar(x, databar(:,i), stdbar(:,i), 'k', 'linestyle', 'none');
 25 | end
 26 | 
 27 | if (metricType==1)
 28 |     ylim([0 5.2])
 29 | elseif (metricType==2)||(metricType==3)
 30 |     ylim([0 5])
 31 | elseif ((metricType==4)||(metricType==5))
 32 |     ylim([-0.5 4.5])
 33 | elseif (metricType==6)
 34 |     ylim([0 1.1])
 35 | elseif (metricType==7)
 36 |     ylim([0 2.2])
 37 | elseif (metricType==8)
 38 |     ylim([0 23.5])
 39 | elseif (metricType==9)
 40 |     ylim([0 25])
 41 | elseif (metricType==10)
 42 |     ylim([-5 16])
 43 | else
 44 |     ylim([0 1])
 45 | end
 46 | 
 47 | 
 48 | % title({'Performance in different noise conditions at various SNR levels'
 49 | %     ['Metric: ',convertStringsToChars(metricNames(metricType))]})
 50 | grid on
 51 | h=suptitle({'Performance in different noise conditions at various SNR levels'
 52 |     ['Metric: ',char(cellstr(metricNames(metricType)))]
 53 |     'Modified RNNoise'});%  --> change this part of the title manually
 54 | set(h,'FontSize',16,'FontWeight','bold')
 55 | 
 56 | HeightScaleFactor = 1.5;
 57 | NewHeight = h.Position(2) * HeightScaleFactor;
 58 | %h.Position(2) = h.Position(2) - (NewHeight - h.Position(4));
 59 | h.Position(2) = NewHeight;
 60 | 
 61 | 
 62 | name={'bus'; 'cafe'; 'living'; 'office'; 'psquare'};
 63 | set(gca,'xticklabel',name,'FontWeight', 'bold','FontSize',12);
 64 | lgd=legend('2.5dB','7.5dB','12.5dB','17.5dB', 'Location', 'northeastoutside');
 65 | title(lgd,'SNR Levels')
 66 | 
 67 | hold off
 68 | 
 69 | %saveas(h1,sprintf(convertCharsToStrings(['Figure',(3*metricType-2)])),'png');
 70 | 
 71 | 
 72 | %% 2nd Figure
 73 | 
 74 | h2=figure (3*metricType-1)
 75 | snrX=[2.5,7.5,12.5,17.5];
 76 | 
 77 | subplot(2,6,[1,2])
 78 | errorbar(snrX,out2(1:4),out6(1:4), 'LineWidth', 1.5);
 79 | xticks([2.5 7.5 12.5 17.5]) 
 80 | xlabel('SNR levels', 'FontSize', 8)
 81 | title('Noise Profile: BUS','FontSize',9, 'FontWeight', 'bold')
 82 | grid on
 83 | set(gca,'FontWeight', 'bold','FontSize',9);
 84 | 
 85 | subplot(2,6,[3,4])
 86 | errorbar(snrX,out2(5:8),out6(5:8), 'LineWidth', 1.5);
 87 | xticks([2.5 7.5 12.5 17.5]) 
 88 | xlabel('SNR levels', 'FontSize', 8)
 89 | title('Noise Profile: CAFE','FontSize',9, 'FontWeight', 'bold')
 90 | grid on
 91 | set(gca,'FontWeight', 'bold','FontSize',9);
 92 | 
 93 | subplot(2,6,[5,6])
 94 | errorbar(snrX,out2(9:12),out6(9:12), 'LineWidth', 1.5);
 95 | xticks([2.5 7.5 12.5 17.5]) 
 96 | xlabel('SNR levels', 'FontSize', 8)
 97 | title('Noise Profile: LIVING','FontSize',9, 'FontWeight', 'bold')
 98 | grid on
 99 | set(gca,'FontWeight', 'bold','FontSize',9);
100 | 
101 | subplot(2,6,[8,9])
102 | errorbar(snrX,out2(13:16),out6(13:16), 'LineWidth', 1.5);
103 | xticks([2.5 7.5 12.5 17.5]) 
104 | xlabel('SNR levels', 'FontSize', 8)
105 | title('Noise Profile: OFFICE','FontSize',9, 'FontWeight', 'bold')
106 | grid on
107 | set(gca,'FontWeight', 'bold','FontSize',9);
108 | 
109 | subplot(2,6,[10,11])
110 | errorbar(snrX,out2(17:20),out6(17:20), 'LineWidth', 1.5);
111 | xticks([2.5 7.5 12.5 17.5]) 
112 | xlabel('SNR levels', 'FontSize', 8)
113 | title('Noise Profile: PSQUARE','FontSize',9, 'FontWeight', 'bold')
114 | grid on
115 | set(gca,'FontWeight', 'bold','FontSize',9);
116 | 
117 | h=suptitle({'System Performance for Specific Noise Conditions at Different SNR Levels'
118 |     ['Metric: ',char(cellstr(metricNames(metricType))), ' - Mofified RNNoise']
119 |     });
120 | set(h,'FontSize',14,'FontWeight','bold')
121 | 
122 | %% 3rd Figure
123 | 
124 | figure (3*metricType)
125 | x=1:5;
126 | 
127 | subplot(2,2,1)
128 | databar1=[out2(1),out2(5),out2(9),out2(13),out2(17)];
129 | stdbar1=[out6(1),out6(5),out6(9),out6(13),out6(17)];
130 | bar(x,databar1,'FaceColor', [0.00 0.45 0.74])
131 | hold on
132 | er1=errorbar(x,databar1,stdbar1);
133 | er1.Color = [0 0 0];                            
134 | er1.LineStyle = 'none';  
135 | name={'bus'; 'cafe'; 'living'; 'office'; 'psquare'};
136 | set(gca,'xticklabel',name,'FontWeight','bold');
137 | hold off
138 | title('SNR Level: 2.5dB')
139 | grid on
140 | 
141 | subplot(2,2,2)
142 | databar2=[out2(2),out2(6),out2(10),out2(14),out2(18)];
143 | stdbar2=[out6(2),out6(6),out6(10),out6(14),out6(18)];
144 | bar(x,databar2,'FaceColor', [0.85 0.33 0.10])
145 | hold on
146 | er2=errorbar(x,databar2,stdbar2);
147 | er2.Color = [0 0 0];                            
148 | er2.LineStyle = 'none';  
149 | name={'bus'; 'cafe'; 'living'; 'office'; 'psquare'};
150 | set(gca,'xticklabel',name,'FontWeight','bold');
151 | hold off
152 | title('SNR Level: 7.5dB')
153 | grid on
154 | 
155 | subplot(2,2,3)
156 | databar3=[out2(3),out2(7),out2(11),out2(15),out2(19)];
157 | stdbar3=[out6(3),out6(7),out6(11),out6(15),out6(19)];
158 | bar(x,databar3,'FaceColor', [0.93 0.69 0.13])
159 | hold on
160 | er3=errorbar(x,databar3,stdbar3);
161 | er3.Color = [0 0 0];                            
162 | er3.LineStyle = 'none';  
163 | name={'bus'; 'cafe'; 'living'; 'office'; 'psquare'};
164 | set(gca,'xticklabel',name,'FontWeight','bold');
165 | hold off
166 | title('SNR Level: 12.5dB')
167 | grid on
168 | 
169 | subplot(2,2,4)
170 | databar4=[out2(4),out2(8),out2(12),out2(16),out2(20)];
171 | stdbar4=[out6(4),out6(8),out6(12),out6(16),out6(20)];
172 | bar(x,databar4,'FaceColor', [0.49 0.18 0.56])
173 | hold on
174 | er4=errorbar(x,databar4,stdbar4);
175 | er4.Color = [0 0 0];                            
176 | er4.LineStyle = 'none';  
177 | name={'bus'; 'cafe'; 'living'; 'office'; 'psquare'};
178 | set(gca,'xticklabel',name,'FontWeight','bold');
179 | hold off
180 | title('SNR Level: 17.5dB')
181 | grid on
182 | 
183 | h=suptitle({'System Performance at Specific SNR leves for Different Noise Profiles'
184 |     ['Metric: ',char(cellstr(metricNames(metricType))),' - Modified RNNoise']
185 |     });
186 | set(h,'FontSize',14,'FontWeight','bold')
187 | 


--------------------------------------------------------------------------------
/evaluation/EvalStatFun.m:
--------------------------------------------------------------------------------
 1 | function [out1, out2, out3, out4]=EvalStatFun(Vmatrix,metricType,statType)
 2 | %% Instructions
 3 | % statType:
 4 | % 1 for mean
 5 | % 2 for std
 6 | 
 7 | % metricsType
 8 | % 1 for Csig
 9 | % 2 for Cbak
10 | % 3 for Covl
11 | % 4 for narrowband PESQ
12 | % 5 for wideband PESQ
13 | % 6 for STOI
14 | % 7 for LLR
15 | % 8 for fwSNRseg
16 | % 9 for Snr_mean
17 | % 10 for SegSNR_mean
18 | % 11 for Pers
19 | 
20 | %% How did the system do overall? (all noise types at all SNRs)
21 | Allvalues=Vmatrix(:);
22 | if (statType==1)
23 |     out1=mean(nonzeros(Allvalues));
24 | else
25 |     out1=std(nonzeros(Allvalues));
26 | end
27 | 
28 | %% How did the system do for avery kind of noise for every different SNR level?
29 | 
30 | out2 = zeros(size(Vmatrix,2),1);
31 | for i=1:size(Vmatrix,2)
32 |     if(statType==1)
33 |         out2(i) = mean(nonzeros(Vmatrix(:,i)));
34 |     else
35 |         out2(i) = std(nonzeros(Vmatrix(:,i)));
36 |     end
37 | end
38 | 
39 | %% How did the system do for each SNR level, taking into consideration all types of noise
40 | %BAR??
41 | 
42 | SNR2_5=[Vmatrix(:,1)',Vmatrix(:,5)',Vmatrix(:,9)',Vmatrix(:,13)',Vmatrix(:,17)'];
43 | SNR7_5=[Vmatrix(:,2)',Vmatrix(:,6)',Vmatrix(:,10)',Vmatrix(:,14)',Vmatrix(:,18)'];
44 | SNR12_5=[Vmatrix(:,3)',Vmatrix(:,7)',Vmatrix(:,11)',Vmatrix(:,15)',Vmatrix(:,19)'];
45 | SNR17_5=[Vmatrix(:,4)',Vmatrix(:,8)',Vmatrix(:,12)',Vmatrix(:,16)',Vmatrix(:,20)'];
46 | 
47 | if(statType==1)
48 |     out3=[mean(nonzeros(SNR2_5));mean(nonzeros(SNR7_5));mean(nonzeros(SNR12_5));mean(nonzeros(SNR17_5))];
49 | else
50 |     out3=[std(nonzeros(SNR2_5));std(nonzeros(SNR7_5));std(nonzeros(SNR12_5));std(nonzeros(SNR17_5))];
51 | end
52 | 
53 | %% How did the system do for each noise type, taking into consideration all SNRs
54 | %LINEEE
55 | 
56 | if((metricType~=8)&&(metricType~=9)&&(metricType~=10))
57 |     
58 |     bus=[Vmatrix(:,1)',Vmatrix(:,2)',Vmatrix(:,3)',Vmatrix(:,4)'];
59 |     cafe=[Vmatrix(:,5)',Vmatrix(:,6)',Vmatrix(:,7)',Vmatrix(:,8)'];
60 |     living=[Vmatrix(:,9)',Vmatrix(:,10)',Vmatrix(:,11)',Vmatrix(:,12)'];
61 |     office=[Vmatrix(:,13)',Vmatrix(:,14)',Vmatrix(:,15)',Vmatrix(:,16)'];
62 |     psquare=[Vmatrix(:,17)',Vmatrix(:,18)',Vmatrix(:,19)',Vmatrix(:,20)'];
63 |     
64 |     if(statType==1)
65 |         out4=[mean(nonzeros(bus));mean(nonzeros(cafe));mean(nonzeros(living));mean(nonzeros(office));mean(nonzeros(psquare))];
66 |     else
67 |         out4=[std(nonzeros(bus));std(nonzeros(cafe));std(nonzeros(living));std(nonzeros(office));std(nonzeros(psquare))];
68 |     end
69 |     
70 | else
71 |     out4=zeros(5,1);
72 | end


--------------------------------------------------------------------------------
/evaluation/README.txt:
--------------------------------------------------------------------------------
 1 | %RNNoise and modified
 2 | (1) Downsample noisy/cleaned samples
 3 | 	Downsample_WAV.m
 4 | 	(have to change folder to noisy_wav or clean_wav)
 5 | 	(commented part is for folder "clean" containing original clean data)
 6 | 
 7 | (2) Evaluation
 8 | 	EVALUATION_script.m (uses EvalPlotFun.m and EvalStatFun.m) 
 9 | 	(change folder to store results - end of file)
10 | 
11 | %Classic Methods
12 | (1) CDownsample_WAV_Noisy.m
13 | 	('_down16.wav': creates 16KHz downsampled noisy data)
14 | 
15 | (2) Denoise with Classic Methods
16 | 	CDenoiseClassicMethods.m
17 | 	(choose folder to store results using variable "technic". Also change function in for loop to change method)
18 | 
19 | (3) CDownsample_WAV_Classic.m
20 | 	('_down16_AfterDen.wav': downsampled denoised signals for PESQ and composite)
21 | 
22 | (4) CEVALUATION_script_Classic.m (uses EvalPlotFun.m and EvalStatFun.m)
23 | 	(change folder to store results - end of file and also variable "technic")
24 | 
25 | % We used Loizou's functions (comp_fwseg,comp_llr,comp_snr,composite,logmmse,pesq,wiener_as) from https://www.crcpress.com/downloads/K14513/K14513_CD_Files.zip
26 | %  and some functions (v_readwav,v_writewav,wavread,wavwrite) for audio I/O from VOICEBOX that is under GNU Public Licence (http://www.ee.ic.ac.uk/hp/staff/dmb/voicebox/voicebox.html)
27 | %  The abovementioned functions can be found on the link given.
28 | % The function stoi in the folder is taken from http://insy.ewi.tudelft.nl/content/short-time-objective-intelligibility-measure


--------------------------------------------------------------------------------
/evaluation/spectrograms.m:
--------------------------------------------------------------------------------
 1 | clear all;
 2 | 
 3 | %% 1
 4 | 
 5 | % [y1,fs] = audioread('p232_019clean.wav');
 6 | % y2 = audioread('p232_019noisy.wav');
 7 | % y3 = audioread('p232_019rnnoise.wav');
 8 | % y4 = audioread('p232_019wiener.wav');
 9 | 
10 | 
11 | %% 2
12 | 
13 | % [y1,fs] = audioread('p232_006clean.wav');
14 | % y2 = audioread('p232_006noisy.wav');
15 | % y3 = audioread('p232_006rnnoise2.5.wav');
16 | % y4 = audioread('p232_006wiener.wav');
17 | 
18 | 
19 | %% 3
20 | % [y1,fs] = audioread('p232_019clean.wav');
21 | % y2 = audioread('p232_019noisy.wav');
22 | % y3 = audioread('p232_019rnnoise.wav');
23 | % y4 = audioread('p232_019logmmse.wav');
24 | 
25 | %% 4
26 | [y1,fs] = audioread('p232_006clean.wav');
27 | y2 = audioread('p232_006noisy.wav');
28 | y3 = audioread('p232_006rnnoise.wav');
29 | y4 = audioread('p232_006logmmse.wav');
30 | 
31 | %% 5
32 | % [y1,fs] = audioread('p232_003_clean.wav');
33 | % y2 = audioread('p232_003_bus7.5_noisy.wav');
34 | % y3 = audioread('p232_003_bus7.5_reference.wav');
35 | % y4 = audioread('p232_003_bus7.5_modified.wav');
36 | 
37 | 
38 | %% plot
39 | figure
40 | subplot(221)
41 | spectrogram(y1,2048,1024,[],fs,'yaxis')
42 | ax1 = gca;
43 | ax1.YScale = 'log';
44 | title("Clean Speech")
45 | subplot(222)
46 | spectrogram(y2,2048,1024,[],fs,'yaxis')
47 | ax2 = gca;
48 | ax2.YScale = 'log';
49 | title("Noisy Speech")
50 | subplot(223)
51 | spectrogram(y3,2048,1024,[],fs,'yaxis')
52 | ax3 = gca;
53 | ax3.YScale = 'log';
54 | title("Denoised Speech using RNNoise")
55 | %title("Denoised Speech using Reference RNNoise")
56 | subplot(224)
57 | spectrogram(y4,2048,1024,[],fs,'yaxis')
58 | ax4 = gca;
59 | ax4.YScale = 'log';
60 | %title("Denoised Speech using Wiener Filter")
61 | title("Denoised Speech using logMMSE Method")
62 | %title("Denoised Speech using Modified RNNoise")
63 | % suptitle({"Output Signal Comparison"
64 | %    "Noise Type: OFFICE, SNR: 2.5 dB"})
65 |  suptitle({"Output Signal Comparison"
66 |    "Noise Type: CAFE, SNR: 17.5 dB"})
67 | %  suptitle({"Output Signal Comparison"
68 | %     "Noise Type: BUS, SNR: 7.5 dB"})


--------------------------------------------------------------------------------
/evaluation/stoi.m:
--------------------------------------------------------------------------------
  1 | function d = stoi(x, y, fs_signal)
  2 | %   d = stoi(x, y, fs_signal) returns the output of the short-time
  3 | %   objective intelligibility (STOI) measure described in [1, 2], where x 
  4 | %   and y denote the clean and processed speech, respectively, with sample
  5 | %   rate fs_signal in Hz. The output d is expected to have a monotonic 
  6 | %   relation with the subjective speech-intelligibility, where a higher d 
  7 | %   denotes better intelligible speech. See [1, 2] for more details.
  8 | %
  9 | %   References:
 10 | %      [1] C.H.Taal, R.C.Hendriks, R.Heusdens, J.Jensen 'A Short-Time
 11 | %      Objective Intelligibility Measure for Time-Frequency Weighted Noisy
 12 | %      Speech', ICASSP 2010, Texas, Dallas.
 13 | %
 14 | %      [2] C.H.Taal, R.C.Hendriks, R.Heusdens, J.Jensen 'An Algorithm for 
 15 | %      Intelligibility Prediction of Time-Frequency Weighted Noisy Speech', 
 16 | %      IEEE Transactions on Audio, Speech, and Language Processing, 2011. 
 17 | %
 18 | %
 19 | % Copyright 2009: Delft University of Technology, Signal & Information
 20 | % Processing Lab. The software is free for non-commercial use. This program
 21 | % comes WITHOUT ANY WARRANTY.
 22 | %
 23 | %
 24 | %
 25 | % Updates:
 26 | % 2011-04-26 Using the more efficient 'taa_corr' instead of 'corr'
 27 | 
 28 | if length(x)~=length(y)
 29 |     error('x and y should have the same length');
 30 | end
 31 | 
 32 | % initialization
 33 | x           = x(:);                             % clean speech column vector
 34 | y           = y(:);                             % processed speech column vector
 35 | 
 36 | fs          = 10000;                            % sample rate of proposed intelligibility measure
 37 | N_frame    	= 256;                              % window support
 38 | K           = 512;                              % FFT size
 39 | J           = 15;                               % Number of 1/3 octave bands
 40 | mn          = 150;                              % Center frequency of first 1/3 octave band in Hz.
 41 | H           = thirdoct(fs, K, J, mn);           % Get 1/3 octave band matrix
 42 | N           = 30;                               % Number of frames for intermediate intelligibility measure (Length analysis window)
 43 | Beta        = -15;                           	% lower SDR-bound
 44 | dyn_range   = 40;                               % speech dynamic range
 45 | 
 46 | % resample signals if other samplerate is used than fs
 47 | if fs_signal ~= fs
 48 |     x	= resample(x, fs, fs_signal);
 49 |     y 	= resample(y, fs, fs_signal);
 50 | end
 51 | 
 52 | % remove silent frames
 53 | [x y] = removeSilentFrames(x, y, dyn_range, N_frame, N_frame/2);
 54 | 
 55 | % apply 1/3 octave band TF-decomposition
 56 | x_hat     	= stdft(x, N_frame, N_frame/2, K); 	% apply short-time DFT to clean speech
 57 | y_hat     	= stdft(y, N_frame, N_frame/2, K); 	% apply short-time DFT to processed speech
 58 | 
 59 | x_hat       = x_hat(:, 1:(K/2+1)).';         	% take clean single-sided spectrum
 60 | y_hat       = y_hat(:, 1:(K/2+1)).';        	% take processed single-sided spectrum
 61 | 
 62 | X           = zeros(J, size(x_hat, 2));         % init memory for clean speech 1/3 octave band TF-representation 
 63 | Y           = zeros(J, size(y_hat, 2));         % init memory for processed speech 1/3 octave band TF-representation 
 64 | 
 65 | for i = 1:size(x_hat, 2)
 66 |     X(:, i)	= sqrt(H*abs(x_hat(:, i)).^2);      % apply 1/3 octave bands as described in Eq.(1) [1]
 67 |     Y(:, i)	= sqrt(H*abs(y_hat(:, i)).^2);
 68 | end
 69 | 
 70 | % loop al segments of length N and obtain intermediate intelligibility measure for all TF-regions
 71 | d_interm  	= zeros(J, length(N:size(X, 2)));                               % init memory for intermediate intelligibility measure
 72 | c           = 10^(-Beta/20);                                                % constant for clipping procedure
 73 | 
 74 | for m = N:size(X, 2)
 75 |     X_seg  	= X(:, (m-N+1):m);                                              % region with length N of clean TF-units for all j
 76 |     Y_seg  	= Y(:, (m-N+1):m);                                              % region with length N of processed TF-units for all j
 77 |     alpha   = sqrt(sum(X_seg.^2, 2)./sum(Y_seg.^2, 2));                     % obtain scale factor for normalizing processed TF-region for all j
 78 |     aY_seg 	= Y_seg.*repmat(alpha, [1 N]);                               	% obtain \alpha*Y_j(n) from Eq.(2) [1]
 79 |     for j = 1:J
 80 |       	Y_prime             = min(aY_seg(j, :), X_seg(j, :)+X_seg(j, :)*c); % apply clipping from Eq.(3)   	
 81 |         d_interm(j, m-N+1)  = taa_corr(X_seg(j, :).', Y_prime(:));          % obtain correlation coeffecient from Eq.(4) [1]
 82 |     end
 83 | end
 84 |         
 85 | d = mean(d_interm(:));                                                      % combine all intermediate intelligibility measures as in Eq.(4) [1]
 86 | 
 87 | %%
 88 | function  [A cf] = thirdoct(fs, N_fft, numBands, mn)
 89 | %   [A CF] = THIRDOCT(FS, N_FFT, NUMBANDS, MN) returns 1/3 octave band matrix
 90 | %   inputs:
 91 | %       FS:         samplerate 
 92 | %       N_FFT:      FFT size
 93 | %       NUMBANDS:   number of bands
 94 | %       MN:         center frequency of first 1/3 octave band
 95 | %   outputs:
 96 | %       A:          octave band matrix
 97 | %       CF:         center frequencies
 98 | 
 99 | f               = linspace(0, fs, N_fft+1);
100 | f               = f(1:(N_fft/2+1));
101 | k               = 0:(numBands-1); 
102 | cf              = 2.^(k/3)*mn;
103 | fl              = sqrt((2.^(k/3)*mn).*2.^((k-1)/3)*mn);
104 | fr              = sqrt((2.^(k/3)*mn).*2.^((k+1)/3)*mn);
105 | A               = zeros(numBands, length(f));
106 | 
107 | for i = 1:(length(cf))
108 |     [a b]                   = min((f-fl(i)).^2);
109 |     fl(i)                   = f(b);
110 |     fl_ii                   = b;
111 | 
112 | 	[a b]                   = min((f-fr(i)).^2);
113 |     fr(i)                   = f(b);
114 |     fr_ii                   = b;
115 |     A(i,fl_ii:(fr_ii-1))	= 1;
116 | end
117 | 
118 | rnk         = sum(A, 2);
119 | numBands  	= find((rnk(2:end)>=rnk(1:(end-1))) & (rnk(2:end)~=0)~=0, 1, 'last' )+1;
120 | A           = A(1:numBands, :);
121 | cf          = cf(1:numBands);
122 | 
123 | %%
124 | function x_stdft = stdft(x, N, K, N_fft)
125 | %   X_STDFT = X_STDFT(X, N, K, N_FFT) returns the short-time
126 | %	hanning-windowed dft of X with frame-size N, overlap K and DFT size
127 | %   N_FFT. The columns and rows of X_STDFT denote the frame-index and
128 | %   dft-bin index, respectively.
129 | 
130 | frames      = 1:K:(length(x)-N);
131 | x_stdft     = zeros(length(frames), N_fft);
132 | 
133 | w           = hanning(N);
134 | x           = x(:);
135 | 
136 | for i = 1:length(frames)
137 |     ii              = frames(i):(frames(i)+N-1);
138 | 	x_stdft(i, :) 	= fft(x(ii).*w, N_fft);
139 | end
140 | 
141 | %%
142 | function [x_sil y_sil] = removeSilentFrames(x, y, range, N, K)
143 | %   [X_SIL Y_SIL] = REMOVESILENTFRAMES(X, Y, RANGE, N, K) X and Y
144 | %   are segmented with frame-length N and overlap K, where the maximum energy
145 | %   of all frames of X is determined, say X_MAX. X_SIL and Y_SIL are the
146 | %   reconstructed signals, excluding the frames, where the energy of a frame
147 | %   of X is smaller than X_MAX-RANGE
148 | 
149 | x       = x(:);
150 | y       = y(:);
151 | 
152 | frames  = 1:K:(length(x)-N);
153 | w       = hanning(N);
154 | msk     = zeros(size(frames));
155 | 
156 | for j = 1:length(frames)
157 |     jj      = frames(j):(frames(j)+N-1);
158 |     msk(j) 	= 20*log10(norm(x(jj).*w)./sqrt(N));
159 | end
160 | 
161 | msk     = (msk-max(msk)+range)>0;
162 | count   = 1;
163 | 
164 | x_sil   = zeros(size(x));
165 | y_sil   = zeros(size(y));
166 | 
167 | for j = 1:length(frames)
168 |     if msk(j)
169 |         jj_i            = frames(j):(frames(j)+N-1);
170 |         jj_o            = frames(count):(frames(count)+N-1);
171 |         x_sil(jj_o)     = x_sil(jj_o) + x(jj_i).*w;
172 |         y_sil(jj_o)  	= y_sil(jj_o) + y(jj_i).*w;
173 |         count           = count+1;
174 |     end
175 | end
176 | 
177 | x_sil = x_sil(1:jj_o(end));
178 | y_sil = y_sil(1:jj_o(end));
179 | 
180 | %%
181 | function rho = taa_corr(x, y)
182 | %   RHO = TAA_CORR(X, Y) Returns correlation coeffecient between column
183 | %   vectors x and y. Gives same results as 'corr' from statistics toolbox.
184 | xn    	= x-mean(x);
185 | xn  	= xn/sqrt(sum(xn.^2));
186 | yn   	= y-mean(y);
187 | yn    	= yn/sqrt(sum(yn.^2));
188 | rho   	= sum(xn.*yn);


--------------------------------------------------------------------------------
/evaluation_batch.sh:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | # Bash script to evaluate a given set of audio files using RNNoise
 3 | 
 4 | # Convert all files to raw
 5 | cd noisy_testset_wav
 6 | while IFS=, read -r file scene db; 
 7 | do 
 8 |     # Print currently processed file
 9 |     echo "[$(date +%T)]: Processing $file";
10 | 
11 |     # Convert desired audio track to raw
12 |     sox $file.wav $file.raw
13 | 
14 |     # Extract RNNoise features using the feature extractor
15 |     ../featureExtractor/feature_extractor $file.raw $file.features.f32
16 | 
17 |     # Extract extended features
18 |     python3.8 ../src/featureExtraction.py testing $file.wav $file.extendedFeatures.bin
19 | 
20 |     # Join feature sets using bin2hdf5
21 |     python3.8 ../training/bin2hdf5.py $file.features.f32 $file.extendedFeatures.bin $file.fullFeatures.h5 testing
22 | done < ../log_testset.csv
23 | 
24 | # Clean up work directory
25 | # rm *.f32; rm *.raw; rm *.h5; rm *.bin;
26 | 
27 | cd ..
28 | 
29 | # Run RNNoise
30 | python3.8 ./training/evaluation_batch.py noisy_testset_wav model_ex.hdf5
31 | 
32 | # Categorize files
33 | while IFS=, read -r file scene db; 
34 | do 
35 |     echo "[$(date +%T)]: Processing $file";
36 |     # Create Directories
37 |     mkdir -p $scene/"$db DB"/noisy_wav/
38 |     mkdir -p $scene/"$db DB"/clean_wav/
39 |     # Categorize based on log file
40 |     cp noisy_testset_wav/"$file.wav" $scene/"$db DB"/noisy_wav/"$file.wav"
41 |     cp noisy_testset_wav/"$file.clean.wav" $scene/"$db DB"/clean_wav/"$file.wav"
42 | done < log_testset.csv
43 | 
44 | rm *.clean.wav


--------------------------------------------------------------------------------
/featureExtractor/.deps/.dirstamp:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CedArctic/rnnoise-ex/16affa8f70ec84196b411f042a5682802c6857f2/featureExtractor/.deps/.dirstamp


--------------------------------------------------------------------------------
/featureExtractor/.deps/rnnoise_demo.Po:
--------------------------------------------------------------------------------
 1 | examples/rnnoise_demo.o: examples/rnnoise_demo.c \
 2 |  /usr/include/stdc-predef.h /usr/include/stdio.h \
 3 |  /usr/include/bits/libc-header-start.h /usr/include/features.h \
 4 |  /usr/include/sys/cdefs.h /usr/include/bits/wordsize.h \
 5 |  /usr/include/bits/long-double.h /usr/include/gnu/stubs.h \
 6 |  /usr/include/gnu/stubs-64.h \
 7 |  /usr/lib/gcc/x86_64-pc-linux-gnu/10.2.0/include/stddef.h \
 8 |  /usr/lib/gcc/x86_64-pc-linux-gnu/10.2.0/include/stdarg.h \
 9 |  /usr/include/bits/types.h /usr/include/bits/timesize.h \
10 |  /usr/include/bits/typesizes.h /usr/include/bits/time64.h \
11 |  /usr/include/bits/types/__fpos_t.h /usr/include/bits/types/__mbstate_t.h \
12 |  /usr/include/bits/types/__fpos64_t.h /usr/include/bits/types/__FILE.h \
13 |  /usr/include/bits/types/FILE.h /usr/include/bits/types/struct_FILE.h \
14 |  /usr/include/bits/stdio_lim.h /usr/include/bits/floatn.h \
15 |  /usr/include/bits/floatn-common.h /usr/include/bits/stdio.h \
16 |  include/rnnoise.h
17 | /usr/include/stdc-predef.h:
18 | /usr/include/stdio.h:
19 | /usr/include/bits/libc-header-start.h:
20 | /usr/include/features.h:
21 | /usr/include/sys/cdefs.h:
22 | /usr/include/bits/wordsize.h:
23 | /usr/include/bits/long-double.h:
24 | /usr/include/gnu/stubs.h:
25 | /usr/include/gnu/stubs-64.h:
26 | /usr/lib/gcc/x86_64-pc-linux-gnu/10.2.0/include/stddef.h:
27 | /usr/lib/gcc/x86_64-pc-linux-gnu/10.2.0/include/stdarg.h:
28 | /usr/include/bits/types.h:
29 | /usr/include/bits/timesize.h:
30 | /usr/include/bits/typesizes.h:
31 | /usr/include/bits/time64.h:
32 | /usr/include/bits/types/__fpos_t.h:
33 | /usr/include/bits/types/__mbstate_t.h:
34 | /usr/include/bits/types/__fpos64_t.h:
35 | /usr/include/bits/types/__FILE.h:
36 | /usr/include/bits/types/FILE.h:
37 | /usr/include/bits/types/struct_FILE.h:
38 | /usr/include/bits/stdio_lim.h:
39 | /usr/include/bits/floatn.h:
40 | /usr/include/bits/floatn-common.h:
41 | /usr/include/bits/stdio.h:
42 | include/rnnoise.h:
43 | 


--------------------------------------------------------------------------------
/featureExtractor/.dirstamp:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CedArctic/rnnoise-ex/16affa8f70ec84196b411f042a5682802c6857f2/featureExtractor/.dirstamp


--------------------------------------------------------------------------------
/featureExtractor/.libs/lt-rnnoise_demo:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CedArctic/rnnoise-ex/16affa8f70ec84196b411f042a5682802c6857f2/featureExtractor/.libs/lt-rnnoise_demo


--------------------------------------------------------------------------------
/featureExtractor/.libs/rnnoise_demo:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CedArctic/rnnoise-ex/16affa8f70ec84196b411f042a5682802c6857f2/featureExtractor/.libs/rnnoise_demo


--------------------------------------------------------------------------------
/featureExtractor/denoise.c:
--------------------------------------------------------------------------------
  1 | /* Copyright (c) 2018 Gregor Richards
  2 |  * Copyright (c) 2017 Mozilla */
  3 | /*
  4 |    Redistribution and use in source and binary forms, with or without
  5 |    modification, are permitted provided that the following conditions
  6 |    are met:
  7 | 
  8 |    - Redistributions of source code must retain the above copyright
  9 |    notice, this list of conditions and the following disclaimer.
 10 | 
 11 |    - Redistributions in binary form must reproduce the above copyright
 12 |    notice, this list of conditions and the following disclaimer in the
 13 |    documentation and/or other materials provided with the distribution.
 14 | 
 15 |    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 16 |    ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 17 |    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 18 |    A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR
 19 |    CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 20 |    EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 21 |    PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
 22 |    PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
 23 |    LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 24 |    NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 25 |    SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 26 | */
 27 | 
 28 | #ifdef HAVE_CONFIG_H
 29 | #include "config.h"
 30 | #endif
 31 | 
 32 | #include "denoise.h"
 33 | 
 34 | 
 35 | /* The built-in model, used if no file is given as input */
 36 | extern const struct RNNModel rnnoise_model_orig;
 37 | 
 38 | 
 39 | static const opus_int16 eband5ms[] = {
 40 | /*0  200 400 600 800  1k 1.2 1.4 1.6  2k 2.4 2.8 3.2  4k 4.8 5.6 6.8  8k 9.6 12k 15.6 20k*/
 41 |   0,  1,  2,  3,  4,  5,  6,  7,  8, 10, 12, 14, 16, 20, 24, 28, 34, 40, 48, 60, 78, 100
 42 | };
 43 | 
 44 | 
 45 | typedef struct {
 46 |   int init;
 47 |   kiss_fft_state *kfft;
 48 |   float half_window[FRAME_SIZE];
 49 |   float dct_table[NB_BANDS*NB_BANDS];
 50 | } CommonState;
 51 | 
 52 | void compute_band_energy(float *bandE, const kiss_fft_cpx *X) {
 53 |   int i;
 54 |   float sum[NB_BANDS] = {0};
 55 |   for (i=0;i<NB_BANDS-1;i++)
 56 |   {
 57 |     int j;
 58 |     int band_size;
 59 |     band_size = (eband5ms[i+1]-eband5ms[i])<<FRAME_SIZE_SHIFT;
 60 |     for (j=0;j<band_size;j++) {
 61 |       float tmp;
 62 |       float frac = (float)j/band_size;
 63 |       tmp = SQUARE(X[(eband5ms[i]<<FRAME_SIZE_SHIFT) + j].r);
 64 |       tmp += SQUARE(X[(eband5ms[i]<<FRAME_SIZE_SHIFT) + j].i);
 65 |       sum[i] += (1-frac)*tmp;
 66 |       sum[i+1] += frac*tmp;
 67 |     }
 68 |   }
 69 |   sum[0] *= 2;
 70 |   sum[NB_BANDS-1] *= 2;
 71 |   for (i=0;i<NB_BANDS;i++)
 72 |   {
 73 |     bandE[i] = sum[i];
 74 |   }
 75 | }
 76 | 
 77 | void compute_band_corr(float *bandE, const kiss_fft_cpx *X, const kiss_fft_cpx *P) {
 78 |   int i;
 79 |   float sum[NB_BANDS] = {0};
 80 |   for (i=0;i<NB_BANDS-1;i++)
 81 |   {
 82 |     int j;
 83 |     int band_size;
 84 |     band_size = (eband5ms[i+1]-eband5ms[i])<<FRAME_SIZE_SHIFT;
 85 |     for (j=0;j<band_size;j++) {
 86 |       float tmp;
 87 |       float frac = (float)j/band_size;
 88 |       tmp = X[(eband5ms[i]<<FRAME_SIZE_SHIFT) + j].r * P[(eband5ms[i]<<FRAME_SIZE_SHIFT) + j].r;
 89 |       tmp += X[(eband5ms[i]<<FRAME_SIZE_SHIFT) + j].i * P[(eband5ms[i]<<FRAME_SIZE_SHIFT) + j].i;
 90 |       sum[i] += (1-frac)*tmp;
 91 |       sum[i+1] += frac*tmp;
 92 |     }
 93 |   }
 94 |   sum[0] *= 2;
 95 |   sum[NB_BANDS-1] *= 2;
 96 |   for (i=0;i<NB_BANDS;i++)
 97 |   {
 98 |     bandE[i] = sum[i];
 99 |   }
100 | }
101 | 
102 | void interp_band_gain(float *g, const float *bandE) {
103 |   int i;
104 |   memset(g, 0, FREQ_SIZE);
105 |   for (i=0;i<NB_BANDS-1;i++)
106 |   {
107 |     int j;
108 |     int band_size;
109 |     band_size = (eband5ms[i+1]-eband5ms[i])<<FRAME_SIZE_SHIFT;
110 |     for (j=0;j<band_size;j++) {
111 |       float frac = (float)j/band_size;
112 |       g[(eband5ms[i]<<FRAME_SIZE_SHIFT) + j] = (1-frac)*bandE[i] + frac*bandE[i+1];
113 |     }
114 |   }
115 | }
116 | 
117 | 
118 | CommonState common;
119 | 
120 | static void check_init() {
121 |   int i;
122 |   if (common.init) return;
123 |   common.kfft = opus_fft_alloc_twiddles(2*FRAME_SIZE, NULL, NULL, NULL, 0);
124 |   for (i=0;i<FRAME_SIZE;i++)
125 |     common.half_window[i] = sin(.5*M_PI*sin(.5*M_PI*(i+.5)/FRAME_SIZE) * sin(.5*M_PI*(i+.5)/FRAME_SIZE));
126 |   for (i=0;i<NB_BANDS;i++) {
127 |     int j;
128 |     for (j=0;j<NB_BANDS;j++) {
129 |       common.dct_table[i*NB_BANDS + j] = cos((i+.5)*j*M_PI/NB_BANDS);
130 |       if (j==0) common.dct_table[i*NB_BANDS + j] *= sqrt(.5);
131 |     }
132 |   }
133 |   common.init = 1;
134 | }
135 | 
136 | static void dct(float *out, const float *in) {
137 |   int i;
138 |   check_init();
139 |   for (i=0;i<NB_BANDS;i++) {
140 |     int j;
141 |     float sum = 0;
142 |     for (j=0;j<NB_BANDS;j++) {
143 |       sum += in[j] * common.dct_table[j*NB_BANDS + i];
144 |     }
145 |     out[i] = sum*sqrt(2./22);
146 |   }
147 | }
148 | 
149 | #if 0
150 | static void idct(float *out, const float *in) {
151 |   int i;
152 |   check_init();
153 |   for (i=0;i<NB_BANDS;i++) {
154 |     int j;
155 |     float sum = 0;
156 |     for (j=0;j<NB_BANDS;j++) {
157 |       sum += in[j] * common.dct_table[i*NB_BANDS + j];
158 |     }
159 |     out[i] = sum*sqrt(2./22);
160 |   }
161 | }
162 | #endif
163 | 
164 | static void forward_transform(kiss_fft_cpx *out, const float *in) {
165 |   int i;
166 |   kiss_fft_cpx x[WINDOW_SIZE];
167 |   kiss_fft_cpx y[WINDOW_SIZE];
168 |   check_init();
169 |   for (i=0;i<WINDOW_SIZE;i++) {
170 |     x[i].r = in[i];
171 |     x[i].i = 0;
172 |   }
173 |   opus_fft(common.kfft, x, y, 0);
174 |   for (i=0;i<FREQ_SIZE;i++) {
175 |     out[i] = y[i];
176 |   }
177 | }
178 | 
179 | static void inverse_transform(float *out, const kiss_fft_cpx *in) {
180 |   int i;
181 |   kiss_fft_cpx x[WINDOW_SIZE];
182 |   kiss_fft_cpx y[WINDOW_SIZE];
183 |   check_init();
184 |   for (i=0;i<FREQ_SIZE;i++) {
185 |     x[i] = in[i];
186 |   }
187 |   for (;i<WINDOW_SIZE;i++) {
188 |     x[i].r = x[WINDOW_SIZE - i].r;
189 |     x[i].i = -x[WINDOW_SIZE - i].i;
190 |   }
191 |   opus_fft(common.kfft, x, y, 0);
192 |   /* output in reverse order for IFFT. */
193 |   out[0] = WINDOW_SIZE*y[0].r;
194 |   for (i=1;i<WINDOW_SIZE;i++) {
195 |     out[i] = WINDOW_SIZE*y[WINDOW_SIZE - i].r;
196 |   }
197 | }
198 | 
199 | static void apply_window(float *x) {
200 |   int i;
201 |   check_init();
202 |   for (i=0;i<FRAME_SIZE;i++) {
203 |     x[i] *= common.half_window[i];
204 |     x[WINDOW_SIZE - 1 - i] *= common.half_window[i];
205 |   }
206 | }
207 | 
208 | int rnnoise_get_size() {
209 |   return sizeof(DenoiseState);
210 | }
211 | 
212 | int rnnoise_init(DenoiseState *st, RNNModel *model) {
213 |   memset(st, 0, sizeof(*st));
214 |   if (model)
215 |     st->rnn.model = model;
216 |   else
217 |     st->rnn.model = &rnnoise_model_orig;
218 |   st->rnn.vad_gru_state = calloc(sizeof(float), st->rnn.model->vad_gru_size);
219 |   st->rnn.noise_gru_state = calloc(sizeof(float), st->rnn.model->noise_gru_size);
220 |   st->rnn.denoise_gru_state = calloc(sizeof(float), st->rnn.model->denoise_gru_size);
221 |   return 0;
222 | }
223 | 
224 | DenoiseState *rnnoise_create(RNNModel *model) {
225 |   DenoiseState *st;
226 |   st = malloc(rnnoise_get_size());
227 |   rnnoise_init(st, model);
228 |   return st;
229 | }
230 | 
231 | void rnnoise_destroy(DenoiseState *st) {
232 |   free(st->rnn.vad_gru_state);
233 |   free(st->rnn.noise_gru_state);
234 |   free(st->rnn.denoise_gru_state);
235 |   free(st);
236 | }
237 | 
238 | #if TRAINING
239 | int lowpass = FREQ_SIZE;
240 | int band_lp = NB_BANDS;
241 | #endif
242 | 
243 | static void frame_analysis(DenoiseState *st, kiss_fft_cpx *X, float *Ex, const float *in) {
244 |   int i;
245 |   float x[WINDOW_SIZE];
246 |   RNN_COPY(x, st->analysis_mem, FRAME_SIZE);
247 |   for (i=0;i<FRAME_SIZE;i++) x[FRAME_SIZE + i] = in[i];
248 |   RNN_COPY(st->analysis_mem, in, FRAME_SIZE);
249 | 
250 |   // Apply Vorbis window
251 |   apply_window(x);
252 | 
253 |   // Fourier Transform FFT
254 |   forward_transform(X, x);
255 | #if TRAINING
256 |   for (i=lowpass;i<FREQ_SIZE;i++)
257 |     X[i].r = X[i].i = 0;
258 | #endif
259 | 
260 |   // Compute band energy
261 |   compute_band_energy(Ex, X);
262 | }
263 | 
264 | int compute_frame_features(DenoiseState *st, kiss_fft_cpx *X, kiss_fft_cpx *P,
265 |                                   float *Ex, float *Ep, float *Exp, float *features, const float *in) {
266 |   int i;
267 |   float E = 0;
268 |   float *ceps_0, *ceps_1, *ceps_2;
269 |   float spec_variability = 0;
270 |   float Ly[NB_BANDS];
271 |   float p[WINDOW_SIZE];
272 |   float pitch_buf[PITCH_BUF_SIZE>>1];
273 |   int pitch_index;
274 |   float gain;
275 |   float *(pre[1]);
276 |   // float** pre;
277 |   float tmp[NB_BANDS];
278 |   float follow, logMax;
279 | 
280 |   // FFT and energy
281 |   frame_analysis(st, X, Ex, in);
282 | 
283 |   // Copy last part of pitch buffer to the beginning and fill the rest with noisy frame
284 |   RNN_MOVE(st->pitch_buf, &st->pitch_buf[FRAME_SIZE], PITCH_BUF_SIZE-FRAME_SIZE);
285 |   RNN_COPY(&st->pitch_buf[PITCH_BUF_SIZE-FRAME_SIZE], in, FRAME_SIZE);
286 | 
287 |   // Pointer to pitch buffer
288 |   pre[0] = &st->pitch_buf[0];
289 | 
290 |   // Find pitch
291 |   pitch_downsample(pre, pitch_buf, PITCH_BUF_SIZE, 1);
292 |   pitch_search(pitch_buf+(PITCH_MAX_PERIOD>>1), pitch_buf, PITCH_FRAME_SIZE,
293 |                PITCH_MAX_PERIOD-3*PITCH_MIN_PERIOD, &pitch_index);
294 |   pitch_index = PITCH_MAX_PERIOD-pitch_index;
295 | 
296 | 
297 |   gain = remove_doubling(pitch_buf, PITCH_MAX_PERIOD, PITCH_MIN_PERIOD,
298 |           PITCH_FRAME_SIZE, &pitch_index, st->last_period, st->last_gain);
299 |   st->last_period = pitch_index;
300 |   st->last_gain = gain;
301 | 
302 |   // x(n) = x(n-T)
303 |   for (i=0;i<WINDOW_SIZE;i++)
304 |     p[i] = st->pitch_buf[PITCH_BUF_SIZE-WINDOW_SIZE-pitch_index+i];
305 |     // p[i] = st->pitch_buf[pitch_index_old + i];
306 | 
307 | 
308 |   apply_window(p);
309 |   forward_transform(P, p);
310 |   compute_band_energy(Ep, P);
311 |   compute_band_corr(Exp, X, P);
312 |   for (i=0;i<NB_BANDS;i++) Exp[i] = Exp[i]/sqrt(.001+Ex[i]*Ep[i]);
313 |   
314 |   dct(tmp, Exp);
315 |   for (i=0;i<NB_DELTA_CEPS;i++) features[NB_BANDS+2*NB_DELTA_CEPS+i] = tmp[i];
316 |   features[NB_BANDS+2*NB_DELTA_CEPS] -= 1.3;
317 |   features[NB_BANDS+2*NB_DELTA_CEPS+1] -= 0.9;
318 |   features[NB_BANDS+3*NB_DELTA_CEPS] = .01*(pitch_index-300);
319 |   logMax = -2;
320 |   follow = -2;
321 |   for (i=0;i<NB_BANDS;i++) {
322 |     Ly[i] = log10(1e-2+Ex[i]);
323 |     Ly[i] = MAX16(logMax-7, MAX16(follow-1.5, Ly[i]));
324 |     logMax = MAX16(logMax, Ly[i]);
325 |     follow = MAX16(follow-1.5, Ly[i]);
326 |     E += Ex[i];
327 |   }
328 |   if (!TRAINING && E < 0.04) {
329 |     /* If there's no audio, avoid messing up the state. */
330 |     RNN_CLEAR(features, NB_FEATURES);
331 |     return 1;
332 |   }
333 |   dct(features, Ly);
334 |   features[0] -= 12;
335 |   features[1] -= 4;
336 |   ceps_0 = st->cepstral_mem[st->memid];
337 |   ceps_1 = (st->memid < 1) ? st->cepstral_mem[CEPS_MEM+st->memid-1] : st->cepstral_mem[st->memid-1];
338 |   ceps_2 = (st->memid < 2) ? st->cepstral_mem[CEPS_MEM+st->memid-2] : st->cepstral_mem[st->memid-2];
339 |   for (i=0;i<NB_BANDS;i++) ceps_0[i] = features[i];
340 |   st->memid++;
341 |   for (i=0;i<NB_DELTA_CEPS;i++) {
342 |     features[i] = ceps_0[i] + ceps_1[i] + ceps_2[i];
343 |     features[NB_BANDS+i] = ceps_0[i] - ceps_2[i];
344 |     features[NB_BANDS+NB_DELTA_CEPS+i] =  ceps_0[i] - 2*ceps_1[i] + ceps_2[i];
345 |   }
346 |   /* Spectral variability features. */
347 |   if (st->memid == CEPS_MEM) st->memid = 0;
348 |   for (i=0;i<CEPS_MEM;i++)
349 |   {
350 |     int j;
351 |     float mindist = 1e15f;
352 |     for (j=0;j<CEPS_MEM;j++)
353 |     {
354 |       int k;
355 |       float dist=0;
356 |       for (k=0;k<NB_BANDS;k++)
357 |       {
358 |         float tmp;
359 |         tmp = st->cepstral_mem[i][k] - st->cepstral_mem[j][k];
360 |         dist += tmp*tmp;
361 |       }
362 |       if (j!=i)
363 |         mindist = MIN32(mindist, dist);
364 |     }
365 |     spec_variability += mindist;
366 |   }
367 |   features[NB_BANDS+3*NB_DELTA_CEPS+1] = spec_variability/CEPS_MEM-2.1;
368 |   return TRAINING && E < 0.1;
369 | }
370 | 
371 | static void frame_synthesis(DenoiseState *st, float *out, const kiss_fft_cpx *y) {
372 |   float x[WINDOW_SIZE];
373 |   int i;
374 |   inverse_transform(x, y);
375 |   apply_window(x);
376 |   for (i=0;i<FRAME_SIZE;i++) out[i] = x[i] + st->synthesis_mem[i];
377 |   RNN_COPY(st->synthesis_mem, &x[FRAME_SIZE], FRAME_SIZE);
378 | }
379 | 
380 | void biquad(float *y, float mem[2], const float *x, const float *b, const float *a, int N) {
381 |   int i;
382 |   for (i=0;i<N;i++) {
383 |     float xi, yi;
384 |     xi = x[i];
385 |     yi = x[i] + mem[0];
386 |     mem[0] = mem[1] + (b[0]*(double)xi - a[0]*(double)yi);
387 |     mem[1] = (b[1]*(double)xi - a[1]*(double)yi);
388 |     y[i] = yi;
389 |   }
390 | }
391 | 
392 | void pitch_filter(kiss_fft_cpx *X, const kiss_fft_cpx *P, const float *Ex, const float *Ep,
393 |                   const float *Exp, const float *g) {
394 |   int i;
395 |   float r[NB_BANDS];
396 |   float rf[FREQ_SIZE] = {0};
397 |   for (i=0;i<NB_BANDS;i++) {
398 | #if 0
399 |     if (Exp[i]>g[i]) r[i] = 1;
400 |     else r[i] = Exp[i]*(1-g[i])/(.001 + g[i]*(1-Exp[i]));
401 |     r[i] = MIN16(1, MAX16(0, r[i]));
402 | #else
403 |     if (Exp[i]>g[i]) r[i] = 1;
404 |     else r[i] = SQUARE(Exp[i])*(1-SQUARE(g[i]))/(.001 + SQUARE(g[i])*(1-SQUARE(Exp[i])));
405 |     r[i] = sqrt(MIN16(1, MAX16(0, r[i])));
406 | #endif
407 |     r[i] *= sqrt(Ex[i]/(1e-8+Ep[i]));
408 |   }
409 |   interp_band_gain(rf, r);
410 |   for (i=0;i<FREQ_SIZE;i++) {
411 |     X[i].r += rf[i]*P[i].r;
412 |     X[i].i += rf[i]*P[i].i;
413 |   }
414 |   float newE[NB_BANDS];
415 |   compute_band_energy(newE, X);
416 |   float norm[NB_BANDS];
417 |   float normf[FREQ_SIZE]={0};
418 |   for (i=0;i<NB_BANDS;i++) {
419 |     norm[i] = sqrt(Ex[i]/(1e-8+newE[i]));
420 |   }
421 |   interp_band_gain(normf, norm);
422 |   for (i=0;i<FREQ_SIZE;i++) {
423 |     X[i].r *= normf[i];
424 |     X[i].i *= normf[i];
425 |   }
426 | }
427 | 
428 | float rnnoise_process_frame(DenoiseState *st, float *out, const float *in) {
429 |   int i;
430 |   kiss_fft_cpx X[FREQ_SIZE];
431 |   kiss_fft_cpx P[WINDOW_SIZE];
432 |   float x[FRAME_SIZE];
433 |   float Ex[NB_BANDS], Ep[NB_BANDS];
434 |   float Exp[NB_BANDS];
435 |   float features[NB_FEATURES];
436 |   float g[NB_BANDS];
437 |   float gf[FREQ_SIZE]={1};
438 |   float vad_prob = 0;
439 |   int silence;
440 |   static const float a_hp[2] = {-1.99599, 0.99600};
441 |   static const float b_hp[2] = {-2, 1};
442 |   biquad(x, st->mem_hp_x, in, b_hp, a_hp, FRAME_SIZE);
443 | 
444 |   silence = compute_frame_features(st, X, P, Ex, Ep, Exp, features, x);
445 | 
446 |   if (!silence) {
447 |     
448 |     compute_rnn(&st->rnn, g, &vad_prob, features);
449 | 
450 |     pitch_filter(X, P, Ex, Ep, Exp, g);
451 |     for (i=0;i<NB_BANDS;i++) {
452 |       float alpha = .6f;
453 |       g[i] = MAX16(g[i], alpha*st->lastg[i]);
454 |       st->lastg[i] = g[i];
455 |     }
456 |     interp_band_gain(gf, g);
457 | #if 1
458 |     for (i=0;i<FREQ_SIZE;i++) {
459 |       X[i].r *= gf[i];
460 |       X[i].i *= gf[i];
461 |     }
462 | #endif
463 |   }
464 | 
465 |   frame_synthesis(st, out, X);
466 |   return vad_prob;
467 | }
468 | 
469 | #if TRAINING
470 | 
471 | static float uni_rand() {
472 |   return rand()/(double)RAND_MAX-.5;
473 | }
474 | 
475 | static void rand_resp(float *a, float *b) {
476 |   a[0] = .75*uni_rand();
477 |   a[1] = .75*uni_rand();
478 |   b[0] = .75*uni_rand();
479 |   b[1] = .75*uni_rand();
480 | }
481 | #endif
482 | 


--------------------------------------------------------------------------------
/featureExtractor/denoise.h:
--------------------------------------------------------------------------------
 1 | #ifndef DENOISE_H
 2 | #define DENOISE_H
 3 | 
 4 | #define FRAME_SIZE_SHIFT 2
 5 | #define FRAME_SIZE (120<<FRAME_SIZE_SHIFT)
 6 | #define WINDOW_SIZE (2*FRAME_SIZE)
 7 | #define FREQ_SIZE (FRAME_SIZE + 1)
 8 | 
 9 | #define PITCH_MIN_PERIOD 60
10 | #define PITCH_MAX_PERIOD 768
11 | #define PITCH_FRAME_SIZE 960
12 | #define PITCH_BUF_SIZE (PITCH_MAX_PERIOD+PITCH_FRAME_SIZE)
13 | 
14 | #define SQUARE(x) ((x)*(x))
15 | 
16 | #define NB_BANDS 22
17 | 
18 | #define CEPS_MEM 8
19 | #define NB_DELTA_CEPS 6
20 | 
21 | #define NB_FEATURES (NB_BANDS+3*NB_DELTA_CEPS+2)
22 | 
23 | 
24 | #ifndef TRAINING
25 | #define TRAINING 0
26 | #endif
27 | 
28 | #include <stdlib.h>
29 | #include <string.h>
30 | #include <stdio.h>
31 | #include "../src/kiss_fft.h"
32 | #include "../src/common.h"
33 | #include <math.h>
34 | #include "rnnoise.h"
35 | #include "../src/pitch.h"
36 | #include "../src/arch.h"
37 | #include "../src/rnn.h"
38 | #include "../src/rnn_data.h"
39 | 
40 | struct DenoiseState {
41 |   float analysis_mem[FRAME_SIZE];
42 |   float cepstral_mem[CEPS_MEM][NB_BANDS];
43 |   int memid;
44 |   float synthesis_mem[FRAME_SIZE];
45 |   float pitch_buf[PITCH_BUF_SIZE];
46 |   float pitch_enh_buf[PITCH_BUF_SIZE];
47 |   float last_gain;
48 |   int last_period;
49 |   float mem_hp_x[2];
50 |   float lastg[NB_BANDS];
51 |   RNNState rnn;
52 | };
53 | 
54 | int compute_frame_features(DenoiseState *st, kiss_fft_cpx *X, kiss_fft_cpx *P,
55 |                                   float *Ex, float *Ep, float *Exp, float *features, const float *in);
56 | 
57 | void biquad(float *y, float mem[2], const float *x, const float *b, const float *a, int N);
58 | 
59 | #endif


--------------------------------------------------------------------------------
/featureExtractor/denoise.o:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CedArctic/rnnoise-ex/16affa8f70ec84196b411f042a5682802c6857f2/featureExtractor/denoise.o


--------------------------------------------------------------------------------
/featureExtractor/feature_extractor:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/CedArctic/rnnoise-ex/16affa8f70ec84196b411f042a5682802c6857f2/featureExtractor/feature_extractor


--------------------------------------------------------------------------------
/featureExtractor/feature_extractor.c:
--------------------------------------------------------------------------------
  1 | /* Copyright (c) 2018 Gregor Richards
  2 |  * Copyright (c) 2017 Mozilla */
  3 | /*
  4 |    Redistribution and use in source and binary forms, with or without
  5 |    modification, are permitted provided that the following conditions
  6 |    are met:
  7 | 
  8 |    - Redistributions of source code must retain the above copyright
  9 |    notice, this list of conditions and the following disclaimer.
 10 | 
 11 |    - Redistributions in binary form must reproduce the above copyright
 12 |    notice, this list of conditions and the following disclaimer in the
 13 |    documentation and/or other materials provided with the distribution.
 14 | 
 15 |    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 16 |    ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 17 |    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 18 |    A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR
 19 |    CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 20 |    EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 21 |    PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
 22 |    PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
 23 |    LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 24 |    NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 25 |    SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 26 | */
 27 | 
 28 | #include <stdio.h>
 29 | #include <stdlib.h>
 30 | #include <string.h>
 31 | #include <math.h>
 32 | 
 33 | #include "rnnoise.h"
 34 | #include "../config.h"
 35 | #include "../src/kiss_fft.h"
 36 | #include "../src/common.h"
 37 | #include "../src/pitch.h"
 38 | #include "../src/arch.h"
 39 | #include "../src/rnn.h"
 40 | #include "../src/rnn_data.h"
 41 | #include "../src/denoise.h"
 42 | 
 43 | 
 44 | #define NB_BANDS 22
 45 | #define NB_DELTA_CEPS 6
 46 | #define NB_FEATURES (NB_BANDS+3*NB_DELTA_CEPS+2)
 47 | #define FRAME_SIZE_SHIFT 2
 48 | #define FRAME_SIZE (120<<FRAME_SIZE_SHIFT)
 49 | #define WINDOW_SIZE (2*FRAME_SIZE)
 50 | #define FREQ_SIZE (FRAME_SIZE + 1)
 51 | 
 52 | void feature_extraction(DenoiseState *st, float *out, const float *in, float* features) {
 53 |   kiss_fft_cpx X[FREQ_SIZE];
 54 |   kiss_fft_cpx P[WINDOW_SIZE];
 55 |   float x[FRAME_SIZE];
 56 |   float Ex[NB_BANDS], Ep[NB_BANDS];
 57 |   float Exp[NB_BANDS];
 58 |   
 59 |   static const float a_hp[2] = {-1.99599, 0.99600};
 60 |   static const float b_hp[2] = {-2, 1};
 61 | 
 62 |   biquad(x, st->mem_hp_x, in, b_hp, a_hp, FRAME_SIZE);
 63 | 
 64 |   compute_frame_features(st, X, P, Ex, Ep, Exp, features, x);
 65 | 
 66 |   return;
 67 | }
 68 | 
 69 | int main(int argc, char **argv) {
 70 | 
 71 |   float x[FRAME_SIZE];
 72 |   FILE *f1, *fout;
 73 | 
 74 |   float features[NB_FEATURES];
 75 | 
 76 |   // Create and allocate state
 77 |   DenoiseState *st;
 78 |   st = rnnoise_create(NULL);
 79 | 
 80 |   // Invalid input error message
 81 |   if (argc!=3) {
 82 |     fprintf(stderr, "usage: %s <noisy speech.raw> <features output.f32>\n", argv[0]);
 83 |     return 1;
 84 |   }
 85 | 
 86 |   // Open file I/O streams
 87 |   f1 = fopen(argv[1], "rb");
 88 |   fout = fopen(argv[2], "wb");
 89 | 
 90 |   while (1) {
 91 | 
 92 |     // Allocate temporary space for the current frame and load it to memory
 93 |     short tmp[FRAME_SIZE];
 94 |     fread(tmp, sizeof(short), FRAME_SIZE, f1);
 95 |     
 96 |     // Check if we reached end of file
 97 |     if (feof(f1)) break;
 98 |     
 99 |     // Copy frame to x[]
100 |     for (int i=0;i<FRAME_SIZE;i++){
101 |       x[i] = tmp[i];
102 |     }
103 |     
104 |     // Process frame with RNNoise
105 |     // rnnoise_process_frame(st, x, x);
106 |     feature_extraction(st, x, x, features);
107 | 
108 |     // Write out features
109 |     fwrite(features, sizeof(float), NB_FEATURES, fout);
110 |   }
111 | 
112 |   // Destructors & File I/O close
113 |   rnnoise_destroy(st);
114 |   fclose(f1);
115 |   fclose(fout);
116 |   return 0;
117 | }


--------------------------------------------------------------------------------
/log_testset.csv:
--------------------------------------------------------------------------------
  1 | p257_001, bus, 17.5,
  2 | p257_002, bus, 12.5,
  3 | p257_003, bus, 7.5,
  4 | p257_004, bus, 2.5,
  5 | p257_006, cafe, 17.5,
  6 | p257_007, cafe, 12.5,
  7 | p257_008, cafe, 7.5,
  8 | p257_009, cafe, 2.5,
  9 | p257_010, living, 17.5,
 10 | p257_011, living, 12.5,
 11 | p257_012, living, 7.5,
 12 | p257_013, living, 2.5,
 13 | p257_014, office, 17.5,
 14 | p257_015, office, 12.5,
 15 | p257_016, office, 7.5,
 16 | p257_017, office, 2.5,
 17 | p257_018, psquare, 17.5,
 18 | p257_019, psquare, 12.5,
 19 | p257_020, psquare, 7.5,
 20 | p257_022, psquare, 2.5,
 21 | p257_023, bus, 17.5,
 22 | p257_024, bus, 12.5,
 23 | p257_025, bus, 7.5,
 24 | p257_026, bus, 2.5,
 25 | p257_027, cafe, 17.5,
 26 | p257_028, cafe, 12.5,
 27 | p257_029, cafe, 7.5,
 28 | p257_030, cafe, 2.5,
 29 | p257_031, living, 17.5,
 30 | p257_032, living, 12.5,
 31 | p257_033, living, 7.5,
 32 | p257_034, living, 2.5,
 33 | p257_035, office, 17.5,
 34 | p257_036, office, 12.5,
 35 | p257_037, office, 7.5,
 36 | p257_038, office, 2.5,
 37 | p257_039, psquare, 17.5,
 38 | p257_040, psquare, 12.5,
 39 | p257_041, psquare, 7.5,
 40 | p257_042, psquare, 2.5,
 41 | p257_043, bus, 17.5,
 42 | p257_044, bus, 12.5,
 43 | p257_045, bus, 7.5,
 44 | p257_046, bus, 2.5,
 45 | p257_047, cafe, 17.5,
 46 | p257_048, cafe, 12.5,
 47 | p257_049, cafe, 7.5,
 48 | p257_050, cafe, 2.5,
 49 | p257_051, living, 17.5,
 50 | p257_052, living, 12.5,
 51 | p257_053, living, 7.5,
 52 | p257_054, living, 2.5,
 53 | p257_055, office, 17.5,
 54 | p257_056, office, 12.5,
 55 | p257_057, office, 7.5,
 56 | p257_058, office, 2.5,
 57 | p257_059, psquare, 17.5,
 58 | p257_060, psquare, 12.5,
 59 | p257_061, psquare, 7.5,
 60 | p257_062, psquare, 2.5,
 61 | p257_063, bus, 17.5,
 62 | p257_064, bus, 12.5,
 63 | p257_065, bus, 7.5,
 64 | p257_066, bus, 2.5,
 65 | p257_067, cafe, 17.5,
 66 | p257_068, cafe, 12.5,
 67 | p257_069, cafe, 7.5,
 68 | p257_070, cafe, 2.5,
 69 | p257_071, living, 17.5,
 70 | p257_072, living, 12.5,
 71 | p257_073, living, 7.5,
 72 | p257_074, living, 2.5,
 73 | p257_075, office, 17.5,
 74 | p257_076, office, 12.5,
 75 | p257_077, office, 7.5,
 76 | p257_078, office, 2.5,
 77 | p257_079, psquare, 17.5,
 78 | p257_080, psquare, 12.5,
 79 | p257_081, psquare, 7.5,
 80 | p257_082, psquare, 2.5,
 81 | p257_083, bus, 17.5,
 82 | p257_084, bus, 12.5,
 83 | p257_085, bus, 7.5,
 84 | p257_086, bus, 2.5,
 85 | p257_087, cafe, 17.5,
 86 | p257_088, cafe, 12.5,
 87 | p257_089, cafe, 7.5,
 88 | p257_090, cafe, 2.5,
 89 | p257_091, living, 17.5,
 90 | p257_092, living, 12.5,
 91 | p257_093, living, 7.5,
 92 | p257_094, living, 2.5,
 93 | p257_095, office, 17.5,
 94 | p257_096, office, 12.5,
 95 | p257_097, office, 7.5,
 96 | p257_098, office, 2.5,
 97 | p257_099, psquare, 17.5,
 98 | p257_100, psquare, 12.5,
 99 | p257_101, psquare, 7.5,
100 | p257_102, psquare, 2.5,
101 | p257_103, bus, 17.5,
102 | p257_104, bus, 12.5,
103 | p257_105, bus, 7.5,
104 | p257_106, bus, 2.5,
105 | p257_107, cafe, 17.5,
106 | p257_108, cafe, 12.5,
107 | p257_109, cafe, 7.5,
108 | p257_110, cafe, 2.5,
109 | p257_111, living, 17.5,
110 | p257_112, living, 12.5,
111 | p257_113, living, 7.5,
112 | p257_114, living, 2.5,
113 | p257_115, office, 17.5,
114 | p257_116, office, 12.5,
115 | p257_117, office, 7.5,
116 | p257_118, office, 2.5,
117 | p257_119, psquare, 17.5,
118 | p257_120, psquare, 12.5,
119 | p257_121, psquare, 7.5,
120 | p257_122, psquare, 2.5,
121 | p257_123, bus, 17.5,
122 | p257_124, bus, 12.5,
123 | p257_125, bus, 7.5,
124 | p257_126, bus, 2.5,
125 | p257_127, cafe, 17.5,
126 | p257_128, cafe, 12.5,
127 | p257_129, cafe, 7.5,
128 | p257_130, cafe, 2.5,
129 | p257_131, living, 17.5,
130 | p257_132, living, 12.5,
131 | p257_133, living, 7.5,
132 | p257_135, living, 2.5,
133 | p257_136, office, 17.5,
134 | p257_137, office, 12.5,
135 | p257_138, office, 7.5,
136 | p257_139, office, 2.5,
137 | p257_140, psquare, 17.5,
138 | p257_141, psquare, 12.5,
139 | p257_142, psquare, 7.5,
140 | p257_143, psquare, 2.5,
141 | p257_144, bus, 17.5,
142 | p257_145, bus, 12.5,
143 | p257_146, bus, 7.5,
144 | p257_147, bus, 2.5,
145 | p257_148, cafe, 17.5,
146 | p257_149, cafe, 12.5,
147 | p257_150, cafe, 7.5,
148 | p257_151, cafe, 2.5,
149 | p257_152, living, 17.5,
150 | p257_153, living, 12.5,
151 | p257_154, living, 7.5,
152 | p257_155, living, 2.5,
153 | p257_156, office, 17.5,
154 | p257_157, office, 12.5,
155 | p257_158, office, 7.5,
156 | p257_159, office, 2.5,
157 | p257_160, psquare, 17.5,
158 | p257_161, psquare, 12.5,
159 | p257_162, psquare, 7.5,
160 | p257_163, psquare, 2.5,
161 | p257_164, bus, 17.5,
162 | p257_165, bus, 12.5,
163 | p257_166, bus, 7.5,
164 | p257_167, bus, 2.5,
165 | p257_168, cafe, 17.5,
166 | p257_169, cafe, 12.5,
167 | p257_170, cafe, 7.5,
168 | p257_171, cafe, 2.5,
169 | p257_172, living, 17.5,
170 | p257_173, living, 12.5,
171 | p257_174, living, 7.5,
172 | p257_175, living, 2.5,
173 | p257_176, office, 17.5,
174 | p257_177, office, 12.5,
175 | p257_178, office, 7.5,
176 | p257_179, office, 2.5,
177 | p257_180, psquare, 17.5,
178 | p257_181, psquare, 12.5,
179 | p257_182, psquare, 7.5,
180 | p257_183, psquare, 2.5,
181 | p257_184, bus, 17.5,
182 | p257_185, bus, 12.5,
183 | p257_186, bus, 7.5,
184 | p257_187, bus, 2.5,
185 | p257_188, cafe, 17.5,
186 | p257_189, cafe, 12.5,
187 | p257_190, cafe, 7.5,
188 | p257_191, cafe, 2.5,
189 | p257_192, living, 17.5,
190 | p257_193, living, 12.5,
191 | p257_194, living, 7.5,
192 | p257_195, living, 2.5,
193 | p257_196, office, 17.5,
194 | p257_197, office, 12.5,
195 | p257_198, office, 7.5,
196 | p257_199, office, 2.5,
197 | p257_200, psquare, 17.5,
198 | p257_201, psquare, 12.5,
199 | p257_202, psquare, 7.5,
200 | p257_203, psquare, 2.5,
201 | p257_204, bus, 17.5,
202 | p257_205, bus, 12.5,
203 | p257_206, bus, 7.5,
204 | p257_207, bus, 2.5,
205 | p257_208, cafe, 17.5,
206 | p257_209, cafe, 12.5,
207 | p257_210, cafe, 7.5,
208 | p257_211, cafe, 2.5,
209 | p257_212, living, 17.5,
210 | p257_213, living, 12.5,
211 | p257_214, living, 7.5,
212 | p257_215, living, 2.5,
213 | p257_216, office, 17.5,
214 | p257_217, office, 12.5,
215 | p257_218, office, 7.5,
216 | p257_219, office, 2.5,
217 | p257_220, psquare, 17.5,
218 | p257_221, psquare, 12.5,
219 | p257_222, psquare, 7.5,
220 | p257_223, psquare, 2.5,
221 | p257_224, bus, 17.5,
222 | p257_225, bus, 12.5,
223 | p257_226, bus, 7.5,
224 | p257_227, bus, 2.5,
225 | p257_228, cafe, 17.5,
226 | p257_229, cafe, 12.5,
227 | p257_230, cafe, 7.5,
228 | p257_231, cafe, 2.5,
229 | p257_232, living, 17.5,
230 | p257_233, living, 12.5,
231 | p257_234, living, 7.5,
232 | p257_235, living, 2.5,
233 | p257_236, office, 17.5,
234 | p257_237, office, 12.5,
235 | p257_238, office, 7.5,
236 | p257_239, office, 2.5,
237 | p257_240, psquare, 17.5,
238 | p257_241, psquare, 12.5,
239 | p257_242, psquare, 7.5,
240 | p257_243, psquare, 2.5,
241 | p257_244, bus, 17.5,
242 | p257_245, bus, 12.5,
243 | p257_246, bus, 7.5,
244 | p257_247, bus, 2.5,
245 | p257_248, cafe, 17.5,
246 | p257_249, cafe, 12.5,
247 | p257_250, cafe, 7.5,
248 | p257_251, cafe, 2.5,
249 | p257_252, living, 17.5,
250 | p257_253, living, 12.5,
251 | p257_254, living, 7.5,
252 | p257_255, living, 2.5,
253 | p257_256, office, 17.5,
254 | p257_257, office, 12.5,
255 | p257_258, office, 7.5,
256 | p257_259, office, 2.5,
257 | p257_260, psquare, 17.5,
258 | p257_261, psquare, 12.5,
259 | p257_262, psquare, 7.5,
260 | p257_263, psquare, 2.5,
261 | p257_264, bus, 17.5,
262 | p257_265, bus, 12.5,
263 | p257_266, bus, 7.5,
264 | p257_267, bus, 2.5,
265 | p257_268, cafe, 17.5,
266 | p257_269, cafe, 12.5,
267 | p257_270, cafe, 7.5,
268 | p257_271, cafe, 2.5,
269 | p257_272, living, 17.5,
270 | p257_273, living, 12.5,
271 | p257_274, living, 7.5,
272 | p257_275, living, 2.5,
273 | p257_276, office, 17.5,
274 | p257_277, office, 12.5,
275 | p257_278, office, 7.5,
276 | p257_279, office, 2.5,
277 | p257_280, psquare, 17.5,
278 | p257_281, psquare, 12.5,
279 | p257_282, psquare, 7.5,
280 | p257_283, psquare, 2.5,
281 | p257_284, bus, 17.5,
282 | p257_285, bus, 12.5,
283 | p257_286, bus, 7.5,
284 | p257_287, bus, 2.5,
285 | p257_288, cafe, 17.5,
286 | p257_289, cafe, 12.5,
287 | p257_290, cafe, 7.5,
288 | p257_291, cafe, 2.5,
289 | p257_292, living, 17.5,
290 | p257_293, living, 12.5,
291 | p257_294, living, 7.5,
292 | p257_295, living, 2.5,
293 | p257_296, office, 17.5,
294 | p257_297, office, 12.5,
295 | p257_298, office, 7.5,
296 | p257_299, office, 2.5,
297 | p257_300, psquare, 17.5,
298 | p257_301, psquare, 12.5,
299 | p257_302, psquare, 7.5,
300 | p257_303, psquare, 2.5,
301 | p257_304, bus, 17.5,
302 | p257_305, bus, 12.5,
303 | p257_306, bus, 7.5,
304 | p257_307, bus, 2.5,
305 | p257_308, cafe, 17.5,
306 | p257_309, cafe, 12.5,
307 | p257_310, cafe, 7.5,
308 | p257_311, cafe, 2.5,
309 | p257_312, living, 17.5,
310 | p257_313, living, 12.5,
311 | p257_314, living, 7.5,
312 | p257_315, living, 2.5,
313 | p257_316, office, 17.5,
314 | p257_317, office, 12.5,
315 | p257_318, office, 7.5,
316 | p257_319, office, 2.5,
317 | p257_320, psquare, 17.5,
318 | p257_321, psquare, 12.5,
319 | p257_322, psquare, 7.5,
320 | p257_323, psquare, 2.5,
321 | p257_324, bus, 17.5,
322 | p257_325, bus, 12.5,
323 | p257_326, bus, 7.5,
324 | p257_327, bus, 2.5,
325 | p257_328, cafe, 17.5,
326 | p257_329, cafe, 12.5,
327 | p257_330, cafe, 7.5,
328 | p257_331, cafe, 2.5,
329 | p257_332, living, 17.5,
330 | p257_333, living, 12.5,
331 | p257_334, living, 7.5,
332 | p257_335, living, 2.5,
333 | p257_336, office, 17.5,
334 | p257_337, office, 12.5,
335 | p257_338, office, 7.5,
336 | p257_339, office, 2.5,
337 | p257_340, psquare, 17.5,
338 | p257_341, psquare, 12.5,
339 | p257_342, psquare, 7.5,
340 | p257_343, psquare, 2.5,
341 | p257_344, bus, 17.5,
342 | p257_345, bus, 12.5,
343 | p257_346, bus, 7.5,
344 | p257_347, bus, 2.5,
345 | p257_348, cafe, 17.5,
346 | p257_349, cafe, 12.5,
347 | p257_350, cafe, 7.5,
348 | p257_351, cafe, 2.5,
349 | p257_352, living, 17.5,
350 | p257_353, living, 12.5,
351 | p257_354, living, 7.5,
352 | p257_355, living, 2.5,
353 | p257_356, office, 17.5,
354 | p257_357, office, 12.5,
355 | p257_358, office, 7.5,
356 | p257_359, office, 2.5,
357 | p257_360, psquare, 17.5,
358 | p257_361, psquare, 12.5,
359 | p257_362, psquare, 7.5,
360 | p257_363, psquare, 2.5,
361 | p257_364, bus, 17.5,
362 | p257_365, bus, 12.5,
363 | p257_366, bus, 7.5,
364 | p257_367, bus, 2.5,
365 | p257_368, cafe, 17.5,
366 | p257_369, cafe, 12.5,
367 | p257_370, cafe, 7.5,
368 | p257_371, cafe, 2.5,
369 | p257_372, living, 17.5,
370 | p257_373, living, 12.5,
371 | p257_374, living, 7.5,
372 | p257_375, living, 2.5,
373 | p257_376, office, 17.5,
374 | p257_377, office, 12.5,
375 | p257_378, office, 7.5,
376 | p257_379, office, 2.5,
377 | p257_380, psquare, 17.5,
378 | p257_381, psquare, 12.5,
379 | p257_382, psquare, 7.5,
380 | p257_383, psquare, 2.5,
381 | p257_384, bus, 17.5,
382 | p257_385, bus, 12.5,
383 | p257_386, bus, 7.5,
384 | p257_387, bus, 2.5,
385 | p257_388, cafe, 17.5,
386 | p257_389, cafe, 12.5,
387 | p257_390, cafe, 7.5,
388 | p257_391, cafe, 2.5,
389 | p257_392, living, 17.5,
390 | p257_393, living, 12.5,
391 | p257_394, living, 7.5,
392 | p257_395, living, 2.5,
393 | p257_396, office, 17.5,
394 | p257_397, office, 12.5,
395 | p257_398, office, 7.5,
396 | p257_399, office, 2.5,
397 | p257_400, psquare, 17.5,
398 | p257_401, psquare, 12.5,
399 | p257_402, psquare, 7.5,
400 | p257_403, psquare, 2.5,
401 | p257_404, bus, 17.5,
402 | p257_405, bus, 12.5,
403 | p257_406, bus, 7.5,
404 | p257_407, bus, 2.5,
405 | p257_408, cafe, 17.5,
406 | p257_409, cafe, 12.5,
407 | p257_410, cafe, 7.5,
408 | p257_411, cafe, 2.5,
409 | p257_412, living, 17.5,
410 | p257_413, living, 12.5,
411 | p257_414, living, 7.5,
412 | p257_415, living, 2.5,
413 | p257_416, office, 17.5,
414 | p257_417, office, 12.5,
415 | p257_418, office, 7.5,
416 | p257_419, office, 2.5,
417 | p257_420, psquare, 17.5,
418 | p257_421, psquare, 12.5,
419 | p257_422, psquare, 7.5,
420 | p257_423, psquare, 2.5,
421 | p257_424, psquare, 17.5,
422 | p257_425, bus, 7.5,
423 | p257_426, living, 2.5,
424 | p257_427, cafe, 2.5,
425 | p257_428, bus, 12.5,
426 | p257_429, living, 12.5,
427 | p257_430, living, 7.5,
428 | p257_431, psquare, 7.5,
429 | p257_432, cafe, 12.5,
430 | p257_433, office, 12.5,
431 | p257_434, bus, 2.5,
432 | p232_001, bus, 17.5,
433 | p232_002, bus, 12.5,
434 | p232_003, bus, 7.5,
435 | p232_005, bus, 2.5,
436 | p232_006, cafe, 17.5,
437 | p232_007, cafe, 12.5,
438 | p232_009, cafe, 7.5,
439 | p232_010, cafe, 2.5,
440 | p232_011, living, 17.5,
441 | p232_012, living, 12.5,
442 | p232_013, living, 7.5,
443 | p232_014, living, 2.5,
444 | p232_015, office, 17.5,
445 | p232_016, office, 12.5,
446 | p232_017, office, 7.5,
447 | p232_019, office, 2.5,
448 | p232_020, psquare, 17.5,
449 | p232_021, psquare, 12.5,
450 | p232_022, psquare, 7.5,
451 | p232_023, psquare, 2.5,
452 | p232_024, bus, 17.5,
453 | p232_025, bus, 12.5,
454 | p232_027, bus, 7.5,
455 | p232_028, bus, 2.5,
456 | p232_029, cafe, 17.5,
457 | p232_030, cafe, 12.5,
458 | p232_031, cafe, 7.5,
459 | p232_032, cafe, 2.5,
460 | p232_033, living, 17.5,
461 | p232_034, living, 12.5,
462 | p232_035, living, 7.5,
463 | p232_036, living, 2.5,
464 | p232_037, office, 17.5,
465 | p232_038, office, 12.5,
466 | p232_039, office, 7.5,
467 | p232_040, office, 2.5,
468 | p232_041, psquare, 17.5,
469 | p232_042, psquare, 12.5,
470 | p232_043, psquare, 7.5,
471 | p232_044, psquare, 2.5,
472 | p232_045, bus, 17.5,
473 | p232_046, bus, 12.5,
474 | p232_047, bus, 7.5,
475 | p232_048, bus, 2.5,
476 | p232_049, cafe, 17.5,
477 | p232_050, cafe, 12.5,
478 | p232_051, cafe, 7.5,
479 | p232_052, cafe, 2.5,
480 | p232_053, living, 17.5,
481 | p232_054, living, 12.5,
482 | p232_055, living, 7.5,
483 | p232_056, living, 2.5,
484 | p232_057, office, 17.5,
485 | p232_058, office, 12.5,
486 | p232_059, office, 7.5,
487 | p232_060, office, 2.5,
488 | p232_061, psquare, 17.5,
489 | p232_062, psquare, 12.5,
490 | p232_063, psquare, 7.5,
491 | p232_064, psquare, 2.5,
492 | p232_065, bus, 17.5,
493 | p232_066, bus, 12.5,
494 | p232_067, bus, 7.5,
495 | p232_068, bus, 2.5,
496 | p232_069, cafe, 17.5,
497 | p232_070, cafe, 12.5,
498 | p232_071, cafe, 7.5,
499 | p232_072, cafe, 2.5,
500 | p232_073, living, 17.5,
501 | p232_074, living, 12.5,
502 | p232_075, living, 7.5,
503 | p232_076, living, 2.5,
504 | p232_077, office, 17.5,
505 | p232_078, office, 12.5,
506 | p232_079, office, 7.5,
507 | p232_080, office, 2.5,
508 | p232_081, psquare, 17.5,
509 | p232_082, psquare, 12.5,
510 | p232_083, psquare, 7.5,
511 | p232_084, psquare, 2.5,
512 | p232_085, bus, 17.5,
513 | p232_086, bus, 12.5,
514 | p232_087, bus, 7.5,
515 | p232_088, bus, 2.5,
516 | p232_089, cafe, 17.5,
517 | p232_090, cafe, 12.5,
518 | p232_091, cafe, 7.5,
519 | p232_092, cafe, 2.5,
520 | p232_093, living, 17.5,
521 | p232_094, living, 12.5,
522 | p232_095, living, 7.5,
523 | p232_096, living, 2.5,
524 | p232_097, office, 17.5,
525 | p232_098, office, 12.5,
526 | p232_099, office, 7.5,
527 | p232_100, office, 2.5,
528 | p232_101, psquare, 17.5,
529 | p232_102, psquare, 12.5,
530 | p232_103, psquare, 7.5,
531 | p232_104, psquare, 2.5,
532 | p232_105, bus, 17.5,
533 | p232_106, bus, 12.5,
534 | p232_107, bus, 7.5,
535 | p232_108, bus, 2.5,
536 | p232_109, cafe, 17.5,
537 | p232_110, cafe, 12.5,
538 | p232_112, cafe, 7.5,
539 | p232_113, cafe, 2.5,
540 | p232_114, living, 17.5,
541 | p232_115, living, 12.5,
542 | p232_116, living, 7.5,
543 | p232_117, living, 2.5,
544 | p232_118, office, 17.5,
545 | p232_119, office, 12.5,
546 | p232_120, office, 7.5,
547 | p232_121, office, 2.5,
548 | p232_123, psquare, 17.5,
549 | p232_124, psquare, 12.5,
550 | p232_125, psquare, 7.5,
551 | p232_126, psquare, 2.5,
552 | p232_127, bus, 17.5,
553 | p232_128, bus, 12.5,
554 | p232_129, bus, 7.5,
555 | p232_130, bus, 2.5,
556 | p232_131, cafe, 17.5,
557 | p232_132, cafe, 12.5,
558 | p232_133, cafe, 7.5,
559 | p232_134, cafe, 2.5,
560 | p232_135, living, 17.5,
561 | p232_136, living, 12.5,
562 | p232_137, living, 7.5,
563 | p232_138, living, 2.5,
564 | p232_139, office, 17.5,
565 | p232_140, office, 12.5,
566 | p232_141, office, 7.5,
567 | p232_142, office, 2.5,
568 | p232_143, psquare, 17.5,
569 | p232_144, psquare, 12.5,
570 | p232_145, psquare, 7.5,
571 | p232_146, psquare, 2.5,
572 | p232_147, bus, 17.5,
573 | p232_148, bus, 12.5,
574 | p232_150, bus, 7.5,
575 | p232_151, bus, 2.5,
576 | p232_152, cafe, 17.5,
577 | p232_153, cafe, 12.5,
578 | p232_154, cafe, 7.5,
579 | p232_155, cafe, 2.5,
580 | p232_156, living, 17.5,
581 | p232_158, living, 12.5,
582 | p232_159, living, 7.5,
583 | p232_160, living, 2.5,
584 | p232_161, office, 17.5,
585 | p232_162, office, 12.5,
586 | p232_163, office, 7.5,
587 | p232_164, office, 2.5,
588 | p232_165, psquare, 17.5,
589 | p232_167, psquare, 12.5,
590 | p232_169, psquare, 7.5,
591 | p232_170, psquare, 2.5,
592 | p232_171, bus, 17.5,
593 | p232_172, bus, 12.5,
594 | p232_173, bus, 7.5,
595 | p232_174, bus, 2.5,
596 | p232_175, cafe, 17.5,
597 | p232_176, cafe, 12.5,
598 | p232_177, cafe, 7.5,
599 | p232_178, cafe, 2.5,
600 | p232_179, living, 17.5,
601 | p232_180, living, 12.5,
602 | p232_181, living, 7.5,
603 | p232_182, living, 2.5,
604 | p232_183, office, 17.5,
605 | p232_184, office, 12.5,
606 | p232_185, office, 7.5,
607 | p232_186, office, 2.5,
608 | p232_187, psquare, 17.5,
609 | p232_188, psquare, 12.5,
610 | p232_189, psquare, 7.5,
611 | p232_190, psquare, 2.5,
612 | p232_191, bus, 17.5,
613 | p232_193, bus, 12.5,
614 | p232_194, bus, 7.5,
615 | p232_195, bus, 2.5,
616 | p232_196, cafe, 17.5,
617 | p232_197, cafe, 12.5,
618 | p232_198, cafe, 7.5,
619 | p232_199, cafe, 2.5,
620 | p232_200, living, 17.5,
621 | p232_201, living, 12.5,
622 | p232_202, living, 7.5,
623 | p232_203, living, 2.5,
624 | p232_204, office, 17.5,
625 | p232_205, office, 12.5,
626 | p232_206, office, 7.5,
627 | p232_207, office, 2.5,
628 | p232_208, psquare, 17.5,
629 | p232_209, psquare, 12.5,
630 | p232_210, psquare, 7.5,
631 | p232_211, psquare, 2.5,
632 | p232_213, bus, 17.5,
633 | p232_214, bus, 12.5,
634 | p232_215, bus, 7.5,
635 | p232_216, bus, 2.5,
636 | p232_217, cafe, 17.5,
637 | p232_218, cafe, 12.5,
638 | p232_219, cafe, 7.5,
639 | p232_220, cafe, 2.5,
640 | p232_221, living, 17.5,
641 | p232_223, living, 12.5,
642 | p232_224, living, 7.5,
643 | p232_225, living, 2.5,
644 | p232_226, office, 17.5,
645 | p232_227, office, 12.5,
646 | p232_228, office, 7.5,
647 | p232_229, office, 2.5,
648 | p232_230, psquare, 17.5,
649 | p232_231, psquare, 12.5,
650 | p232_232, psquare, 7.5,
651 | p232_234, psquare, 2.5,
652 | p232_235, bus, 17.5,
653 | p232_236, bus, 12.5,
654 | p232_237, bus, 7.5,
655 | p232_238, bus, 2.5,
656 | p232_239, cafe, 17.5,
657 | p232_240, cafe, 12.5,
658 | p232_241, cafe, 7.5,
659 | p232_242, cafe, 2.5,
660 | p232_243, living, 17.5,
661 | p232_244, living, 12.5,
662 | p232_245, living, 7.5,
663 | p232_246, living, 2.5,
664 | p232_247, office, 17.5,
665 | p232_248, office, 12.5,
666 | p232_249, office, 7.5,
667 | p232_250, office, 2.5,
668 | p232_251, psquare, 17.5,
669 | p232_252, psquare, 12.5,
670 | p232_253, psquare, 7.5,
671 | p232_254, psquare, 2.5,
672 | p232_255, bus, 17.5,
673 | p232_256, bus, 12.5,
674 | p232_257, bus, 7.5,
675 | p232_258, bus, 2.5,
676 | p232_259, cafe, 17.5,
677 | p232_260, cafe, 12.5,
678 | p232_261, cafe, 7.5,
679 | p232_263, cafe, 2.5,
680 | p232_264, living, 17.5,
681 | p232_265, living, 12.5,
682 | p232_266, living, 7.5,
683 | p232_267, living, 2.5,
684 | p232_268, office, 17.5,
685 | p232_269, office, 12.5,
686 | p232_270, office, 7.5,
687 | p232_271, office, 2.5,
688 | p232_272, psquare, 17.5,
689 | p232_273, psquare, 12.5,
690 | p232_274, psquare, 7.5,
691 | p232_275, psquare, 2.5,
692 | p232_276, bus, 17.5,
693 | p232_277, bus, 12.5,
694 | p232_278, bus, 7.5,
695 | p232_279, bus, 2.5,
696 | p232_280, cafe, 17.5,
697 | p232_281, cafe, 12.5,
698 | p232_282, cafe, 7.5,
699 | p232_283, cafe, 2.5,
700 | p232_284, living, 17.5,
701 | p232_285, living, 12.5,
702 | p232_286, living, 7.5,
703 | p232_287, living, 2.5,
704 | p232_288, office, 17.5,
705 | p232_289, office, 12.5,
706 | p232_290, office, 7.5,
707 | p232_291, office, 2.5,
708 | p232_292, psquare, 17.5,
709 | p232_293, psquare, 12.5,
710 | p232_294, psquare, 7.5,
711 | p232_295, psquare, 2.5,
712 | p232_296, bus, 17.5,
713 | p232_297, bus, 12.5,
714 | p232_298, bus, 7.5,
715 | p232_299, bus, 2.5,
716 | p232_300, cafe, 17.5,
717 | p232_301, cafe, 12.5,
718 | p232_302, cafe, 7.5,
719 | p232_303, cafe, 2.5,
720 | p232_305, living, 17.5,
721 | p232_306, living, 12.5,
722 | p232_307, living, 7.5,
723 | p232_308, living, 2.5,
724 | p232_309, office, 17.5,
725 | p232_310, office, 12.5,
726 | p232_311, office, 7.5,
727 | p232_312, office, 2.5,
728 | p232_313, psquare, 17.5,
729 | p232_314, psquare, 12.5,
730 | p232_315, psquare, 7.5,
731 | p232_316, psquare, 2.5,
732 | p232_317, bus, 17.5,
733 | p232_318, bus, 12.5,
734 | p232_319, bus, 7.5,
735 | p232_320, bus, 2.5,
736 | p232_321, cafe, 17.5,
737 | p232_322, cafe, 12.5,
738 | p232_323, cafe, 7.5,
739 | p232_324, cafe, 2.5,
740 | p232_325, living, 17.5,
741 | p232_326, living, 12.5,
742 | p232_327, living, 7.5,
743 | p232_328, living, 2.5,
744 | p232_329, office, 17.5,
745 | p232_330, office, 12.5,
746 | p232_331, office, 7.5,
747 | p232_332, office, 2.5,
748 | p232_333, psquare, 17.5,
749 | p232_334, psquare, 12.5,
750 | p232_335, psquare, 7.5,
751 | p232_336, psquare, 2.5,
752 | p232_337, bus, 17.5,
753 | p232_338, bus, 12.5,
754 | p232_339, bus, 7.5,
755 | p232_340, bus, 2.5,
756 | p232_341, cafe, 17.5,
757 | p232_342, cafe, 12.5,
758 | p232_343, cafe, 7.5,
759 | p232_344, cafe, 2.5,
760 | p232_346, living, 17.5,
761 | p232_347, living, 12.5,
762 | p232_348, living, 7.5,
763 | p232_349, living, 2.5,
764 | p232_350, office, 17.5,
765 | p232_351, office, 12.5,
766 | p232_352, office, 7.5,
767 | p232_353, office, 2.5,
768 | p232_354, psquare, 17.5,
769 | p232_355, psquare, 12.5,
770 | p232_356, psquare, 7.5,
771 | p232_357, psquare, 2.5,
772 | p232_358, bus, 17.5,
773 | p232_359, bus, 12.5,
774 | p232_360, bus, 7.5,
775 | p232_361, bus, 2.5,
776 | p232_362, cafe, 17.5,
777 | p232_363, cafe, 12.5,
778 | p232_364, cafe, 7.5,
779 | p232_365, cafe, 2.5,
780 | p232_366, living, 17.5,
781 | p232_367, living, 12.5,
782 | p232_368, living, 7.5,
783 | p232_369, living, 2.5,
784 | p232_370, office, 17.5,
785 | p232_371, office, 12.5,
786 | p232_372, office, 7.5,
787 | p232_373, office, 2.5,
788 | p232_374, psquare, 17.5,
789 | p232_375, psquare, 12.5,
790 | p232_377, psquare, 7.5,
791 | p232_378, psquare, 2.5,
792 | p232_379, bus, 17.5,
793 | p232_380, bus, 12.5,
794 | p232_381, bus, 7.5,
795 | p232_382, bus, 2.5,
796 | p232_383, cafe, 17.5,
797 | p232_384, cafe, 12.5,
798 | p232_385, cafe, 7.5,
799 | p232_386, cafe, 2.5,
800 | p232_387, living, 17.5,
801 | p232_388, living, 12.5,
802 | p232_389, living, 7.5,
803 | p232_390, living, 2.5,
804 | p232_391, office, 17.5,
805 | p232_392, office, 12.5,
806 | p232_393, office, 7.5,
807 | p232_394, office, 2.5,
808 | p232_396, psquare, 17.5,
809 | p232_397, psquare, 12.5,
810 | p232_398, psquare, 7.5,
811 | p232_399, psquare, 2.5,
812 | p232_400, psquare, 17.5,
813 | p232_402, living, 7.5,
814 | p232_403, living, 12.5,
815 | p232_404, cafe, 17.5,
816 | p232_405, psquare, 7.5,
817 | p232_407, living, 2.5,
818 | p232_409, psquare, 2.5,
819 | p232_410, office, 7.5,
820 | p232_411, office, 17.5,
821 | p232_412, psquare, 12.5,
822 | p232_413, office, 12.5,
823 | p232_414, bus, 2.5,
824 | p232_415, cafe, 2.5,
825 | 


--------------------------------------------------------------------------------
/src/denoise.c:
--------------------------------------------------------------------------------
  1 | /* Copyright (c) 2018 Gregor Richards
  2 |  * Copyright (c) 2017 Mozilla */
  3 | /*
  4 |    Redistribution and use in source and binary forms, with or without
  5 |    modification, are permitted provided that the following conditions
  6 |    are met:
  7 | 
  8 |    - Redistributions of source code must retain the above copyright
  9 |    notice, this list of conditions and the following disclaimer.
 10 | 
 11 |    - Redistributions in binary form must reproduce the above copyright
 12 |    notice, this list of conditions and the following disclaimer in the
 13 |    documentation and/or other materials provided with the distribution.
 14 | 
 15 |    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 16 |    ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
 17 |    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
 18 |    A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR
 19 |    CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 20 |    EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 21 |    PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
 22 |    PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
 23 |    LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 24 |    NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 25 |    SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 26 | */
 27 | 
 28 | #ifdef HAVE_CONFIG_H
 29 | #include "config.h"
 30 | #endif
 31 | 
 32 | #include <stdlib.h>
 33 | #include <string.h>
 34 | #include <stdio.h>
 35 | #include "kiss_fft.h"
 36 | #include "common.h"
 37 | #include <math.h>
 38 | #include "rnnoise.h"
 39 | #include "pitch.h"
 40 | #include "arch.h"
 41 | #include "rnn.h"
 42 | #include "rnn_data.h"
 43 | #include "denoise.h"
 44 | 
 45 | 
 46 | /* The built-in model, used if no file is given as input */
 47 | extern const struct RNNModel rnnoise_model_orig;
 48 | 
 49 | 
 50 | static const opus_int16 eband5ms[] = {
 51 | /*0  200 400 600 800  1k 1.2 1.4 1.6  2k 2.4 2.8 3.2  4k 4.8 5.6 6.8  8k 9.6 12k 15.6 20k*/
 52 |   0,  1,  2,  3,  4,  5,  6,  7,  8, 10, 12, 14, 16, 20, 24, 28, 34, 40, 48, 60, 78, 100
 53 | };
 54 | 
 55 | 
 56 | typedef struct {
 57 |   int init;
 58 |   kiss_fft_state *kfft;
 59 |   float half_window[FRAME_SIZE];
 60 |   float dct_table[NB_BANDS*NB_BANDS];
 61 | } CommonState;
 62 | 
 63 | void compute_band_energy(float *bandE, const kiss_fft_cpx *X) {
 64 |   int i;
 65 |   float sum[NB_BANDS] = {0};
 66 |   for (i=0;i<NB_BANDS-1;i++)
 67 |   {
 68 |     int j;
 69 |     int band_size;
 70 |     band_size = (eband5ms[i+1]-eband5ms[i])<<FRAME_SIZE_SHIFT;
 71 |     for (j=0;j<band_size;j++) {
 72 |       float tmp;
 73 |       float frac = (float)j/band_size;
 74 |       tmp = SQUARE(X[(eband5ms[i]<<FRAME_SIZE_SHIFT) + j].r);
 75 |       tmp += SQUARE(X[(eband5ms[i]<<FRAME_SIZE_SHIFT) + j].i);
 76 |       sum[i] += (1-frac)*tmp;
 77 |       sum[i+1] += frac*tmp;
 78 |     }
 79 |   }
 80 |   sum[0] *= 2;
 81 |   sum[NB_BANDS-1] *= 2;
 82 |   for (i=0;i<NB_BANDS;i++)
 83 |   {
 84 |     bandE[i] = sum[i];
 85 |   }
 86 | }
 87 | 
 88 | void compute_band_corr(float *bandE, const kiss_fft_cpx *X, const kiss_fft_cpx *P) {
 89 |   int i;
 90 |   float sum[NB_BANDS] = {0};
 91 |   for (i=0;i<NB_BANDS-1;i++)
 92 |   {
 93 |     int j;
 94 |     int band_size;
 95 |     band_size = (eband5ms[i+1]-eband5ms[i])<<FRAME_SIZE_SHIFT;
 96 |     for (j=0;j<band_size;j++) {
 97 |       float tmp;
 98 |       float frac = (float)j/band_size;
 99 |       tmp = X[(eband5ms[i]<<FRAME_SIZE_SHIFT) + j].r * P[(eband5ms[i]<<FRAME_SIZE_SHIFT) + j].r;
100 |       tmp += X[(eband5ms[i]<<FRAME_SIZE_SHIFT) + j].i * P[(eband5ms[i]<<FRAME_SIZE_SHIFT) + j].i;
101 |       sum[i] += (1-frac)*tmp;
102 |       sum[i+1] += frac*tmp;
103 |     }
104 |   }
105 |   sum[0] *= 2;
106 |   sum[NB_BANDS-1] *= 2;
107 |   for (i=0;i<NB_BANDS;i++)
108 |   {
109 |     bandE[i] = sum[i];
110 |   }
111 | }
112 | 
113 | void interp_band_gain(float *g, const float *bandE) {
114 |   int i;
115 |   memset(g, 0, FREQ_SIZE);
116 |   for (i=0;i<NB_BANDS-1;i++)
117 |   {
118 |     int j;
119 |     int band_size;
120 |     band_size = (eband5ms[i+1]-eband5ms[i])<<FRAME_SIZE_SHIFT;
121 |     for (j=0;j<band_size;j++) {
122 |       float frac = (float)j/band_size;
123 |       g[(eband5ms[i]<<FRAME_SIZE_SHIFT) + j] = (1-frac)*bandE[i] + frac*bandE[i+1];
124 |     }
125 |   }
126 | }
127 | 
128 | 
129 | CommonState common;
130 | 
131 | static void check_init() {
132 |   int i;
133 |   if (common.init) return;
134 |   common.kfft = opus_fft_alloc_twiddles(2*FRAME_SIZE, NULL, NULL, NULL, 0);
135 |   for (i=0;i<FRAME_SIZE;i++)
136 |     common.half_window[i] = sin(.5*M_PI*sin(.5*M_PI*(i+.5)/FRAME_SIZE) * sin(.5*M_PI*(i+.5)/FRAME_SIZE));
137 |   for (i=0;i<NB_BANDS;i++) {
138 |     int j;
139 |     for (j=0;j<NB_BANDS;j++) {
140 |       common.dct_table[i*NB_BANDS + j] = cos((i+.5)*j*M_PI/NB_BANDS);
141 |       if (j==0) common.dct_table[i*NB_BANDS + j] *= sqrt(.5);
142 |     }
143 |   }
144 |   common.init = 1;
145 | }
146 | 
147 | static void dct(float *out, const float *in) {
148 |   int i;
149 |   check_init();
150 |   for (i=0;i<NB_BANDS;i++) {
151 |     int j;
152 |     float sum = 0;
153 |     for (j=0;j<NB_BANDS;j++) {
154 |       sum += in[j] * common.dct_table[j*NB_BANDS + i];
155 |     }
156 |     out[i] = sum*sqrt(2./22);
157 |   }
158 | }
159 | 
160 | #if 0
161 | static void idct(float *out, const float *in) {
162 |   int i;
163 |   check_init();
164 |   for (i=0;i<NB_BANDS;i++) {
165 |     int j;
166 |     float sum = 0;
167 |     for (j=0;j<NB_BANDS;j++) {
168 |       sum += in[j] * common.dct_table[i*NB_BANDS + j];
169 |     }
170 |     out[i] = sum*sqrt(2./22);
171 |   }
172 | }
173 | #endif
174 | 
175 | static void forward_transform(kiss_fft_cpx *out, const float *in) {
176 |   int i;
177 |   kiss_fft_cpx x[WINDOW_SIZE];
178 |   kiss_fft_cpx y[WINDOW_SIZE];
179 |   check_init();
180 |   for (i=0;i<WINDOW_SIZE;i++) {
181 |     x[i].r = in[i];
182 |     x[i].i = 0;
183 |   }
184 |   opus_fft(common.kfft, x, y, 0);
185 |   for (i=0;i<FREQ_SIZE;i++) {
186 |     out[i] = y[i];
187 |   }
188 | }
189 | 
190 | static void inverse_transform(float *out, const kiss_fft_cpx *in) {
191 |   int i;
192 |   kiss_fft_cpx x[WINDOW_SIZE];
193 |   kiss_fft_cpx y[WINDOW_SIZE];
194 |   check_init();
195 |   for (i=0;i<FREQ_SIZE;i++) {
196 |     x[i] = in[i];
197 |   }
198 |   for (;i<WINDOW_SIZE;i++) {
199 |     x[i].r = x[WINDOW_SIZE - i].r;
200 |     x[i].i = -x[WINDOW_SIZE - i].i;
201 |   }
202 |   opus_fft(common.kfft, x, y, 0);
203 |   /* output in reverse order for IFFT. */
204 |   out[0] = WINDOW_SIZE*y[0].r;
205 |   for (i=1;i<WINDOW_SIZE;i++) {
206 |     out[i] = WINDOW_SIZE*y[WINDOW_SIZE - i].r;
207 |   }
208 | }
209 | 
210 | static void apply_window(float *x) {
211 |   int i;
212 |   check_init();
213 |   for (i=0;i<FRAME_SIZE;i++) {
214 |     x[i] *= common.half_window[i];
215 |     x[WINDOW_SIZE - 1 - i] *= common.half_window[i];
216 |   }
217 | }
218 | 
219 | int rnnoise_get_size() {
220 |   return sizeof(DenoiseState);
221 | }
222 | 
223 | int rnnoise_init(DenoiseState *st, RNNModel *model) {
224 |   memset(st, 0, sizeof(*st));
225 |   if (model)
226 |     st->rnn.model = model;
227 |   else
228 |     st->rnn.model = &rnnoise_model_orig;
229 |   st->rnn.vad_gru_state = calloc(sizeof(float), st->rnn.model->vad_gru_size);
230 |   st->rnn.noise_gru_state = calloc(sizeof(float), st->rnn.model->noise_gru_size);
231 |   st->rnn.denoise_gru_state = calloc(sizeof(float), st->rnn.model->denoise_gru_size);
232 |   return 0;
233 | }
234 | 
235 | DenoiseState *rnnoise_create(RNNModel *model) {
236 |   DenoiseState *st;
237 |   st = malloc(rnnoise_get_size());
238 |   rnnoise_init(st, model);
239 |   return st;
240 | }
241 | 
242 | void rnnoise_destroy(DenoiseState *st) {
243 |   free(st->rnn.vad_gru_state);
244 |   free(st->rnn.noise_gru_state);
245 |   free(st->rnn.denoise_gru_state);
246 |   free(st);
247 | }
248 | 
249 | #if TRAINING
250 | int lowpass = FREQ_SIZE;
251 | int band_lp = NB_BANDS;
252 | #endif
253 | 
254 | static void frame_analysis(DenoiseState *st, kiss_fft_cpx *X, float *Ex, const float *in) {
255 |   int i;
256 |   float x[WINDOW_SIZE];
257 |   RNN_COPY(x, st->analysis_mem, FRAME_SIZE);
258 |   for (i=0;i<FRAME_SIZE;i++) x[FRAME_SIZE + i] = in[i];
259 |   RNN_COPY(st->analysis_mem, in, FRAME_SIZE);
260 | 
261 |   // Apply Vorbis window
262 |   apply_window(x);
263 | 
264 |   // Fourier Transform FFT
265 |   forward_transform(X, x);
266 | #if TRAINING
267 |   for (i=lowpass;i<FREQ_SIZE;i++)
268 |     X[i].r = X[i].i = 0;
269 | #endif
270 | 
271 |   // Compute band energy
272 |   compute_band_energy(Ex, X);
273 | }
274 | 
275 | int compute_frame_features(DenoiseState *st, kiss_fft_cpx *X, kiss_fft_cpx *P,
276 |                                   float *Ex, float *Ep, float *Exp, float *features, const float *in) {
277 |   int i;
278 |   float E = 0;
279 |   float *ceps_0, *ceps_1, *ceps_2;
280 |   float spec_variability = 0;
281 |   float Ly[NB_BANDS];
282 |   float p[WINDOW_SIZE];
283 |   float pitch_buf[PITCH_BUF_SIZE>>1];
284 |   int pitch_index;
285 |   float gain;
286 |   float *(pre[1]);
287 |   // float** pre;
288 |   float tmp[NB_BANDS];
289 |   float follow, logMax;
290 | 
291 |   // FFT and energy
292 |   frame_analysis(st, X, Ex, in);
293 | 
294 |   // Copy last part of pitch buffer to the beginning and fill the rest with noisy frame
295 |   RNN_MOVE(st->pitch_buf, &st->pitch_buf[FRAME_SIZE], PITCH_BUF_SIZE-FRAME_SIZE);
296 |   RNN_COPY(&st->pitch_buf[PITCH_BUF_SIZE-FRAME_SIZE], in, FRAME_SIZE);
297 | 
298 |   // Pointer to pitch buffer
299 |   pre[0] = &st->pitch_buf[0];
300 | 
301 |   // Find pitch
302 |   pitch_downsample(pre, pitch_buf, PITCH_BUF_SIZE, 1);
303 |   pitch_search(pitch_buf+(PITCH_MAX_PERIOD>>1), pitch_buf, PITCH_FRAME_SIZE,
304 |                PITCH_MAX_PERIOD-3*PITCH_MIN_PERIOD, &pitch_index);
305 |   pitch_index = PITCH_MAX_PERIOD-pitch_index;
306 | 
307 | 
308 |   gain = remove_doubling(pitch_buf, PITCH_MAX_PERIOD, PITCH_MIN_PERIOD,
309 |           PITCH_FRAME_SIZE, &pitch_index, st->last_period, st->last_gain);
310 |   st->last_period = pitch_index;
311 |   st->last_gain = gain;
312 | 
313 |   // x(n) = x(n-T)
314 |   for (i=0;i<WINDOW_SIZE;i++)
315 |     p[i] = st->pitch_buf[PITCH_BUF_SIZE-WINDOW_SIZE-pitch_index+i];
316 |     // p[i] = st->pitch_buf[pitch_index_old + i];
317 | 
318 | 
319 |   apply_window(p);
320 |   forward_transform(P, p);
321 |   compute_band_energy(Ep, P);
322 |   compute_band_corr(Exp, X, P);
323 |   for (i=0;i<NB_BANDS;i++) Exp[i] = Exp[i]/sqrt(.001+Ex[i]*Ep[i]);
324 |   
325 |   dct(tmp, Exp);
326 |   for (i=0;i<NB_DELTA_CEPS;i++) features[NB_BANDS+2*NB_DELTA_CEPS+i] = tmp[i];
327 |   features[NB_BANDS+2*NB_DELTA_CEPS] -= 1.3;
328 |   features[NB_BANDS+2*NB_DELTA_CEPS+1] -= 0.9;
329 |   features[NB_BANDS+3*NB_DELTA_CEPS] = .01*(pitch_index-300);
330 |   logMax = -2;
331 |   follow = -2;
332 |   for (i=0;i<NB_BANDS;i++) {
333 |     Ly[i] = log10(1e-2+Ex[i]);
334 |     Ly[i] = MAX16(logMax-7, MAX16(follow-1.5, Ly[i]));
335 |     logMax = MAX16(logMax, Ly[i]);
336 |     follow = MAX16(follow-1.5, Ly[i]);
337 |     E += Ex[i];
338 |   }
339 |   if (!TRAINING && E < 0.04) {
340 |     /* If there's no audio, avoid messing up the state. */
341 |     RNN_CLEAR(features, NB_FEATURES);
342 |     return 1;
343 |   }
344 |   dct(features, Ly);
345 |   features[0] -= 12;
346 |   features[1] -= 4;
347 |   ceps_0 = st->cepstral_mem[st->memid];
348 |   ceps_1 = (st->memid < 1) ? st->cepstral_mem[CEPS_MEM+st->memid-1] : st->cepstral_mem[st->memid-1];
349 |   ceps_2 = (st->memid < 2) ? st->cepstral_mem[CEPS_MEM+st->memid-2] : st->cepstral_mem[st->memid-2];
350 |   for (i=0;i<NB_BANDS;i++) ceps_0[i] = features[i];
351 |   st->memid++;
352 |   for (i=0;i<NB_DELTA_CEPS;i++) {
353 |     features[i] = ceps_0[i] + ceps_1[i] + ceps_2[i];
354 |     features[NB_BANDS+i] = ceps_0[i] - ceps_2[i];
355 |     features[NB_BANDS+NB_DELTA_CEPS+i] =  ceps_0[i] - 2*ceps_1[i] + ceps_2[i];
356 |   }
357 |   /* Spectral variability features. */
358 |   if (st->memid == CEPS_MEM) st->memid = 0;
359 |   for (i=0;i<CEPS_MEM;i++)
360 |   {
361 |     int j;
362 |     float mindist = 1e15f;
363 |     for (j=0;j<CEPS_MEM;j++)
364 |     {
365 |       int k;
366 |       float dist=0;
367 |       for (k=0;k<NB_BANDS;k++)
368 |       {
369 |         float tmp;
370 |         tmp = st->cepstral_mem[i][k] - st->cepstral_mem[j][k];
371 |         dist += tmp*tmp;
372 |       }
373 |       if (j!=i)
374 |         mindist = MIN32(mindist, dist);
375 |     }
376 |     spec_variability += mindist;
377 |   }
378 |   features[NB_BANDS+3*NB_DELTA_CEPS+1] = spec_variability/CEPS_MEM-2.1;
379 |   return TRAINING && E < 0.1;
380 | }
381 | 
382 | static void frame_synthesis(DenoiseState *st, float *out, const kiss_fft_cpx *y) {
383 |   float x[WINDOW_SIZE];
384 |   int i;
385 |   inverse_transform(x, y);
386 |   apply_window(x);
387 |   for (i=0;i<FRAME_SIZE;i++) out[i] = x[i] + st->synthesis_mem[i];
388 |   RNN_COPY(st->synthesis_mem, &x[FRAME_SIZE], FRAME_SIZE);
389 | }
390 | 
391 | void biquad(float *y, float mem[2], const float *x, const float *b, const float *a, int N) {
392 |   int i;
393 |   for (i=0;i<N;i++) {
394 |     float xi, yi;
395 |     xi = x[i];
396 |     yi = x[i] + mem[0];
397 |     mem[0] = mem[1] + (b[0]*(double)xi - a[0]*(double)yi);
398 |     mem[1] = (b[1]*(double)xi - a[1]*(double)yi);
399 |     y[i] = yi;
400 |   }
401 | }
402 | 
403 | void pitch_filter(kiss_fft_cpx *X, const kiss_fft_cpx *P, const float *Ex, const float *Ep,
404 |                   const float *Exp, const float *g) {
405 |   int i;
406 |   float r[NB_BANDS];
407 |   float rf[FREQ_SIZE] = {0};
408 |   for (i=0;i<NB_BANDS;i++) {
409 | #if 0
410 |     if (Exp[i]>g[i]) r[i] = 1;
411 |     else r[i] = Exp[i]*(1-g[i])/(.001 + g[i]*(1-Exp[i]));
412 |     r[i] = MIN16(1, MAX16(0, r[i]));
413 | #else
414 |     if (Exp[i]>g[i]) r[i] = 1;
415 |     else r[i] = SQUARE(Exp[i])*(1-SQUARE(g[i]))/(.001 + SQUARE(g[i])*(1-SQUARE(Exp[i])));
416 |     r[i] = sqrt(MIN16(1, MAX16(0, r[i])));
417 | #endif
418 |     r[i] *= sqrt(Ex[i]/(1e-8+Ep[i]));
419 |   }
420 |   interp_band_gain(rf, r);
421 |   for (i=0;i<FREQ_SIZE;i++) {
422 |     X[i].r += rf[i]*P[i].r;
423 |     X[i].i += rf[i]*P[i].i;
424 |   }
425 |   float newE[NB_BANDS];
426 |   compute_band_energy(newE, X);
427 |   float norm[NB_BANDS];
428 |   float normf[FREQ_SIZE]={0};
429 |   for (i=0;i<NB_BANDS;i++) {
430 |     norm[i] = sqrt(Ex[i]/(1e-8+newE[i]));
431 |   }
432 |   interp_band_gain(normf, norm);
433 |   for (i=0;i<FREQ_SIZE;i++) {
434 |     X[i].r *= normf[i];
435 |     X[i].i *= normf[i];
436 |   }
437 | }
438 | 
439 | float rnnoise_process_frame(DenoiseState *st, float *out, const float *in) {
440 |   int i;
441 |   kiss_fft_cpx X[FREQ_SIZE];
442 |   kiss_fft_cpx P[WINDOW_SIZE];
443 |   float x[FRAME_SIZE];
444 |   float Ex[NB_BANDS], Ep[NB_BANDS];
445 |   float Exp[NB_BANDS];
446 |   float features[NB_FEATURES];
447 |   float g[NB_BANDS];
448 |   float gf[FREQ_SIZE]={1};
449 |   float vad_prob = 0;
450 |   int silence;
451 |   static const float a_hp[2] = {-1.99599, 0.99600};
452 |   static const float b_hp[2] = {-2, 1};
453 |   biquad(x, st->mem_hp_x, in, b_hp, a_hp, FRAME_SIZE);
454 | 
455 |   silence = compute_frame_features(st, X, P, Ex, Ep, Exp, features, x);
456 | 
457 |   if (!silence) {
458 |     
459 |     compute_rnn(&st->rnn, g, &vad_prob, features);
460 | 
461 |     pitch_filter(X, P, Ex, Ep, Exp, g);
462 |     for (i=0;i<NB_BANDS;i++) {
463 |       float alpha = .6f;
464 |       g[i] = MAX16(g[i], alpha*st->lastg[i]);
465 |       st->lastg[i] = g[i];
466 |     }
467 |     interp_band_gain(gf, g);
468 | #if 1
469 |     for (i=0;i<FREQ_SIZE;i++) {
470 |       X[i].r *= gf[i];
471 |       X[i].i *= gf[i];
472 |     }
473 | #endif
474 |   }
475 | 
476 |   frame_synthesis(st, out, X);
477 |   return vad_prob;
478 | }
479 | 
480 | #if TRAINING
481 | 
482 | static float uni_rand() {
483 |   return rand()/(double)RAND_MAX-.5;
484 | }
485 | 
486 | static void rand_resp(float *a, float *b) {
487 |   a[0] = .75*uni_rand();
488 |   a[1] = .75*uni_rand();
489 |   b[0] = .75*uni_rand();
490 |   b[1] = .75*uni_rand();
491 | }
492 | 
493 | int main(int argc, char **argv) {
494 |   int i;
495 |   int count=0;
496 |   static const float a_hp[2] = {-1.99599, 0.99600};
497 |   static const float b_hp[2] = {-2, 1};
498 |   float a_noise[2] = {0};
499 |   float b_noise[2] = {0};
500 |   float a_sig[2] = {0};
501 |   float b_sig[2] = {0};
502 |   float mem_hp_x[2]={0};
503 |   float mem_hp_n[2]={0};
504 |   float mem_resp_x[2]={0};
505 |   float mem_resp_n[2]={0};
506 |   float x[FRAME_SIZE];
507 |   float n[FRAME_SIZE];
508 |   float xn[FRAME_SIZE];
509 |   int vad_cnt=0;
510 |   int gain_change_count=0;
511 |   float speech_gain = 1, noise_gain = 1;
512 | 
513 |   // File IO pointers
514 |   FILE *f1, *f2, *fout;
515 | 
516 |   int maxCount;
517 |   DenoiseState *st;
518 |   DenoiseState *noise_state;
519 |   DenoiseState *noisy;
520 |   st = rnnoise_create(NULL);
521 |   noise_state = rnnoise_create(NULL);
522 |   noisy = rnnoise_create(NULL);
523 |   if (argc!=4) {
524 |     fprintf(stderr, "usage: %s <speech> <noise> <count>\n", argv[0]);
525 |     return 1;
526 |   }
527 | 
528 |   // Speech file pointer
529 |   f1 = fopen(argv[1], "r");
530 | 
531 |   // Noise file pointer
532 |   f2 = fopen(argv[2], "r");
533 | 
534 |   // Noisy speech output pointer
535 |   fout = fopen("noisySpeechSamples.raw", "w");
536 | 
537 |   // Declare number of samples
538 |   maxCount = atoi(argv[3]);
539 | 
540 |   // Read 150 noise frames
541 |   for(i=0;i<150;i++) {
542 |     short tmp[FRAME_SIZE];
543 |     fread(tmp, sizeof(short), FRAME_SIZE, f2);
544 |   }
545 | 
546 |   while (1) {
547 | 
548 |     // Declare arrays & variables
549 |     kiss_fft_cpx X[FREQ_SIZE], Y[FREQ_SIZE], N[FREQ_SIZE], P[WINDOW_SIZE];
550 |     float Ex[NB_BANDS], Ey[NB_BANDS], En[NB_BANDS], Ep[NB_BANDS];
551 |     float Exp[NB_BANDS];
552 |     float Ln[NB_BANDS];
553 |     float features[NB_FEATURES];
554 |     float g[NB_BANDS];
555 |     short tmp[FRAME_SIZE];
556 |     float vad=0;
557 |     float E=0;
558 | 
559 |     // End while loop when number of samples has been satisfied
560 |     if (count==maxCount) break;
561 | 
562 |     // Print progress
563 |     if ((count%1000)==0) fprintf(stderr, "%d\r", count);
564 | 
565 | 
566 |     if (++gain_change_count > 2821) {
567 |       speech_gain = pow(10., (-40+(rand()%60))/20.);
568 |       noise_gain = pow(10., (-30+(rand()%50))/20.);
569 |       if (rand()%10==0) noise_gain = 0;
570 |       noise_gain *= speech_gain;
571 |       if (rand()%10==0) speech_gain = 0;
572 |       gain_change_count = 0;
573 |       rand_resp(a_noise, b_noise);
574 |       rand_resp(a_sig, b_sig);
575 |       lowpass = FREQ_SIZE * 3000./24000. * pow(50., rand()/(double)RAND_MAX);
576 |       for (i=0;i<NB_BANDS;i++) {
577 |         if (eband5ms[i]<<FRAME_SIZE_SHIFT > lowpass) {
578 |           band_lp = i;
579 |           break;
580 |         }
581 |       }
582 |     }
583 | 
584 |     if (speech_gain != 0) {
585 |       fread(tmp, sizeof(short), FRAME_SIZE, f1);
586 |       if (feof(f1)) {
587 |         rewind(f1);
588 |         fread(tmp, sizeof(short), FRAME_SIZE, f1);
589 |       }
590 |       for (i=0;i<FRAME_SIZE;i++) x[i] = speech_gain*tmp[i];
591 |       for (i=0;i<FRAME_SIZE;i++) E += tmp[i]*(float)tmp[i];
592 |     } else {
593 |       for (i=0;i<FRAME_SIZE;i++) x[i] = 0;
594 |       E = 0;
595 |     }
596 |     if (noise_gain!=0) {
597 |       fread(tmp, sizeof(short), FRAME_SIZE, f2);
598 |       if (feof(f2)) {
599 |         rewind(f2);
600 |         fread(tmp, sizeof(short), FRAME_SIZE, f2);
601 |       }
602 |       for (i=0;i<FRAME_SIZE;i++) n[i] = noise_gain*tmp[i];
603 |     } else {
604 |       for (i=0;i<FRAME_SIZE;i++) n[i] = 0;
605 |     }
606 |     biquad(x, mem_hp_x, x, b_hp, a_hp, FRAME_SIZE);
607 |     biquad(x, mem_resp_x, x, b_sig, a_sig, FRAME_SIZE);
608 |     biquad(n, mem_hp_n, n, b_hp, a_hp, FRAME_SIZE);
609 |     biquad(n, mem_resp_n, n, b_noise, a_noise, FRAME_SIZE);
610 | 
611 |     // Join noise and sound samples
612 |     for (i=0;i<FRAME_SIZE;i++) xn[i] = x[i] + n[i];
613 | 
614 |     // Cast xn to short
615 |     for (i=0;i<FRAME_SIZE;i++) tmp[i] = xn[i];
616 | 
617 |     // Write noisy samples to file
618 |     fwrite(tmp, sizeof(short), FRAME_SIZE, fout);
619 |     // fwrite(xn, sizeof(float), FRAME_SIZE, fout);
620 | 
621 |     if (E > 1e9f) {
622 |       vad_cnt=0;
623 |     } else if (E > 1e8f) {
624 |       vad_cnt -= 5;
625 |     } else if (E > 1e7f) {
626 |       vad_cnt++;
627 |     } else {
628 |       vad_cnt+=2;
629 |     }
630 |     if (vad_cnt < 0) vad_cnt = 0;
631 |     if (vad_cnt > 15) vad_cnt = 15;
632 | 
633 |     if (vad_cnt >= 10) vad = 0;
634 |     else if (vad_cnt > 0) vad = 0.5f;
635 |     else vad = 1.f;
636 | 
637 |     frame_analysis(st, Y, Ey, x);
638 |     frame_analysis(noise_state, N, En, n);
639 | 
640 |     for (i=0;i<NB_BANDS;i++) Ln[i] = log10(1e-2+En[i]);
641 |     int silence = compute_frame_features(noisy, X, P, Ex, Ep, Exp, features, xn);
642 |     pitch_filter(X, P, Ex, Ep, Exp, g);
643 |     //printf("%f %d\n", noisy->last_gain, noisy->last_period);
644 | 
645 |     for (i=0;i<NB_BANDS;i++) {
646 |       g[i] = sqrt((Ey[i]+1e-3)/(Ex[i]+1e-3));
647 |       if (g[i] > 1) g[i] = 1;
648 |       if (silence || i > band_lp) g[i] = -1;
649 |       if (Ey[i] < 5e-2 && Ex[i] < 5e-2) g[i] = -1;
650 |       if (vad==0 && noise_gain==0) g[i] = -1;
651 |     }
652 | 
653 |     count++;
654 | #if 1
655 |     fwrite(features, sizeof(float), NB_FEATURES, stdout);
656 |     fwrite(g, sizeof(float), NB_BANDS, stdout);
657 |     fwrite(Ln, sizeof(float), NB_BANDS, stdout);
658 |     fwrite(&vad, sizeof(float), 1, stdout);
659 | #endif
660 |   }
661 |   fprintf(stderr, "matrix size: %d x %d\n", count, NB_FEATURES + 2*NB_BANDS + 1);
662 |   fclose(f1);
663 |   fclose(f2);
664 |   return 0;
665 | }
666 | 
667 | #endif
668 | 


--------------------------------------------------------------------------------
/src/denoise.h:
--------------------------------------------------------------------------------
 1 | #ifndef DENOISE_H
 2 | #define DENOISE_H
 3 | 
 4 | #define FRAME_SIZE_SHIFT 2
 5 | #define FRAME_SIZE (120<<FRAME_SIZE_SHIFT)
 6 | #define WINDOW_SIZE (2*FRAME_SIZE)
 7 | #define FREQ_SIZE (FRAME_SIZE + 1)
 8 | 
 9 | #define PITCH_MIN_PERIOD 60
10 | #define PITCH_MAX_PERIOD 768
11 | #define PITCH_FRAME_SIZE 960
12 | #define PITCH_BUF_SIZE (PITCH_MAX_PERIOD+PITCH_FRAME_SIZE)
13 | 
14 | #define SQUARE(x) ((x)*(x))
15 | 
16 | #define NB_BANDS 22
17 | 
18 | #define CEPS_MEM 8
19 | #define NB_DELTA_CEPS 6
20 | 
21 | #define NB_FEATURES (NB_BANDS+3*NB_DELTA_CEPS+2)
22 | 
23 | 
24 | #ifndef TRAINING
25 | #define TRAINING 0
26 | #endif
27 | 
28 | #include <stdlib.h>
29 | #include <string.h>
30 | #include <stdio.h>
31 | #include "../src/kiss_fft.h"
32 | #include "../src/common.h"
33 | #include <math.h>
34 | #include "rnnoise.h"
35 | #include "../src/pitch.h"
36 | #include "../src/arch.h"
37 | #include "../src/rnn.h"
38 | #include "../src/rnn_data.h"
39 | 
40 | struct DenoiseState {
41 |   float analysis_mem[FRAME_SIZE];
42 |   float cepstral_mem[CEPS_MEM][NB_BANDS];
43 |   int memid;
44 |   float synthesis_mem[FRAME_SIZE];
45 |   float pitch_buf[PITCH_BUF_SIZE];
46 |   float pitch_enh_buf[PITCH_BUF_SIZE];
47 |   float last_gain;
48 |   int last_period;
49 |   float mem_hp_x[2];
50 |   float lastg[NB_BANDS];
51 |   RNNState rnn;
52 | };
53 | 
54 | int compute_frame_features(DenoiseState *st, kiss_fft_cpx *X, kiss_fft_cpx *P,
55 |                                   float *Ex, float *Ep, float *Exp, float *features, const float *in);
56 | 
57 | void biquad(float *y, float mem[2], const float *x, const float *b, const float *a, int N);
58 | 
59 | #endif


--------------------------------------------------------------------------------
/src/featureExtraction.py:
--------------------------------------------------------------------------------
  1 | # Feature extraction example
  2 | import numpy as np
  3 | import librosa
  4 | import soundfile as sf
  5 | import sys
  6 | import h5py
  7 | 
  8 | # Sampling rate
  9 | samplingRate = 48000
 10 | 
 11 | # Set up for training
 12 | if sys.argv[1] == 'training':
 13 |     # Total number of 10ms frames that will be processed
 14 |     totalFrames = int(sys.argv[4])
 15 | 
 16 |     # Number of samples to process per batch
 17 |     batch_size = 1000 * samplingRate
 18 | 
 19 |     # Βatches of samples. Division by 100 is due to the 10ms duration of each frame in totalFrames
 20 |     batches = int(totalFrames * samplingRate / (batch_size * 100))
 21 | 
 22 |     # Number of RNNoise frames to which each batch of samples maps to
 23 |     frames_per_batch = int(totalFrames / batches)
 24 | 
 25 |     # Initiallize vectors
 26 |     spectral_centroid = np.zeros(shape=(totalFrames))
 27 |     spectral_bandwidth = np.zeros(shape=(totalFrames))
 28 |     spectral_rolloff = np.zeros(shape=(totalFrames))
 29 | 
 30 | # Set up for testing
 31 | else:
 32 |     # Load the sample wav file with its sampling rate
 33 |     y, sr = sf.read(sys.argv[2])
 34 | 
 35 |     # Each frame is 10ms
 36 |     totalFrames = int(len(y) / 48000) * 100 
 37 | 
 38 |     batch_size = len(y)
 39 | 
 40 |     batches = 1
 41 | 
 42 |     frames_per_batch = totalFrames
 43 | 
 44 | 
 45 | # Open an h5 file for output
 46 | hf = h5py.File(sys.argv[3], 'w')
 47 | 
 48 | # Process batches
 49 | for batch_num in range(batches):
 50 | 
 51 |     print("Processing batch", batch_num, "out of", batches)
 52 | 
 53 |     # Load input
 54 |     if sys.argv[1] == 'training':
 55 |         # For training:
 56 |         y, sr = sf.read(sys.argv[2], channels=1, samplerate=samplingRate, subtype='PCM_16', start=batch_num*batch_size, frames=batch_size)
 57 |     else:
 58 |         # Load the sample wav file with its sampling rate
 59 |         y, sr = sf.read(sys.argv[2])
 60 | 
 61 |     # Split to 20ms overlaping frames
 62 |     # 960 is 20ms for 48000 sampling rate, 480 adds 10ms overlap with the previous frame
 63 |     # frames = librosa.util.frame(x=y, frame_length=960, hop_length=480, axis=0)
 64 | 
 65 |     # Root Mean Square
 66 |     #rms = librosa.feature.rms(y=y, frame_length=960, hop_length=480)
 67 | 
 68 |     # Spectral centroid
 69 |     spectral_centroid_t = librosa.feature.spectral_centroid(y=y, sr=sr, n_fft=960, hop_length=480)
 70 |     spectral_centroid_t = np.reshape(spectral_centroid_t, newshape=(-1))
 71 |     if sys.argv[1] == 'training':
 72 |         spectral_centroid[batch_num*frames_per_batch:(batch_num+1)*frames_per_batch] = spectral_centroid_t[:-1]
 73 |     else:
 74 |         spectral_centroid = spectral_centroid_t[:-1]
 75 | 
 76 |     # Spectral bandwidth
 77 |     spectral_bandwidth_t = librosa.feature.spectral_bandwidth(y=y, sr=sr, n_fft=960, hop_length=480)
 78 |     spectral_bandwidth_t = np.reshape(spectral_bandwidth_t, newshape=(-1))
 79 |     if sys.argv[1] == 'training':
 80 |         spectral_bandwidth[batch_num*frames_per_batch:(batch_num+1)*frames_per_batch] = spectral_bandwidth_t[:-1]
 81 |     else:
 82 |         spectral_bandwidth = spectral_bandwidth_t[:-1]
 83 | 
 84 |     # Spectral flatness
 85 |     #spectral_flatness = librosa.feature.spectral_flatness(y=y, n_fft=960, hop_length=480)
 86 | 
 87 |     # Spectral roll-off frequency
 88 |     spectral_rolloff_t = librosa.feature.spectral_rolloff(y=y, sr=sr, n_fft=960, hop_length=480)
 89 |     spectral_rolloff_t = np.reshape(spectral_rolloff_t, newshape=(-1))
 90 |     if sys.argv[1] == 'training':
 91 |         spectral_rolloff[batch_num*frames_per_batch:(batch_num+1)*frames_per_batch] = spectral_rolloff_t[:-1]
 92 |     else:
 93 |         spectral_rolloff = spectral_rolloff_t[:-1]
 94 | 
 95 | 
 96 | 
 97 | # Normalize and save data
 98 | if sys.argv[1] == 'training':
 99 |     spectral_centroid_std = np.std(spectral_centroid)
100 |     spectral_centroid_mean = np.mean(spectral_centroid)
101 |     spectral_centroid -= spectral_centroid_mean
102 |     spectral_centroid /= spectral_centroid_std
103 |     print('Spectral Centroid Std:', spectral_centroid_std)
104 |     print('Spectral Centroid Mean:', spectral_centroid_mean)
105 | else:
106 |     spectral_centroid -= 4112.5994
107 |     spectral_centroid /= 2842.3116
108 | hf.create_dataset('centroid', data=spectral_centroid)
109 | 
110 | if sys.argv[1] == 'training':
111 |     spectral_bandwidth_std = np.std(spectral_bandwidth)
112 |     spectral_bandwidth_mean = np.mean(spectral_bandwidth)
113 |     spectral_bandwidth -= spectral_bandwidth_mean
114 |     spectral_bandwidth /= spectral_bandwidth_std
115 |     print('Spectral Bandwidth Std:', spectral_bandwidth_std)
116 |     print('Spectral Bandwidth Mean:', spectral_bandwidth_mean)
117 | else:
118 |     spectral_bandwidth -= 4952.0481
119 |     spectral_bandwidth /= 1936.2998
120 | hf.create_dataset('bandwidth', data=spectral_bandwidth)
121 | 
122 | if sys.argv[1] == 'training':
123 |     spectral_rolloff_std = np.std(spectral_rolloff)
124 |     spectral_rolloff_mean = np.mean(spectral_rolloff)
125 |     spectral_rolloff -= spectral_rolloff_mean
126 |     spectral_rolloff /= spectral_rolloff_std
127 |     print('Spectral Rolloff Std:', spectral_rolloff_std)
128 |     print('Spectral Rolloff Mean:', spectral_rolloff_mean)
129 | else:
130 |     spectral_rolloff -= 8670.3725
131 |     spectral_rolloff /= 6298.5521
132 | hf.create_dataset('rolloff', data=spectral_rolloff)
133 | 
134 | # Close h5
135 | hf.close()


--------------------------------------------------------------------------------
/training/bin2hdf5.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/python
 2 | 
 3 | from __future__ import print_function
 4 | 
 5 | import numpy as np
 6 | import h5py
 7 | import sys
 8 | 
 9 | if sys.argv[4] == 'training':
10 |     ORIGINAL_FEATURES = 87
11 | else:
12 |     ORIGINAL_FEATURES = 42
13 | 
14 | # Load original RNNoise features
15 | vdata = np.fromfile(sys.argv[1], dtype='float32')
16 | vdataY = int(vdata.size/ORIGINAL_FEATURES)
17 | vdata = np.reshape(vdata, (vdataY, ORIGINAL_FEATURES))
18 | 
19 | # Load extended features
20 | exFeatFile = h5py.File(sys.argv[2], 'r')
21 | exdata = np.array([exFeatFile.get('centroid'), exFeatFile.get('bandwidth'), exFeatFile.get('rolloff')]).T
22 | exFeatFile.close()
23 | 
24 | # Concatenate the matrices
25 | data = np.concatenate((vdata[:,:ORIGINAL_FEATURES], exdata, vdata[:,ORIGINAL_FEATURES:]), axis=1)
26 | 
27 | # Write feature file
28 | h5f = h5py.File(sys.argv[3], 'w')
29 | h5f.create_dataset('data', data=data)
30 | h5f.close()
31 | 


--------------------------------------------------------------------------------
/training/evaluation.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/python
  2 | 
  3 | from __future__ import print_function
  4 | 
  5 | import sys
  6 | import h5py
  7 | import numpy as np
  8 | import librosa
  9 | import soundfile as sf
 10 | import math
 11 | from tensorflow import keras
 12 | from tensorflow.keras.constraints import Constraint
 13 | 
 14 | # Global constants declarations
 15 | NB_BANDS = 22
 16 | FRAME_SIZE = 480
 17 | WINDOW_SIZE = FRAME_SIZE * 2
 18 | FREQ_SIZE = FRAME_SIZE + 1
 19 | MAX_PITCH = 768
 20 | FRAME_SIZE_SHIFT = 2
 21 | eband5ms= np.array([0,  1,  2,  3,  4,  5,  6,  7,  8, 10, 12, 14, 16, 20, 24, 28, 34, 40, 48, 60, 78, 100])
 22 | 
 23 | # Calculate energy for each subband
 24 | def energy(X):
 25 |   sum = [0] * NB_BANDS
 26 |   for i in range(NB_BANDS - 1):
 27 |     band_size = (eband5ms[i+1]-eband5ms[i]) << FRAME_SIZE_SHIFT
 28 |     for j in range(band_size):
 29 |       frac = j/band_size
 30 |       tmp = (X[(eband5ms[i]<<FRAME_SIZE_SHIFT) + j].real)**2+(X[(eband5ms[i]<<FRAME_SIZE_SHIFT) + j].imag)**2
 31 |       sum[i] += (1-frac)*tmp
 32 |       sum[i+1] += frac*tmp
 33 |   
 34 |   sum[0] *= 2
 35 |   sum[NB_BANDS - 1] *= 2
 36 |   Ex = [0] * NB_BANDS
 37 |   for i in range(NB_BANDS):
 38 |     Ex[i] = sum[i]
 39 | 
 40 |   return Ex
 41 | 
 42 | # Apply vorbis window
 43 | def vorbis_window(x):
 44 |     dt = np.linspace(1, 960, 960)
 45 |     win = np.sin((np.pi/2)*np.sin(np.pi*dt/960)**2)
 46 |     x = np.multiply(x,win)
 47 |     return x
 48 | 
 49 | # Interpolate band gain
 50 | def interp_band_gain(bandE):
 51 |     
 52 |     g = [0] * FREQ_SIZE
 53 | 
 54 |     for i in range(NB_BANDS - 1):
 55 |         band_size = (eband5ms[i+1]-eband5ms[i])<<FRAME_SIZE_SHIFT
 56 |         for j in range(band_size):
 57 |             frac = j/band_size
 58 |             g[(eband5ms[i]<<FRAME_SIZE_SHIFT) + j] = (1-frac)*bandE[i] + frac*bandE[i+1]
 59 | 
 60 |     return g
 61 | 
 62 | # Pitch Filter
 63 | def pitch_filter(X, P, Ex, Ep, Exp, g):
 64 |     
 65 |     r = [0] * NB_BANDS
 66 | 
 67 |     resultX = []
 68 |     
 69 |     for i in range(NB_BANDS):
 70 |         if Exp[i]>g[i] :
 71 |             r[i] = 1
 72 |         else:
 73 |             r[i] = ((Exp[i])**2) *(1-(g[i])**2)/(0.001 + (g[i]**2) * (1-(Exp[i])**2))
 74 |         
 75 |         r[i] = math.sqrt(min(1, max(0, r[i])))
 76 |         r[i] *= math.sqrt(Ex[i]/(1e-8+Ep[i]))
 77 |     
 78 |     rf = interp_band_gain(r)
 79 | 
 80 |     for i in range(FREQ_SIZE):
 81 |         resultX.append(complex((X[i].real + rf[i]*P[i].real),(X[i].imag + rf[i]*P[i].imag)))
 82 |         # resultX[i].real += rf[i]*P[i].real
 83 |         # resultX[i].imag += rf[i]*P[i].imag
 84 |     
 85 |     newE = energy(resultX)
 86 | 
 87 |     norm = [(math.sqrt(Ex[i]/(1e-8+newE[i]))) for i in range(NB_BANDS)]
 88 | 
 89 |     normf = interp_band_gain(norm)
 90 | 
 91 |     for i in range(FREQ_SIZE):
 92 |         resultX[i] = complex(resultX[i].real * normf[i], resultX[i].imag * normf[i])
 93 |         # resultX[i].real *= normf[i]
 94 |         # resultX[i].imag *= normf[i]
 95 | 
 96 |     return resultX
 97 | 
 98 | # Compute band correlation
 99 | def compute_band_corr(X, P):
100 | 
101 |     sum = [0] * NB_BANDS
102 | 
103 |     for i in range(NB_BANDS - 1):
104 |         band_size = (eband5ms[i+1]-eband5ms[i]) << FRAME_SIZE_SHIFT
105 |         for j in range(band_size):
106 |             frac = j / band_size
107 |             tmp = X[(eband5ms[i]<<FRAME_SIZE_SHIFT) + j].real * P[(eband5ms[i]<<FRAME_SIZE_SHIFT) + j].real
108 |             tmp += X[(eband5ms[i]<<FRAME_SIZE_SHIFT) + j].imag * P[(eband5ms[i]<<FRAME_SIZE_SHIFT) + j].imag
109 |             sum[i] += (1-frac)*tmp
110 |             sum[i+1] += frac*tmp
111 |     
112 |     sum[0] *= 2
113 |     sum[NB_BANDS-1] *= 2
114 | 
115 |     return sum
116 | 
117 | # Model functions
118 | def my_crossentropy(y_true, y_pred):
119 |     return K.mean(2*K.abs(y_true-0.5) * K.binary_crossentropy(y_pred, y_true), axis=-1)
120 | 
121 | def mymask(y_true):
122 |     return K.minimum(y_true+1., 1.)
123 | 
124 | def msse(y_true, y_pred):
125 |     return K.mean(mymask(y_true) * K.square(K.sqrt(y_pred) - K.sqrt(y_true)), axis=-1)
126 | 
127 | def mycost(y_true, y_pred):
128 |     return K.mean(mymask(y_true) * (10*K.square(K.square(K.sqrt(y_pred) - K.sqrt(y_true))) + K.square(K.sqrt(y_pred) - K.sqrt(y_true)) + 0.01*K.binary_crossentropy(y_pred, y_true)), axis=-1)
129 | 
130 | def my_accuracy(y_true, y_pred):
131 |     return K.mean(2*K.abs(y_true-0.5) * K.equal(y_true, K.round(y_pred)), axis=-1)
132 | 
133 | # Custom object in tensorflow model
134 | class WeightClip(Constraint):
135 |     #Clips the weights incident to each hidden unit to be inside a range
136 |     
137 |     def __init__(self, c=2, **kwargs):
138 |         self.c = c
139 | 
140 |     def __call__(self, p):
141 |         return K.clip(p, -self.c, self.c)
142 | 
143 |     def get_config(self):
144 |         return {'name': self.__class__.__name__,
145 |             'c': self.c}
146 | 
147 | 
148 | # Load model
149 | model = keras.models.load_model(filepath=sys.argv[1], custom_objects={'WeightClip': WeightClip, 
150 | 'mycost':mycost, 'my_crossentropy':my_crossentropy, 'mymask':mymask, 'msse': msse, 'my_accuracy': my_accuracy})
151 | 
152 | # Load & reshape features data
153 | print('Loading data...')
154 | with h5py.File(sys.argv[2], 'r') as hf:
155 |     all_data = hf['data'][:]
156 | print('done.')
157 | 
158 | window_size = 1
159 | nb_sequences = len(all_data)//window_size
160 | print(nb_sequences, ' sequences')
161 | x_input = all_data[:nb_sequences*window_size, :all_data.shape[1]]
162 | x_input = np.reshape(x_input, (nb_sequences, window_size, all_data.shape[1]))
163 | 
164 | # Get output
165 | output = model.predict(x_input)
166 | vadOutput = output[1]
167 | gainsOutput = output[0]
168 | 
169 | # Transform results data
170 | gainsOutput = np.reshape(gainsOutput, (nb_sequences * window_size, 22))
171 | vadOutput = np.reshape(vadOutput, (nb_sequences * window_size, 1))
172 | 
173 | # Transform input features for convenience
174 | x_input = np.reshape(x_input, (nb_sequences * window_size, all_data.shape[1]))
175 | 
176 | # Load audio data
177 | y, sr = sf.read(sys.argv[3])
178 | 
179 | # Split to 20ms overlaping frames
180 | # 960 is 20ms for 48000 sampling rate, 480 adds 10ms overlap at the beginning
181 | inWindows = librosa.util.frame(x=y, frame_length=960, hop_length=480, axis=0)
182 | 
183 | # Calculate pitches using the input features
184 | pitches = [round(MAX_PITCH-(x/0.01 + 300)) for x in x_input[:,40]]
185 | 
186 | # Declare gains used
187 | finalGains = np.zeros(NB_BANDS)
188 | 
189 | # Declare output array
190 | outData = np.zeros(len(y))
191 | 
192 | windowIndex = 0
193 | # Fourier transform each window, apply gains and inverse FFT
194 | for window in inWindows:
195 | 
196 |     # Print currently processed window number
197 |     #print(windowIndex)
198 | 
199 |     # Apply vorbis window to window
200 |     vWindow = vorbis_window(window)
201 | 
202 |     # FFT the window
203 |     fftWindow = np.fft.fft(vWindow, n=WINDOW_SIZE)
204 | 
205 |     # Slice FFT Window on FREQ_SIZE
206 |     fftWindow = fftWindow[:FREQ_SIZE]
207 | 
208 |     # Energy of fftWindow
209 |     EvWindow = energy(fftWindow)
210 | 
211 |     # Create the pitch buffer based on the last 3 windows
212 |     pitchBuffer = []
213 |     if windowIndex > 1:
214 |         # [max_pitch-frame_size][frame][frame][frame]
215 |         pitchBuffer = np.concatenate((inWindows[windowIndex - 2][(WINDOW_SIZE - MAX_PITCH):FRAME_SIZE], inWindows[windowIndex - 1][:FRAME_SIZE], window), axis=0)
216 |     elif windowIndex == 1:
217 |         pitchBuffer = np.concatenate((np.zeros(MAX_PITCH - FRAME_SIZE), inWindows[windowIndex - 1][:FRAME_SIZE], window), axis=0)
218 |     elif windowIndex == 0:
219 |         pitchBuffer = np.concatenate((np.zeros(MAX_PITCH), window), axis=0)
220 | 
221 |     # Calculate p buffer
222 |     p = pitchBuffer[pitches[windowIndex] : pitches[windowIndex] + WINDOW_SIZE]
223 | 
224 |     # Apply vorbis window on p
225 |     vP = vorbis_window(p)
226 | 
227 |     # FFT of vP
228 |     fftP = np.fft.fft(vP, n=WINDOW_SIZE)
229 | 
230 |     # Zero padding on fftP
231 |     fftP[FRAME_SIZE+1:] = 0
232 | 
233 |     # Energy of vP
234 |     EvP = energy(fftP)
235 | 
236 |     # Compute ExP
237 |     ExP = compute_band_corr(fftWindow, fftP)
238 | 
239 |     # Normalize ExP
240 |     for i in range(NB_BANDS):
241 |         ExP[i] = ExP[i]/math.sqrt(0.001+EvWindow[i]*EvP[i])
242 | 
243 |     # Apply pitch filter
244 |     X = pitch_filter(fftWindow, fftP, EvWindow, EvP, ExP, gainsOutput[windowIndex,:])
245 |     # Disable pitch filter
246 |     #X = fftWindow
247 | 
248 |     # Gain Smoothing
249 |     alpha = 0.6
250 |     for i in range(NB_BANDS):
251 |         finalGains[i] = max(gainsOutput[windowIndex,i], alpha * finalGains[i])
252 | 
253 |     # Interpolate band gains
254 |     gf = interp_band_gain(finalGains)
255 | 
256 |     # Apply gains
257 |     for i in range(FREQ_SIZE):
258 |         X[i] = complex((X[i].real * gf[i]), (X[i].imag * gf[i]))
259 | 
260 |     # Synthesize frames
261 |     X = np.concatenate([X, np.zeros(FRAME_SIZE-1)])
262 |     for i in range(FREQ_SIZE, WINDOW_SIZE):
263 |         X[i] = np.complex(X[WINDOW_SIZE-i].real, -X[WINDOW_SIZE-i].imag)
264 |     x = np.fft.ifft(X, n=WINDOW_SIZE)
265 |     vx = vorbis_window(x)
266 |     outData[(windowIndex * FRAME_SIZE):((windowIndex + 2) * FRAME_SIZE)] = np.add(outData[(windowIndex * FRAME_SIZE):((windowIndex + 2) * FRAME_SIZE)], vx)
267 | 
268 |     # Increment frame index
269 |     windowIndex += 1
270 | 
271 | # Normalize energy
272 | # eClean = 0
273 | # for element in outData:
274 | #     eClean += element ** 2
275 | # eNoisy = 0
276 | # for element in y:
277 | #     eNoisy += element ** 2
278 | # ratio = (eNoisy / eClean)
279 | # outData = outData * ratio
280 | # outData /= max(outData)
281 | 
282 | # Write output file
283 | sf.write(sys.argv[4], outData, sr, subtype='PCM_16')


--------------------------------------------------------------------------------
/training/evaluation_batch.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/python
  2 | 
  3 | from __future__ import print_function
  4 | 
  5 | import sys
  6 | import h5py
  7 | import numpy as np
  8 | import librosa
  9 | import soundfile as sf
 10 | import math
 11 | from tensorflow import keras
 12 | from tensorflow.keras.constraints import Constraint
 13 | import os
 14 | from os.path import splitext
 15 | import glob
 16 | 
 17 | # Global constants declarations
 18 | NB_BANDS = 22
 19 | FRAME_SIZE = 480
 20 | WINDOW_SIZE = FRAME_SIZE * 2
 21 | FREQ_SIZE = FRAME_SIZE + 1
 22 | MAX_PITCH = 768
 23 | FRAME_SIZE_SHIFT = 2
 24 | eband5ms= np.array([0,  1,  2,  3,  4,  5,  6,  7,  8, 10, 12, 14, 16, 20, 24, 28, 34, 40, 48, 60, 78, 100])
 25 | 
 26 | # Calculate energy for each subband
 27 | def energy(X):
 28 |   sum = [0] * NB_BANDS
 29 |   for i in range(NB_BANDS - 1):
 30 |     band_size = (eband5ms[i+1]-eband5ms[i]) << FRAME_SIZE_SHIFT
 31 |     for j in range(band_size):
 32 |       frac = j/band_size
 33 |       tmp = (X[(eband5ms[i]<<FRAME_SIZE_SHIFT) + j].real)**2+(X[(eband5ms[i]<<FRAME_SIZE_SHIFT) + j].imag)**2
 34 |       sum[i] += (1-frac)*tmp
 35 |       sum[i+1] += frac*tmp
 36 |   
 37 |   sum[0] *= 2
 38 |   sum[NB_BANDS - 1] *= 2
 39 |   Ex = [0] * NB_BANDS
 40 |   for i in range(NB_BANDS):
 41 |     Ex[i] = sum[i]
 42 | 
 43 |   return Ex
 44 | 
 45 | # Apply vorbis window
 46 | def vorbis_window(x):
 47 |     dt = np.linspace(1, 960, 960)
 48 |     win = np.sin((np.pi/2)*np.sin(np.pi*dt/960)**2)
 49 |     x = np.multiply(x,win)
 50 |     return x
 51 | 
 52 | # Interpolate band gain
 53 | def interp_band_gain(bandE):
 54 |     
 55 |     g = [0] * FREQ_SIZE
 56 | 
 57 |     for i in range(NB_BANDS - 1):
 58 |         band_size = (eband5ms[i+1]-eband5ms[i])<<FRAME_SIZE_SHIFT
 59 |         for j in range(band_size):
 60 |             frac = j/band_size
 61 |             g[(eband5ms[i]<<FRAME_SIZE_SHIFT) + j] = (1-frac)*bandE[i] + frac*bandE[i+1]
 62 | 
 63 |     return g
 64 | 
 65 | # Pitch Filter
 66 | def pitch_filter(X, P, Ex, Ep, Exp, g):
 67 |     
 68 |     r = [0] * NB_BANDS
 69 | 
 70 |     resultX = []
 71 |     
 72 |     for i in range(NB_BANDS):
 73 |         if Exp[i]>g[i] :
 74 |             r[i] = 1
 75 |         else:
 76 |             r[i] = ((Exp[i])**2) *(1-(g[i])**2)/(0.001 + (g[i]**2) * (1-(Exp[i])**2))
 77 |         
 78 |         r[i] = math.sqrt(min(1, max(0, r[i])))
 79 |         r[i] *= math.sqrt(Ex[i]/(1e-8+Ep[i]))
 80 |     
 81 |     rf = interp_band_gain(r)
 82 | 
 83 |     for i in range(FREQ_SIZE):
 84 |         resultX.append(complex((X[i].real + rf[i]*P[i].real),(X[i].imag + rf[i]*P[i].imag)))
 85 |         # resultX[i].real += rf[i]*P[i].real
 86 |         # resultX[i].imag += rf[i]*P[i].imag
 87 |     
 88 |     newE = energy(resultX)
 89 | 
 90 |     norm = [(math.sqrt(Ex[i]/(1e-8+newE[i]))) for i in range(NB_BANDS)]
 91 | 
 92 |     normf = interp_band_gain(norm)
 93 | 
 94 |     for i in range(FREQ_SIZE):
 95 |         resultX[i] = complex(resultX[i].real * normf[i], resultX[i].imag * normf[i])
 96 |         # resultX[i].real *= normf[i]
 97 |         # resultX[i].imag *= normf[i]
 98 | 
 99 |     return resultX
100 | 
101 | # Compute band correlation
102 | def compute_band_corr(X, P):
103 | 
104 |     sum = [0] * NB_BANDS
105 | 
106 |     for i in range(NB_BANDS - 1):
107 |         band_size = (eband5ms[i+1]-eband5ms[i]) << FRAME_SIZE_SHIFT
108 |         for j in range(band_size):
109 |             frac = j / band_size
110 |             tmp = X[(eband5ms[i]<<FRAME_SIZE_SHIFT) + j].real * P[(eband5ms[i]<<FRAME_SIZE_SHIFT) + j].real
111 |             tmp += X[(eband5ms[i]<<FRAME_SIZE_SHIFT) + j].imag * P[(eband5ms[i]<<FRAME_SIZE_SHIFT) + j].imag
112 |             sum[i] += (1-frac)*tmp
113 |             sum[i+1] += frac*tmp
114 |     
115 |     sum[0] *= 2
116 |     sum[NB_BANDS-1] *= 2
117 | 
118 |     return sum
119 | 
120 | # Model functions
121 | def my_crossentropy(y_true, y_pred):
122 |     return K.mean(2*K.abs(y_true-0.5) * K.binary_crossentropy(y_pred, y_true), axis=-1)
123 | 
124 | def mymask(y_true):
125 |     return K.minimum(y_true+1., 1.)
126 | 
127 | def msse(y_true, y_pred):
128 |     return K.mean(mymask(y_true) * K.square(K.sqrt(y_pred) - K.sqrt(y_true)), axis=-1)
129 | 
130 | def mycost(y_true, y_pred):
131 |     return K.mean(mymask(y_true) * (10*K.square(K.square(K.sqrt(y_pred) - K.sqrt(y_true))) + K.square(K.sqrt(y_pred) - K.sqrt(y_true)) + 0.01*K.binary_crossentropy(y_pred, y_true)), axis=-1)
132 | 
133 | def my_accuracy(y_true, y_pred):
134 |     return K.mean(2*K.abs(y_true-0.5) * K.equal(y_true, K.round(y_pred)), axis=-1)
135 | 
136 | # Custom object in tensorflow model
137 | class WeightClip(Constraint):
138 |     #Clips the weights incident to each hidden unit to be inside a range
139 |     
140 |     def __init__(self, c=2, **kwargs):
141 |         self.c = c
142 | 
143 |     def __call__(self, p):
144 |         return K.clip(p, -self.c, self.c)
145 | 
146 |     def get_config(self):
147 |         return {'name': self.__class__.__name__,
148 |             'c': self.c}
149 | 
150 | 
151 | # Load model
152 | model = keras.models.load_model(filepath=sys.argv[2], custom_objects={'WeightClip': WeightClip, 
153 | 'mycost':mycost, 'my_crossentropy':my_crossentropy, 'mymask':mymask, 'msse': msse, 'my_accuracy': my_accuracy})
154 | 
155 | # Get list of audio files
156 | audioFilesPath = sys.argv[1]
157 | audioFiles = [f for f in glob.glob(audioFilesPath + "*.wav")]
158 | 
159 | fileCounter = 0
160 | 
161 | for audioFile in audioFiles:
162 | 
163 |     print("Processing file: ", fileCounter)
164 | 
165 |     # Load & reshape features data
166 |     print('Loading data...')
167 |     with h5py.File(os.path.splitext(audioFile)[0] + ".fullFeatures.h5", 'r') as hf:
168 |         all_data = hf['data'][:]
169 |     print('done.')
170 | 
171 |     window_size = 1
172 |     nb_sequences = len(all_data)//window_size
173 |     print(nb_sequences, ' sequences')
174 |     x_input = all_data[:nb_sequences*window_size, :all_data.shape[1]]
175 |     x_input = np.reshape(x_input, (nb_sequences, window_size, all_data.shape[1]))
176 | 
177 |     # Get output
178 |     output = model.predict(x_input)
179 |     vadOutput = output[1]
180 |     gainsOutput = output[0]
181 | 
182 |     # Transform results data
183 |     gainsOutput = np.reshape(gainsOutput, (nb_sequences * window_size, 22))
184 |     vadOutput = np.reshape(vadOutput, (nb_sequences * window_size, 1))
185 | 
186 |     # Transform input features for convenience
187 |     x_input = np.reshape(x_input, (nb_sequences * window_size, all_data.shape[1]))
188 | 
189 |     # Load audio data
190 |     y, sr = sf.read(audioFile)
191 | 
192 |     # Split to 20ms overlaping frames
193 |     # 960 is 20ms for 48000 sampling rate, 480 adds 10ms overlap at the beginning
194 |     inWindows = librosa.util.frame(x=y, frame_length=960, hop_length=480, axis=0)
195 | 
196 |     # Calculate pitches using the input features
197 |     pitches = [round(MAX_PITCH-(x/0.01 + 300)) for x in x_input[:,40]]
198 | 
199 |     # Declare gains used
200 |     finalGains = np.zeros(NB_BANDS)
201 | 
202 |     # Declare output array
203 |     outData = np.zeros(len(y))
204 | 
205 |     windowIndex = 0
206 |     # Fourier transform each window, apply gains and inverse FFT
207 |     for window in inWindows:
208 | 
209 |         # Print currently processed window number
210 |         #print(windowIndex)
211 | 
212 |         # Apply vorbis window to window
213 |         vWindow = vorbis_window(window)
214 | 
215 |         # FFT the window
216 |         fftWindow = np.fft.fft(vWindow, n=WINDOW_SIZE)
217 | 
218 |         # Slice FFT Window on FREQ_SIZE
219 |         fftWindow = fftWindow[:FREQ_SIZE]
220 | 
221 |         # Energy of fftWindow
222 |         EvWindow = energy(fftWindow)
223 | 
224 |         # Create the pitch buffer based on the last 3 windows
225 |         pitchBuffer = []
226 |         if windowIndex > 1:
227 |             # [max_pitch-frame_size][frame][frame][frame]
228 |             pitchBuffer = np.concatenate((inWindows[windowIndex - 2][(WINDOW_SIZE - MAX_PITCH):FRAME_SIZE], inWindows[windowIndex - 1][:FRAME_SIZE], window), axis=0)
229 |         elif windowIndex == 1:
230 |             pitchBuffer = np.concatenate((np.zeros(MAX_PITCH - FRAME_SIZE), inWindows[windowIndex - 1][:FRAME_SIZE], window), axis=0)
231 |         elif windowIndex == 0:
232 |             pitchBuffer = np.concatenate((np.zeros(MAX_PITCH), window), axis=0)
233 | 
234 |         # Calculate p buffer
235 |         p = pitchBuffer[pitches[windowIndex] : pitches[windowIndex] + WINDOW_SIZE]
236 | 
237 |         # Apply vorbis window on p
238 |         vP = vorbis_window(p)
239 | 
240 |         # FFT of vP
241 |         fftP = np.fft.fft(vP, n=WINDOW_SIZE)
242 | 
243 |         # Zero padding on fftP
244 |         fftP[FRAME_SIZE+1:] = 0
245 | 
246 |         # Energy of vP
247 |         EvP = energy(fftP)
248 | 
249 |         # Compute ExP
250 |         ExP = compute_band_corr(fftWindow, fftP)
251 | 
252 |         # Normalize ExP
253 |         for i in range(NB_BANDS):
254 |             ExP[i] = ExP[i]/math.sqrt(0.001+EvWindow[i]*EvP[i])
255 | 
256 |         # Apply pitch filter
257 |         X = pitch_filter(fftWindow, fftP, EvWindow, EvP, ExP, gainsOutput[windowIndex,:])
258 |         # Disable pitch filter
259 |         #X = fftWindow
260 | 
261 |         # Gain Smoothing
262 |         alpha = 0.6
263 |         for i in range(NB_BANDS):
264 |             finalGains[i] = max(gainsOutput[windowIndex,i], alpha * finalGains[i])
265 | 
266 |         # Interpolate band gains
267 |         gf = interp_band_gain(finalGains)
268 | 
269 |         # Apply gains
270 |         for i in range(FREQ_SIZE):
271 |             X[i] = complex((X[i].real * gf[i]), (X[i].imag * gf[i]))
272 | 
273 |         # Synthesize frames
274 |         X = np.concatenate([X, np.zeros(FRAME_SIZE-1)])
275 |         for i in range(FREQ_SIZE, WINDOW_SIZE):
276 |             X[i] = np.complex(X[WINDOW_SIZE-i].real, -X[WINDOW_SIZE-i].imag)
277 |         x = np.fft.ifft(X, n=WINDOW_SIZE)
278 |         vx = vorbis_window(x)
279 |         outData[(windowIndex * FRAME_SIZE):((windowIndex + 2) * FRAME_SIZE)] = np.add(outData[(windowIndex * FRAME_SIZE):((windowIndex + 2) * FRAME_SIZE)], vx)
280 | 
281 |         # Increment frame index
282 |         windowIndex += 1
283 | 
284 |     # Normalize energy
285 |     # eClean = 0
286 |     # for element in outData:
287 |     #     eClean += element ** 2
288 |     # eNoisy = 0
289 |     # for element in y:
290 |     #     eNoisy += element ** 2
291 |     # ratio = (eNoisy / eClean)
292 |     # outData = outData * ratio
293 |     # outData /= max(outData)
294 | 
295 |     # Write output file
296 |     sf.write(os.path.splitext(audioFile)[0] + ".clean.wav", outData, sr, subtype='PCM_16')
297 | 
298 |     fileCounter += 1


--------------------------------------------------------------------------------
/training/rnn_train.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/python
  2 | 
  3 | from __future__ import print_function
  4 | 
  5 | import keras
  6 | from keras.models import Sequential
  7 | from keras.models import Model
  8 | from keras.layers import Input
  9 | from keras.layers import Dense
 10 | from keras.layers import LSTM
 11 | from keras.layers import GRU
 12 | from keras.layers import SimpleRNN
 13 | from keras.layers import Dropout
 14 | from keras.layers import concatenate
 15 | from keras import losses
 16 | from keras import regularizers
 17 | from keras.constraints import min_max_norm
 18 | import h5py
 19 | 
 20 | from keras.constraints import Constraint
 21 | from keras import backend as K
 22 | import numpy as np
 23 | import sys
 24 | 
 25 | #import tensorflow as tf
 26 | #from keras.backend.tensorflow_backend import set_session
 27 | #config = tf.ConfigProto()
 28 | #config.gpu_options.per_process_gpu_memory_fraction = 0.42
 29 | #set_session(tf.Session(config=config))
 30 | 
 31 | 
 32 | def my_crossentropy(y_true, y_pred):
 33 |     return K.mean(2*K.abs(y_true-0.5) * K.binary_crossentropy(y_pred, y_true), axis=-1)
 34 | 
 35 | def mymask(y_true):
 36 |     return K.minimum(y_true+1., 1.)
 37 | 
 38 | def msse(y_true, y_pred):
 39 |     return K.mean(mymask(y_true) * K.square(K.sqrt(y_pred) - K.sqrt(y_true)), axis=-1)
 40 | 
 41 | def mycost(y_true, y_pred):
 42 |     return K.mean(mymask(y_true) * (10*K.square(K.square(K.sqrt(y_pred) - K.sqrt(y_true))) + K.square(K.sqrt(y_pred) - K.sqrt(y_true)) + 0.01*K.binary_crossentropy(y_pred, y_true)), axis=-1)
 43 | 
 44 | def my_accuracy(y_true, y_pred):
 45 |     return K.mean(2*K.abs(y_true-0.5) * K.equal(y_true, K.round(y_pred)), axis=-1)
 46 | 
 47 | class WeightClip(Constraint):
 48 |     '''Clips the weights incident to each hidden unit to be inside a range
 49 |     '''
 50 |     def __init__(self, c=2):
 51 |         self.c = c
 52 | 
 53 |     def __call__(self, p):
 54 |         return K.clip(p, -self.c, self.c)
 55 | 
 56 |     def get_config(self):
 57 |         return {'name': self.__class__.__name__,
 58 |             'c': self.c}
 59 | 
 60 | reg = 0.000001
 61 | constraint = WeightClip(0.499)
 62 | 
 63 | print('Loading data...')
 64 | with h5py.File(sys.argv[1], 'r') as hf:
 65 |     all_data = hf['data'][:]
 66 | print('done.')
 67 | 
 68 | extraFeatures = 3
 69 | 
 70 | print('Build model...')
 71 | main_input = Input(shape=(None, 42+extraFeatures), name='main_input')
 72 | tmp = Dense(24, activation='tanh', name='input_dense', kernel_constraint=constraint, bias_constraint=constraint)(main_input)
 73 | vad_gru = GRU(24, activation='tanh', recurrent_activation='sigmoid', return_sequences=True, name='vad_gru', kernel_regularizer=regularizers.l2(reg), recurrent_regularizer=regularizers.l2(reg), kernel_constraint=constraint, recurrent_constraint=constraint, bias_constraint=constraint)(tmp)
 74 | vad_output = Dense(1, activation='sigmoid', name='vad_output', kernel_constraint=constraint, bias_constraint=constraint)(vad_gru)
 75 | noise_input = keras.layers.concatenate([tmp, vad_gru, main_input])
 76 | noise_gru = GRU(48, activation='relu', recurrent_activation='sigmoid', return_sequences=True, name='noise_gru', kernel_regularizer=regularizers.l2(reg), recurrent_regularizer=regularizers.l2(reg), kernel_constraint=constraint, recurrent_constraint=constraint, bias_constraint=constraint)(noise_input)
 77 | denoise_input = keras.layers.concatenate([vad_gru, noise_gru, main_input])
 78 | 
 79 | denoise_gru = GRU(96, activation='tanh', recurrent_activation='sigmoid', return_sequences=True, name='denoise_gru', kernel_regularizer=regularizers.l2(reg), recurrent_regularizer=regularizers.l2(reg), kernel_constraint=constraint, recurrent_constraint=constraint, bias_constraint=constraint)(denoise_input)
 80 | 
 81 | denoise_output = Dense(22, activation='sigmoid', name='denoise_output', kernel_constraint=constraint, bias_constraint=constraint)(denoise_gru)
 82 | 
 83 | model = Model(inputs=main_input, outputs=[denoise_output, vad_output])
 84 | 
 85 | model.compile(loss=[mycost, my_crossentropy],
 86 |               metrics=[msse],
 87 |               optimizer='adam', loss_weights=[10, 0.5])
 88 | 
 89 | 
 90 | batch_size = 32
 91 | 
 92 | window_size = 2000
 93 | 
 94 | nb_sequences = len(all_data)//window_size
 95 | print(nb_sequences, ' sequences')
 96 | x_train = all_data[:nb_sequences*window_size, :42+extraFeatures]
 97 | x_train = np.reshape(x_train, (nb_sequences, window_size, 42+extraFeatures))
 98 | 
 99 | y_train = np.copy(all_data[:nb_sequences*window_size, 42+extraFeatures:42+extraFeatures + 22])
100 | y_train = np.reshape(y_train, (nb_sequences, window_size, 22))
101 | 
102 | noise_train = np.copy(all_data[:nb_sequences*window_size, 42+extraFeatures + 22:42+extraFeatures + 44])
103 | noise_train = np.reshape(noise_train, (nb_sequences, window_size, 22))
104 | 
105 | vad_train = np.copy(all_data[:nb_sequences*window_size, 42+extraFeatures + 44:42+extraFeatures + 45])
106 | vad_train = np.reshape(vad_train, (nb_sequences, window_size, 1))
107 | 
108 | all_data = 0
109 | #x_train = x_train.astype('float32')
110 | #y_train = y_train.astype('float32')
111 | 
112 | print(len(x_train), 'train sequences. x shape =', x_train.shape, 'y shape = ', y_train.shape)
113 | 
114 | print('Train...')
115 | model.fit(x_train, [y_train, vad_train],
116 |           batch_size=batch_size,
117 |           epochs=120,
118 |           validation_split=0.1)
119 | 
120 | # Save full model
121 | model.save(sys.argv[2])
122 | 


--------------------------------------------------------------------------------