├── .gitattributes
├── .gitignore
├── LICENSE
├── README.md
├── audio_features.py
├── audio_processing.py
├── av_speech_enhancement.py
├── av_sync.py
├── create_dataset_tfrecords.py
├── dataset_reader.py
├── docs
    ├── Gemfile
    ├── Gemfile.lock
    ├── _config.yml
    ├── _site
    │   ├── assets
    │   │   └── css
    │   │   │   └── style.css
    │   ├── index.html
    │   └── videos
    │   │   ├── grid_2spk.mp4
    │   │   ├── grid_3spk.mp4
    │   │   ├── timit_2spk.mp4
    │   │   └── timit_3spk.mp4
    ├── index.md
    └── videos
    │   ├── grid_2spk.mp4
    │   ├── grid_3spk.mp4
    │   ├── timit_2spk.mp4
    │   └── timit_3spk.mp4
├── enhancement_model.py
├── eval_metrics.py
├── face_landmarks.py
├── mixed_speech_generator.py
├── requirements.txt
├── target_binary_mask.py
├── testing.py
└── training.py


/.gitattributes:
--------------------------------------------------------------------------------
 1 | ###############################################################################
 2 | # Set default behavior to automatically normalize line endings.
 3 | ###############################################################################
 4 | * text=auto
 5 | 
 6 | ###############################################################################
 7 | # Set default behavior for command prompt diff.
 8 | #
 9 | # This is need for earlier builds of msysgit that does not have it on by
10 | # default for csharp files.
11 | # Note: This is only used by command line
12 | ###############################################################################
13 | #*.cs     diff=csharp
14 | 
15 | ###############################################################################
16 | # Set the merge driver for project and solution files
17 | #
18 | # Merging from the command prompt will add diff markers to the files if there
19 | # are conflicts (Merging from VS is not affected by the settings below, in VS
20 | # the diff markers are never inserted). Diff markers may cause the following 
21 | # file extensions to fail to load in VS. An alternative would be to treat
22 | # these files as binary and thus will always conflict and require user
23 | # intervention with every merge. To do so, just uncomment the entries below
24 | ###############################################################################
25 | #*.sln       merge=binary
26 | #*.csproj    merge=binary
27 | #*.vbproj    merge=binary
28 | #*.vcxproj   merge=binary
29 | #*.vcproj    merge=binary
30 | #*.dbproj    merge=binary
31 | #*.fsproj    merge=binary
32 | #*.lsproj    merge=binary
33 | #*.wixproj   merge=binary
34 | #*.modelproj merge=binary
35 | #*.sqlproj   merge=binary
36 | #*.wwaproj   merge=binary
37 | 
38 | ###############################################################################
39 | # behavior for image files
40 | #
41 | # image files are treated as binary by default.
42 | ###############################################################################
43 | #*.jpg   binary
44 | #*.png   binary
45 | #*.gif   binary
46 | 
47 | ###############################################################################
48 | # diff behavior for common document formats
49 | # 
50 | # Convert binary document formats to text before diffing them. This feature
51 | # is only available from the command line. Turn it on by uncommenting the 
52 | # entries below.
53 | ###############################################################################
54 | #*.doc   diff=astextplain
55 | #*.DOC   diff=astextplain
56 | #*.docx  diff=astextplain
57 | #*.DOCX  diff=astextplain
58 | #*.dot   diff=astextplain
59 | #*.DOT   diff=astextplain
60 | #*.pdf   diff=astextplain
61 | #*.PDF   diff=astextplain
62 | #*.rtf   diff=astextplain
63 | #*.RTF   diff=astextplain
64 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | visual_speech_mapping/*
2 | __pycache__/*


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
  1 |                                  Apache License
  2 |                            Version 2.0, January 2004
  3 |                         http://www.apache.org/licenses/
  4 | 
  5 |    TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
  6 | 
  7 |    1. Definitions.
  8 | 
  9 |       "License" shall mean the terms and conditions for use, reproduction,
 10 |       and distribution as defined by Sections 1 through 9 of this document.
 11 | 
 12 |       "Licensor" shall mean the copyright owner or entity authorized by
 13 |       the copyright owner that is granting the License.
 14 | 
 15 |       "Legal Entity" shall mean the union of the acting entity and all
 16 |       other entities that control, are controlled by, or are under common
 17 |       control with that entity. For the purposes of this definition,
 18 |       "control" means (i) the power, direct or indirect, to cause the
 19 |       direction or management of such entity, whether by contract or
 20 |       otherwise, or (ii) ownership of fifty percent (50%) or more of the
 21 |       outstanding shares, or (iii) beneficial ownership of such entity.
 22 | 
 23 |       "You" (or "Your") shall mean an individual or Legal Entity
 24 |       exercising permissions granted by this License.
 25 | 
 26 |       "Source" form shall mean the preferred form for making modifications,
 27 |       including but not limited to software source code, documentation
 28 |       source, and configuration files.
 29 | 
 30 |       "Object" form shall mean any form resulting from mechanical
 31 |       transformation or translation of a Source form, including but
 32 |       not limited to compiled object code, generated documentation,
 33 |       and conversions to other media types.
 34 | 
 35 |       "Work" shall mean the work of authorship, whether in Source or
 36 |       Object form, made available under the License, as indicated by a
 37 |       copyright notice that is included in or attached to the work
 38 |       (an example is provided in the Appendix below).
 39 | 
 40 |       "Derivative Works" shall mean any work, whether in Source or Object
 41 |       form, that is based on (or derived from) the Work and for which the
 42 |       editorial revisions, annotations, elaborations, or other modifications
 43 |       represent, as a whole, an original work of authorship. For the purposes
 44 |       of this License, Derivative Works shall not include works that remain
 45 |       separable from, or merely link (or bind by name) to the interfaces of,
 46 |       the Work and Derivative Works thereof.
 47 | 
 48 |       "Contribution" shall mean any work of authorship, including
 49 |       the original version of the Work and any modifications or additions
 50 |       to that Work or Derivative Works thereof, that is intentionally
 51 |       submitted to Licensor for inclusion in the Work by the copyright owner
 52 |       or by an individual or Legal Entity authorized to submit on behalf of
 53 |       the copyright owner. For the purposes of this definition, "submitted"
 54 |       means any form of electronic, verbal, or written communication sent
 55 |       to the Licensor or its representatives, including but not limited to
 56 |       communication on electronic mailing lists, source code control systems,
 57 |       and issue tracking systems that are managed by, or on behalf of, the
 58 |       Licensor for the purpose of discussing and improving the Work, but
 59 |       excluding communication that is conspicuously marked or otherwise
 60 |       designated in writing by the copyright owner as "Not a Contribution."
 61 | 
 62 |       "Contributor" shall mean Licensor and any individual or Legal Entity
 63 |       on behalf of whom a Contribution has been received by Licensor and
 64 |       subsequently incorporated within the Work.
 65 | 
 66 |    2. Grant of Copyright License. Subject to the terms and conditions of
 67 |       this License, each Contributor hereby grants to You a perpetual,
 68 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 69 |       copyright license to reproduce, prepare Derivative Works of,
 70 |       publicly display, publicly perform, sublicense, and distribute the
 71 |       Work and such Derivative Works in Source or Object form.
 72 | 
 73 |    3. Grant of Patent License. Subject to the terms and conditions of
 74 |       this License, each Contributor hereby grants to You a perpetual,
 75 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 76 |       (except as stated in this section) patent license to make, have made,
 77 |       use, offer to sell, sell, import, and otherwise transfer the Work,
 78 |       where such license applies only to those patent claims licensable
 79 |       by such Contributor that are necessarily infringed by their
 80 |       Contribution(s) alone or by combination of their Contribution(s)
 81 |       with the Work to which such Contribution(s) was submitted. If You
 82 |       institute patent litigation against any entity (including a
 83 |       cross-claim or counterclaim in a lawsuit) alleging that the Work
 84 |       or a Contribution incorporated within the Work constitutes direct
 85 |       or contributory patent infringement, then any patent licenses
 86 |       granted to You under this License for that Work shall terminate
 87 |       as of the date such litigation is filed.
 88 | 
 89 |    4. Redistribution. You may reproduce and distribute copies of the
 90 |       Work or Derivative Works thereof in any medium, with or without
 91 |       modifications, and in Source or Object form, provided that You
 92 |       meet the following conditions:
 93 | 
 94 |       (a) You must give any other recipients of the Work or
 95 |           Derivative Works a copy of this License; and
 96 | 
 97 |       (b) You must cause any modified files to carry prominent notices
 98 |           stating that You changed the files; and
 99 | 
100 |       (c) You must retain, in the Source form of any Derivative Works
101 |           that You distribute, all copyright, patent, trademark, and
102 |           attribution notices from the Source form of the Work,
103 |           excluding those notices that do not pertain to any part of
104 |           the Derivative Works; and
105 | 
106 |       (d) If the Work includes a "NOTICE" text file as part of its
107 |           distribution, then any Derivative Works that You distribute must
108 |           include a readable copy of the attribution notices contained
109 |           within such NOTICE file, excluding those notices that do not
110 |           pertain to any part of the Derivative Works, in at least one
111 |           of the following places: within a NOTICE text file distributed
112 |           as part of the Derivative Works; within the Source form or
113 |           documentation, if provided along with the Derivative Works; or,
114 |           within a display generated by the Derivative Works, if and
115 |           wherever such third-party notices normally appear. The contents
116 |           of the NOTICE file are for informational purposes only and
117 |           do not modify the License. You may add Your own attribution
118 |           notices within Derivative Works that You distribute, alongside
119 |           or as an addendum to the NOTICE text from the Work, provided
120 |           that such additional attribution notices cannot be construed
121 |           as modifying the License.
122 | 
123 |       You may add Your own copyright statement to Your modifications and
124 |       may provide additional or different license terms and conditions
125 |       for use, reproduction, or distribution of Your modifications, or
126 |       for any such Derivative Works as a whole, provided Your use,
127 |       reproduction, and distribution of the Work otherwise complies with
128 |       the conditions stated in this License.
129 | 
130 |    5. Submission of Contributions. Unless You explicitly state otherwise,
131 |       any Contribution intentionally submitted for inclusion in the Work
132 |       by You to the Licensor shall be under the terms and conditions of
133 |       this License, without any additional terms or conditions.
134 |       Notwithstanding the above, nothing herein shall supersede or modify
135 |       the terms of any separate license agreement you may have executed
136 |       with Licensor regarding such Contributions.
137 | 
138 |    6. Trademarks. This License does not grant permission to use the trade
139 |       names, trademarks, service marks, or product names of the Licensor,
140 |       except as required for reasonable and customary use in describing the
141 |       origin of the Work and reproducing the content of the NOTICE file.
142 | 
143 |    7. Disclaimer of Warranty. Unless required by applicable law or
144 |       agreed to in writing, Licensor provides the Work (and each
145 |       Contributor provides its Contributions) on an "AS IS" BASIS,
146 |       WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147 |       implied, including, without limitation, any warranties or conditions
148 |       of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149 |       PARTICULAR PURPOSE. You are solely responsible for determining the
150 |       appropriateness of using or redistributing the Work and assume any
151 |       risks associated with Your exercise of permissions under this License.
152 | 
153 |    8. Limitation of Liability. In no event and under no legal theory,
154 |       whether in tort (including negligence), contract, or otherwise,
155 |       unless required by applicable law (such as deliberate and grossly
156 |       negligent acts) or agreed to in writing, shall any Contributor be
157 |       liable to You for damages, including any direct, indirect, special,
158 |       incidental, or consequential damages of any character arising as a
159 |       result of this License or out of the use or inability to use the
160 |       Work (including but not limited to damages for loss of goodwill,
161 |       work stoppage, computer failure or malfunction, or any and all
162 |       other commercial damages or losses), even if such Contributor
163 |       has been advised of the possibility of such damages.
164 | 
165 |    9. Accepting Warranty or Additional Liability. While redistributing
166 |       the Work or Derivative Works thereof, You may choose to offer,
167 |       and charge a fee for, acceptance of support, warranty, indemnity,
168 |       or other liability obligations and/or rights consistent with this
169 |       License. However, in accepting such obligations, You may act only
170 |       on Your own behalf and on Your sole responsibility, not on behalf
171 |       of any other Contributor, and only if You agree to indemnify,
172 |       defend, and hold each Contributor harmless for any liability
173 |       incurred by, or claims asserted against, such Contributor by reason
174 |       of your accepting any such warranty or additional liability.
175 | 
176 |    END OF TERMS AND CONDITIONS
177 | 
178 |    APPENDIX: How to apply the Apache License to your work.
179 | 
180 |       To apply the Apache License to your work, attach the following
181 |       boilerplate notice, with the fields enclosed by brackets "[]"
182 |       replaced with your own identifying information. (Don't include
183 |       the brackets!)  The text should be enclosed in the appropriate
184 |       comment syntax for the file format. We also recommend that a
185 |       file or class name and description of purpose be included on the
186 |       same "printed page" as the copyright notice for easier
187 |       identification within third-party archives.
188 | 
189 |    Copyright [yyyy] [name of copyright owner]
190 | 
191 |    Licensed under the Apache License, Version 2.0 (the "License");
192 |    you may not use this file except in compliance with the License.
193 |    You may obtain a copy of the License at
194 | 
195 |        http://www.apache.org/licenses/LICENSE-2.0
196 | 
197 |    Unless required by applicable law or agreed to in writing, software
198 |    distributed under the License is distributed on an "AS IS" BASIS,
199 |    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200 |    See the License for the specific language governing permissions and
201 |    limitations under the License.
202 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # Face Landmark-based Speaker-Independent Audio-Visual Speech Enhancement in Multi-Talker Environments
  2 | Implementation of the audio-visual speech enhancement system described in the paper [Face Landmark-based Speaker-Independent Audio-Visual Speech Enhancement in Multi-Talker Environments](https://arxiv.org/abs/1811.02480) by University of Modena and Reggio Emilia and Istituto Italiano di Tecnologia.
  3 | 
  4 | If you are interested in this work check out the [project page](https://dr-pato.github.io/audio_visual_speech_enhancement/).
  5 | 
  6 | ## Getting Started
  7 | ### Install requirements
  8 | All code is written for Python 3. Create a virtual environment (optional) and install all the requirements running:
  9 | ```
 10 | pip install -r requirements.txt
 11 | ```
 12 | ### Usage
 13 | The main program is ```av_speech_enhancement.py```. You can get a list of subcommands typing ```av_speech_enhancement.py -h```.  Try ```av_speech_enhancement.py <subcommand> -h``` for more information about a subcommand.
 14 | The audio-visual dataset must have the following directory structure:
 15 | ```
 16 | s1
 17 |   /audio
 18 | 	/file1.waw
 19 | 	/file2.wav
 20 | 	...
 21 |   /video
 22 | 	/file1.mpg
 23 | 	/file2.mpg
 24 | 	...
 25 | s2
 26 |   /audio
 27 | 	/file1.wav
 28 | 	/file2.wav
 29 | 	...
 30 |   /video
 31 | 	/file1.mpg
 32 | 	/file2.mpg
 33 | 	...
 34 | ...
 35 | ```
 36 | #### Mixed-speech generation
 37 | Generate mixed-speech for training, validation and test sets separately:
 38 | ```
 39 | av_speech_enhancement.py mixed_speech_generator
 40 | 	--data_dir <data_dir>
 41 | 	--base_speaker_ids <spk1> <spk2> <...>
 42 | 	[--noisy_speaker_ids <spk1> <spk2> <...>]
 43 | 	--audio_dir <audio_dir>
 44 | 	--dest_dir <dest_dir>
 45 | 	--num_samples <num_samples>
 46 | 	--num_mix <num_mix>
 47 | 	--num_mix_speakers <num_mix_speakers> {1,2}
 48 | ```
 49 | The generated files are organized as follow:
 50 | ```
 51 | TRAINING_SET
 52 | 	    /s1
 53 | 	       /file1_with_s2_file2.wav
 54 | 	       /file2_with_s10_file4.wav
 55 | 	       ...
 56 | 	    /s2
 57 | 	       /file1_with_s12_file5.wav
 58 | 	       /file2_with_s1_file1.wav
 59 | 	       ...
 60 | 	...
 61 | VALIDATION_SET
 62 | 	...
 63 | TEST_SET
 64 | 	...
 65 | ```
 66 | #### Audio pre-processing
 67 | Compute power-law compressed spectrograms of mixed-speech audio samples. Repeat this operation for training, validation and test sets. Files are saved in NPY format.
 68 | ```
 69 | av_speech_enhancement.py audio_preprocessing
 70 | 	--data_dir <data_dir>
 71 | 	--speaker_ids <spk1> <spk2> <...>
 72 | 	--audio_dir <audio_dir>
 73 | 	--dest_dir <dest_dir>
 74 | 	--sample_rate <sample_rate>
 75 | 	--max_wav_length <max_wav_length>
 76 | ```
 77 | #### Video pre-processing
 78 | Extract face landmarks from video using Dlib face detector and face landmark extractor. Files are saved in TXT format (each row has 136 values that represents the flattened x-y values of 68 face landmarks).
 79 | ```
 80 | av_speech_enhancement.py video_preprocessing
 81 | 	--data_dir <data_dir>
 82 | 	--speaker_ids <spk1> <spk2> <...>
 83 | 	--video_dir <video_dir>
 84 | 	--dest_dir <dest_dir>
 85 | 	--shape_predictor <shape_predictor_file>
 86 | 	--ext <video_file_extension>
 87 | ```
 88 | ```<shape_predictor_file>```  contains the parameters of the face landmark extractor model. You can download a pre-trained model file [here](http://dlib.net/files/shape_predictor_68_face_landmarks.dat.bz2).
 89 | 
 90 | If you want to check the result of the face landmark extractor type:
 91 | ```
 92 | av_speech_enhancement.py show_face_landmarks
 93 | 	--video <video_file>
 94 | 	--fps <fps>
 95 | 	--shape_predictor <shape_predictor_file>
 96 | ```
 97 | 
 98 | #### Computing Target Binary Masks
 99 | Compute TBMs from clean audio samples. For each speaker Long-Term Average Speech Spectrum (LTASS) is computed and then the threshold is applied to all clean audio samples in ```<audio_dir>```.
100 | ```
101 | av_speech_enhancement.py tbm_computation
102 | 	--data_dir <data_dir>
103 | 	--speaker_ids <spk1> <spk2> <...>
104 | 	--audio_dir <audio_dir>
105 | 	--dest_dir <dest_dir>
106 | 	--sample_rate <sample_rate>
107 | 	--max_wav_length <max_wav_length>
108 | ```
109 | 
110 | #### TFRecords generation
111 | Before training you have to generate TFRecords of mixed-speech dataset. ```<data_dir>/<mix_dir>``` must have three subdirectories named ```TRAINING_SET```, ```VALIDATION_SET``` and ```TEST_SET``` created with ```<mixed_speech_generator>``` subcommand. Pre-computed spectrogram (NPY format) must be located in the same directory of audio file.
112 | Set ```<tfrecords_mode>```  to "fixed" if samples of the dataset all have the same length (as in GRID corpus), otherwise use "var" (as in TCD-TIMIT corpus).
113 | ```
114 | av_speech_enhancement.py tfrecords_generator
115 | 	--data_dir <data_dir>
116 | 	--num_speakers <number_speakers_mixed> {2,3}
117 | 	--mode <tfrecords_mode> {fixed,var}
118 | 	--dest_dir <dest_dir>
119 | 	--base_audio_dir <base_audio_dir>
120 | 	--video_dir <video_dir>
121 | 	--tbm_dir <tbm_dir>
122 | 	--mix_audio_dir <mix_audio_dir>
123 | 	--delta <delta_video_feat> {0,1,2]
124 | 	--norm_data_dir <normalization_data_dir>
125 | ```
126 | #### Training
127 | Train an audio-visual speech enhancement model described. You can choose between VL2M, VL2M_ref, Audio-Visual Concat and Audio-Visual Concat-ref models.
128 | ```
129 | av_speech_enhancement.py training
130 | 	--data_dir <data_dir>
131 | 	--train_set <training_set_subdir>
132 | 	--validation_set <validation_set_subdir>
133 | 	--exp <experiment_id>
134 | 	--mode <tfrecords_mode> {fixed,var}
135 | 	--audio_dim <audio_frame_dimension>
136 | 	--video_dim <video_frame_dimension>
137 | 	--num_audio_samples <num_audio_samples>
138 | 	--model <model_selection> {vl2m,vl2m_ref,av_concat_mask,av_concat_mask_ref}
139 | 	--opt <optimizer_choice> {sgd,adam,momentum}
140 | 	--learning_rate <learning_rate>
141 | 	--updating_step <updating_step>
142 | 	--learning_decay <learning_decay>
143 | 	--batch_size <batch_size>
144 | 	--epochs <num_epochs>
145 | 	--hidden_units <num_hidden_lstm_units>
146 | 	--layers <num_lstm_layers>
147 | 	--dropout <dropout_rate>
148 | 	--regularization <regularization_weight>
149 | ```
150 | #### Testing
151 | Test your trained model. Enhanced speech samples and estimated masks are saved in ```<data_dir>/<output_dir>```. Estimated masks are saved  in subdirectories ```<mask_dir>``` of each speaker directory.
152 | ```
153 | av_speech_enhancement.py testing
154 | 	--data_dir <data_dir>
155 | 	--test_set <training_set_subdir>
156 | 	--exp <experiment_id>
157 | 	--ckp <model_checkpoint>
158 | 	--mode <tfrecords_mode> {fixed,var}
159 | 	--audio_dim <audio_frame_dimension>
160 | 	--video_dim <video_frame_dimension>
161 | 	--num_audio_samples <num_audio_samples>
162 | 	--output_dir <output_dir>
163 | 	--mask_dir <mask_dir>
164 | ```
165 | ## Reference
166 | If this project is useful for your research, please cite:
167 | ```
168 | @inproceedings{morrone2019face,
169 |   title={Face Landmark-based Speaker-Independent Audio-Visual Speech Enhancement in Multi-Talker Environments},
170 |   author={Morrone, Giovanni and Bergamaschi, Sonia and Pasa, Luca and Fadiga, Luciano and Tikhanoff, Vadim and Badino, Leonardo},
171 |   booktitle={2019 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)},
172 |   pages={6900-6904},
173 |   year={2019},
174 |   organization={IEEE}
175 | }
176 | ```
177 | 


--------------------------------------------------------------------------------
/audio_features.py:
--------------------------------------------------------------------------------
 1 | import os
 2 | from os.path import join
 3 | from glob import glob
 4 | import numpy as np
 5 | import tensorflow as tf
 6 | from scipy import signal
 7 | from scipy.io import wavfile
 8 | import math
 9 | 
10 | # Avoid printing TF log messages
11 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
12 | 
13 | # TF constants definition
14 | TF_INTRA_OP_PT = int(os.getenv('TF_INTRA_OP', 0))
15 | TF_INTER_OP_PT = int(os.getenv('TF_INTER_OP', 0))
16 | 
17 | 
18 | def downsampling(samples, sample_rate, downsample_rate):
19 |     secs = len(samples) / float(sample_rate)
20 |     num_samples = int(downsample_rate * secs)
21 | 
22 |     return signal.resample(samples, num_samples)
23 | 
24 | 
25 | def compute_spectrograms(audio_folder, max_audio_length=48000, sample_rate=16e3, n_fft=512, window_size=25, step_size=10):
26 |     audio_filenames = sorted(glob(join(audio_folder, '*.wav')))
27 |     num_frames = np.zeros(len(audio_filenames), dtype=np.int32)
28 |     
29 |     window_frame_size = int(round(window_size / 1e3 * sample_rate))
30 |     step_frame_size = int(round(step_size  / 1e3 * sample_rate))
31 |     
32 |     audio_samples = np.zeros((len(audio_filenames), max_audio_length + n_fft//2))
33 |     
34 |     for i, wav_file in enumerate(audio_filenames):
35 |         rate, samples = wavfile.read(wav_file)
36 |         samples = downsampling(samples, rate, sample_rate)
37 |         audio_samples[i, n_fft//2: len(samples) + n_fft//2] = samples
38 |         num_frames[i] = math.ceil(float(len(samples) + n_fft//2) / step_frame_size)
39 |     
40 |     # Create Graph
41 |     with tf.Graph().as_default():
42 |         samples_tensor = tf.constant(audio_samples, dtype=tf.float32)
43 |         # Compute STFT
44 |         specs_tensor = tf.contrib.signal.stft(samples_tensor, frame_length=window_frame_size, frame_step=step_frame_size,
45 |                                               fft_length=n_fft, pad_end=True)
46 |         # Apply power-law compression
47 |         specs_tensor = tf.abs(specs_tensor) ** 0.3
48 |     
49 |         # Start session
50 |         with tf.Session(config=tf.ConfigProto(allow_soft_placement=True,
51 |                                               log_device_placement=False,
52 |                                               intra_op_parallelism_threads=TF_INTRA_OP_PT,
53 |                                               inter_op_parallelism_threads=TF_INTER_OP_PT)) as sess:
54 |             specs = sess.run(specs_tensor)
55 | 
56 |     return audio_filenames, specs, num_frames
57 | 
58 | 
59 | def save_spectrograms_speaker(audio_folder, dest_folder, sample_rate=16e3, max_audio_length=48000):
60 |     # Create destination directory if not exists
61 |     if not os.path.isdir(dest_folder):
62 |         os.makedirs(dest_folder)
63 |     
64 |     audio_filenames, specs, num_frames = compute_spectrograms(audio_folder, sample_rate=sample_rate, max_audio_length=max_audio_length)
65 | 
66 |     if not os.path.exists(dest_folder):
67 |         os.makedirs(dest_folder)
68 | 
69 |     for a_file, spec, nf in zip(audio_filenames, specs, num_frames):
70 |         s_file = os.path.join(dest_folder, os.path.basename(a_file).replace('.wav', '.npy'))
71 |         np.save(s_file, spec[:nf])
72 |             
73 |     print('Done. Spectrogram generated: %d.' % len(audio_filenames))
74 | 
75 | 
76 | def save_spectrograms(dataset_path, list_of_speakers, audio_dir, dest_dir, sample_rate=16e3, max_audio_length=48000):
77 |     for s in list_of_speakers:
78 |         print('Computing spectrograms of speaker {:d}...'.format(s))
79 |         audio_path = os.path.join(dataset_path, 's' + str(s), audio_dir)
80 |         dest_path = os.path.join(dataset_path, 's' + str(s), dest_dir)
81 |         
82 |         save_spectrograms_speaker(audio_path, dest_path, sample_rate, max_audio_length)
83 | 
84 |         print('Speaker {:d} completed.'.format(s))


--------------------------------------------------------------------------------
/audio_processing.py:
--------------------------------------------------------------------------------
 1 | import tensorflow as tf
 2 | 
 3 | 
 4 | def compute_stft(sources, window_size=25, step_size=10, out_shape=None):
 5 |     """Compute STFT"""
 6 |     # Explicit padding at start for correct reconstruction
 7 |     paddings = [[0, 0], [256, 0]]
 8 |     sources_pad = tf.pad(sources, paddings)
 9 |     
10 |     window_frame_size = int(round(window_size / 1e3 * 16e3))
11 |     step_frame_size = int(round(step_size  / 1e3 * 16e3))
12 |         
13 |     # Compute STFTs
14 |     specs = tf.contrib.signal.stft(sources_pad, frame_length=window_frame_size,
15 |                                    frame_step=step_frame_size, pad_end=True)
16 |     
17 |     if out_shape is not None:
18 |         specs = tf.slice(specs, begin=[0, 0, 0], size=out_shape)
19 |     
20 |     return specs
21 | 
22 | 
23 | def reconstruct_sources(specs, num_samples=48000, sample_rate=16e3, window_size=25, step_size=10):
24 |     """Compute inverse STFT"""
25 |     window_frame_size = int(round(window_size / 1e3 * sample_rate))
26 |     step_frame_size = int(round(step_size  / 1e3 * sample_rate))
27 |     reconstructed_sources = tf.contrib.signal.inverse_stft(specs, frame_length=window_frame_size,
28 |                                                            frame_step=step_frame_size,
29 |                                                            window_fn=tf.contrib.signal.inverse_stft_window_fn(step_frame_size))
30 | 
31 |     if num_samples > 0:
32 |         return tf.slice(reconstructed_sources, begin=[0,0], size=[tf.shape(specs)[0], num_samples])
33 |     else:
34 |         return reconstructed_sources
35 | 
36 | 
37 | def get_sources(enh_mag_specs, rec_ang_specs, num_samples=48000, sample_rate=16e3, window_size=25, step_size=10):
38 |     """Get waveform from magnitude and phase of STFT"""
39 |     enh_specs = tf.complex(real=enh_mag_specs * tf.cos(rec_ang_specs), imag=enh_mag_specs * tf.sin(rec_ang_specs))
40 |     
41 |     return reconstruct_sources(enh_specs, num_samples, sample_rate, window_size, step_size)
42 | 
43 | 
44 | def get_oracle_iam(target_stft, mixed_stft, clip_value=10):
45 |     """Get oracle Ideal Amplitude Mask (IAM) from target and mixed spectrograms"""
46 |     target_specs_mag = tf.abs(target_stft) ** 0.3
47 |     mixed_specs_mag = tf.abs(mixed_stft) ** 0.3
48 | 
49 |     return tf.cast(tf.clip_by_value(target_specs_mag / mixed_specs_mag, 0, clip_value), tf.float64)


--------------------------------------------------------------------------------
/av_speech_enhancement.py:
--------------------------------------------------------------------------------
  1 | import sys
  2 | import argparse
  3 | 
  4 | from mixed_speech_generator import create_mixed_tracks_data
  5 | from audio_features import save_spectrograms
  6 | from face_landmarks import save_face_landmarks, show_face_landmarks
  7 | from target_binary_mask import save_target_binary_masks
  8 | from create_dataset_tfrecords import create_dataset
  9 | from training import train, Configuration
 10 | from testing import test
 11 | 
 12 | 
 13 | def parse_args():
 14 |     parser = argparse.ArgumentParser(description="""Audio-visual speech enhancement system.
 15 |     Try 'av_speech_enhancement <subcommand> --help' for more information about subcommands.
 16 |     A full list of subcommands is shown below (positional arguments).""")
 17 |     subparsers = parser.add_subparsers(dest='subparser_name')
 18 | 
 19 |     # Mixed-speech generator
 20 |     mixed_speech_parser = subparsers.add_parser('mixed_speech_generator', description="""Generate mixed-speech samples.
 21 |     For each <speaker_id> in <base_speaker_ids> are randomly chosen <num_samples> wavs from directory <data_dir>/s<speaker_id>/<audio_dir>.
 22 |     For each chosen sample are generated <num_mix> mixed-speech samples (<num_mix_speakers> noisy samples are selected from directories
 23 |     of <noisy_speaker_ids>. Mixed-speech wavs are saved in <data_dir>/<dest_dir>/<speaker_id>.""")
 24 |     mixed_speech_parser.add_argument('-data', '--data_dir', required=True, help='The base pathname of dataset')
 25 |     mixed_speech_parser.add_argument('-bs', '--base_speaker_ids', nargs='+', type=int, required=True,
 26 |                                      help='Speaker IDs of base wav')
 27 |     mixed_speech_parser.add_argument('-ns', '--noisy_speaker_ids', nargs='*', type=int, default=[],
 28 |                                      help='Speaker IDs of noisy wavs (if omitted are the same of base speaker IDs)')
 29 |     mixed_speech_parser.add_argument('-a', '--audio_dir', required=True,
 30 |                                      help='The subdirectory that contains wav files to be processed')
 31 |     mixed_speech_parser.add_argument('-d', '--dest_dir', required=True,
 32 |                                      help='The directory where mixed-speech wavs are saved')
 33 |     mixed_speech_parser.add_argument('-num', '--num_samples', type=int, required=True,
 34 |                                      help='Number of randomly chosen base wavs for each speaker')
 35 |     mixed_speech_parser.add_argument('-mix', '--num_mix', type=int, required=True,
 36 |                                      help='Number of mixed-speech wavs generated for each base wav')
 37 |     mixed_speech_parser.add_argument('-ms', '--num_mix_speakers', type=int, choices=[1, 2], required=True,
 38 |                                      help='Number of wavs of noisy speakers mixed with base wav')
 39 |     
 40 |     # Compute audio spectrograms
 41 |     audio_preprocessing_parser = subparsers.add_parser('audio_preprocessing', description="""Compute audio spectrograms.
 42 |     For each <speaker_id> in <speaker_ids> power-law compressed spectrograms are computed for all wavs in
 43 |     <data_dir>/s<speaker_id>/<audio_dir>. The spectrograms are saved in NPY format in <data_dir>/s<speaker_id>/<dest_dir>.""")
 44 |     audio_preprocessing_parser.add_argument('-data', '--data_dir', required=True, help='The base pathname of dataset')
 45 |     audio_preprocessing_parser.add_argument('-s', '--speaker_ids', nargs='+', type=int, required=True,
 46 |                                             help='Speaker IDs of wavs to be processed')
 47 |     audio_preprocessing_parser.add_argument('-a', '--audio_dir', required=True,
 48 |                                             help='The subdirectory that contains wav files to be processed')
 49 |     audio_preprocessing_parser.add_argument('-d', '--dest_dir', required=True,
 50 |                                             help='The subdirectory where spectrograms are saved')
 51 |     audio_preprocessing_parser.add_argument('-sr', '--sample_rate', type=int, default=16000,
 52 |                                             help='Desired sample rate (in Hz)')
 53 |     audio_preprocessing_parser.add_argument('-ml', '--max_wav_length', type=int, required=True,
 54 |                                             help='Set this value to the maximum length (in samples with desidered sample rate) of single wav .')
 55 | 
 56 |     # Compute face landmarks
 57 |     video_preprocessing_parser = subparsers.add_parser('video_preprocessing', description="""Compute face landmarks.
 58 |     For each <speaker_id> in <speaker_ids> face landmarks are computed for all videos in
 59 |     <data_dir>/s<speaker_id>/<video_dir>. The face landmarks are saved in TXT format in <data_dir>/s<speaker_id>/<dest_dir>.""")
 60 |     video_preprocessing_parser.add_argument('-data', '--data_dir', required=True, help='The base pathname of dataset')
 61 |     video_preprocessing_parser.add_argument('-s', '--speaker_ids', nargs='+', type=int, required=True,
 62 |                                             help='Speaker IDs of videos to be processed')
 63 |     video_preprocessing_parser.add_argument('-v', '--video_dir', required=True,
 64 |                                             help='The subdirectory that contains video files to be processed')
 65 |     video_preprocessing_parser.add_argument('-d', '--dest_dir', required=True,
 66 |                                             help='The subdirectory where output files are saved')
 67 |     video_preprocessing_parser.add_argument('-sp', '--shape_predictor', required=True,
 68 |                                             help='Path of the file that contains the parameters of Dlib face landmark extractor')
 69 |     video_preprocessing_parser.add_argument('-e', '--ext', required=True, default='mpg',
 70 |                                             help='The extension of video files')
 71 | 
 72 |     # Show face landmarks
 73 |     show_landmarks_parser = subparsers.add_parser('show_face_landmarks',
 74 |                                                   description='Show face landmarks along with face bounding boxes')
 75 |     show_landmarks_parser.add_argument('-v', '--video', required=True,
 76 |                                        help='The video file to be processed')
 77 |     show_landmarks_parser.add_argument('--fps', type=float, default=25.0,
 78 |                                        help='Video frame rate. If it set to zero, you have to press any key to show the video frame-by-frame')
 79 |     group = show_landmarks_parser.add_mutually_exclusive_group(required=True)
 80 |     group.add_argument('-sp', '--shape_predictor', default='',
 81 |                        help='The file that contains the parameters of Dlib face landmark extractor')
 82 |     show_landmarks_parser.add_argument('--full', action='store_const', const=True, default=False,
 83 |                                        help='Draw connected lines of face landmarks points')
 84 | 
 85 |     # Compute target binary masks
 86 |     tbm_parser = subparsers.add_parser('tbm_computation', description="""Compute Target Binary Masks (TBM).
 87 |     For each <speaker_id> in <speaker_ids> TBMs are computed for all wavs in
 88 |     <data_dir>/s<speaker_id>/<audio_dir>. The spectrograms are saved in NPY format in <data_dir>/s<speaker_id>/<dest_dir>.""")
 89 |     tbm_parser.add_argument('-data', '--data_dir', required=True, help='The base pathname of dataset')
 90 |     tbm_parser.add_argument('-s', '--speaker_ids', nargs='+', type=int, required=True,
 91 |                             help='Speaker IDs of wavs to be processed')
 92 |     tbm_parser.add_argument('-a', '--audio_dir', required=True,
 93 |                             help='The subdirectory that contains wav files to be processed')
 94 |     tbm_parser.add_argument('-d', '--dest_dir', required=True,
 95 |                             help='The subdirectory where masks are saved')
 96 |     tbm_parser.add_argument('-mf', '--mask_factor', type=float, default=0.6,
 97 |                             help='A value that controls the amount of T-F units assigned to target speaker (higher values assign more T-F units to noise/silence)')
 98 |     tbm_parser.add_argument('-sr', '--sample_rate', type=int, default=16000,
 99 |                             help='Desired sample rate (in Hz)')
100 |     tbm_parser.add_argument('-ml', '--max_wav_length', type=int, required=True,
101 |                             help='Set this value to the maximum length (in samples with desired sample rate) of single wav.')
102 |     tbm_parser.add_argument('-nl', '--num_ltass', type=int, default=1000,
103 |                             help='Number of samples of target speaker used to compute LTASS')
104 | 
105 |     # Create TFRecords
106 |     tfrecords_parser = subparsers.add_parser('tfrecords_generator', description="""Create TFRecords of dataset.
107 |     <data_dir>/<mix_dir> must have the following structure. There are three directories named TRAINING_SET, VALIDATION_SET and
108 |     TEST_SET. These directories cointains subdirectories s<speaker_id> for each speaker. A directory of a speaker
109 |     contains mixed-speech samples created with <mixed_speech_generator> command along with associated pre-computed
110 |     spectrograms (NPY format). <data_dir>/s<speaker_id>/<audio_dir|video_dir|tbm_dir> contains files associated to each
111 |     mixed-speech samples to create a single TFRecord.
112 |     """)
113 |     tfrecords_parser.add_argument('-data', '--data_dir', required=True, help='The base pathname of dataset')
114 |     tfrecords_parser.add_argument('-n', '--num_speakers', type=int, choices=[2, 3], default=2,
115 |                                   help='Numbers of speakers in the mixture (default: 2)')
116 |     tfrecords_parser.add_argument('-m', '--mode', default='fixed', choices=['fixed', 'var'],
117 |                                   help='"fixed" (default) if wavs have all the same length, "var" otherwise')
118 |     tfrecords_parser.add_argument('-d', '--dest_dir', required=True,
119 |                                   help='The subdirectory where TFRecords are saved')
120 |     tfrecords_parser.add_argument('-b', '--base_audio_dir', required=True,
121 |                                   help='The subdirectory that contains base-speech wavs')
122 |     tfrecords_parser.add_argument('-v', '--video_dir', required=True,
123 |                                   help='The subdirectory that contains pre-computed face landmarks (TXT format)')
124 |     tfrecords_parser.add_argument('-tbm', '--tbm_dir', required=True,
125 |                                   help='The subdirectory that contains pre-computed TBM (NPY format)')
126 |     tfrecords_parser.add_argument('-mix', '--mix_audio_dir', required=True,
127 |                                   help='The subdirectory that contains mixed-speech wavs and pre-computed spectrograms (NPY format)')
128 |     tfrecords_parser.add_argument('--delta', type=int, choices=[0, 1, 2], default=1,
129 |                                   help='Select video features. 0: raw face landmarks; 1: motion vectors of face landmarks (default); 2: motion of motion vectors')
130 |     tfrecords_parser.add_argument('-norm', '--norm_data_dir', required=True,
131 |                                   help='The directory where mean and standard deviation of audio and visual features are saved')
132 |     
133 |     # Train audio-visual speech enhancement model
134 |     training_parser = subparsers.add_parser('training', description="""Train an audio-visual speech enhancement model.""")
135 |     training_parser.add_argument('-data', '--data_dir', required=True, help='The base pathname of dataset')
136 |     training_parser.add_argument('-ts', '--train_set', required=True, help='Subdirectory with TFRecords of training set')
137 |     training_parser.add_argument('-vs', '--val_set', required=True, help='Subdirectory with TFRecords of validation set')
138 |     training_parser.add_argument('-e', '--exp', required=True, help='Experiment identifier')
139 |     training_parser.add_argument('-m', '--mode', default='fixed', choices=['fixed', 'var'],
140 |                                  help='"TFRecord type: fixed" (default) or "var"')
141 |     training_parser.add_argument('-vd', '--video_dim', type=int, default=136,
142 |                                  help='Size of a single video frame (default: 136)')
143 |     training_parser.add_argument('-ad', '--audio_dim', type=int, default=257,
144 |                                  help='Size of a single audio frame (default: 257)')
145 |     training_parser.add_argument('-ns', '--num_audio_samples', type=int,
146 |                                  help='Number of samples of audio wav if <mode> is "fixed" (otherwise it is ignored)')
147 |     training_parser.add_argument('--model', choices=['vl2m', 'vl2m_ref', 'av_concat_mask', 'av_concat_mask_ref'],
148 |                                  help='Model type. "av_concat_spec" is the "av_concat_mask" model w/o masking')
149 |     training_parser.add_argument('-o', '--opt', required=True, choices=['sgd', 'adam', 'momentum'],
150 |                                  help='Training optimizer.')
151 |     training_parser.add_argument('-lr', '--learning_rate', type=float, required=True, help='Initial learning rate')
152 |     training_parser.add_argument('-us', '--updating_step', type=int, default=1000,
153 |                                  help='Frequency (in training steps) of updates of learning rate ("sgd" and "momentum" only)')
154 |     training_parser.add_argument('-lc', '--learning_decay', type=float, default=1.0,
155 |                                  help='Learning rate decay ("sgd" and "momentum" only)')
156 |     training_parser.add_argument('-bs', '--batch_size', type=int, help='Training batch size')
157 |     training_parser.add_argument('-ep', '--epochs', type=int, help='Number of training epochs')
158 |     training_parser.add_argument('-nh', '--hidden_units', type=int, help='Number of units of BLSTM cells')
159 |     training_parser.add_argument('-nl', '--layers', type=int, help='Number of stacked BLSTM cells')
160 |     training_parser.add_argument('-d', '--dropout', type=float, default=1,
161 |                                  help='Dropout rate (default: 1)')
162 |     training_parser.add_argument('-r', '--regularization', type=float, default=0,
163 |                                  help='Weights regularization hyperparameter (default: 0)')
164 |     training_parser.add_argument('-mt', '--mask_threshold', type=float, default=-1,
165 |                                  help='Threshold on estimated TBM for reconstruction ("vl2m" model only). If -1 (default) thresholding is not applied')
166 | 
167 |     # Eval audio-visual speech enhancement model
168 |     testing_parser = subparsers.add_parser('testing', description="""Test an audio-visual speech enhancement model.""")
169 |     testing_parser.add_argument('-data', '--data_dir', required=True, help='The base pathname of dataset')
170 |     testing_parser.add_argument('-ts', '--test_set', required=True, help='Subdirectory with TFRecords of test set')
171 |     testing_parser.add_argument('-e', '--exp', required=True, help='Experiment name to be evaluated')
172 |     testing_parser.add_argument('-c', '--ckp', required=True,
173 |                                 help='Model checkpoint to be restored. The format is <n_epoch>_<n_step>')
174 |     testing_parser.add_argument('-m', '--mode', default='fixed', choices=['fixed', 'var'],
175 |                                 help='TFRecord type: "fixed" (default) or "var"')
176 |     testing_parser.add_argument('-vd', '--video_dim', type=int, default=136,
177 |                                 help='Size of a single video frame (default: 136)')
178 |     testing_parser.add_argument('-ad', '--audio_dim', type=int, default=257,
179 |                                 help='Size of a single audio frame (default: 257)')
180 |     testing_parser.add_argument('-ns', '--num_audio_samples', type=int,
181 |                                 help='Number of samples of audio wav if <mode> is "fixed" (otherwise it is ignored)')
182 |     testing_parser.add_argument('-mt', '--mask_threshold', type=float, default=-1,
183 |                                 help='Threshold on estimated TBM for reconstruction. If -1 (default) thresholding is not applied')
184 |     testing_parser.add_argument('-me', '--mix_eval', action='store_const', const=True, default=False, 
185 |                                 help='If it is set mixed-speech wavs are evaluated')
186 |     testing_parser.add_argument('-od', '--output_dir', default='',
187 |                                 help='Directory where are saved enhanced, mixed and target wavs. If empty string no wavs are saved')
188 |     testing_parser.add_argument('-md', '--mask_dir', default='',
189 |                                 help='Subdirectory <data_dir>/s<base_speaker_id>/<mask_dir> where estimated masks (spectrograms for "av_concat_spec" model')
190 |     
191 |     return parser.parse_args()
192 | 
193 | 
194 | def main():
195 |     args = parse_args()
196 |     
197 |     if args.subparser_name == 'mixed_speech_generator':
198 |         create_mixed_tracks_data(args.data_dir, args.base_speaker_ids, args.noisy_speaker_ids, args.audio_dir,
199 |                                  args.dest_dir, args.num_samples, args.num_mix, args.num_mix_speakers)
200 |     elif args.subparser_name == 'audio_preprocessing':
201 |         save_spectrograms(args.data_dir, args.speaker_ids, args.audio_dir, args.dest_dir,
202 |                           args.sample_rate, args.max_wav_length)
203 |     elif args.subparser_name == 'video_preprocessing':
204 |         save_face_landmarks(args.data_dir, args.speaker_ids, args.video_dir, args.dest_dir, args.shape_predictor, args.ext)
205 |     elif args.subparser_name == 'show_face_landmarks':
206 |         show_face_landmarks(args.video, fps=args.fps, predictor_params=args.shape_predictor, full_draw=args.full)
207 |     elif args.subparser_name == 'tbm_computation':
208 |         save_target_binary_masks(args.data_dir, args.speaker_ids, args.audio_dir, args.dest_dir, args.mask_factor,
209 |                                  args.sample_rate, args.max_wav_length, args.num_ltass)
210 |     elif args.subparser_name == 'tfrecords_generator':
211 |         create_dataset(args.data_dir, args.num_speakers, args.video_dir, args.tbm_dir, args.base_audio_dir, args.mix_audio_dir, args.norm_data_dir,
212 |                        args.dest_dir, args.mode, args.delta)
213 |     elif args.subparser_name == 'training':
214 |         config = Configuration(args.learning_rate, args.updating_step, args.learning_decay, args.dropout, args.batch_size,
215 |                                args.opt, args.video_dim, args.audio_dim, args.num_audio_samples, args.epochs, args.hidden_units,
216 |                                args.layers, args.regularization, args.mask_threshold)
217 |         train(args.model, args.data_dir, args.train_set, args.val_set, config, args.exp, args.mode)
218 |     elif args.subparser_name == 'testing':
219 |         test(args.data_dir, args.test_set, args.exp, args.ckp, args.video_dim, args.audio_dim, args.mode,
220 |              args.num_audio_samples, args.mask_threshold, args.mix_eval, args.output_dir, args.mask_dir)
221 |     else:
222 |         print("Bad subcommand name.")
223 |         sys.exit(1)
224 | 
225 | 
226 | if __name__ == '__main__':
227 |     main()
228 | 


--------------------------------------------------------------------------------
/av_sync.py:
--------------------------------------------------------------------------------
 1 | from __future__ import division
 2 | 
 3 | import numpy as np
 4 | from scipy.interpolate import interp2d
 5 | 
 6 | 
 7 | def inc_fps(frames, target_len):
 8 |     x = np.arange(frames.shape[1])
 9 |     y = np.arange(frames.shape[0])
10 |     y_inc = np.linspace(0, len(frames) * (1 - 1 / target_len), target_len)
11 |     
12 |     return interp2d(x, y, frames, kind='linear')(x, y_inc)
13 | 
14 | 
15 | def sync_audio_visual_features(mask_filename, video_filename, tot_frames=None, min_frames=None, pad='start'):
16 |     """
17 |     Upsample video frames to the same frame rate of mask.
18 |     tot_frames is the number of frames of full video.
19 |     min_frames is the minimum number of video frames required (otherwise the video is overly corrupted).
20 |     pad is used to add padding at start or at end when the number of vifro frames is smaller than tot_frames
21 |     """
22 |     # Get the binary mask
23 |     mask = np.load(mask_filename)
24 |     
25 |     # Get face landmarks
26 |     video_features = np.loadtxt(video_filename, dtype=np.int32)
27 | 
28 |     # Skip highly corrupted files and correct other ones
29 |     if len(video_features.shape) != 2 or (min_frames is not None and video_features.shape[0] < min_frames):
30 |         return (None, None)
31 |     elif tot_frames is not None and video_features.shape[0] < tot_frames:
32 |         # Replication of first frame at beginning
33 |         n_rep = tot_frames - video_features.shape[0]
34 |         if pad == 'start':
35 |             video_features = np.vstack((np.tile(video_features[0], (n_rep, 1)), video_features))
36 |         elif pad == 'end':
37 |             video_features = np.vstack((video_features, np.tile(video_features[0], (n_rep, 1))))
38 | 
39 |     video_features = inc_fps(video_features, len(mask))
40 | 
41 |     # Check dimensions
42 |     if len(mask) == len(video_features):
43 |         return mask.astype(np.float32), video_features
44 |     else:
45 |         return (None, None)
46 | 
47 | 


--------------------------------------------------------------------------------
/create_dataset_tfrecords.py:
--------------------------------------------------------------------------------
  1 | from glob import glob
  2 | import os
  3 | import random
  4 | import tensorflow as tf
  5 | import numpy as np
  6 | from scipy.io import wavfile
  7 | from scipy import signal
  8 | import time
  9 | from av_sync import sync_audio_visual_features
 10 | from audio_features import downsampling
 11 | 
 12 | 
 13 | def adjust_samples(reference_len, adjusted):
 14 |     if len(adjusted) < reference_len:
 15 |         adjusted_samples = np.zeros(reference_len)
 16 |         shift = (reference_len - len(adjusted)) // 2
 17 |         adjusted_samples[shift: shift + len(adjusted)] = adjusted
 18 |         return adjusted_samples
 19 |     else:
 20 |         return adjusted[: reference_len]
 21 | 
 22 | 
 23 | def get_video_filenames(data_path, video_folder, speaker, n_samples=1000):
 24 |     features_file_list = []
 25 | 
 26 |     speaker_file_list = glob(os.path.join(data_path, 's' + str(speaker), video_folder, '*.txt'))
 27 |     random.shuffle(speaker_file_list)
 28 |         
 29 |     return sorted(speaker_file_list[:n_samples])
 30 | 
 31 | 
 32 | def get_mix_filenames(data_path, mix_audio_folder, speaker, n_samples):
 33 |     file_list = []
 34 | 
 35 |     speaker_file_list = glob(os.path.join(data_path, 's' + str(speaker), mix_audio_folder, '*.wav'))
 36 |     random.shuffle(speaker_file_list)
 37 |         
 38 |     return sorted(speaker_file_list[:n_samples])
 39 | 
 40 | 
 41 | def video_normalization(features_file_list, delta=0):
 42 |     print('Total number of files = {}'.format(len(features_file_list)))
 43 | 
 44 |     features = []
 45 | 
 46 |     for file_index, txt_file in enumerate(features_file_list):
 47 |         data = np.loadtxt(txt_file)
 48 |         if len(data.shape) == 2:
 49 |             if delta > 0:
 50 |                 data = data[1:] - data[:-1]
 51 |                 if delta == 2:
 52 |                     data = data[1:] - data[:-1]
 53 |             features.append(data)
 54 | 
 55 |     features = np.vstack(features)
 56 |     print(features.shape)
 57 | 
 58 |     dataset_mean = np.mean(features, axis=0)
 59 |     dataset_stdev = np.std(features, axis=0)
 60 | 
 61 |     print('Mean:')
 62 |     print(dataset_mean.shape)
 63 |     print(dataset_mean)
 64 |     print('Standard deviation:')
 65 |     print(dataset_stdev.shape)
 66 |     print(dataset_stdev)
 67 | 
 68 |     return dataset_mean, dataset_stdev
 69 | 
 70 | 
 71 | def audio_normalization(features_file_list):
 72 |     print('Total number of files = {}'.format(len(features_file_list)))
 73 | 
 74 |     features = []
 75 | 
 76 |     for file_index, wav_file in enumerate(features_file_list):
 77 |         npy_file = wav_file.replace('.wav', '.npy')
 78 |         data = np.load(npy_file)
 79 |         if len(data.shape) == 2:
 80 |             features.append(data)
 81 | 
 82 |     features = np.vstack(features)
 83 |     print(features.shape)
 84 | 
 85 |     dataset_mean = np.mean(features, axis=0)
 86 |     dataset_stdev = np.std(features, axis=0)
 87 | 
 88 |     print('Mean:')
 89 |     print(dataset_mean.shape)
 90 |     print(dataset_mean)
 91 |     print('Standard deviation:')
 92 |     print(dataset_stdev.shape)
 93 |     print(dataset_stdev)
 94 | 
 95 |     return dataset_mean, dataset_stdev
 96 | 
 97 | def serialize_sample_fixed(video_sequence, tbm, audio_samples, audio_paths, mix_spec_norm):
 98 |     # Video_sequence and audio_sequence must have the same length
 99 |     # The object we return
100 |     example = tf.train.SequenceExample()
101 | 
102 |     # Non-sequential features of our example
103 |     example.context.feature['sequence_length'].int64_list.value.append(len(video_sequence))
104 |     example.context.feature['base_audio_wav'].float_list.value.extend(audio_samples[0])
105 |     example.context.feature['other_audio_wav'].float_list.value.extend(audio_samples[1])
106 |     example.context.feature['mix_audio_wav'].float_list.value.extend(audio_samples[2])
107 |     example.context.feature['base_audio_path'].bytes_list.value.append(audio_paths[0].encode())
108 |     example.context.feature['other_audio_path'].bytes_list.value.append(audio_paths[1].encode())
109 |     example.context.feature['mix_audio_path'].bytes_list.value.append(audio_paths[2].encode())
110 | 
111 |     # Feature lists for the sequential features of our example
112 |     fl_tbm = example.feature_lists.feature_list['tbm']
113 |     fl_video_feat = example.feature_lists.feature_list['video_features']
114 |     fl_mix_spec = example.feature_lists.feature_list['mixed_spectrogram']
115 | 
116 |     for tbm_feat in tbm:
117 |         fl_tbm.feature.add().float_list.value.extend(tbm_feat)
118 |     
119 |     for video_feat in video_sequence:
120 |         fl_video_feat.feature.add().float_list.value.extend(video_feat)
121 | 
122 |     for mix_spec_feat in mix_spec_norm:
123 |         fl_mix_spec.feature.add().float_list.value.extend(mix_spec_feat)
124 |     
125 |     return example
126 | 
127 | 
128 | def serialize_sample_var(video_sequence, tbm, audio_samples, audio_paths, mix_spec_norm):
129 |     # Video_sequence and audio_sequence must have the same length
130 |     # The object we return
131 |     example = tf.train.SequenceExample()
132 |     
133 |     # Non-sequential features of our example
134 |     example.context.feature['sequence_length'].int64_list.value.append(len(video_sequence))
135 |     
136 |     # Feature lists for the sequential features of our example
137 |     fl_tbm = example.feature_lists.feature_list['tbm']
138 |     fl_video_feat = example.feature_lists.feature_list['video_features']
139 |     fl_mix_spec = example.feature_lists.feature_list['mixed_spectrogram']
140 |     fl_base_audio_wav = example.feature_lists.feature_list['base_audio_wav']
141 |     fl_other_audio_wav = example.feature_lists.feature_list['other_audio_wav']
142 |     fl_mix_audio_wav = example.feature_lists.feature_list['mix_audio_wav']
143 |     fl_base_audio_path = example.feature_lists.feature_list['base_audio_path']
144 |     fl_other_audio_path = example.feature_lists.feature_list['other_audio_path']
145 |     fl_mix_audio_path = example.feature_lists.feature_list['mix_audio_path']
146 |     
147 |     for tbm_feat in tbm:
148 |         fl_tbm.feature.add().float_list.value.extend(tbm_feat)
149 |     
150 |     for video_feat in video_sequence:
151 |         fl_video_feat.feature.add().float_list.value.extend(video_feat)
152 |     
153 |     for mix_spec_feat in mix_spec_norm:
154 |         fl_mix_spec.feature.add().float_list.value.extend(mix_spec_feat)
155 |     
156 |     for base_audio_sample in audio_samples[0]:
157 |         fl_base_audio_wav.feature.add().float_list.value.append(base_audio_sample)
158 |     
159 |     for other_audio_sample in audio_samples[1]:
160 |         fl_other_audio_wav.feature.add().float_list.value.append(other_audio_sample)
161 | 
162 |     for mix_audio_sample in audio_samples[2]:
163 |         fl_mix_audio_wav.feature.add().float_list.value.append(mix_audio_sample)
164 |     
165 |     for base_path_feat in audio_paths[0]:
166 |         fl_base_audio_path.feature.add().int64_list.value.append(ord(base_path_feat))
167 |     
168 |     for other_path_feat in audio_paths[1]:
169 |         fl_other_audio_path.feature.add().int64_list.value.append(ord(other_path_feat))
170 | 
171 |     for mix_path_feat in audio_paths[2]:
172 |         fl_mix_audio_path.feature.add().int64_list.value.append(ord(mix_path_feat))
173 |     
174 |     return example
175 | 
176 | 
177 | def get_filenames_2spk(data_path, mix_audio_folder, base_audio_folder, video_folder, tbm_folder):
178 |     speaker = os.path.basename(mix_audio_folder)
179 |     mix_audio_file_list = glob(os.path.join(data_path, mix_audio_folder, '*.wav'))
180 |     base_audio_file_list = []
181 |     other_audio_file_list = []
182 |     video_file_list = []
183 |     tbm_file_list = []
184 | 
185 |     for mix_file in mix_audio_file_list:
186 |         mix_split = os.path.basename(mix_file).split('_')
187 |         base_audio_file_list.append(os.path.join(data_path, speaker, base_audio_folder, mix_split[0] + '.wav'))
188 |         tbm_file_list.append(os.path.join(data_path, speaker, tbm_folder, mix_split[0] + '.npy'))
189 |         video_file_list.append(os.path.join(data_path, speaker, video_folder, mix_split[0] + '.txt'))
190 |         other_audio_file_list.append(os.path.join(data_path, mix_split[2], base_audio_folder, mix_split[3]))
191 | 
192 |     return mix_audio_file_list, base_audio_file_list, tbm_file_list, video_file_list, other_audio_file_list
193 | 
194 | 
195 | def get_filenames_3spk(data_path, mix_audio_folder, base_audio_folder, video_folder, tbm_folder):
196 |     speaker = os.path.basename(mix_audio_folder)
197 |     mix_audio_file_list = glob(os.path.join(data_path, mix_audio_folder, '*.wav'))
198 |     base_audio_file_list = []
199 |     other1_audio_file_list = []
200 |     other2_audio_file_list = []
201 |     video_file_list = []
202 |     tbm_file_list = []
203 | 
204 |     for mix_file in mix_audio_file_list:
205 |         mix_split = os.path.basename(mix_file).split('_')
206 |         base_audio_file_list.append(os.path.join(data_path, speaker, base_audio_folder, mix_split[0] + '.wav'))
207 |         tbm_file_list.append(os.path.join(data_path, speaker, tbm_folder, mix_split[0] + '.npy'))
208 |         video_file_list.append(os.path.join(data_path, speaker, video_folder, mix_split[0] + '.txt'))
209 |         other1_audio_file_list.append(os.path.join(data_path, mix_split[2], base_audio_folder, mix_split[3] + '.wav'))
210 |         other2_audio_file_list.append(os.path.join(data_path, mix_split[4], base_audio_folder, mix_split[5]))
211 | 
212 |     return mix_audio_file_list, base_audio_file_list, tbm_file_list, video_file_list, other1_audio_file_list, other2_audio_file_list
213 | 
214 | 
215 | def create_tfrecords_speaker_2spk(data_path, mix_audio_folder, video_folder, tbm_folder, base_audio_folder, dest_dir,
216 |                                   delta, dataset_norm_folder, tfrecord_mode='fixed', file_counter=0):
217 |     # get speaker code
218 |     speaker = os.path.basename(mix_audio_folder)
219 |     
220 |     # get associated files from file_list
221 |     mix_audio_file_list, base_audio_file_list, tbm_file_list, video_file_list, other_audio_file_list = \
222 |         get_filenames_2spk(data_path, mix_audio_folder, base_audio_folder, video_folder, tbm_folder)
223 |     
224 |     # normalization data filenames
225 |     dataset_video_mean_file = os.path.join(dataset_norm_folder, speaker + '_video_mean.npy')
226 |     dataset_video_std_file = os.path.join(dataset_norm_folder, speaker + '_video_std.npy')
227 |     dataset_audio_mean_file = os.path.join(dataset_norm_folder, speaker + '_audio_mean.npy')
228 |     dataset_audio_std_file = os.path.join(dataset_norm_folder, speaker + '_audio_std.npy')
229 | 
230 |     # load dataset video mean and std
231 |     try:
232 |         dataset_video_mean = np.load(dataset_video_mean_file)
233 |         dataset_video_std = np.load(dataset_video_std_file)
234 |     except IOError:
235 |         print('Computing mean and std of video features of speaker %s...' % speaker)
236 |         dataset_video_mean, dataset_video_std = video_normalization(video_file_list, delta=delta)
237 |         np.save(dataset_video_mean_file, dataset_video_mean)
238 |         np.save(dataset_video_std_file, dataset_video_std)
239 | 
240 |     # load dataset audio mean and std
241 |     try:
242 |         dataset_audio_mean = np.load(dataset_audio_mean_file)
243 |         dataset_audio_std = np.load(dataset_audio_std_file)
244 |     except IOError:
245 |         print('Computing mean and std of audio features of speaker %s...' % speaker)
246 |         dataset_audio_mean, dataset_audio_std = audio_normalization(mix_audio_file_list)
247 |         np.save(dataset_audio_mean_file, dataset_audio_mean)
248 |         np.save(dataset_audio_std_file, dataset_audio_std)
249 | 
250 |     
251 |     for file_video, file_tbm, file_base_audio, file_mix_audio, file_other_audio in zip(video_file_list, tbm_file_list, base_audio_file_list, mix_audio_file_list, other_audio_file_list):
252 |         # get sync features
253 |         tbm, features_video = sync_audio_visual_features(file_tbm, file_video)
254 |         mix_spec = np.load(file_mix_audio.replace('.wav', '.npy'))
255 | 
256 |         # Check return value
257 |         if features_video is not None:
258 |             print('TBM: {:s} - Video: {:s} - Audio: {:s}'.format(file_tbm, file_video, file_mix_audio))
259 | 
260 |             # Normalize video features
261 |             if not delta:
262 |                 features_video = np.subtract(features_video, dataset_video_mean) / dataset_video_std
263 |             else:
264 |                 # Add delta video features
265 |                 delta_features_video = np.zeros_like(features_video)
266 |                 delta_features_video[1:] = features_video[1:] - features_video[:-1]
267 |                 if delta == 1:
268 |                     delta_features_video_norm = np.subtract(delta_features_video, dataset_video_mean) / dataset_video_std
269 |                 elif delta == 2:
270 |                     delta_features_video[2:] = delta_features_video[2:] - delta_features_video[1:-1]
271 |                     delta_features_video_norm = np.subtract(delta_features_video, dataset_video_mean) / dataset_video_std
272 |                 features_video = delta_features_video_norm
273 |         
274 |             # Normalize spectrograms
275 |             mix_spec_norm = np.subtract(mix_spec, dataset_audio_mean) / dataset_audio_std
276 |   
277 |             # read audio files
278 |             sample_rate, base_audio_wav = wavfile.read(file_base_audio)
279 |             _, other_audio_wav = wavfile.read(file_other_audio)
280 |             _, mix_audio_wav = wavfile.read(file_mix_audio)
281 |             # downsampling
282 |             base_audio_wav = downsampling(base_audio_wav, sample_rate, 16e3)
283 |             other_audio_wav = downsampling(other_audio_wav, sample_rate, 16e3)
284 |             mix_audio_wav = downsampling(mix_audio_wav, sample_rate, 16e3)
285 |             # adjust samples to match audio lengths
286 |             base_audio_wav = base_audio_wav[: len(mix_audio_wav)]
287 |             other_audio_wav = adjust_samples(len(mix_audio_wav), other_audio_wav)
288 | 
289 |             # get filenames relative paths
290 |             base_audio_path = file_base_audio.replace(data_path, '')
291 |             other_audio_path = file_other_audio.replace(data_path, '')
292 |             mix_audio_path = file_mix_audio.replace(data_path, '')
293 | 
294 |             # create TFRecord
295 |             sample_file = os.path.join(dest_dir, 'sample_{:05d}.tfrecords'.format(file_counter))
296 |             file_counter += 1
297 | 
298 |             with open(sample_file, 'w') as fp:
299 |                 writer = tf.python_io.TFRecordWriter(fp.name)
300 |                 audio_wavs = [base_audio_wav, other_audio_wav, mix_audio_wav]
301 |                 audio_paths = [base_audio_path, other_audio_path, mix_audio_path]
302 |                 if tfrecord_mode == 'fixed':
303 |                     serialized_sample = serialize_sample_fixed(features_video, tbm, audio_wavs, audio_paths, mix_spec_norm)
304 |                 elif tfrecord_mode == 'var':
305 |                     serialized_sample = serialize_sample_var(features_video, tbm, audio_wavs, audio_paths, mix_spec_norm)
306 |                 writer.write(serialized_sample.SerializeToString())
307 |                 writer.close()
308 |         else:
309 |             print('Skipped -> TBM: {:s} - Video: {:s}'.format(file_tbm, file_video))
310 | 
311 |     return file_counter
312 | 
313 | 
314 | def create_tfrecords_speaker_3spk(data_path, mix_audio_folder, video_folder, tbm_folder, base_audio_folder, dest_dir,
315 |                                   delta, dataset_norm_folder, tfrecord_mode='fixed', file_counter=0):
316 |     # get speaker code
317 |     speaker = os.path.basename(mix_audio_folder)
318 |     
319 |     # get associated files from file_list
320 |     mix_audio_file_list, base_audio_file_list, tbm_file_list, video_file_list, other1_audio_file_list, other2_audio_file_list = \
321 |         get_filenames_3spk(data_path, mix_audio_folder, base_audio_folder, video_folder, tbm_folder)
322 | 
323 |     # normalization data filenames
324 |     dataset_video_mean_file = os.path.join(dataset_norm_folder, speaker + '_video_mean.npy')
325 |     dataset_video_std_file = os.path.join(dataset_norm_folder, speaker + '_video_std.npy')
326 |     dataset_audio_mean_file = os.path.join(dataset_norm_folder, speaker + '_audio_mean.npy')
327 |     dataset_audio_std_file = os.path.join(dataset_norm_folder, speaker + '_audio_std.npy')
328 | 
329 |     # load dataset video mean and std
330 |     try:
331 |         dataset_video_mean = np.load(dataset_video_mean_file)
332 |         dataset_video_std = np.load(dataset_video_std_file)
333 |     except IOError:
334 |         print('Computing mean and std of video features of speaker %s...' % speaker)
335 |         dataset_video_mean, dataset_video_std = video_normalization(video_file_list, delta=delta)
336 |         np.save(dataset_video_mean_file, dataset_video_mean)
337 |         np.save(dataset_video_std_file, dataset_video_std)
338 | 
339 |     # load dataset audio mean and std
340 |     try:
341 |         dataset_audio_mean = np.load(dataset_audio_mean_file)
342 |         dataset_audio_std = np.load(dataset_audio_std_file)
343 |     except IOError:
344 |         print('Computing mean and std of audio features of speaker %s...' % speaker)
345 |         dataset_audio_mean, dataset_audio_std = audio_normalization(mix_audio_file_list)
346 |         np.save(dataset_audio_mean_file, dataset_audio_mean)
347 |         np.save(dataset_audio_std_file, dataset_audio_std)
348 | 
349 | 
350 |     for file_video, file_tbm, file_base_audio, file_mix_audio, file_other_audio1, file_other_audio2 in zip(video_file_list, tbm_file_list, base_audio_file_list, mix_audio_file_list, other1_audio_file_list, other2_audio_file_list):
351 |         # get sync features
352 |         tbm, features_video = sync_audio_visual_features(file_tbm, file_video)
353 |         mix_spec = np.load(file_mix_audio.replace('.wav', '.npy'))
354 | 
355 |         # Check return value
356 |         if features_video is not None:
357 |             print('TBM: {:s} - Video: {:s} - Audio: {:s}'.format(file_tbm, file_video, file_mix_audio))
358 | 
359 |             # Normalize video features
360 |             if not delta:
361 |                 features_video = np.subtract(features_video, dataset_video_mean) / dataset_video_std
362 |             else:
363 |                 # Add delta video features
364 |                 delta_features_video = np.zeros_like(features_video)
365 |                 delta_features_video[1:] = features_video[1:] - features_video[:-1]
366 |                 if delta == 1:
367 |                     delta_features_video_norm = np.subtract(delta_features_video, dataset_video_mean) / dataset_video_std
368 |                 elif delta == 2:
369 |                     delta_features_video[2:] = delta_features_video[2:] - delta_features_video[1:-1]
370 |                     delta_features_video_norm = np.subtract(delta_features_video, dataset_video_mean) / dataset_video_std
371 |                 features_video = delta_features_video_norm
372 |         
373 |             # Normalize spectrograms
374 |             mix_spec_norm = np.subtract(mix_spec, dataset_audio_mean) / dataset_audio_std
375 |   
376 |             # read audio files
377 |             sample_rate, base_audio_wav = wavfile.read(file_base_audio)
378 |             _, other1_audio_wav = wavfile.read(file_other_audio1)
379 |             _, other2_audio_wav = wavfile.read(file_other_audio2)
380 |             _, mix_audio_wav = wavfile.read(file_mix_audio)
381 |             # downsampling
382 |             base_audio_wav = downsampling(base_audio_wav, sample_rate, 16e3)
383 |             other1_audio_wav = downsampling(other1_audio_wav, sample_rate, 16e3)
384 |             other2_audio_wav = downsampling(other2_audio_wav, sample_rate, 16e3)
385 |             mix_audio_wav = downsampling(mix_audio_wav, sample_rate, 16e3)
386 |             # adjust samples to match audio lengths
387 |             base_audio_wav = base_audio_wav[: len(mix_audio_wav)]
388 |             other1_audio_wav = adjust_samples(len(mix_audio_wav), other1_audio_wav)
389 |             other2_audio_wav = adjust_samples(len(mix_audio_wav), other2_audio_wav)
390 |             other_audio_wav = other1_audio_wav + other2_audio_wav
391 | 
392 |             # get filenames relative paths
393 |             base_audio_path = file_base_audio.replace(data_path, '')
394 |             other_audio_path = '_'.join(os.path.basename(file_mix_audio).split('_')[2:])[:-4]
395 |             mix_audio_path = file_mix_audio.replace(data_path, '')
396 | 
397 |             # create TFRecord
398 |             sample_file = os.path.join(dest_dir, 'sample_{:05d}.tfrecords'.format(file_counter))
399 |             file_counter += 1
400 | 
401 |             with open(sample_file, 'w') as fp:
402 |                 writer = tf.python_io.TFRecordWriter(fp.name)
403 |                 audio_wavs = [base_audio_wav, other_audio_wav, mix_audio_wav]
404 |                 audio_paths = [base_audio_path, other_audio_path, mix_audio_path]
405 |                 if tfrecord_mode == 'fixed':
406 |                     serialized_sample = serialize_sample_fixed(features_video, tbm, audio_wavs, audio_paths, mix_spec_norm)
407 |                 elif tfrecord_mode == 'var':
408 |                     serialized_sample = serialize_sample_var(features_video, tbm, audio_wavs, audio_paths, mix_spec_norm)
409 |                 writer.write(serialized_sample.SerializeToString())
410 |                 writer.close()
411 |         else:
412 |             print('Skipped -> TBM: {:s} - Video: {:s}'.format(file_tbm, file_video))
413 | 
414 |     return file_counter
415 | 
416 | 
417 | def create_tfrecords(data_path, n_speakers, mix_audio_folder, video_folder, tbm_folder, base_audio_folder, dest_dir,
418 |                      delta, dataset_norm_folder, tfrecord_mode='fixed'):
419 |     file_counter = 0
420 |     for mix_speaker_dir in glob(os.path.join(data_path, mix_audio_folder, '*')):
421 |         if n_speakers == 2:
422 |             file_counter = create_tfrecords_speaker_2spk(data_path, mix_speaker_dir, video_folder, tbm_folder, base_audio_folder,
423 |                                                          dest_dir, delta, dataset_norm_folder, tfrecord_mode, file_counter)
424 |         elif n_speakers == 3:
425 |             file_counter = create_tfrecords_speaker_3spk(data_path, mix_speaker_dir, video_folder, tbm_folder, base_audio_folder,
426 |                                                          dest_dir, delta, dataset_norm_folder, tfrecord_mode, file_counter)
427 |     return file_counter
428 | 
429 | 
430 | def create_dataset(dataset_dir, n_speakers, video_dir, tbm_dir, base_audio_dir, mix_audio_dir, norm_data_dir, save_dir, tfrecord_mode='fixed', delta=1):
431 |     train_dir = os.path.join(dataset_dir, save_dir, 'TRAINING_SET')
432 |     val_dir = os.path.join(dataset_dir, save_dir, 'VALIDATION_SET')
433 |     test_dir = os.path.join(dataset_dir, save_dir, 'TEST_SET')
434 | 
435 |     # create tfrecord directories if they not exist
436 |     for folder in (train_dir, val_dir, test_dir):
437 |         if not os.path.exists(folder):
438 |             os.makedirs(folder)
439 |     
440 |     # create normalization folder if not exists
441 |     if not os.path.exists(norm_data_dir):
442 |         os.makedirs(norm_data_dir)
443 |     
444 |     # generate TFRecords
445 |     print('Creating training TFRecords...')
446 |     train_mix_path = os.path.join(mix_audio_dir, 'TRAINING_SET')
447 |     num_train_samples = create_tfrecords(dataset_dir, n_speakers, train_mix_path, video_dir, tbm_dir, base_audio_dir,
448 |                                          train_dir, delta, norm_data_dir, tfrecord_mode)
449 | 
450 |     print('Creating validation TFRecords...')
451 |     val_mix_path = os.path.join(mix_audio_dir, 'VALIDATION_SET')
452 |     num_val_samples = create_tfrecords(dataset_dir, n_speakers, val_mix_path, video_dir, tbm_dir, base_audio_dir,
453 |                                        val_dir, delta, norm_data_dir, tfrecord_mode)
454 | 
455 |     print('Creating test TFRecords...')
456 |     test_mix_path = os.path.join(mix_audio_dir, 'TEST_SET')
457 |     num_test_samples = create_tfrecords(dataset_dir, n_speakers, test_mix_path, video_dir, tbm_dir, base_audio_dir,
458 |                                         test_dir, delta, norm_data_dir, tfrecord_mode)
459 | 
460 |     print('')
461 |     print('Samples successfully generated:')
462 |     print('-> Training:', num_train_samples)
463 |     print('-> Validation:', num_val_samples)
464 |     print('-> Test:', num_test_samples)


--------------------------------------------------------------------------------
/dataset_reader.py:
--------------------------------------------------------------------------------
 1 | import tensorflow as tf
 2 | from os import listdir
 3 | from os.path import isfile, join
 4 | import random
 5 | import numpy as np
 6 | 
 7 | 
 8 | class DataManager:
 9 |     """Utilities to read TFRecords"""
10 | 
11 |     def __init__(self, single_audio_frame_size, single_video_frame_size, num_audio_samples=48000, buffer_size=1000, mode='fixed'):
12 |         self.num_audio_samples = num_audio_samples
13 |         self.single_audio_frame_size = single_audio_frame_size
14 |         self.single_video_frame_size = single_video_frame_size
15 |         self.buffer_size = buffer_size
16 |         self.mode = mode
17 | 
18 | 
19 |     def get_dataset(self, folder_path):
20 |         file_list = [join(folder_path, f) for f in listdir(folder_path) if isfile(join(folder_path, f))]
21 |         random.shuffle(file_list)
22 |         dataset = tf.data.TFRecordDataset(file_list)
23 | 
24 |         if self.mode == 'fixed':
25 |             dataset = dataset.map(self.read_data_format_fixed)
26 |         elif self.mode == 'var':
27 |             dataset = dataset.map(self.read_data_format_var)
28 | 
29 |         n_samples = len(file_list)
30 |         
31 |         return dataset, n_samples
32 | 
33 | 
34 |     def get_iterator(self, dataset, batch_size=16, n_epochs=None, train=True):
35 |         if train:
36 |             dataset = dataset.shuffle(self.buffer_size)
37 |         
38 |         dataset = dataset.repeat(n_epochs)
39 |         
40 |         if self.mode == 'fixed':
41 |             batch_dataset = dataset.batch(batch_size)
42 |         elif self.mode == 'var':
43 |             batch_dataset = dataset.padded_batch(batch_size, padded_shapes=([], [None, None], [None, None], [None, None], [None], [None], [None], [None], [None], [None]))
44 |              
45 |         iterator = batch_dataset.make_initializable_iterator()
46 |         
47 |         return batch_dataset, iterator
48 | 
49 | 
50 |     def read_data_format_fixed(self, sample):
51 |         context_parsed, sequence_parsed = \
52 |                     tf.parse_single_sequence_example(sample,
53 |                                                      context_features={
54 |                                                       'sequence_length': tf.FixedLenFeature([], dtype=tf.int64),
55 |                                                       'base_audio_wav': tf.FixedLenFeature([self.num_audio_samples], dtype=tf.float32),
56 |                                                       'other_audio_wav': tf.FixedLenFeature([self.num_audio_samples], dtype=tf.float32),
57 |                                                       'mix_audio_wav': tf.FixedLenFeature([self.num_audio_samples], dtype=tf.float32),
58 |                                                       'base_audio_path': tf.VarLenFeature(dtype=tf.string),
59 |                                                       'other_audio_path': tf.VarLenFeature(dtype=tf.string),
60 |                                                       'mix_audio_path': tf.VarLenFeature(dtype=tf.string)
61 |                                                       },
62 |                                                      sequence_features={
63 |                                                       'tbm': tf.FixedLenSequenceFeature([self.single_audio_frame_size], dtype=tf.float32),
64 |                                                       'video_features': tf.FixedLenSequenceFeature([self.single_video_frame_size], dtype=tf.float32),
65 |                                                       'mixed_spectrogram': tf.FixedLenSequenceFeature([self.single_audio_frame_size], dtype=tf.float32)
66 |                                                       })
67 | 
68 |         #sequence_parsed['mixed_spectrogram'],
69 |         return tf.to_int32(context_parsed['sequence_length']), \
70 |                sequence_parsed['tbm'], sequence_parsed['video_features'], sequence_parsed['mixed_spectrogram'], \
71 |                context_parsed['base_audio_wav'], context_parsed['other_audio_wav'], context_parsed['mix_audio_wav'], \
72 |                context_parsed['base_audio_path'], context_parsed['other_audio_path'], context_parsed['mix_audio_path']
73 |     
74 | 
75 |     def read_data_format_var(self, sample):
76 |         context_parsed, sequence_parsed = \
77 |                     tf.parse_single_sequence_example(sample,
78 |                                                      context_features={
79 |                                                       'sequence_length': tf.FixedLenFeature([], dtype=tf.int64)
80 |                                                       },
81 |                                                      sequence_features={
82 |                                                       'tbm': tf.FixedLenSequenceFeature([self.single_audio_frame_size], dtype=tf.float32),
83 |                                                       'video_features': tf.FixedLenSequenceFeature([self.single_video_frame_size], dtype=tf.float32),
84 |                                                       'mixed_spectrogram': tf.FixedLenSequenceFeature([self.single_audio_frame_size], dtype=tf.float32),
85 |                                                       'base_audio_wav': tf.FixedLenSequenceFeature([], dtype=tf.float32),
86 |                                                       'other_audio_wav': tf.FixedLenSequenceFeature([], dtype=tf.float32),
87 |                                                       'mix_audio_wav': tf.FixedLenSequenceFeature([], dtype=tf.float32),
88 |                                                       'base_audio_path': tf.FixedLenSequenceFeature([], dtype=tf.int64),
89 |                                                       'other_audio_path': tf.FixedLenSequenceFeature([], dtype=tf.int64),
90 |                                                       'mix_audio_path': tf.FixedLenSequenceFeature([], dtype=tf.int64)
91 |                                                      })
92 | 
93 |         return tf.to_int32(context_parsed['sequence_length']), \
94 |                sequence_parsed['tbm'], sequence_parsed['video_features'], sequence_parsed['mixed_spectrogram'], \
95 |                sequence_parsed['base_audio_wav'], sequence_parsed['other_audio_wav'], sequence_parsed['mix_audio_wav'], \
96 |                sequence_parsed['base_audio_path'], sequence_parsed['other_audio_path'], sequence_parsed['mix_audio_path']


--------------------------------------------------------------------------------
/docs/Gemfile:
--------------------------------------------------------------------------------
1 | source 'https://rubygems.org'
2 | gem 'github-pages', group: :jekyll_plugins


--------------------------------------------------------------------------------
/docs/Gemfile.lock:
--------------------------------------------------------------------------------
  1 | GEM
  2 |   remote: https://rubygems.org/
  3 |   specs:
  4 |     activesupport (4.2.10)
  5 |       i18n (~> 0.7)
  6 |       minitest (~> 5.1)
  7 |       thread_safe (~> 0.3, >= 0.3.4)
  8 |       tzinfo (~> 1.1)
  9 |     addressable (2.5.2)
 10 |       public_suffix (>= 2.0.2, < 4.0)
 11 |     coffee-script (2.4.1)
 12 |       coffee-script-source
 13 |       execjs
 14 |     coffee-script-source (1.11.1)
 15 |     colorator (1.1.0)
 16 |     commonmarker (0.17.13)
 17 |       ruby-enum (~> 0.5)
 18 |     concurrent-ruby (1.1.1)
 19 |     dnsruby (1.61.2)
 20 |       addressable (~> 2.5)
 21 |     em-websocket (0.5.1)
 22 |       eventmachine (>= 0.12.9)
 23 |       http_parser.rb (~> 0.6.0)
 24 |     ethon (0.11.0)
 25 |       ffi (>= 1.3.0)
 26 |     eventmachine (1.2.7-x64-mingw32)
 27 |     execjs (2.7.0)
 28 |     faraday (0.15.3)
 29 |       multipart-post (>= 1.2, < 3)
 30 |     ffi (1.9.25-x64-mingw32)
 31 |     forwardable-extended (2.6.0)
 32 |     gemoji (3.0.0)
 33 |     github-pages (192)
 34 |       activesupport (= 4.2.10)
 35 |       github-pages-health-check (= 1.8.1)
 36 |       jekyll (= 3.7.4)
 37 |       jekyll-avatar (= 0.6.0)
 38 |       jekyll-coffeescript (= 1.1.1)
 39 |       jekyll-commonmark-ghpages (= 0.1.5)
 40 |       jekyll-default-layout (= 0.1.4)
 41 |       jekyll-feed (= 0.10.0)
 42 |       jekyll-gist (= 1.5.0)
 43 |       jekyll-github-metadata (= 2.9.4)
 44 |       jekyll-mentions (= 1.4.1)
 45 |       jekyll-optional-front-matter (= 0.3.0)
 46 |       jekyll-paginate (= 1.1.0)
 47 |       jekyll-readme-index (= 0.2.0)
 48 |       jekyll-redirect-from (= 0.14.0)
 49 |       jekyll-relative-links (= 0.5.3)
 50 |       jekyll-remote-theme (= 0.3.1)
 51 |       jekyll-sass-converter (= 1.5.2)
 52 |       jekyll-seo-tag (= 2.5.0)
 53 |       jekyll-sitemap (= 1.2.0)
 54 |       jekyll-swiss (= 0.4.0)
 55 |       jekyll-theme-architect (= 0.1.1)
 56 |       jekyll-theme-cayman (= 0.1.1)
 57 |       jekyll-theme-dinky (= 0.1.1)
 58 |       jekyll-theme-hacker (= 0.1.1)
 59 |       jekyll-theme-leap-day (= 0.1.1)
 60 |       jekyll-theme-merlot (= 0.1.1)
 61 |       jekyll-theme-midnight (= 0.1.1)
 62 |       jekyll-theme-minimal (= 0.1.1)
 63 |       jekyll-theme-modernist (= 0.1.1)
 64 |       jekyll-theme-primer (= 0.5.3)
 65 |       jekyll-theme-slate (= 0.1.1)
 66 |       jekyll-theme-tactile (= 0.1.1)
 67 |       jekyll-theme-time-machine (= 0.1.1)
 68 |       jekyll-titles-from-headings (= 0.5.1)
 69 |       jemoji (= 0.10.1)
 70 |       kramdown (= 1.17.0)
 71 |       liquid (= 4.0.0)
 72 |       listen (= 3.1.5)
 73 |       mercenary (~> 0.3)
 74 |       minima (= 2.5.0)
 75 |       nokogiri (>= 1.8.2, < 2.0)
 76 |       rouge (= 2.2.1)
 77 |       terminal-table (~> 1.4)
 78 |     github-pages-health-check (1.8.1)
 79 |       addressable (~> 2.3)
 80 |       dnsruby (~> 1.60)
 81 |       octokit (~> 4.0)
 82 |       public_suffix (~> 2.0)
 83 |       typhoeus (~> 1.3)
 84 |     html-pipeline (2.8.4)
 85 |       activesupport (>= 2)
 86 |       nokogiri (>= 1.4)
 87 |     http_parser.rb (0.6.0)
 88 |     i18n (0.9.5)
 89 |       concurrent-ruby (~> 1.0)
 90 |     jekyll (3.7.4)
 91 |       addressable (~> 2.4)
 92 |       colorator (~> 1.0)
 93 |       em-websocket (~> 0.5)
 94 |       i18n (~> 0.7)
 95 |       jekyll-sass-converter (~> 1.0)
 96 |       jekyll-watch (~> 2.0)
 97 |       kramdown (~> 1.14)
 98 |       liquid (~> 4.0)
 99 |       mercenary (~> 0.3.3)
100 |       pathutil (~> 0.9)
101 |       rouge (>= 1.7, < 4)
102 |       safe_yaml (~> 1.0)
103 |     jekyll-avatar (0.6.0)
104 |       jekyll (~> 3.0)
105 |     jekyll-coffeescript (1.1.1)
106 |       coffee-script (~> 2.2)
107 |       coffee-script-source (~> 1.11.1)
108 |     jekyll-commonmark (1.2.0)
109 |       commonmarker (~> 0.14)
110 |       jekyll (>= 3.0, < 4.0)
111 |     jekyll-commonmark-ghpages (0.1.5)
112 |       commonmarker (~> 0.17.6)
113 |       jekyll-commonmark (~> 1)
114 |       rouge (~> 2)
115 |     jekyll-default-layout (0.1.4)
116 |       jekyll (~> 3.0)
117 |     jekyll-feed (0.10.0)
118 |       jekyll (~> 3.3)
119 |     jekyll-gist (1.5.0)
120 |       octokit (~> 4.2)
121 |     jekyll-github-metadata (2.9.4)
122 |       jekyll (~> 3.1)
123 |       octokit (~> 4.0, != 4.4.0)
124 |     jekyll-mentions (1.4.1)
125 |       html-pipeline (~> 2.3)
126 |       jekyll (~> 3.0)
127 |     jekyll-optional-front-matter (0.3.0)
128 |       jekyll (~> 3.0)
129 |     jekyll-paginate (1.1.0)
130 |     jekyll-readme-index (0.2.0)
131 |       jekyll (~> 3.0)
132 |     jekyll-redirect-from (0.14.0)
133 |       jekyll (~> 3.3)
134 |     jekyll-relative-links (0.5.3)
135 |       jekyll (~> 3.3)
136 |     jekyll-remote-theme (0.3.1)
137 |       jekyll (~> 3.5)
138 |       rubyzip (>= 1.2.1, < 3.0)
139 |     jekyll-sass-converter (1.5.2)
140 |       sass (~> 3.4)
141 |     jekyll-seo-tag (2.5.0)
142 |       jekyll (~> 3.3)
143 |     jekyll-sitemap (1.2.0)
144 |       jekyll (~> 3.3)
145 |     jekyll-swiss (0.4.0)
146 |     jekyll-theme-architect (0.1.1)
147 |       jekyll (~> 3.5)
148 |       jekyll-seo-tag (~> 2.0)
149 |     jekyll-theme-cayman (0.1.1)
150 |       jekyll (~> 3.5)
151 |       jekyll-seo-tag (~> 2.0)
152 |     jekyll-theme-dinky (0.1.1)
153 |       jekyll (~> 3.5)
154 |       jekyll-seo-tag (~> 2.0)
155 |     jekyll-theme-hacker (0.1.1)
156 |       jekyll (~> 3.5)
157 |       jekyll-seo-tag (~> 2.0)
158 |     jekyll-theme-leap-day (0.1.1)
159 |       jekyll (~> 3.5)
160 |       jekyll-seo-tag (~> 2.0)
161 |     jekyll-theme-merlot (0.1.1)
162 |       jekyll (~> 3.5)
163 |       jekyll-seo-tag (~> 2.0)
164 |     jekyll-theme-midnight (0.1.1)
165 |       jekyll (~> 3.5)
166 |       jekyll-seo-tag (~> 2.0)
167 |     jekyll-theme-minimal (0.1.1)
168 |       jekyll (~> 3.5)
169 |       jekyll-seo-tag (~> 2.0)
170 |     jekyll-theme-modernist (0.1.1)
171 |       jekyll (~> 3.5)
172 |       jekyll-seo-tag (~> 2.0)
173 |     jekyll-theme-primer (0.5.3)
174 |       jekyll (~> 3.5)
175 |       jekyll-github-metadata (~> 2.9)
176 |       jekyll-seo-tag (~> 2.0)
177 |     jekyll-theme-slate (0.1.1)
178 |       jekyll (~> 3.5)
179 |       jekyll-seo-tag (~> 2.0)
180 |     jekyll-theme-tactile (0.1.1)
181 |       jekyll (~> 3.5)
182 |       jekyll-seo-tag (~> 2.0)
183 |     jekyll-theme-time-machine (0.1.1)
184 |       jekyll (~> 3.5)
185 |       jekyll-seo-tag (~> 2.0)
186 |     jekyll-titles-from-headings (0.5.1)
187 |       jekyll (~> 3.3)
188 |     jekyll-watch (2.1.2)
189 |       listen (~> 3.0)
190 |     jemoji (0.10.1)
191 |       gemoji (~> 3.0)
192 |       html-pipeline (~> 2.2)
193 |       jekyll (~> 3.0)
194 |     kramdown (1.17.0)
195 |     liquid (4.0.0)
196 |     listen (3.1.5)
197 |       rb-fsevent (~> 0.9, >= 0.9.4)
198 |       rb-inotify (~> 0.9, >= 0.9.7)
199 |       ruby_dep (~> 1.2)
200 |     mercenary (0.3.6)
201 |     mini_portile2 (2.3.0)
202 |     minima (2.5.0)
203 |       jekyll (~> 3.5)
204 |       jekyll-feed (~> 0.9)
205 |       jekyll-seo-tag (~> 2.1)
206 |     minitest (5.11.3)
207 |     multipart-post (2.0.0)
208 |     nokogiri (1.8.5-x64-mingw32)
209 |       mini_portile2 (~> 2.3.0)
210 |     octokit (4.13.0)
211 |       sawyer (~> 0.8.0, >= 0.5.3)
212 |     pathutil (0.16.2)
213 |       forwardable-extended (~> 2.6)
214 |     public_suffix (2.0.5)
215 |     rb-fsevent (0.10.3)
216 |     rb-inotify (0.9.10)
217 |       ffi (>= 0.5.0, < 2)
218 |     rouge (2.2.1)
219 |     ruby-enum (0.7.2)
220 |       i18n
221 |     ruby_dep (1.5.0)
222 |     rubyzip (1.2.2)
223 |     safe_yaml (1.0.4)
224 |     sass (3.6.0)
225 |       sass-listen (~> 4.0.0)
226 |     sass-listen (4.0.0)
227 |       rb-fsevent (~> 0.9, >= 0.9.4)
228 |       rb-inotify (~> 0.9, >= 0.9.7)
229 |     sawyer (0.8.1)
230 |       addressable (>= 2.3.5, < 2.6)
231 |       faraday (~> 0.8, < 1.0)
232 |     terminal-table (1.8.0)
233 |       unicode-display_width (~> 1.1, >= 1.1.1)
234 |     thread_safe (0.3.6)
235 |     typhoeus (1.3.0)
236 |       ethon (>= 0.9.0)
237 |     tzinfo (1.2.5)
238 |       thread_safe (~> 0.1)
239 |     unicode-display_width (1.4.0)
240 | 
241 | PLATFORMS
242 |   x64-mingw32
243 | 
244 | DEPENDENCIES
245 |   github-pages
246 | 
247 | BUNDLED WITH
248 |    1.17.1
249 | 


--------------------------------------------------------------------------------
/docs/_config.yml:
--------------------------------------------------------------------------------
1 | theme: jekyll-theme-cayman
2 | title: Face Landmark-based Speaker-Independent Audio-Visual Speech Enhancement in Multi-Talker Environments
3 | description: Giovanni Morrone, Luca Pasa, Vadim Tikhanoff, Sonia Bergamaschi, Luciano Fadiga, Leonardo Badino <br> University of Modena and Reggio Emilia and Istituto Italiano di Tecnologia
4 | repository: dr-pato/audio_visual_speech_enhancement
5 | 


--------------------------------------------------------------------------------
/docs/_site/assets/css/style.css:
--------------------------------------------------------------------------------
  1 | /*! normalize.css v3.0.2 | MIT License | git.io/normalize */
  2 | /** 1. Set default font family to sans-serif. 2. Prevent iOS text size adjust after orientation change, without disabling user zoom. */
  3 | @import url("https://fonts.googleapis.com/css?family=Open+Sans:400,700");
  4 | html { font-family: sans-serif; /* 1 */ -ms-text-size-adjust: 100%; /* 2 */ -webkit-text-size-adjust: 100%; /* 2 */ }
  5 | 
  6 | /** Remove default margin. */
  7 | body { margin: 0; }
  8 | 
  9 | /* HTML5 display definitions ========================================================================== */
 10 | /** Correct `block` display not defined for any HTML5 element in IE 8/9. Correct `block` display not defined for `details` or `summary` in IE 10/11 and Firefox. Correct `block` display not defined for `main` in IE 11. */
 11 | article, aside, details, figcaption, figure, footer, header, hgroup, main, menu, nav, section, summary { display: block; }
 12 | 
 13 | /** 1. Correct `inline-block` display not defined in IE 8/9. 2. Normalize vertical alignment of `progress` in Chrome, Firefox, and Opera. */
 14 | audio, canvas, progress, video { display: inline-block; /* 1 */ vertical-align: baseline; /* 2 */ }
 15 | 
 16 | /** Prevent modern browsers from displaying `audio` without controls. Remove excess height in iOS 5 devices. */
 17 | audio:not([controls]) { display: none; height: 0; }
 18 | 
 19 | /** Address `[hidden]` styling not present in IE 8/9/10. Hide the `template` element in IE 8/9/11, Safari, and Firefox < 22. */
 20 | [hidden], template { display: none; }
 21 | 
 22 | /* Links ========================================================================== */
 23 | /** Remove the gray background color from active links in IE 10. */
 24 | a { background-color: transparent; }
 25 | 
 26 | /** Improve readability when focused and also mouse hovered in all browsers. */
 27 | a:active, a:hover { outline: 0; }
 28 | 
 29 | /* Text-level semantics ========================================================================== */
 30 | /** Address styling not present in IE 8/9/10/11, Safari, and Chrome. */
 31 | abbr[title] { border-bottom: 1px dotted; }
 32 | 
 33 | /** Address style set to `bolder` in Firefox 4+, Safari, and Chrome. */
 34 | b, strong { font-weight: bold; }
 35 | 
 36 | /** Address styling not present in Safari and Chrome. */
 37 | dfn { font-style: italic; }
 38 | 
 39 | /** Address variable `h1` font-size and margin within `section` and `article` contexts in Firefox 4+, Safari, and Chrome. */
 40 | h1 { font-size: 2em; margin: 0.67em 0; }
 41 | 
 42 | /** Address styling not present in IE 8/9. */
 43 | mark { background: #ff0; color: #000; }
 44 | 
 45 | /** Address inconsistent and variable font size in all browsers. */
 46 | small { font-size: 80%; }
 47 | 
 48 | /** Prevent `sub` and `sup` affecting `line-height` in all browsers. */
 49 | sub, sup { font-size: 75%; line-height: 0; position: relative; vertical-align: baseline; }
 50 | 
 51 | sup { top: -0.5em; }
 52 | 
 53 | sub { bottom: -0.25em; }
 54 | 
 55 | /* Embedded content ========================================================================== */
 56 | /** Remove border when inside `a` element in IE 8/9/10. */
 57 | img { border: 0; }
 58 | 
 59 | /** Correct overflow not hidden in IE 9/10/11. */
 60 | svg:not(:root) { overflow: hidden; }
 61 | 
 62 | /* Grouping content ========================================================================== */
 63 | /** Address margin not present in IE 8/9 and Safari. */
 64 | figure { margin: 1em 40px; }
 65 | 
 66 | /** Address differences between Firefox and other browsers. */
 67 | hr { box-sizing: content-box; height: 0; }
 68 | 
 69 | /** Contain overflow in all browsers. */
 70 | pre { overflow: auto; }
 71 | 
 72 | /** Address odd `em`-unit font size rendering in all browsers. */
 73 | code, kbd, pre, samp { font-family: monospace, monospace; font-size: 1em; }
 74 | 
 75 | /* Forms ========================================================================== */
 76 | /** Known limitation: by default, Chrome and Safari on OS X allow very limited styling of `select`, unless a `border` property is set. */
 77 | /** 1. Correct color not being inherited. Known issue: affects color of disabled elements. 2. Correct font properties not being inherited. 3. Address margins set differently in Firefox 4+, Safari, and Chrome. */
 78 | button, input, optgroup, select, textarea { color: inherit; /* 1 */ font: inherit; /* 2 */ margin: 0; /* 3 */ }
 79 | 
 80 | /** Address `overflow` set to `hidden` in IE 8/9/10/11. */
 81 | button { overflow: visible; }
 82 | 
 83 | /** Address inconsistent `text-transform` inheritance for `button` and `select`. All other form control elements do not inherit `text-transform` values. Correct `button` style inheritance in Firefox, IE 8/9/10/11, and Opera. Correct `select` style inheritance in Firefox. */
 84 | button, select { text-transform: none; }
 85 | 
 86 | /** 1. Avoid the WebKit bug in Android 4.0.* where (2) destroys native `audio` and `video` controls. 2. Correct inability to style clickable `input` types in iOS. 3. Improve usability and consistency of cursor style between image-type `input` and others. */
 87 | button, html input[type="button"], input[type="reset"], input[type="submit"] { -webkit-appearance: button; /* 2 */ cursor: pointer; /* 3 */ }
 88 | 
 89 | /** Re-set default cursor for disabled elements. */
 90 | button[disabled], html input[disabled] { cursor: default; }
 91 | 
 92 | /** Remove inner padding and border in Firefox 4+. */
 93 | button::-moz-focus-inner, input::-moz-focus-inner { border: 0; padding: 0; }
 94 | 
 95 | /** Address Firefox 4+ setting `line-height` on `input` using `!important` in the UA stylesheet. */
 96 | input { line-height: normal; }
 97 | 
 98 | /** It's recommended that you don't attempt to style these elements. Firefox's implementation doesn't respect box-sizing, padding, or width.  1. Address box sizing set to `content-box` in IE 8/9/10. 2. Remove excess padding in IE 8/9/10. */
 99 | input[type="checkbox"], input[type="radio"] { box-sizing: border-box; /* 1 */ padding: 0; /* 2 */ }
100 | 
101 | /** Fix the cursor style for Chrome's increment/decrement buttons. For certain `font-size` values of the `input`, it causes the cursor style of the decrement button to change from `default` to `text`. */
102 | input[type="number"]::-webkit-inner-spin-button, input[type="number"]::-webkit-outer-spin-button { height: auto; }
103 | 
104 | /** 1. Address `appearance` set to `searchfield` in Safari and Chrome. 2. Address `box-sizing` set to `border-box` in Safari and Chrome (include `-moz` to future-proof). */
105 | input[type="search"] { -webkit-appearance: textfield; /* 1 */ /* 2 */ box-sizing: content-box; }
106 | 
107 | /** Remove inner padding and search cancel button in Safari and Chrome on OS X. Safari (but not Chrome) clips the cancel button when the search input has padding (and `textfield` appearance). */
108 | input[type="search"]::-webkit-search-cancel-button, input[type="search"]::-webkit-search-decoration { -webkit-appearance: none; }
109 | 
110 | /** Define consistent border, margin, and padding. */
111 | fieldset { border: 1px solid #c0c0c0; margin: 0 2px; padding: 0.35em 0.625em 0.75em; }
112 | 
113 | /** 1. Correct `color` not being inherited in IE 8/9/10/11. 2. Remove padding so people aren't caught out if they zero out fieldsets. */
114 | legend { border: 0; /* 1 */ padding: 0; /* 2 */ }
115 | 
116 | /** Remove default vertical scrollbar in IE 8/9/10/11. */
117 | textarea { overflow: auto; }
118 | 
119 | /** Don't inherit the `font-weight` (applied by a rule above). NOTE: the default cannot safely be changed in Chrome and Safari on OS X. */
120 | optgroup { font-weight: bold; }
121 | 
122 | /* Tables ========================================================================== */
123 | /** Remove most spacing between table cells. */
124 | table { border-collapse: collapse; border-spacing: 0; }
125 | 
126 | td, th { padding: 0; }
127 | 
128 | .highlight table td { padding: 5px; }
129 | 
130 | .highlight table pre { margin: 0; }
131 | 
132 | .highlight .cm { color: #999988; font-style: italic; }
133 | 
134 | .highlight .cp { color: #999999; font-weight: bold; }
135 | 
136 | .highlight .c1 { color: #999988; font-style: italic; }
137 | 
138 | .highlight .cs { color: #999999; font-weight: bold; font-style: italic; }
139 | 
140 | .highlight .c, .highlight .cd { color: #999988; font-style: italic; }
141 | 
142 | .highlight .err { color: #a61717; background-color: #e3d2d2; }
143 | 
144 | .highlight .gd { color: #000000; background-color: #ffdddd; }
145 | 
146 | .highlight .ge { color: #000000; font-style: italic; }
147 | 
148 | .highlight .gr { color: #aa0000; }
149 | 
150 | .highlight .gh { color: #999999; }
151 | 
152 | .highlight .gi { color: #000000; background-color: #ddffdd; }
153 | 
154 | .highlight .go { color: #888888; }
155 | 
156 | .highlight .gp { color: #555555; }
157 | 
158 | .highlight .gs { font-weight: bold; }
159 | 
160 | .highlight .gu { color: #aaaaaa; }
161 | 
162 | .highlight .gt { color: #aa0000; }
163 | 
164 | .highlight .kc { color: #000000; font-weight: bold; }
165 | 
166 | .highlight .kd { color: #000000; font-weight: bold; }
167 | 
168 | .highlight .kn { color: #000000; font-weight: bold; }
169 | 
170 | .highlight .kp { color: #000000; font-weight: bold; }
171 | 
172 | .highlight .kr { color: #000000; font-weight: bold; }
173 | 
174 | .highlight .kt { color: #445588; font-weight: bold; }
175 | 
176 | .highlight .k, .highlight .kv { color: #000000; font-weight: bold; }
177 | 
178 | .highlight .mf { color: #009999; }
179 | 
180 | .highlight .mh { color: #009999; }
181 | 
182 | .highlight .il { color: #009999; }
183 | 
184 | .highlight .mi { color: #009999; }
185 | 
186 | .highlight .mo { color: #009999; }
187 | 
188 | .highlight .m, .highlight .mb, .highlight .mx { color: #009999; }
189 | 
190 | .highlight .sb { color: #d14; }
191 | 
192 | .highlight .sc { color: #d14; }
193 | 
194 | .highlight .sd { color: #d14; }
195 | 
196 | .highlight .s2 { color: #d14; }
197 | 
198 | .highlight .se { color: #d14; }
199 | 
200 | .highlight .sh { color: #d14; }
201 | 
202 | .highlight .si { color: #d14; }
203 | 
204 | .highlight .sx { color: #d14; }
205 | 
206 | .highlight .sr { color: #009926; }
207 | 
208 | .highlight .s1 { color: #d14; }
209 | 
210 | .highlight .ss { color: #990073; }
211 | 
212 | .highlight .s { color: #d14; }
213 | 
214 | .highlight .na { color: #008080; }
215 | 
216 | .highlight .bp { color: #999999; }
217 | 
218 | .highlight .nb { color: #0086B3; }
219 | 
220 | .highlight .nc { color: #445588; font-weight: bold; }
221 | 
222 | .highlight .no { color: #008080; }
223 | 
224 | .highlight .nd { color: #3c5d5d; font-weight: bold; }
225 | 
226 | .highlight .ni { color: #800080; }
227 | 
228 | .highlight .ne { color: #990000; font-weight: bold; }
229 | 
230 | .highlight .nf { color: #990000; font-weight: bold; }
231 | 
232 | .highlight .nl { color: #990000; font-weight: bold; }
233 | 
234 | .highlight .nn { color: #555555; }
235 | 
236 | .highlight .nt { color: #000080; }
237 | 
238 | .highlight .vc { color: #008080; }
239 | 
240 | .highlight .vg { color: #008080; }
241 | 
242 | .highlight .vi { color: #008080; }
243 | 
244 | .highlight .nv { color: #008080; }
245 | 
246 | .highlight .ow { color: #000000; font-weight: bold; }
247 | 
248 | .highlight .o { color: #000000; font-weight: bold; }
249 | 
250 | .highlight .w { color: #bbbbbb; }
251 | 
252 | .highlight { background-color: #f8f8f8; }
253 | 
254 | * { box-sizing: border-box; }
255 | 
256 | body { padding: 0; margin: 0; font-family: "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif; font-size: 16px; line-height: 1.5; color: #606c71; }
257 | 
258 | a { color: #1e6bb8; text-decoration: none; }
259 | a:hover { text-decoration: underline; }
260 | 
261 | .btn { display: inline-block; margin-bottom: 1rem; color: rgba(255, 255, 255, 0.7); background-color: rgba(255, 255, 255, 0.08); border-color: rgba(255, 255, 255, 0.2); border-style: solid; border-width: 1px; border-radius: 0.3rem; transition: color 0.2s, background-color 0.2s, border-color 0.2s; }
262 | .btn:hover { color: rgba(255, 255, 255, 0.8); text-decoration: none; background-color: rgba(255, 255, 255, 0.2); border-color: rgba(255, 255, 255, 0.3); }
263 | .btn + .btn { margin-left: 1rem; }
264 | @media screen and (min-width: 64em) { .btn { padding: 0.75rem 1rem; } }
265 | @media screen and (min-width: 42em) and (max-width: 64em) { .btn { padding: 0.6rem 0.9rem; font-size: 0.9rem; } }
266 | @media screen and (max-width: 42em) { .btn { display: block; width: 100%; padding: 0.75rem; font-size: 0.9rem; }
267 |   .btn + .btn { margin-top: 1rem; margin-left: 0; } }
268 | 
269 | .page-header { color: #fff; text-align: center; background-color: #159957; background-image: linear-gradient(120deg, #155799, #159957); }
270 | @media screen and (min-width: 64em) { .page-header { padding: 5rem 6rem; } }
271 | @media screen and (min-width: 42em) and (max-width: 64em) { .page-header { padding: 3rem 4rem; } }
272 | @media screen and (max-width: 42em) { .page-header { padding: 2rem 1rem; } }
273 | 
274 | .project-name { margin-top: 0; margin-bottom: 0.1rem; }
275 | @media screen and (min-width: 64em) { .project-name { font-size: 3.25rem; } }
276 | @media screen and (min-width: 42em) and (max-width: 64em) { .project-name { font-size: 2.25rem; } }
277 | @media screen and (max-width: 42em) { .project-name { font-size: 1.75rem; } }
278 | 
279 | .project-tagline { margin-bottom: 2rem; font-weight: normal; opacity: 0.7; }
280 | @media screen and (min-width: 64em) { .project-tagline { font-size: 1.25rem; } }
281 | @media screen and (min-width: 42em) and (max-width: 64em) { .project-tagline { font-size: 1.15rem; } }
282 | @media screen and (max-width: 42em) { .project-tagline { font-size: 1rem; } }
283 | 
284 | .main-content { word-wrap: break-word; }
285 | .main-content :first-child { margin-top: 0; }
286 | @media screen and (min-width: 64em) { .main-content { max-width: 64rem; padding: 2rem 6rem; margin: 0 auto; font-size: 1.1rem; } }
287 | @media screen and (min-width: 42em) and (max-width: 64em) { .main-content { padding: 2rem 4rem; font-size: 1.1rem; } }
288 | @media screen and (max-width: 42em) { .main-content { padding: 2rem 1rem; font-size: 1rem; } }
289 | .main-content img { max-width: 100%; }
290 | .main-content h1, .main-content h2, .main-content h3, .main-content h4, .main-content h5, .main-content h6 { margin-top: 2rem; margin-bottom: 1rem; font-weight: normal; color: #159957; }
291 | .main-content p { margin-bottom: 1em; }
292 | .main-content code { padding: 2px 4px; font-family: Consolas, "Liberation Mono", Menlo, Courier, monospace; font-size: 0.9rem; color: #567482; background-color: #f3f6fa; border-radius: 0.3rem; }
293 | .main-content pre { padding: 0.8rem; margin-top: 0; margin-bottom: 1rem; font: 1rem Consolas, "Liberation Mono", Menlo, Courier, monospace; color: #567482; word-wrap: normal; background-color: #f3f6fa; border: solid 1px #dce6f0; border-radius: 0.3rem; }
294 | .main-content pre > code { padding: 0; margin: 0; font-size: 0.9rem; color: #567482; word-break: normal; white-space: pre; background: transparent; border: 0; }
295 | .main-content .highlight { margin-bottom: 1rem; }
296 | .main-content .highlight pre { margin-bottom: 0; word-break: normal; }
297 | .main-content .highlight pre, .main-content pre { padding: 0.8rem; overflow: auto; font-size: 0.9rem; line-height: 1.45; border-radius: 0.3rem; -webkit-overflow-scrolling: touch; }
298 | .main-content pre code, .main-content pre tt { display: inline; max-width: initial; padding: 0; margin: 0; overflow: initial; line-height: inherit; word-wrap: normal; background-color: transparent; border: 0; }
299 | .main-content pre code:before, .main-content pre code:after, .main-content pre tt:before, .main-content pre tt:after { content: normal; }
300 | .main-content ul, .main-content ol { margin-top: 0; }
301 | .main-content blockquote { padding: 0 1rem; margin-left: 0; color: #819198; border-left: 0.3rem solid #dce6f0; }
302 | .main-content blockquote > :first-child { margin-top: 0; }
303 | .main-content blockquote > :last-child { margin-bottom: 0; }
304 | .main-content table { display: block; width: 100%; overflow: auto; word-break: normal; word-break: keep-all; -webkit-overflow-scrolling: touch; }
305 | .main-content table th { font-weight: bold; }
306 | .main-content table th, .main-content table td { padding: 0.5rem 1rem; border: 1px solid #e9ebec; }
307 | .main-content dl { padding: 0; }
308 | .main-content dl dt { padding: 0; margin-top: 1rem; font-size: 1rem; font-weight: bold; }
309 | .main-content dl dd { padding: 0; margin-bottom: 1rem; }
310 | .main-content hr { height: 2px; padding: 0; margin: 1rem 0; background-color: #eff0f1; border: 0; }
311 | 
312 | .site-footer { padding-top: 2rem; margin-top: 2rem; border-top: solid 1px #eff0f1; }
313 | @media screen and (min-width: 64em) { .site-footer { font-size: 1rem; } }
314 | @media screen and (min-width: 42em) and (max-width: 64em) { .site-footer { font-size: 1rem; } }
315 | @media screen and (max-width: 42em) { .site-footer { font-size: 0.9rem; } }
316 | 
317 | .site-footer-owner { display: block; font-weight: bold; }
318 | 
319 | .site-footer-credits { color: #819198; }
320 | 


--------------------------------------------------------------------------------
/docs/_site/index.html:
--------------------------------------------------------------------------------
  1 | <!DOCTYPE html>
  2 | <html lang="en-US">
  3 |   <head>
  4 |     <meta charset="UTF-8">
  5 | 
  6 | <!-- Begin Jekyll SEO tag v2.5.0 -->
  7 | <title>Face Landmark-based Speaker-Independent Audio-Visual Speech Enhancement in Multi-Talker Environments | Giovanni Morrone, Luca Pasa, Vadim Tikhanoff, Sonia Bergamaschi, Luciano Fadiga, Leonardo Badino University of Modena and Reggio Emilia and Istituto Italiano di Tecnologia</title>
  8 | <meta name="generator" content="Jekyll v3.7.4" />
  9 | <meta property="og:title" content="Face Landmark-based Speaker-Independent Audio-Visual Speech Enhancement in Multi-Talker Environments" />
 10 | <meta property="og:locale" content="en_US" />
 11 | <meta name="description" content="Giovanni Morrone, Luca Pasa, Vadim Tikhanoff, Sonia Bergamaschi, Luciano Fadiga, Leonardo Badino University of Modena and Reggio Emilia and Istituto Italiano di Tecnologia" />
 12 | <meta property="og:description" content="Giovanni Morrone, Luca Pasa, Vadim Tikhanoff, Sonia Bergamaschi, Luciano Fadiga, Leonardo Badino University of Modena and Reggio Emilia and Istituto Italiano di Tecnologia" />
 13 | <link rel="canonical" href="http://localhost:4000/" />
 14 | <meta property="og:url" content="http://localhost:4000/" />
 15 | <meta property="og:site_name" content="Face Landmark-based Speaker-Independent Audio-Visual Speech Enhancement in Multi-Talker Environments" />
 16 | <script type="application/ld+json">
 17 | {"headline":"Face Landmark-based Speaker-Independent Audio-Visual Speech Enhancement in Multi-Talker Environments","url":"http://localhost:4000/","name":"Face Landmark-based Speaker-Independent Audio-Visual Speech Enhancement in Multi-Talker Environments","description":"Giovanni Morrone, Luca Pasa, Vadim Tikhanoff, Sonia Bergamaschi, Luciano Fadiga, Leonardo Badino University of Modena and Reggio Emilia and Istituto Italiano di Tecnologia","@type":"WebSite","@context":"http://schema.org"}</script>
 18 | <!-- End Jekyll SEO tag -->
 19 | 
 20 |     <meta name="viewport" content="width=device-width, initial-scale=1">
 21 |     <meta name="theme-color" content="#157878">
 22 |     <link rel="stylesheet" href="/assets/css/style.css?v=587c6138b332665eee05b2dcf48d3084cc54d1b6">
 23 |   </head>
 24 |   <body>
 25 |     <section class="page-header">
 26 |       <h1 class="project-name">Face Landmark-based Speaker-Independent Audio-Visual Speech Enhancement in Multi-Talker Environments</h1>
 27 |       <h2 class="project-tagline">Giovanni Morrone, Luca Pasa, Vadim Tikhanoff, Sonia Bergamaschi, Luciano Fadiga, Leonardo Badino <br> University of Modena and Reggio Emilia and Istituto Italiano di Tecnologia</h2>
 28 |       
 29 |         <a href="http://github.com/dr-pato/audio_visual_speech_enhancement" class="btn">View on GitHub</a>
 30 |       
 31 |       
 32 |     </section>
 33 | 
 34 |     <section class="main-content">
 35 |       <h2 id="abstract">Abstract</h2>
 36 | <p>We address the problem of enhancing the speech of a speaker of interest in a cocktail party scenario when visual information of the speaker of interest is available. Contrary to most previous studies, we do not learn visual features on the typically small audio-visual datasets, but use an already available face landmark detector (trained on a separate image dataset). The landmarks are used by LSTM-based models to generate time-frequency masks which are applied to the acoustic mixed-speech spectrogram. Results show that: (i) landmark motion features are very effective features for this task, (ii) similarly to previous work, reconstruction of the target speaker’s spectrogram mediated by masking is significantly more accurate than direct spectrogram reconstruction, and (iii) the best masks depend on both motion landmark features and the input mixed-speech spectrogram. To the best of our knowledge, our proposed models are the first models trained and evaluated on the limited size GRID and TCD-TIMIT datasets, that achieve speaker-independent speech enhancement in a multi-talker setting.</p>
 37 | 
 38 | <div align="center">
 39 | <iframe width="800" height="450" src="https://www.youtube.com/embed/_him5fsvXyM" frameborder="0" allow="accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture" allowfullscreen="">
 40 | </iframe>
 41 | </div>
 42 | 
 43 | <h2 id="demos">Demos</h2>
 44 | <p>The following videos contains severals examples of enhanced speech generated by models proposed in our paper.</p>
 45 | 
 46 | <h3 id="grid-corpus">GRID corpus</h3>
 47 | 
 48 | <div align="center">
 49 | <table id="mytable" border="0">
 50 |   <tr>
 51 |     <th>2-Speakers Mix</th>
 52 |     <th>3-Speakers Mix</th> 
 53 |   </tr>
 54 |   <tr>
 55 |     <td>
 56 | 		<video width="380" height="317" controls="">
 57 | 		<source src="videos/grid_2spk.mp4" type="video/mp4" />
 58 | 		Your browser does not support the video tag.
 59 | 		</video>
 60 | 	</td>
 61 |     <td>
 62 | 		<video width="380" height="317" controls="">
 63 | 		<source src="videos/grid_3spk.mp4" type="video/mp4" />
 64 | 		Your browser does not support the video tag.
 65 | 		</video>
 66 | 	</td> 
 67 |   </tr>
 68 | </table>
 69 | </div>
 70 | 
 71 | <h3 id="tcd-timit-corpus">TCD-TIMIT corpus</h3>
 72 | 
 73 | <div align="center">
 74 | <table>
 75 |   <tr>
 76 |     <th>2-Speakers Mix</th>
 77 |     <th>3-Speakers Mix</th> 
 78 |   </tr>
 79 |   <tr>
 80 |     <td>
 81 | 		<video width="380" height="317" controls="">
 82 | 		<source src="videos/timit_2spk.mp4" type="video/mp4" />
 83 | 		Your browser does not support the video tag.
 84 | 		</video>
 85 | 	</td>
 86 |     <td>
 87 | 		<video width="380" height="317" controls="">
 88 | 		<source src="videos/timit_3spk.mp4" type="video/mp4" />
 89 | 		Your browser does not support the video tag.
 90 | 		</video>
 91 | 	</td>
 92 |   </tr>
 93 | </table>
 94 | </div>
 95 | 
 96 | 
 97 |       <footer class="site-footer">
 98 |         
 99 |           <span class="site-footer-owner"><a href="http://github.com/dr-pato/audio_visual_speech_enhancement">audio_visual_speech_enhancement</a> is maintained by <a href="http://github.com/dr-pato">dr-pato</a>.</span>
100 |         
101 |         <span class="site-footer-credits">This page was generated by <a href="https://pages.github.com">GitHub Pages</a>.</span>
102 |       </footer>
103 |     </section>
104 | 
105 |     
106 |   </body>
107 | </html>
108 | 


--------------------------------------------------------------------------------
/docs/_site/videos/grid_2spk.mp4:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/dr-pato/audio_visual_speech_enhancement/78f68801813acee648316925903d4312142d61a4/docs/_site/videos/grid_2spk.mp4


--------------------------------------------------------------------------------
/docs/_site/videos/grid_3spk.mp4:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/dr-pato/audio_visual_speech_enhancement/78f68801813acee648316925903d4312142d61a4/docs/_site/videos/grid_3spk.mp4


--------------------------------------------------------------------------------
/docs/_site/videos/timit_2spk.mp4:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/dr-pato/audio_visual_speech_enhancement/78f68801813acee648316925903d4312142d61a4/docs/_site/videos/timit_2spk.mp4


--------------------------------------------------------------------------------
/docs/_site/videos/timit_3spk.mp4:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/dr-pato/audio_visual_speech_enhancement/78f68801813acee648316925903d4312142d61a4/docs/_site/videos/timit_3spk.mp4


--------------------------------------------------------------------------------
/docs/index.md:
--------------------------------------------------------------------------------
 1 | ---
 2 | title: Face Landmark-based Speaker-Independent Audio-Visual Speech Enhancement in Multi-Talker Environments
 3 | layout: default
 4 | ---
 5 | 
 6 | ## Abstract
 7 | We address the problem of enhancing the speech of a speaker of interest in a cocktail party scenario when visual information of the speaker of interest is available. Contrary to most previous studies, we do not learn visual features on the typically small audio-visual datasets, but use an already available face landmark detector (trained on a separate image dataset). The landmarks are used by LSTM-based models to generate time-frequency masks which are applied to the acoustic mixed-speech spectrogram. Results show that: (i) landmark motion features are very effective features for this task, (ii) similarly to previous work, reconstruction of the target speaker’s spectrogram mediated by masking is significantly more accurate than direct spectrogram reconstruction, and (iii) the best masks depend on both motion landmark features and the input mixed-speech spectrogram. To the best of our knowledge, our proposed models are the first models trained and evaluated on the limited size GRID and TCD-TIMIT datasets, that achieve speaker-independent speech enhancement in a multi-talker setting.
 8 | 
 9 | <div align="center">
10 | <iframe
11 | width="800" height="450" src="https://www.youtube.com/embed/YQ0q-OFphKM" frameborder="0" allow="accelerometer; autoplay; encrypted-media; gyroscope; picture-in-picture" allowfullscreen>
12 | </iframe>
13 | </div>
14 | 
15 | ## Demos
16 | The following videos contain several examples of enhanced speech generated by models proposed in our paper.
17 | 
18 | ### GRID corpus
19 | 
20 | <div align="center">
21 | <table id="mytable" border="0">
22 |   <tr>
23 |     <th>2-Speakers Mix</th>
24 |     <th>3-Speakers Mix</th> 
25 |   </tr>
26 |   <tr>
27 |     <td>
28 | 		<video width="380" height="317" controls>
29 | 		<source src="videos/grid_2spk.mp4" type="video/mp4">
30 | 		Your browser does not support the video tag.
31 | 		</video>
32 | 	</td>
33 |     <td>
34 | 		<video width="380" height="317" controls>
35 | 		<source src="videos/grid_3spk.mp4" type="video/mp4">
36 | 		Your browser does not support the video tag.
37 | 		</video>
38 | 	</td> 
39 |   </tr>
40 | </table>
41 | </div>
42 | 
43 | ### TCD-TIMIT corpus
44 | 
45 | <div align="center">
46 | <table>
47 |   <tr>
48 |     <th>2-Speakers Mix</th>
49 |     <th>3-Speakers Mix</th> 
50 |   </tr>
51 |   <tr>
52 |     <td>
53 | 		<video width="380" height="317" controls>
54 | 		<source src="videos/timit_2spk.mp4" type="video/mp4">
55 | 		Your browser does not support the video tag.
56 | 		</video>
57 | 	</td>
58 |     <td>
59 | 		<video width="380" height="317" controls>
60 | 		<source src="videos/timit_3spk.mp4" type="video/mp4">
61 | 		Your browser does not support the video tag.
62 | 		</video>
63 | 	</td>
64 |   </tr>
65 | </table>
66 | </div>
67 | 
68 | ## Paper
69 | The paper is available [here](https://arxiv.org/abs/1811.02480). If this project is useful for your research, please cite:
70 | ```
71 | @inproceedings{morrone2019face,
72 |   title={Face Landmark-based Speaker-Independent Audio-Visual Speech Enhancement in Multi-Talker Environments},
73 |   author={Morrone, Giovanni and Bergamaschi, Sonia and Pasa, Luca and Fadiga, Luciano and Tikhanoff, Vadim and Badino, Leonardo},
74 |   booktitle={ICASSP 2019 - 2019 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)},
75 |   pages={6900-6904},
76 |   year={2019},
77 |   organization={IEEE}
78 | }
79 | ```
80 | 


--------------------------------------------------------------------------------
/docs/videos/grid_2spk.mp4:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/dr-pato/audio_visual_speech_enhancement/78f68801813acee648316925903d4312142d61a4/docs/videos/grid_2spk.mp4


--------------------------------------------------------------------------------
/docs/videos/grid_3spk.mp4:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/dr-pato/audio_visual_speech_enhancement/78f68801813acee648316925903d4312142d61a4/docs/videos/grid_3spk.mp4


--------------------------------------------------------------------------------
/docs/videos/timit_2spk.mp4:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/dr-pato/audio_visual_speech_enhancement/78f68801813acee648316925903d4312142d61a4/docs/videos/timit_2spk.mp4


--------------------------------------------------------------------------------
/docs/videos/timit_3spk.mp4:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/dr-pato/audio_visual_speech_enhancement/78f68801813acee648316925903d4312142d61a4/docs/videos/timit_3spk.mp4


--------------------------------------------------------------------------------
/enhancement_model.py:
--------------------------------------------------------------------------------
  1 | import tensorflow as tf
  2 | import math
  3 | from audio_processing import compute_stft, get_oracle_iam, get_sources
  4 | 
  5 | class VL2M(object):
  6 |     """
  7 |     VL2M
  8 |     Input: video features (face landmarks).
  9 |     Model: stacked BLSTM.
 10 |     Output: Target Binary Mask (TBM).
 11 |     Loss: binary cross entropy
 12 |     """
 13 | 
 14 |     def __init__(self, input_video, sequence_lengths, tbm, mixed_sources, target_sources, keep_prob, config):
 15 |         self.input_video = input_video
 16 |         self.sequence_lengths = sequence_lengths
 17 |         self.tbm = tbm
 18 |         self.audio_frame_size = config.audio_feat_dim
 19 |         self.video_frame_size = config.video_feat_dim
 20 |         self.num_audio_samples = config.num_audio_samples
 21 |         self.mixed_sources = mixed_sources
 22 |         self.target_sources = target_sources
 23 |         self.mixed_specs = compute_stft(mixed_sources, out_shape=tf.shape(tbm))
 24 |         self.num_units = config.n_hidden
 25 |         self.num_layers = config.num_layers
 26 |         self.optimizer_choice = config.optimizer_choice
 27 |         self.initial_learning_rate = config.learning_rate
 28 |         self.learning_rate = config.learning_rate
 29 |         self.updating_step = config.updating_step
 30 |         self.learning_decay = config.learning_decay
 31 |         self.keep_prob = keep_prob
 32 |         self.regularization = config.reg
 33 |         self.mask_threshold = config.mask_thresh
 34 |         self._inference = None
 35 |         self._loss = None
 36 |         self._train_op = None
 37 |         self._prediction = None
 38 |         self._enhanced_sources = None
 39 |         self._summaries = None
 40 |         self._global_step = None
 41 | 
 42 |     def create_graph(self):
 43 |         """
 44 |         Create the Graph of the model.
 45 |         """
 46 |         self.inference
 47 |         self.loss
 48 |         self.train_op
 49 |         self.prediction
 50 |         self.enhanced_sources
 51 |         self.summaries
 52 | 
 53 |     @property
 54 |     def inference(self):
 55 |         if self._inference is None:
 56 |             max_sequence_length = tf.reduce_max(self.sequence_lengths)
 57 |             with tf.variable_scope('forward'):
 58 |                 forward_cells = []
 59 |                 for i in range(self.num_layers):
 60 |                     lstm_cell = tf.contrib.rnn.LSTMCell(num_units=self.num_units, use_peepholes=True, initializer=tf.random_uniform_initializer(minval=-0.1, maxval=0.1))
 61 |                     forward_cells.append(lstm_cell)
 62 | 
 63 |             with tf.variable_scope('backward'):
 64 |                 backward_cells = []
 65 |                 for i in range(self.num_layers):
 66 |                     lstm_cell = tf.contrib.rnn.LSTMCell(num_units=self.num_units, use_peepholes=True, initializer=tf.random_uniform_initializer(minval=-0.1, maxval=0.1))
 67 |                     backward_cells.append(lstm_cell)
 68 | 
 69 |             with tf.variable_scope('Bi_LSTM'):
 70 |                 rnn_outputs, output_state_fw, output_state_bw = tf.contrib.rnn.stack_bidirectional_dynamic_rnn(cells_fw=forward_cells,
 71 |                   cells_bw=backward_cells,
 72 |                   inputs=self.input_video,
 73 |                   dtype=tf.float64,
 74 |                   sequence_length=self.sequence_lengths)
 75 |             rnn_outputs_res = tf.nn.dropout(tf.reshape(rnn_outputs, [-1, self.num_units * 2]), keep_prob=self.keep_prob)
 76 |             with tf.variable_scope('logits'):
 77 |                 weights = tf.Variable(tf.truncated_normal([self.num_units * 2, self.audio_frame_size], stddev=1.0 / math.sqrt(float(self.num_units * 2)),
 78 |                   dtype=tf.float64),
 79 |                   name='weights')
 80 |                 biases = tf.Variable(tf.zeros([self.audio_frame_size], dtype=tf.float64), name='biases')
 81 |                 logits = tf.matmul(rnn_outputs_res, weights) + biases
 82 |             self._inference = tf.reshape(logits, [-1, max_sequence_length, self.audio_frame_size], name='inference')
 83 |         return self._inference
 84 | 
 85 |     @property
 86 |     def prediction(self):
 87 |         if self._prediction is None:
 88 |             self._prediction = tf.sigmoid(self.inference, name='prediction')
 89 |         return self._prediction
 90 | 
 91 |     @property
 92 |     def loss(self):
 93 |         """
 94 |         Calculate the loss from logits and the labels.
 95 |         """
 96 |         if self._loss is None:
 97 |             tbm_cross_entropy = (tf.nn.sigmoid_cross_entropy_with_logits(labels=self.tbm, logits=self.inference)) * (tf.expand_dims(tf.sequence_mask(self.sequence_lengths, dtype=tf.float64), axis=2))
 98 |             func_loss = tf.reduce_sum(tbm_cross_entropy, name='binary_mask_loss')
 99 |             if self.regularization:
100 |                 l2_loss = tf.reduce_sum([tf.nn.l2_loss(var) for var in tf.trainable_variables()], name='reg_loss')
101 |             else:
102 |                 l2_loss = 0
103 |             self._loss = tf.add(func_loss, self.regularization * l2_loss, name='loss')
104 |         return self._loss
105 | 
106 |     @property
107 |     def train_op(self):
108 |         if self._train_op is None:
109 |             with tf.name_scope('optimizer'):
110 |                 if self.optimizer_choice == 'adam':
111 |                     self.learning_rate = tf.constant(self.initial_learning_rate)
112 |                     optimizer = tf.train.AdamOptimizer(learning_rate=self.initial_learning_rate)
113 |                 else:
114 |                     self.learning_rate = tf.train.exponential_decay(self.learning_rate, self.global_step,
115 |                                                                     self.updating_step, self.learning_decay, staircase=True)
116 |                     if self.optimizer_choice == 'sgd':
117 |                         optimizer = tf.train.GradientDescentOptimizer(learning_rate=self.learning_rate)
118 |                     elif self.optimizer_choice == 'momentum':
119 |                             optimizer = tf.train.MomentumOptimizer(learning_rate=self.learning_rate, momentum=0.9)  
120 |                     else:
121 |                         print('Optimizer must be either sgd, momentum or adam. Closing...')
122 |                         sys.exit(1)
123 |                 
124 |                 self._train_op = optimizer.minimize(self.loss, global_step=self.global_step, name='train_op')
125 |         return self._train_op
126 | 
127 |     @property
128 |     def enhanced_sources(self):
129 |         if self._enhanced_sources is None:
130 |             if self.mask_threshold == -1:
131 |                 masks = self.prediction
132 |             else:
133 |                 masks = self.prediction > self.mask_threshold
134 |             masked_mag_specs = tf.abs(self.mixed_specs) * (tf.cast(masks, dtype=tf.float32))
135 |             self._enhanced_sources = get_sources(masked_mag_specs, tf.angle(self.mixed_specs), num_samples=self.num_audio_samples)
136 |         return tf.identity(self._enhanced_sources, name='enhanced_sources')
137 | 
138 |     @property
139 |     def summaries(self):
140 |         """
141 |         Add summaries for TensorBoard
142 |         """
143 |         if self._summaries is None:
144 |             n_samples = 10
145 |             with tf.name_scope('summary'):
146 |                 oracle_tbm = tf.map_fn(tf.image.flip_up_down, tf.expand_dims(tf.transpose(self.tbm, [0, 2, 1]), axis=3))
147 |                 estimated_tbm = tf.map_fn(tf.image.flip_up_down, tf.expand_dims(tf.transpose(self.prediction, [0, 2, 1]), axis=3))
148 |                 estimated_tbm_round = tf.map_fn(tf.image.flip_up_down, tf.expand_dims(tf.transpose(tf.round(self.prediction), [0, 2, 1]), axis=3))
149 |                 tf.summary.image('Oracle TBM', oracle_tbm, max_outputs=n_samples)
150 |                 tf.summary.image('Estimated TBM', estimated_tbm, max_outputs=n_samples)
151 |                 tf.summary.image('Estimated TBM (rounded)', estimated_tbm_round, max_outputs=n_samples)
152 |                 mixed_sources_max = tf.reshape(tf.reduce_max(tf.abs(self.mixed_sources), axis=1), (-1, 1))
153 |                 target_sources_max = tf.reshape(tf.reduce_max(tf.abs(self.target_sources), axis=1), (-1, 1))
154 |                 enhanced_sources_max = tf.reshape(tf.reduce_max(tf.abs(self.enhanced_sources), axis=1), (-1, 1))
155 |                 tf.summary.audio('Mixed audio', self.mixed_sources / mixed_sources_max, 16000.0, max_outputs=n_samples)
156 |                 tf.summary.audio('Target audio', self.target_sources / target_sources_max, 16000.0, max_outputs=n_samples)
157 |                 tf.summary.audio('Masked audio', self.enhanced_sources / enhanced_sources_max, 16000.0, max_outputs=n_samples)
158 |                 self._summaries = tf.summary.merge_all()
159 |         return self._summaries
160 | 
161 |     @property
162 |     def global_step(self):
163 |         if self._global_step is None:
164 |             self._global_step = tf.Variable(0, trainable=False)
165 |         return self._global_step
166 | 
167 |     @property
168 |     def weights(self):
169 |         if self._weights is None:
170 |             self._weights = tf.trainable_variables()
171 |         return self._weights
172 | 
173 | 
174 | class VL2MRef(object):
175 |     """
176 |     VL2M_ref
177 |     Input: audio features and TBM masks.
178 |     Model: two input BLSTM (audio and TBM) + FC layer (AV fusion) + BLSTM.
179 |     Output: Ideal Amplitude Mask (IAM).
180 |     Loss: L2 loss (target_spec - enhanced_spec)
181 |     """
182 | 
183 |     def __init__(self, input_video, sequence_lengths, tbm, mixed_sources, target_sources, input_mixed_specs, keep_prob, config):
184 |         self.input_video = input_video
185 |         self.sequence_lengths = sequence_lengths
186 |         self.tbm = tbm
187 |         self.audio_frame_size = config.audio_feat_dim
188 |         self.video_frame_size = config.video_feat_dim
189 |         self.num_audio_samples = config.num_audio_samples
190 |         self.mixed_sources = mixed_sources
191 |         self.target_sources = target_sources
192 |         self.mixed_specs = compute_stft(mixed_sources, out_shape=tf.shape(input_mixed_specs))
193 |         self.target_specs = compute_stft(target_sources, out_shape=tf.shape(input_mixed_specs))
194 |         self.input_mixed_specs = input_mixed_specs
195 |         self.num_units = config.n_hidden
196 |         self.num_layers = config.num_layers
197 |         self.optimizer_choice = config.optimizer_choice
198 |         self.initial_learning_rate = config.learning_rate
199 |         self.learning_rate = config.learning_rate
200 |         self.updating_step = config.updating_step
201 |         self.learning_decay = config.learning_decay
202 |         self.keep_prob = keep_prob
203 |         self.regularization = config.reg
204 |         self._inference = None
205 |         self._loss = None
206 |         self._train_op = None
207 |         self._prediction = None
208 |         self._enhanced_sources = None
209 |         self._oracle_iam = None
210 |         self._summaries = None
211 |         self._global_step = None
212 |         self._weights = None
213 |         self.func_loss = None
214 |         self.reg_loss = None
215 | 
216 |     def create_graph(self):
217 |         """
218 |         Create the Graph of the model.
219 |         """
220 |         self.inference
221 |         self.prediction
222 |         self.loss
223 |         self.train_op
224 |         self.enhanced_sources
225 |         self.oracle_iam
226 |         self.summaries
227 | 
228 |     @property
229 |     def inference(self):
230 |         if self._inference is None:
231 |             max_sequence_length = tf.reduce_max(self.sequence_lengths)
232 |             with tf.variable_scope('input_lstm'):
233 |                 with tf.variable_scope('mask'):
234 |                     mask_lstm_outputs = self._VL2MRef__bi_lstm_block(self.tbm, self.num_layers, self.num_units)
235 |                 with tf.variable_scope('mixed'):
236 |                     mix_lstm_outputs = self._VL2MRef__bi_lstm_block(self.input_mixed_specs, self.num_layers, self.num_units)
237 |             with tf.variable_scope('input_lstm_projection'):
238 |                 mask_lstm_outputs_res = tf.nn.dropout(tf.reshape(mask_lstm_outputs, [-1, self.num_units * 2]), keep_prob=self.keep_prob)
239 |                 mix_lstm_outputs_res = tf.nn.dropout(tf.reshape(mix_lstm_outputs, [-1, self.num_units * 2]), keep_prob=self.keep_prob)
240 |                 weights_mask = tf.Variable(tf.truncated_normal([self.num_units * 2, self.num_units], stddev=1.0 / math.sqrt(float(self.num_units * 2)),
241 |                   dtype=tf.float64),
242 |                   name='weights_mask')
243 |                 weights_mix = tf.Variable(tf.truncated_normal([self.num_units * 2, self.num_units], stddev=1.0 / math.sqrt(float(self.num_units * 2)),
244 |                   dtype=tf.float64),
245 |                   name='weights_mix')
246 |                 biases_projection = tf.Variable(tf.zeros([self.num_units], dtype=tf.float64), name='biases')
247 |                 input_projection_res = tf.matmul(mask_lstm_outputs_res, weights_mask) + tf.matmul(mix_lstm_outputs_res, weights_mix) + biases_projection
248 |                 input_projection = tf.reshape(input_projection_res, [-1, max_sequence_length, self.num_units])
249 |             with tf.variable_scope('mask_lstm'):
250 |                 lstm_outputs = self._VL2MRef__bi_lstm_block(input_projection, self.num_layers, self.num_units)
251 |             with tf.variable_scope('logits'):
252 |                 lstm_outputs_res = tf.nn.dropout(tf.reshape(lstm_outputs, [-1, self.num_units * 2]), keep_prob=self.keep_prob)
253 |                 weights_out = tf.Variable(tf.truncated_normal([self.num_units * 2, self.audio_frame_size], stddev=1.0 / math.sqrt(float(self.num_units * 2)),
254 |                   dtype=tf.float64),
255 |                   name='weights')
256 |                 biases_out = tf.Variable(tf.zeros([self.audio_frame_size], dtype=tf.float64), name='biases')
257 |                 logits = tf.matmul(lstm_outputs_res, weights_out) + biases_out
258 |             logits_res = tf.reshape(logits, [-1, max_sequence_length, self.audio_frame_size], name='inference')
259 |             self._inference = logits_res
260 |         return self._inference
261 | 
262 |     def __bi_lstm_block(self, inputs, num_layers, num_units):
263 |         with tf.variable_scope('forward'):
264 |             forward_cells = []
265 |             for i in range(num_layers):
266 |                 lstm_cell = tf.contrib.rnn.LSTMCell(num_units=num_units, use_peepholes=True, initializer=tf.random_uniform_initializer(minval=-0.1, maxval=0.1))
267 |                 forward_cells.append(lstm_cell)
268 | 
269 |         with tf.variable_scope('backward'):
270 |             backward_cells = []
271 |             for i in range(num_layers):
272 |                 lstm_cell = tf.contrib.rnn.LSTMCell(num_units=num_units, use_peepholes=True, initializer=tf.random_uniform_initializer(minval=-0.1, maxval=0.1))
273 |                 backward_cells.append(lstm_cell)
274 | 
275 |         with tf.variable_scope('Bi_LSTM'):
276 |             rnn_outputs, output_state_fw, output_state_bw = tf.contrib.rnn.stack_bidirectional_dynamic_rnn(cells_fw=forward_cells,
277 |               cells_bw=backward_cells,
278 |               inputs=inputs,
279 |               dtype=tf.float64,
280 |               sequence_length=self.sequence_lengths)
281 |         return rnn_outputs
282 | 
283 |     @property
284 |     def prediction(self):
285 |         if self._prediction is None:
286 |             prediction = tf.sigmoid(self.inference) * 10
287 |             prediction = (tf.expand_dims(tf.sequence_mask(self.sequence_lengths, dtype=tf.float64), axis=2)) * prediction
288 |             self._prediction = tf.identity(prediction, name='prediction')
289 |         return self._prediction
290 | 
291 |     @property
292 |     def loss(self):
293 |         if self._loss is None:
294 |             target_specs_mag = tf.cast(tf.abs(self.target_specs) ** 0.3, tf.float64)
295 |             mixed_specs_mag = tf.abs(self.mixed_specs) ** 0.3
296 |             predicted_specs = self.prediction * tf.cast(mixed_specs_mag, tf.float64)
297 |             self.func_loss = tf.nn.l2_loss(target_specs_mag - predicted_specs, name='func_loss')
298 |             if self.regularization:
299 |                 self.reg_loss = tf.reduce_sum([tf.nn.l2_loss(var) for var in self.weights], name='reg_loss')
300 |             else:
301 |                 self.reg_loss = tf.constant(0, dtype=tf.float64)
302 |             self._loss = tf.add(self.func_loss, self.regularization * self.reg_loss, name='loss')
303 |         return self._loss
304 | 
305 |     @property
306 |     def train_op(self):
307 |         if self._train_op is None:
308 |             with tf.name_scope('optimizer'):
309 |                 if self.optimizer_choice == 'adam':
310 |                     self.learning_rate = tf.constant(self.initial_learning_rate)
311 |                     optimizer = tf.train.AdamOptimizer(learning_rate=self.initial_learning_rate)
312 |                 else:
313 |                     self.learning_rate = tf.train.exponential_decay(self.learning_rate, self.global_step,
314 |                                                                     self.updating_step, self.learning_decay, staircase=True)
315 |                     if self.optimizer_choice == 'sgd':
316 |                         optimizer = tf.train.GradientDescentOptimizer(learning_rate=self.learning_rate)
317 |                     elif self.optimizer_choice == 'momentum':
318 |                             optimizer = tf.train.MomentumOptimizer(learning_rate=self.learning_rate, momentum=0.9)  
319 |                     else:
320 |                         print('Optimizer must be either sgd, momentum or adam. Closing...')
321 |                         sys.exit(1)
322 |                 
323 |                 self._train_op = optimizer.minimize(self.loss, global_step=self.global_step, name='train_op')
324 |         return self._train_op
325 | 
326 |     @property
327 |     def enhanced_sources(self):
328 |         if self._enhanced_sources is None:
329 |             mixed_mag_specs = tf.abs(self.mixed_specs) ** 0.3
330 |             masked_mag_specs = (mixed_mag_specs * tf.cast(self.prediction, tf.float32)) ** (1 / 0.3)
331 |             self._enhanced_sources = get_sources(masked_mag_specs, tf.angle(self.mixed_specs), num_samples=self.num_audio_samples)
332 |         return tf.identity(self._enhanced_sources, name='enhanced_sources')
333 | 
334 |     @property
335 |     def oracle_iam(self):
336 |         if self._oracle_iam is None:
337 |             self._oracle_iam = get_oracle_iam(self.target_specs, self.mixed_specs)
338 |         return tf.identity(self._oracle_iam, name='oracle_iam')
339 | 
340 |     @property
341 |     def summaries(self):
342 |         """
343 |         Add summaries for TensorBoard
344 |         """
345 |         if self._summaries is None:
346 |             n_samples = 10
347 |             with tf.name_scope('summary'):
348 |                 tbm = tf.map_fn(tf.image.flip_up_down, tf.expand_dims(tf.transpose(self.tbm, [0, 2, 1]), axis=3))
349 |                 oracle_iam = tf.map_fn(tf.image.flip_up_down, tf.expand_dims(tf.transpose(self.oracle_iam, [0, 2, 1]), axis=3))
350 |                 estimated_iam = tf.map_fn(tf.image.flip_up_down, tf.expand_dims(tf.transpose(self.prediction, [0, 2, 1]), axis=3))
351 |                 mag_mixed_specs = tf.map_fn(tf.image.flip_up_down, tf.expand_dims(tf.transpose(tf.abs(self.mixed_specs) ** 0.3, [0, 2, 1]), axis=3))
352 |                 mag_target_specs = tf.map_fn(tf.image.flip_up_down, tf.expand_dims(tf.transpose(tf.abs(self.target_specs) ** 0.3, [0, 2, 1]), axis=3))
353 |                 mag_masked_specs = tf.map_fn(tf.image.flip_up_down, tf.expand_dims(tf.transpose(tf.abs(self.mixed_specs) ** 0.3 * tf.cast(self.prediction, tf.float32), [0, 2, 1]), axis=3))
354 |                 tf.summary.image('TBM', tbm, max_outputs=n_samples)
355 |                 tf.summary.image('Oracle IAM', oracle_iam, max_outputs=n_samples)
356 |                 tf.summary.image('Estimated IAM', estimated_iam, max_outputs=n_samples)
357 |                 tf.summary.image('Mixed spectrogram', mag_mixed_specs, max_outputs=n_samples)
358 |                 tf.summary.image('Target spectrogram', mag_target_specs, max_outputs=n_samples)
359 |                 tf.summary.image('Masked spectrogram', mag_masked_specs, max_outputs=n_samples)
360 |                 mixed_sources_max = tf.reshape(tf.reduce_max(tf.abs(self.mixed_sources), axis=1), (-1, 1))
361 |                 target_sources_max = tf.reshape(tf.reduce_max(tf.abs(self.target_sources), axis=1), (-1, 1))
362 |                 enhanced_sources_max = tf.reshape(tf.reduce_max(tf.abs(self.enhanced_sources), axis=1), (-1, 1))
363 |                 tf.summary.audio('Mixed audio', self.mixed_sources / mixed_sources_max, 16000.0, max_outputs=n_samples)
364 |                 tf.summary.audio('Target audio', self.target_sources / target_sources_max, 16000.0, max_outputs=n_samples)
365 |                 tf.summary.audio('Masked audio', self.enhanced_sources / enhanced_sources_max, 16000.0, max_outputs=n_samples)
366 |                 self._summaries = tf.summary.merge_all()
367 |         return self._summaries
368 | 
369 |     @property
370 |     def global_step(self):
371 |         if self._global_step is None:
372 |             self._global_step = tf.Variable(0, trainable=False)
373 |         return self._global_step
374 | 
375 |     @property
376 |     def weights(self):
377 |         if self._weights is None:
378 |             self._weights = tf.trainable_variables()
379 |         return self._weights
380 | 
381 | 
382 | class AudioVisualConcatMask(object):
383 |     """
384 |     Audio-Visual concat model (with masking)
385 |     Input: frame-level concatenation of audio and video (face landmarks) features.
386 |     Model: stacked BLSTM.
387 |     Output: Ideal Amplitude Mask (IAM).
388 |     Loss: L2 (target_spec - enhanced_spec)
389 |     """
390 | 
391 |     def __init__(self, input_video, sequence_lengths, mixed_sources, target_sources, input_mixed_specs, keep_prob, config):
392 |         self.input_video = input_video
393 |         self.sequence_lengths = sequence_lengths
394 |         self.audio_frame_size = config.audio_feat_dim
395 |         self.video_frame_size = config.video_feat_dim
396 |         self.num_audio_samples = config.num_audio_samples
397 |         self.mixed_sources = mixed_sources
398 |         self.target_sources = target_sources
399 |         self.mixed_specs = compute_stft(mixed_sources, out_shape=tf.shape(input_mixed_specs))
400 |         self.target_specs = compute_stft(target_sources, out_shape=tf.shape(input_mixed_specs))
401 |         self.input_mixed_specs = input_mixed_specs
402 |         self.num_units = config.n_hidden
403 |         self.num_layers = config.num_layers
404 |         self.optimizer_choice = config.optimizer_choice
405 |         self.initial_learning_rate = config.learning_rate
406 |         self.learning_rate = config.learning_rate
407 |         self.updating_step = config.updating_step
408 |         self.learning_decay = config.learning_decay
409 |         self.keep_prob = keep_prob
410 |         self.regularization = config.reg
411 |         self._inference = None
412 |         self._loss = None
413 |         self._train_op = None
414 |         self._prediction = None
415 |         self._enhanced_sources = None
416 |         self._oracle_iam = None
417 |         self._summaries = None
418 |         self._global_step = None
419 |         self._weights = None
420 |         self.func_loss = None
421 |         self.reg_loss = None
422 | 
423 |     def create_graph(self):
424 |         """
425 |         Create the Graph of the model.
426 |         """
427 |         self.inference
428 |         self.prediction
429 |         self.loss
430 |         self.train_op
431 |         self.enhanced_sources
432 |         self.oracle_iam
433 |         self.summaries
434 | 
435 |     @property
436 |     def inference(self):
437 |         if self._inference is None:
438 |             max_sequence_length = tf.reduce_max(self.sequence_lengths)
439 |             net_input = tf.concat([self.input_video, self.input_mixed_specs], axis=2)
440 |             with tf.variable_scope('forward'):
441 |                 forward_cells = []
442 |                 for i in range(self.num_layers):
443 |                     lstm_cell = tf.contrib.rnn.LSTMCell(num_units=self.num_units, use_peepholes=True, initializer=tf.random_uniform_initializer(minval=-0.1, maxval=0.1))
444 |                     forward_cells.append(lstm_cell)
445 | 
446 |             with tf.variable_scope('backward'):
447 |                 backward_cells = []
448 |                 for i in range(self.num_layers):
449 |                     lstm_cell = tf.contrib.rnn.LSTMCell(num_units=self.num_units, use_peepholes=True, initializer=tf.random_uniform_initializer(minval=-0.1, maxval=0.1))
450 |                     backward_cells.append(lstm_cell)
451 | 
452 |             with tf.variable_scope('Bi_LSTM'):
453 |                 rnn_outputs, output_state_fw, output_state_bw = tf.contrib.rnn.stack_bidirectional_dynamic_rnn(cells_fw=forward_cells,
454 |                   cells_bw=backward_cells,
455 |                   inputs=net_input,
456 |                   dtype=tf.float64,
457 |                   sequence_length=self.sequence_lengths)
458 |             rnn_outputs_res = tf.nn.dropout(tf.reshape(rnn_outputs, [-1, self.num_units * 2]), keep_prob=self.keep_prob)
459 |             with tf.variable_scope('logits'):
460 |                 weights = tf.Variable(tf.truncated_normal([self.num_units * 2, self.audio_frame_size], stddev=1.0 / math.sqrt(float(self.num_units * 2)),
461 |                   dtype=tf.float64),
462 |                   name='weights')
463 |                 biases = tf.Variable(tf.zeros([self.audio_frame_size], dtype=tf.float64), name='biases')
464 |                 logits = tf.matmul(rnn_outputs_res, weights) + biases
465 |             self._inference = tf.reshape(logits, [-1, max_sequence_length, self.audio_frame_size], name='inference')
466 |         return self._inference
467 | 
468 |     @property
469 |     def prediction(self):
470 |         if self._prediction is None:
471 |             prediction = tf.sigmoid(self.inference) * 10
472 |             prediction = (tf.expand_dims(tf.sequence_mask(self.sequence_lengths, dtype=tf.float64), axis=2)) * prediction
473 |             self._prediction = tf.identity(prediction, name='prediction')
474 |         return self._prediction
475 | 
476 |     @property
477 |     def loss(self):
478 |         if self._loss is None:
479 |             target_specs_mag = tf.cast(tf.abs(self.target_specs) ** 0.3, tf.float64)
480 |             mixed_specs_mag = tf.abs(self.mixed_specs) ** 0.3
481 |             estimated_specs = self.prediction * tf.cast(mixed_specs_mag, tf.float64)
482 |             self.func_loss = tf.nn.l2_loss(target_specs_mag - estimated_specs, name='func_loss')
483 |             if self.regularization:
484 |                 self.reg_loss = tf.reduce_sum([tf.nn.l2_loss(var) for var in self.weights], name='reg_loss')
485 |             else:
486 |                 self.reg_loss = tf.constant(0, dtype=tf.float64)
487 |             self._loss = tf.add(self.func_loss, self.regularization * self.reg_loss, name='loss')
488 |         return self._loss
489 | 
490 |     @property
491 |     def train_op(self):
492 |         if self._train_op is None:
493 |             with tf.name_scope('optimizer'):
494 |                 if self.optimizer_choice == 'adam':
495 |                     self.learning_rate = tf.constant(self.initial_learning_rate)
496 |                     optimizer = tf.train.AdamOptimizer(learning_rate=self.initial_learning_rate)
497 |                 else:
498 |                     self.learning_rate = tf.train.exponential_decay(self.learning_rate, self.global_step,
499 |                                                                     self.updating_step, self.learning_decay, staircase=True)
500 |                     if self.optimizer_choice == 'sgd':
501 |                         optimizer = tf.train.GradientDescentOptimizer(learning_rate=self.learning_rate)
502 |                     elif self.optimizer_choice == 'momentum':
503 |                             optimizer = tf.train.MomentumOptimizer(learning_rate=self.learning_rate, momentum=0.9)  
504 |                     else:
505 |                         print('Optimizer must be either sgd, momentum or adam. Closing...')
506 |                         sys.exit(1)
507 |                 
508 |                 self._train_op = optimizer.minimize(self.loss, global_step=self.global_step, name='train_op')
509 |         return self._train_op
510 | 
511 |     @property
512 |     def enhanced_sources(self):
513 |         if self._enhanced_sources is None:
514 |             mixed_mag_specs = tf.abs(self.mixed_specs) ** 0.3
515 |             masked_mag_specs = (mixed_mag_specs * tf.cast(self.prediction, tf.float32)) ** (1 / 0.3)
516 |             self._enhanced_sources = get_sources(masked_mag_specs, tf.angle(self.mixed_specs), num_samples=self.num_audio_samples)
517 |         return tf.identity(self._enhanced_sources, name='enhanced_sources')
518 | 
519 |     @property
520 |     def oracle_iam(self):
521 |         if self._oracle_iam is None:
522 |             self._oracle_iam = get_oracle_iam(self.target_specs, self.mixed_specs)
523 |         return tf.identity(self._oracle_iam, name='oracle_iam')
524 | 
525 |     @property
526 |     def summaries(self):
527 |         """
528 |         Add summaries for TensorBoard
529 |         """
530 |         if self._summaries is None:
531 |             n_samples = 10
532 |             with tf.name_scope('summary'):
533 |                 oracle_iam = tf.map_fn(tf.image.flip_up_down, tf.expand_dims(tf.transpose(self.oracle_iam, [0, 2, 1]), axis=3))
534 |                 estimated_iam = tf.map_fn(tf.image.flip_up_down, tf.expand_dims(tf.transpose(self.prediction, [0, 2, 1]), axis=3))
535 |                 mag_mixed_specs = tf.map_fn(tf.image.flip_up_down, tf.expand_dims(tf.transpose(tf.abs(self.mixed_specs) ** 0.3, [0, 2, 1]), axis=3))
536 |                 mag_target_specs = tf.map_fn(tf.image.flip_up_down, tf.expand_dims(tf.transpose(tf.abs(self.target_specs) ** 0.3, [0, 2, 1]), axis=3))
537 |                 mag_masked_specs = tf.map_fn(tf.image.flip_up_down, tf.expand_dims(tf.transpose(tf.abs(self.mixed_specs) ** 0.3 * tf.cast(self.prediction, tf.float32), [0, 2, 1]), axis=3))
538 |                 tf.summary.image('Oracle IAM', oracle_iam, max_outputs=n_samples)
539 |                 tf.summary.image('Estimated IAM', estimated_iam, max_outputs=n_samples)
540 |                 tf.summary.image('Mixed spectrogram', mag_mixed_specs, max_outputs=n_samples)
541 |                 tf.summary.image('Target spectrogram', mag_target_specs, max_outputs=n_samples)
542 |                 tf.summary.image('Masked spectrogram', mag_masked_specs, max_outputs=n_samples)
543 |                 mixed_sources_max = tf.reshape(tf.reduce_max(tf.abs(self.mixed_sources), axis=1), (-1, 1))
544 |                 target_sources_max = tf.reshape(tf.reduce_max(tf.abs(self.target_sources), axis=1), (-1, 1))
545 |                 enhanced_sources_max = tf.reshape(tf.reduce_max(tf.abs(self.enhanced_sources), axis=1), (-1, 1))
546 |                 tf.summary.audio('Mixed audio', self.mixed_sources / mixed_sources_max, 16000.0, max_outputs=n_samples)
547 |                 tf.summary.audio('Target audio', self.target_sources / target_sources_max, 16000.0, max_outputs=n_samples)
548 |                 tf.summary.audio('Masked audio', self.enhanced_sources / enhanced_sources_max, 16000.0, max_outputs=n_samples)
549 |                 self._summaries = tf.summary.merge_all()
550 |         return self._summaries
551 | 
552 |     @property
553 |     def global_step(self):
554 |         if self._global_step is None:
555 |             self._global_step = tf.Variable(0, trainable=False)
556 |         return self._global_step
557 | 
558 |     @property
559 |     def weights(self):
560 |         if self._weights is None:
561 |             self._weights = tf.trainable_variables()
562 |         return self._weights
563 | 
564 | 
565 | class AudioVisualConcatMaskRef(object):
566 |     """
567 |     Audio-Visual concat model (with masking)
568 |     Input: frame-level concatenation of audio spectrogram and TBM-masked audio spectrogram.
569 |     Model: stacked BLSTM.
570 |     Output: Ideal Amplitude Mask (IAM).
571 |     Loss: L2 (target_spec - enhanced_spec)
572 |     """
573 | 
574 |     def __init__(self, input_video, sequence_lengths, tbm, mixed_sources, target_sources, input_mixed_specs, keep_prob, config):
575 |         self.input_video = input_video
576 |         self.sequence_lengths = sequence_lengths
577 |         self.tbm = tbm
578 |         self.audio_frame_size = config.audio_feat_dim
579 |         self.video_frame_size = config.video_feat_dim
580 |         self.num_audio_samples = config.num_audio_samples
581 |         self.mixed_sources = mixed_sources
582 |         self.target_sources = target_sources
583 |         self.mixed_specs = compute_stft(mixed_sources, out_shape=tf.shape(input_mixed_specs))
584 |         self.target_specs = compute_stft(target_sources, out_shape=tf.shape(input_mixed_specs))
585 |         self.input_mixed_specs = input_mixed_specs
586 |         self.num_units = config.n_hidden
587 |         self.num_layers = config.num_layers
588 |         self.optimizer_choice = config.optimizer_choice
589 |         self.initial_learning_rate = config.learning_rate
590 |         self.learning_rate = config.learning_rate
591 |         self.updating_step = config.updating_step
592 |         self.learning_decay = config.learning_decay
593 |         self.keep_prob = keep_prob
594 |         self.regularization = config.reg
595 |         self._inference = None
596 |         self._loss = None
597 |         self._train_op = None
598 |         self._prediction = None
599 |         self._enhanced_sources = None
600 |         self._oracle_iam = None
601 |         self._summaries = None
602 |         self._global_step = None
603 |         self._weights = None
604 |         self.func_loss = None
605 |         self.reg_loss = None
606 | 
607 |     def create_graph(self):
608 |         """
609 |         Create the Graph of the model.
610 |         """
611 |         self.inference
612 |         self.prediction
613 |         self.loss
614 |         self.train_op
615 |         self.enhanced_sources
616 |         self.oracle_iam
617 |         self.summaries
618 | 
619 |     @property
620 |     def inference(self):
621 |         if self._inference is None:
622 |             max_sequence_length = tf.reduce_max(self.sequence_lengths)
623 |             # TBM masking operation on noisy input spectrogram
624 |             tbm_masked_input = self.input_mixed_specs *  self.tbm
625 |             net_input = tf.concat([tbm_masked_input, self.input_mixed_specs], axis=2)
626 |             with tf.variable_scope('forward'):
627 |                 forward_cells = []
628 |                 for i in range(self.num_layers):
629 |                     lstm_cell = tf.contrib.rnn.LSTMCell(num_units=self.num_units, use_peepholes=True, initializer=tf.random_uniform_initializer(minval=-0.1, maxval=0.1))
630 |                     forward_cells.append(lstm_cell)
631 | 
632 |             with tf.variable_scope('backward'):
633 |                 backward_cells = []
634 |                 for i in range(self.num_layers):
635 |                     lstm_cell = tf.contrib.rnn.LSTMCell(num_units=self.num_units, use_peepholes=True, initializer=tf.random_uniform_initializer(minval=-0.1, maxval=0.1))
636 |                     backward_cells.append(lstm_cell)
637 | 
638 |             with tf.variable_scope('Bi_LSTM'):
639 |                 rnn_outputs, output_state_fw, output_state_bw = tf.contrib.rnn.stack_bidirectional_dynamic_rnn(cells_fw=forward_cells,
640 |                   cells_bw=backward_cells,
641 |                   inputs=net_input,
642 |                   dtype=tf.float64,
643 |                   sequence_length=self.sequence_lengths)
644 |             rnn_outputs_res = tf.nn.dropout(tf.reshape(rnn_outputs, [-1, self.num_units * 2]), keep_prob=self.keep_prob)
645 |             with tf.variable_scope('logits'):
646 |                 weights = tf.Variable(tf.truncated_normal([self.num_units * 2, self.audio_frame_size], stddev=1.0 / math.sqrt(float(self.num_units * 2)),
647 |                   dtype=tf.float64),
648 |                   name='weights')
649 |                 biases = tf.Variable(tf.zeros([self.audio_frame_size], dtype=tf.float64), name='biases')
650 |                 logits = tf.matmul(rnn_outputs_res, weights) + biases
651 |             self._inference = tf.reshape(logits, [-1, max_sequence_length, self.audio_frame_size], name='inference')
652 |         return self._inference
653 | 
654 |     @property
655 |     def prediction(self):
656 |         if self._prediction is None:
657 |             prediction = tf.sigmoid(self.inference) * 10
658 |             prediction = (tf.expand_dims(tf.sequence_mask(self.sequence_lengths, dtype=tf.float64), axis=2)) * prediction
659 |             self._prediction = tf.identity(prediction, name='prediction')
660 |         return self._prediction
661 | 
662 |     @property
663 |     def loss(self):
664 |         if self._loss is None:
665 |             target_specs_mag = tf.cast(tf.abs(self.target_specs) ** 0.3, tf.float64)
666 |             mixed_specs_mag = tf.abs(self.mixed_specs) ** 0.3
667 |             estimated_specs = self.prediction * tf.cast(mixed_specs_mag, tf.float64)
668 |             self.func_loss = tf.nn.l2_loss(target_specs_mag - estimated_specs, name='func_loss')
669 |             if self.regularization:
670 |                 self.reg_loss = tf.reduce_sum([tf.nn.l2_loss(var) for var in self.weights], name='reg_loss')
671 |             else:
672 |                 self.reg_loss = tf.constant(0, dtype=tf.float64)
673 |             self._loss = tf.add(self.func_loss, self.regularization * self.reg_loss, name='loss')
674 |         return self._loss
675 | 
676 |     @property
677 |     def train_op(self):
678 |         if self._train_op is None:
679 |             with tf.name_scope('optimizer'):
680 |                 if self.optimizer_choice == 'adam':
681 |                     self.learning_rate = tf.constant(self.initial_learning_rate)
682 |                     optimizer = tf.train.AdamOptimizer(learning_rate=self.initial_learning_rate)
683 |                 else:
684 |                     self.learning_rate = tf.train.exponential_decay(self.learning_rate, self.global_step,
685 |                                                                     self.updating_step, self.learning_decay, staircase=True)
686 |                     if self.optimizer_choice == 'sgd':
687 |                         optimizer = tf.train.GradientDescentOptimizer(learning_rate=self.learning_rate)
688 |                     elif self.optimizer_choice == 'momentum':
689 |                             optimizer = tf.train.MomentumOptimizer(learning_rate=self.learning_rate, momentum=0.9)  
690 |                     else:
691 |                         print('Optimizer must be either sgd, momentum or adam. Closing...')
692 |                         sys.exit(1)
693 |                 
694 |                 self._train_op = optimizer.minimize(self.loss, global_step=self.global_step, name='train_op')
695 |         return self._train_op
696 | 
697 |     @property
698 |     def enhanced_sources(self):
699 |         if self._enhanced_sources is None:
700 |             mixed_mag_specs = tf.abs(self.mixed_specs) ** 0.3
701 |             masked_mag_specs = (mixed_mag_specs * tf.cast(self.prediction, tf.float32)) ** (1 / 0.3)
702 |             self._enhanced_sources = get_sources(masked_mag_specs, tf.angle(self.mixed_specs), num_samples=self.num_audio_samples)
703 |         return tf.identity(self._enhanced_sources, name='enhanced_sources')
704 | 
705 |     @property
706 |     def oracle_iam(self):
707 |         if self._oracle_iam is None:
708 |             self._oracle_iam = get_oracle_iam(self.target_specs, self.mixed_specs)
709 |         return tf.identity(self._oracle_iam, name='oracle_iam')
710 | 
711 |     @property
712 |     def summaries(self):
713 |         """
714 |         Add summaries for TensorBoard
715 |         """
716 |         if self._summaries is None:
717 |             n_samples = 10
718 |             with tf.name_scope('summary'):
719 |                 oracle_iam = tf.map_fn(tf.image.flip_up_down, tf.expand_dims(tf.transpose(self.oracle_iam, [0, 2, 1]), axis=3))
720 |                 estimated_iam = tf.map_fn(tf.image.flip_up_down, tf.expand_dims(tf.transpose(self.prediction, [0, 2, 1]), axis=3))
721 |                 mag_mixed_specs = tf.map_fn(tf.image.flip_up_down, tf.expand_dims(tf.transpose(tf.abs(self.mixed_specs) ** 0.3, [0, 2, 1]), axis=3))
722 |                 mag_target_specs = tf.map_fn(tf.image.flip_up_down, tf.expand_dims(tf.transpose(tf.abs(self.target_specs) ** 0.3, [0, 2, 1]), axis=3))
723 |                 mag_masked_specs = tf.map_fn(tf.image.flip_up_down, tf.expand_dims(tf.transpose(tf.abs(self.mixed_specs) ** 0.3 * tf.cast(self.prediction, tf.float32), [0, 2, 1]), axis=3))
724 |                 tf.summary.image('Oracle IAM', oracle_iam, max_outputs=n_samples)
725 |                 tf.summary.image('Estimated IAM', estimated_iam, max_outputs=n_samples)
726 |                 tf.summary.image('Mixed spectrogram', mag_mixed_specs, max_outputs=n_samples)
727 |                 tf.summary.image('Target spectrogram', mag_target_specs, max_outputs=n_samples)
728 |                 tf.summary.image('Masked spectrogram', mag_masked_specs, max_outputs=n_samples)
729 |                 mixed_sources_max = tf.reshape(tf.reduce_max(tf.abs(self.mixed_sources), axis=1), (-1, 1))
730 |                 target_sources_max = tf.reshape(tf.reduce_max(tf.abs(self.target_sources), axis=1), (-1, 1))
731 |                 enhanced_sources_max = tf.reshape(tf.reduce_max(tf.abs(self.enhanced_sources), axis=1), (-1, 1))
732 |                 tf.summary.audio('Mixed audio', self.mixed_sources / mixed_sources_max, 16000.0, max_outputs=n_samples)
733 |                 tf.summary.audio('Target audio', self.target_sources / target_sources_max, 16000.0, max_outputs=n_samples)
734 |                 tf.summary.audio('Masked audio', self.enhanced_sources / enhanced_sources_max, 16000.0, max_outputs=n_samples)
735 |                 self._summaries = tf.summary.merge_all()
736 |         return self._summaries
737 | 
738 |     @property
739 |     def global_step(self):
740 |         if self._global_step is None:
741 |             self._global_step = tf.Variable(0, trainable=False)
742 |         return self._global_step
743 | 
744 |     @property
745 |     def weights(self):
746 |         if self._weights is None:
747 |             self._weights = tf.trainable_variables()
748 |         return self._weights


--------------------------------------------------------------------------------
/eval_metrics.py:
--------------------------------------------------------------------------------
 1 | import numpy as np
 2 | from scipy.signal import stft
 3 | from mir_eval.separation import bss_eval_sources
 4 | 
 5 | 
 6 | def sdr_batch_eval(target_sources, noisy_sources, estimated_sources, sample_rate=16e3, step_size=10, sequence_lengths=None):
 7 |     sdr_list = []
 8 |     sir_list = []
 9 |     sar_list = []
10 | 
11 |     n_samples_frame = int(step_size / 1e3 * sample_rate)
12 |     for i, (target, noisy, estimated) in enumerate(zip(target_sources, noisy_sources, estimated_sources)):
13 |         if sequence_lengths is not None:
14 |             target = target[: sequence_lengths[i] * n_samples_frame]
15 |             noisy = noisy[: len(target)]
16 |             estimated = estimated[: len(target)]
17 | 
18 |         # Skip evaluation if estimated sources is all-zero vector
19 |         if np.any(estimated):
20 |             ref_sources = np.vstack([target, noisy])
21 |             est_sources = np.vstack([estimated, np.ones_like(estimated)])                    
22 |             sdr, sir, sar, _ = bss_eval_sources(ref_sources, est_sources, compute_permutation=False)
23 |             sdr_list.append(sdr[0])
24 |             sir_list.append(sir[0])
25 |             sar_list.append(sar[0])
26 |                              
27 |     return np.array(sdr_list), np.array(sir_list), np.array(sar_list)
28 | 
29 | 
30 | def sdr_batch_eval_ss(target_source, estimated_source, sample_rate=16e3, step_size=10, sequence_lengths=None):
31 |     """
32 |     Single source version of SDR, SIR and SDR computation
33 |     """
34 |     sdr_list = []
35 |     sir_list = []
36 |     sar_list = []
37 |     
38 |     n_samples_frame = int(step_size / 1e3 * sample_rate)
39 |     for i, (target, estimated) in enumerate(zip(target_source, estimated_source)):
40 |         if sequence_lengths is not None:
41 |            target = target[: sequence_lengths[i] * n_samples_frame]
42 |            estimated = estimated[: len(target)]
43 | 
44 |         # Skip evaluation if estimated sources is all-zero vector
45 |         if np.any(estimated):
46 |             sdr, sir, sar, _ = bss_eval_sources(np.array([target]), np.array([estimated]), compute_permutation=False)
47 |             sdr_list.append(sdr[0])
48 |             sir_list.append(sir[0])
49 |             sar_list.append(sar[0])
50 |                              
51 |     return np.array(sdr_list), np.array(sir_list), np.array(sar_list)
52 | 
53 | 
54 | def l2_batch_eval(target_sources, estimated_sources, sequence_lengths=None, sample_rate=16e3, window_size=25, overlap_size=15):
55 |     l2_list = []
56 | 
57 |     window_frame_len = int(window_size / 1e3 * sample_rate)
58 |     overlap_frame_len = int(overlap_size / 1e3 * sample_rate) 
59 |     n_samples_frame = window_frame_len - overlap_frame_len
60 |     for i, (target, estimated) in enumerate(zip(target_sources, estimated_sources)):
61 |         if sequence_lengths is not None:
62 |            target = target[: sequence_lengths[i] * n_samples_frame]
63 |            estimated = estimated[: len(target)]
64 | 
65 |         freqs, times, target_spec = stft(target, nperseg=window_frame_len, noverlap=overlap_frame_len, nfft=512)
66 |         freqs, times, estimated_spec = stft(estimated, nperseg=window_frame_len, noverlap=overlap_frame_len, nfft=512)
67 | 
68 |         l2_list.append(np.square(np.subtract(np.abs(target_spec) ** 0.3, np.abs(estimated_spec) ** 0.3)).sum())
69 | 
70 |     return np.array(l2_list)
71 | 
72 | 
73 | def snr_batch_eval(target_sources, estimated_sources, sample_rate=16e3, step_size=10, sequence_lengths=None):
74 |     snr_list = []
75 | 
76 |     n_samples_frame = int(step_size / 1e3 * sample_rate)
77 |     for i, (target, estimated, seq_len) in enumerate(zip(target_sources, estimated_sources, sequence_lengths)):
78 |         target = target[: seq_len * n_samples_frame]
79 |         estimated = estimated[: len(target)]
80 |         snr_list.append(10 * np.log10(np.sum(target ** 2) / np.sum((target - estimated) ** 2)))
81 | 
82 |     return np.array(snr_list)
83 | 


--------------------------------------------------------------------------------
/face_landmarks.py:
--------------------------------------------------------------------------------
  1 | from glob import glob
  2 | import os
  3 | import numpy as np
  4 | from scipy.interpolate import interp2d
  5 | import cv2
  6 | import dlib
  7 | from imutils import face_utils
  8 | 
  9 | 
 10 | FACIAL_LANDMARKS_IDXS = dict([
 11 |     ('mouth', (48, 68)),
 12 |     ('right_eyebrow', (17, 22)),
 13 |     ('left_eyebrow', (22, 27)),
 14 |     ('right_eye', (36, 42)),
 15 |     ('left_eye', (42, 48)),
 16 |     ('nose', (27, 36)),
 17 |     ('jaw', (0, 17))
 18 | ])
 19 | 
 20 | 
 21 | def extract_face_landmarks(video_filename, predictor_params, refresh_size=8):
 22 |     detector = dlib.get_frontal_face_detector()
 23 |     predictor = dlib.shape_predictor(predictor_params)
 24 |     tracker = dlib.correlation_tracker()
 25 | 
 26 |     cap = cv2.VideoCapture(video_filename)
 27 | 
 28 |     tracking_face = False # Keep track if we are using tracker
 29 |     i = 0 # Number of frames without detection
 30 |     landmarks = []
 31 |     face_rects = []
 32 |     rect = None
 33 |     
 34 |     while cap.isOpened():
 35 |         ret, frame = cap.read()
 36 |      
 37 |         if ret == True:
 38 |             gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
 39 | 
 40 |             if tracking_face and i < refresh_size:
 41 |                 tracking_quality = tracker.update(gray)
 42 |                 if tracking_quality >= 8.75:
 43 |                     t_pos = tracker.get_position()
 44 |                     x = int(t_pos.left())
 45 |                     y = int(t_pos.top())
 46 |                     w = int(t_pos.width())
 47 |                     h = int(t_pos.height())
 48 |                     i += 1
 49 |                 else:
 50 |                     tracking_face = False
 51 | 
 52 |             if not (tracking_face and i < refresh_size):
 53 |                 i = 0
 54 |                 rects = detector(gray, 1)
 55 |                 if rects:
 56 |                     rect = rects[0] # We suppose to have a single face detection
 57 |                     tracker.start_track(frame, rect)
 58 |                     tracking_face = True
 59 |         
 60 |             if rect:
 61 |                 # determine the facial landmarks for the face region, then
 62 |                 # convert the facial landmark (x,y)-coordinates to a numpy
 63 |                 # array
 64 |                 shape = predictor(gray, rect)
 65 |                 landmarks.append(face_utils.shape_to_np(shape))
 66 |                 face_rects.append(face_utils.rect_to_bb(rect))
 67 | 
 68 |         else:
 69 |             break
 70 | 
 71 |     cap.release()
 72 | 
 73 |     return np.array(landmarks), np.array(face_rects)
 74 | 
 75 | 
 76 | def show_face_landmarks(video_filename, predictor_params="", full_draw=False, fps=25.0, refresh_size=8):
 77 |     """
 78 |     Draws facial landmarks over original video frames. If full_draw is True connected lines
 79 |     of face landmark points are showed.
 80 |     """
 81 |     # Convert fps in frame len in milliseconds
 82 |     frame_len = int(1000 / fps) if fps > 0 else 0
 83 | 
 84 |     # Extract face landmarks and face bounding boxes
 85 |     landmarks, face_rects = extract_face_landmarks(video_filename, predictor_params, refresh_size)
 86 |     
 87 |     cap = cv2.VideoCapture(video_filename)
 88 | 
 89 |     for shape, rect in zip(landmarks, face_rects):
 90 |         ret, frame = cap.read()
 91 |     
 92 |         if ret == True:
 93 |             # loop over the (x, y)-coordinates for the facial landmarks
 94 |             # and draw them on the image
 95 |             for (x, y) in shape:
 96 |                 cv2.circle(frame, (x, y), 1, (0, 0, 255), -1)
 97 | 
 98 |             # draw face bounding box
 99 |             x, y, w, h = rect
100 |             cv2.rectangle(frame, (x, y), (x + w, y + h), (0, 255, 0), 2)
101 | 
102 |             if full_draw:
103 |                 for (i, name) in enumerate(FACIAL_LANDMARKS_IDXS.keys()):
104 |                     # grab the (x, y)-coordinates associated with the
105 |                     # face landmark
106 |                     (j, k) = FACIAL_LANDMARKS_IDXS[name]
107 |                     pts = shape[j:k]
108 |  
109 |                     if name in ('jaw', 'right_eyebrow', 'left_eyebrow'):
110 |                         # since the jawline is a non-enclosed facial region,
111 |                         # just draw lines between the (x, y)-coordinates
112 |                         for l in range(1, len(pts)):
113 |                             ptA = tuple(pts[l - 1])
114 |                             ptB = tuple(pts[l])
115 |                             cv2.line(frame, ptA, ptB, (0, 255, 0), 1)
116 |                     if name in ('right_eye', 'left_eye'):
117 |                         for l in range(len(pts)):
118 |                             ptA = tuple(pts[l - 1])
119 |                             ptB = tuple(pts[l])
120 |                             cv2.line(frame, ptA, ptB, (0, 255, 0), 1)
121 |                     if name == 'nose':
122 |                         for l in range(1, len(pts)):
123 |                             ptA = tuple(pts[l - 1])
124 |                             ptB = tuple(pts[l])
125 |                             cv2.line(frame, ptA, ptB, (0, 255, 0), 1)
126 |                         cv2.line(frame, tuple(pts[-1]), tuple(pts[3]), (0, 255, 0), 1)
127 |                     if name == 'mouth':
128 |                         for l in range(0, 11):
129 |                             ptA = tuple(pts[l])
130 |                             ptB = tuple(pts[l + 1])
131 |                             cv2.line(frame, ptA, ptB, (0, 255, 0), 1)
132 |                         cv2.line(frame, tuple(pts[0]), tuple(pts[11]), (0, 255, 0), 1)
133 |                         for l in range(12, len(pts) - 1):
134 |                             ptA = tuple(pts[l])
135 |                             ptB = tuple(pts[l + 1])
136 |                             cv2.line(frame, ptA, ptB, (0, 255, 0), 1)
137 |                         cv2.line(frame, tuple(pts[12]), tuple(pts[-1]), (0, 255, 0), 1)
138 |          
139 |             # show the output image with face landmarks
140 |             cv2.imshow("Output", frame)
141 |             cv2.waitKey(frame_len)
142 |         else:
143 |             break
144 | 
145 |     cap.release()
146 |     cv2.destroyAllWindows()
147 | 
148 | 
149 | def save_face_landmarks_speaker(video_path, dest_path, predictor_params, file_ext='mpg', refresh_size=8):
150 |     # Create destination directory if not exists
151 |     if not os.path.isdir(dest_path):
152 |         os.makedirs(dest_path)
153 | 
154 |     video_filenames = glob(os.path.join(video_path, '*.' + file_ext))
155 |     
156 |     count = 0
157 |     for v_file in video_filenames:
158 |         landmarks, _ = extract_face_landmarks(v_file, predictor_params, refresh_size)
159 |         l_file = os.path.join(dest_path, os.path.basename(v_file).replace('.' + file_ext, '.txt'))
160 |         np.savetxt(l_file, landmarks.reshape([-1, 136]), fmt='%d')
161 |         print('{} - Video file: {}'.format(count, v_file))
162 |         count += 1
163 |         
164 |     print('Done. Face landmark files created:', count)
165 | 
166 | 
167 | def save_face_landmarks(dataset_path, list_of_speakers, video_dir, dest_dir, predictor_params, file_ext='mpg', refresh_size=8):
168 |     # Every refresh_size frames a new face detection is forced (the face correlation tracker is ignored).
169 | 
170 |     for s in list_of_speakers:
171 |         print('Computing face landmarks of speaker {:d}...'.format(s))
172 |         video_path = os.path.join(dataset_path, 's' + str(s), video_dir)
173 |         dest_path = os.path.join(dataset_path, 's' + str(s), dest_dir)
174 |         
175 |         save_face_landmarks_speaker(video_path, dest_path, predictor_params, file_ext, refresh_size)
176 | 
177 |         print('Speaker {:d} completed.'.format(s))


--------------------------------------------------------------------------------
/mixed_speech_generator.py:
--------------------------------------------------------------------------------
  1 | import os
  2 | from glob import glob 
  3 | import random
  4 | import shutil
  5 | import numpy as np
  6 | from pydub import AudioSegment
  7 | 
  8 | 
  9 | def two_files_audio_sum(file_1_path, file_2_path, file_sum_name, volume_reduction=0):
 10 |     s1 = AudioSegment.from_file(file_1_path)
 11 |     s2 = AudioSegment.from_file(file_2_path) - volume_reduction # volume_reduction in dB
 12 | 
 13 |     s2_shift = (len(s1)-len(s2)) / 2 if len(s1) > len(s2) else 0
 14 |     
 15 |     audio_sum = s1.overlay(s2, position=s2_shift)
 16 |     audio_sum.export(file_sum_name, format='wav')
 17 | 
 18 |     return np.array(audio_sum.get_array_of_samples())
 19 | 
 20 | 
 21 | def three_files_audio_sum(file_1_path, file_2_path, file_3_path, file_sum_name, volume_reduction=0):
 22 |     s1 = AudioSegment.from_file(file_1_path)
 23 |     s2 = AudioSegment.from_file(file_2_path) - volume_reduction # volume_reduction in dB
 24 |     s3 = AudioSegment.from_file(file_3_path) - volume_reduction
 25 | 
 26 |     s2_shift = (len(s1)-len(s2)) / 2 if len(s1) > len(s2) else 0
 27 |     s3_shift = (len(s1)-len(s3)) / 2 if len(s1) > len(s3) else 0
 28 | 
 29 |     audio_sum = s1.overlay(s2, position=s2_shift)
 30 |     audio_sum = audio_sum.overlay(s3, position=s3_shift)
 31 |     audio_sum.export(file_sum_name, format='wav')
 32 | 
 33 |     return np.array(audio_sum.get_array_of_samples())
 34 | 
 35 | 
 36 | def random_files_selector(folders, n_file=1, exclude_files=[]):
 37 |     dir_file_list = []
 38 |     for dir in folders:
 39 |         dir_file_list.append((dir, glob(os.path.join(dir, '*.wav'))))
 40 |     
 41 |     selected_files = []
 42 |     for i in range(n_file):
 43 |         if exclude_files:
 44 |             condition = False
 45 |             while not condition:
 46 |                 condition = False
 47 |                 random_folder_files = random.choice(dir_file_list)
 48 |                 # Avoid repeating same mixing speaker
 49 |                 s1 = os.path.basename(os.path.dirname(random_folder_files[0].replace('audio', '')))
 50 |                 s2 = os.path.basename(os.path.dirname(exclude_files[i].replace('audio', '')))
 51 |                 if s1 != s2:
 52 |                     condition = True
 53 |         else:
 54 |             random_folder_files = random.choice(dir_file_list)
 55 |         is_not_append = True
 56 |         while is_not_append:
 57 |             random_file = random.choice(random_folder_files[1])
 58 |             if random_folder_files not in selected_files:
 59 |                 selected_files.append(random_file)
 60 |                 is_not_append = False
 61 |     return selected_files
 62 | 
 63 | 
 64 | def compare_lengths(file_1_path, file_2_path, max_duration_diff=1000):
 65 |     # max_duration_diff in milliseconds
 66 |     s1 = AudioSegment.from_file(file_1_path)
 67 |     s2 = AudioSegment.from_file(file_2_path)
 68 |    
 69 |     return len(s1) - len(s2) < max_duration_diff
 70 | 
 71 | 
 72 | def create_mixed_tracks_speaker(base_audio_dir, noisy_audio_dirs, dest_dir, n_samples=0, n_mix_per_sample=1, n_mix_speakers=1):
 73 |     """
 74 |     This function generate a mixed audio track for each sample in base_audio_dir.
 75 |     """
 76 |     # Create destination directory if not exists
 77 |     if not os.path.isdir(dest_dir):
 78 |         os.makedirs(dest_dir)
 79 | 
 80 |     base_speech_list = glob(os.path.join(base_audio_dir, '*.wav'))
 81 |     
 82 |     # if num_samples is > 0 we choice num_samples random samples from base_speech_list
 83 |     if n_samples > 0:
 84 |         random.seed(30)
 85 |         random.shuffle(base_speech_list)
 86 |         base_speech_list = base_speech_list[:n_samples]
 87 |     
 88 |     # now, for each speech we have to create n_mix_per_sample combinations:
 89 |     for base_speech in base_speech_list:
 90 |         condition = False
 91 |         # check if noisy utterance is not 2 sec shorter than base utterance, otherwise we have to re-perform the selection.
 92 |         while not condition:
 93 |             condition = True
 94 |             # to create combinations we have to select n_mix_per_samples from noisy speaker
 95 |             speechs_to_combine_1 = random_files_selector(noisy_audio_dirs, n_file=n_mix_per_sample)
 96 |             for speech_to_combine in speechs_to_combine_1:
 97 |                 if not compare_lengths(base_speech, speech_to_combine, max_duration_diff=2000):
 98 |                     condition = False
 99 |                 
100 |         if n_mix_speakers == 2:
101 |             condition = False
102 |             while not condition:
103 |                 condition = True
104 |                 speechs_to_combine_2 = random_files_selector(noisy_audio_dirs, n_file=n_mix_per_sample, exclude_files=speechs_to_combine_1)
105 |                 for speech_to_combine in speechs_to_combine_2:
106 |                     if not compare_lengths(base_speech, speech_to_combine, max_duration_diff=2000):
107 |                         condition = False
108 | 
109 |         if n_mix_speakers == 1:
110 |             for speech_to_combine in speechs_to_combine_1:
111 |                 # create a mix
112 |                 speaker = os.path.basename(os.path.dirname(os.path.dirname(speech_to_combine)))
113 |                 mix_name = os.path.splitext(os.path.basename(base_speech))[0] + '_with_' + speaker + '_' + \
114 |                            os.path.basename(speech_to_combine)
115 |                 two_files_audio_sum(base_speech, speech_to_combine, os.path.join(dest_dir, mix_name))
116 |         elif n_mix_speakers == 2:
117 |             for speech_to_combine_1, speech_to_combine_2 in zip(speechs_to_combine_1, speechs_to_combine_2):
118 |                 # create a mix
119 |                 speaker_1 = os.path.basename(os.path.dirname(os.path.dirname(speech_to_combine_1)))
120 |                 speaker_2 = os.path.basename(os.path.dirname(os.path.dirname(speech_to_combine_2)))
121 |                 mix_name = os.path.splitext(os.path.basename(base_speech))[0] + '_with_' + \
122 |                                             speaker_1 + '_' + os.path.splitext(os.path.basename(speech_to_combine_1))[0] + '_' + \
123 |                                             speaker_2 + '_' + os.path.basename(speech_to_combine_2)
124 |                 three_files_audio_sum(base_speech, speech_to_combine_1, speech_to_combine_2, os.path.join(dest_dir, mix_name))
125 | 
126 | 
127 | def create_mixed_tracks_data(dataset_dir, base_speakers, noisy_speakers=[], audio_dir='audio', dest_dir='mixed', n_samples=0, n_mix_per_sample=1, n_mix_speakers=1):
128 |     if not noisy_speakers:
129 |         noisy_speakers = base_speakers
130 | 
131 |     for s in base_speakers:
132 |         print('Creating mixed tracks of speaker {:d}...'.format(s))
133 |         base_audio_dir = os.path.join(dataset_dir, 's' + str(s), 'audio')
134 |         noisy_speakers_dirs = [os.path.join(dataset_dir, 's' + str(n_s), 'audio') for n_s in noisy_speakers if n_s != s]
135 |         mixed_dest_dir = os.path.join(dataset_dir, dest_dir, 's' + str(s))
136 |         
137 |         create_mixed_tracks_speaker(base_audio_dir, noisy_speakers_dirs, mixed_dest_dir, n_samples, n_mix_per_sample, n_mix_speakers)
138 |         
139 |         print('Speaker {:d} completed.'.format(s))
140 | 


--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
 1 | dlib>=19.7.0
 2 | imutils>=0.4.5
 3 | mir-eval>=0.4
 4 | numpy>=1.14.0
 5 | opencv-python>=3.3.0
 6 | pydub>=0.21.0
 7 | scipy>=1.0.0
 8 | tensorboard>=1.10.0
 9 | tensorflow>=1.10.0
10 | tensorflow-gpu>=1.10.0


--------------------------------------------------------------------------------
/target_binary_mask.py:
--------------------------------------------------------------------------------
  1 | import os
  2 | from os.path import join
  3 | from glob import glob
  4 | import numpy as np
  5 | import tensorflow as tf
  6 | from scipy import signal
  7 | from scipy.io import wavfile
  8 | import math
  9 | from audio_features import downsampling
 10 | 
 11 | # Avoid printing TF log messages
 12 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
 13 | 
 14 | # TF constants definition
 15 | TF_INTRA_OP_PT = int(os.getenv('TF_INTRA_OP', 0))
 16 | TF_INTER_OP_PT = int(os.getenv('TF_INTER_OP', 0))
 17 | 
 18 | 
 19 | def ltass_speaker(audio_folder, sample_rate=16e3, max_audio_length=48000, window_size=25, step_size=10, n_samples=1000):
 20 |     """
 21 |     Compute the speaker Long-Term Average Speech Spectrum of a speaker.
 22 |     """
 23 |     audio_filenames = sorted(glob(join(audio_folder, '*.wav')))
 24 |     
 25 |     audio_samples = np.zeros((len(audio_filenames), max_audio_length))
 26 |     num_frames = np.zeros(len(audio_filenames), dtype=np.int32)
 27 |     
 28 |     window_frame_size = int(round(window_size / 1e3 * sample_rate))
 29 |     step_frame_size = int(round(step_size  / 1e3 * sample_rate))
 30 |     
 31 |     for i, wav_file in enumerate(audio_filenames):
 32 |         rate, samples = wavfile.read(wav_file)
 33 |         samples = downsampling(samples, rate, sample_rate)
 34 |         audio_samples[i, :len(samples)] = samples
 35 |         num_frames[i] = len(samples) // step_frame_size
 36 |     
 37 |     # Create Graph
 38 |     with tf.Graph().as_default():
 39 |         samples_tensor = tf.constant(audio_samples, dtype=tf.float32)
 40 |         # Compute STFT
 41 |         specs_tensor = tf.contrib.signal.stft(samples_tensor, frame_length=window_frame_size, frame_step=step_frame_size, pad_end=False)
 42 |         # Apply power-law compression
 43 |         specs_tensor = tf.abs(specs_tensor) ** 0.3
 44 |         
 45 |         # Start session
 46 |         with tf.Session(config=tf.ConfigProto(allow_soft_placement=True,
 47 |                                               log_device_placement=False,
 48 |                                               intra_op_parallelism_threads=TF_INTRA_OP_PT,
 49 |                                               inter_op_parallelism_threads=TF_INTER_OP_PT)) as sess:
 50 |             specs = sess.run(specs_tensor)
 51 | 
 52 |     spectrograms = []
 53 |     for spec, nf in zip(specs, num_frames):
 54 |         spectrograms.append(spec[:nf])
 55 |     
 56 |     flat_specs = np.vstack(spectrograms)
 57 |     mean_spec = flat_specs.mean(axis=0)
 58 |     stdev_spec = flat_specs.std(axis=0)
 59 |     
 60 |     return mean_spec, stdev_spec, specs
 61 | 
 62 | 
 63 | def compute_tbm(audio_folder, mask_threshold, sample_rate=16e3, max_audio_length=48000, n_fft=512, window_size=25, step_size=10):
 64 |     """
 65 |     Compute TBMs using LTASS.
 66 |     """
 67 |     audio_filenames = sorted(glob(join(audio_folder, '*.wav')))
 68 |     num_frames = np.zeros(len(audio_filenames), dtype=np.int32)
 69 |     
 70 |     window_frame_size = int(round(window_size / 1e3 * sample_rate))
 71 |     step_frame_size = int(round(step_size  / 1e3 * sample_rate))
 72 |     
 73 |     audio_samples = np.zeros((len(audio_filenames), max_audio_length + n_fft//2))
 74 |     
 75 |     for i, wav_file in enumerate(audio_filenames):
 76 |         rate, samples = wavfile.read(wav_file)
 77 |         samples = downsampling(samples, rate, sample_rate)
 78 |         audio_samples[i, n_fft//2: len(samples) + n_fft//2] = samples
 79 |         num_frames[i] = math.ceil(float(len(samples) + n_fft//2) / step_frame_size)
 80 |     
 81 |     # Create Graph
 82 |     with tf.Graph().as_default():
 83 |         samples_tensor = tf.constant(audio_samples, dtype=tf.float32)
 84 |         # Compute STFT
 85 |         specs_tensor = tf.contrib.signal.stft(samples_tensor, frame_length=window_frame_size, frame_step=step_frame_size, pad_end=True)
 86 |         specs_tensor = tf.abs(specs_tensor)
 87 | 
 88 |         # Start session
 89 |         with tf.Session(config=tf.ConfigProto(allow_soft_placement=True,
 90 |                                               log_device_placement=False,
 91 |                                               intra_op_parallelism_threads=TF_INTRA_OP_PT,
 92 |                                               inter_op_parallelism_threads=TF_INTER_OP_PT)) as sess:
 93 |             specs = sess.run(specs_tensor)
 94 |             
 95 |     masks = specs > mask_threshold
 96 |     
 97 |     return audio_filenames, masks, num_frames
 98 | 
 99 | 
100 | def save_target_binary_masks_speaker(audio_folder, mask_folder, mask_factor=0.5, sample_rate=16e3, max_audio_length=48000, ltass_samples=1000):
101 |     # Create destination directory if not exists
102 |     if not os.path.isdir(mask_folder):
103 |         os.makedirs(mask_folder)
104 |     
105 |     # Compute thresholds and spectrograms
106 |     print('Computing LTASS threshold...')
107 |     threshold_mean, threshold_std, _ = ltass_speaker(audio_folder, sample_rate, max_audio_length=max_audio_length, n_samples=ltass_samples)
108 |     # Denormalize
109 |     threshold_freq = (threshold_mean + threshold_std * mask_factor) ** (1 / 0.3)
110 |     print('done.')
111 |     print('Threshold shape:', threshold_freq.shape)
112 |     
113 |     # Compute binary masks
114 |     audio_filenames, masks, num_frames = compute_tbm(audio_folder, threshold_freq, sample_rate, max_audio_length=max_audio_length)
115 | 
116 |     if not os.path.exists(mask_folder):
117 |         os.makedirs(mask_folder)
118 | 
119 |     for a_file, mask, nf in zip(audio_filenames, masks, num_frames):
120 |         s_file = os.path.join(mask_folder, os.path.basename(a_file).replace('.wav', '.npy'))
121 |         np.save(s_file, mask[:nf])
122 |             
123 |     print('Done. Target Binary Masks generated:', len(audio_filenames))
124 | 
125 | 
126 | def save_target_binary_masks(dataset_path, list_of_speakers, audio_folder, dest_dir, mask_factor=0.5, sample_rate=16e3, max_audio_length=48000, ltass_samples=1000):
127 |     for s in list_of_speakers:
128 |         print('Computing Target Binary Masks of speaker {:d}...'.format(s))
129 |         audio_path = os.path.join(dataset_path, 's' + str(s), audio_folder)
130 |         dest_path = os.path.join(dataset_path, 's' + str(s), dest_dir)
131 |         
132 |         save_target_binary_masks_speaker(audio_path, dest_path, mask_factor, sample_rate, max_audio_length, ltass_samples)
133 | 
134 |         print('Speaker {:d} completed.'.format(s))
135 | 


--------------------------------------------------------------------------------
/testing.py:
--------------------------------------------------------------------------------
  1 | from __future__ import division
  2 | 
  3 | import os
  4 | import numpy as np
  5 | import itertools
  6 | from scipy.io import wavfile
  7 | import tensorflow as tf
  8 | from dataset_reader import DataManager
  9 | from eval_metrics import snr_batch_eval, sdr_batch_eval, l2_batch_eval
 10 | 
 11 | # Avoid printing tensorflow log messages
 12 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
 13 | 
 14 | # TF constants definition
 15 | TF_INTRA_OP_PT = int(os.getenv('TF_INTRA_OP', 0))
 16 | TF_INTER_OP_PT = int(os.getenv('TF_INTER_OP', 0))
 17 | 
 18 | 
 19 | def test(data_path, data_path_test, exp_num, ckp_step, video_feat_dim, audio_feat_dim, tfrecord_mode, num_audio_samples,
 20 |          mask_threshold=-1, mix_eval=True, save_rec_dir='', save_out_dir=''):
 21 |     data_path_test = os.path.join(data_path, data_path_test)
 22 | 
 23 |     checkpoints_dir = os.path.join(data_path, 'logs', 'checkpoints', 'exp_' + exp_num)
 24 |     graph_checkpoint = os.path.join(checkpoints_dir, 'model_epoch_' + str(ckp_step) + '.ckpt.meta')
 25 |     params_checkpoint = os.path.join(checkpoints_dir, 'model_epoch_' + str(ckp_step) + '.ckpt')
 26 | 
 27 |     print('Loading model:')
 28 |     saver = tf.train.import_meta_graph(graph_checkpoint)
 29 |     print('done.\n')
 30 |     graph = tf.get_default_graph()
 31 |     
 32 |     # Create the DataManager that reads TFRecords.
 33 |     with tf.name_scope('test_batch'):
 34 |         test_data_manager = DataManager(audio_feat_dim, video_feat_dim, num_audio_samples, mode=tfrecord_mode)
 35 |         test_dataset, num_examples_test = test_data_manager.get_dataset(data_path_test)
 36 |         test_dataset, test_it = test_data_manager.get_iterator(test_dataset, batch_size=256,
 37 |                                                                n_epochs=1, train=False)
 38 |         next_test_batch = test_it.get_next()
 39 |     
 40 |     # Placeholders
 41 |     input_video_ph = graph.get_tensor_by_name('placeholder/input_video:0')
 42 |     sequence_lengths_ph = graph.get_tensor_by_name('placeholder/sequence_lengths:0')
 43 |     tbm_ph = graph.get_tensor_by_name('placeholder/tbm:0')
 44 |     input_mixed_specs_ph = graph.get_tensor_by_name('placeholder/input_mixed_specs:0')
 45 |     mixed_sources_ph = graph.get_tensor_by_name('placeholder/mixed_sources:0')
 46 |     target_sources_ph = graph.get_tensor_by_name('placeholder/target_sources:0')
 47 |     keep_prob_ph = graph.get_tensor_by_name('placeholder/keep_prob:0')
 48 | 
 49 |     # The inizializer operation.
 50 |     init_op = tf.group(test_it.initializer)
 51 |         
 52 |     # Start session
 53 |     with tf.Session(config=tf.ConfigProto(allow_soft_placement=True,
 54 |                                           log_device_placement=False,
 55 |                                           intra_op_parallelism_threads=TF_INTRA_OP_PT,
 56 |                                           inter_op_parallelism_threads=TF_INTER_OP_PT)) as sess:
 57 |         sess.run(init_op)
 58 | 
 59 |         # Load model weigths
 60 |         print('Restore weigths:')
 61 |         saver.restore(sess, params_checkpoint)
 62 |         print('done.\n')
 63 | 
 64 |         # Get enhanced sources tensor op
 65 |         enhanced_sources_tensor = graph.get_tensor_by_name('model/enhanced_sources:0')
 66 | 
 67 |         # Get output tensor op
 68 |         output_tensor = graph.get_tensor_by_name('model/prediction:0')
 69 |         
 70 |         # Get loss tensor op
 71 |         loss_tensor = graph.get_tensor_by_name('model/loss:0')
 72 | 
 73 |         try:
 74 |             test_cost = 0
 75 |             test_sequence_lengths_list = []
 76 |             test_output_list = []
 77 |             test_base_audio_list = []
 78 |             test_other_audio_list = []
 79 |             test_mixed_audio_list = []
 80 |             test_enhanced_audio_list = []
 81 |             test_base_paths_list = []
 82 |             test_mixed_paths_list = []
 83 |             
 84 |             print('Starting dataset inference...')
 85 |             while True:
 86 |                 # Fetch test samples batch.
 87 |                 test_length, test_tbm, test_video_feature, test_mixed_specs, test_base_audio, test_other_audio, \
 88 |                    test_mixed_audio, test_base_paths, test_other_paths, test_mixed_paths = sess.run(next_test_batch)
 89 |                 
 90 |                 # Mask thresholding (useful if you are using estimated TBMs)
 91 |                 if mask_threshold != -1:
 92 |                     test_tbm = (test_tbm > mask_threshold).astype(np.float32)
 93 | 
 94 |                 # Compute validation loss and enhanced sources
 95 |                 cost, test_output, test_enhanced_audio = sess.run(fetches=[loss_tensor, output_tensor, enhanced_sources_tensor],
 96 |                                                                   feed_dict={
 97 |                                                                       input_video_ph: test_video_feature,
 98 |                                                                       sequence_lengths_ph: test_length,
 99 |                                                                       tbm_ph: test_tbm,
100 |                                                                       input_mixed_specs_ph: test_mixed_specs,
101 |                                                                       mixed_sources_ph: test_mixed_audio,
102 |                                                                       target_sources_ph: test_base_audio,
103 |                                                                       keep_prob_ph: 1.0})
104 |                 
105 |                 test_cost += cost
106 |                 test_sequence_lengths_list.append(test_length)
107 |                 test_output_list.append(test_output)
108 |                 test_base_audio_list.append(test_base_audio)
109 |                 test_other_audio_list.append(test_other_audio)
110 |                 test_mixed_audio_list.append(test_mixed_audio)
111 |                 test_enhanced_audio_list.append(test_enhanced_audio)
112 |                 test_base_paths_list.append(test_base_paths.values if tfrecord_mode == 'fixed' else test_base_paths)
113 |                 test_mixed_paths_list.append(test_mixed_paths.values if tfrecord_mode == 'fixed' else test_mixed_paths)
114 |         except tf.errors.OutOfRangeError:
115 |             print('End dataset inference.')
116 |             test_sequence_lengths_list = np.concatenate(test_sequence_lengths_list)
117 |             test_output_list  = list(itertools.chain(*test_output_list)) 
118 |             test_base_audio_list  = list(itertools.chain(*test_base_audio_list)) 
119 |             test_other_audio_list = list(itertools.chain(*test_other_audio_list))
120 |             test_mixed_audio_list = list(itertools.chain(*test_mixed_audio_list))
121 |             test_enhanced_audio_list = list(itertools.chain(*test_enhanced_audio_list))
122 |             test_base_paths_list = np.concatenate(test_base_paths_list) if tfrecord_mode == 'fixed' else list(itertools.chain(*test_base_paths_list))
123 |             test_mixed_paths_list = np.concatenate(test_mixed_paths_list) if tfrecord_mode == 'fixed' else list(itertools.chain(*test_mixed_paths_list))
124 |             
125 |         # Evaluate enhanced and mixed sources
126 |         print('')
127 |         print('Computing dataset evaluation metrics...')
128 |         snr_enhanced = snr_batch_eval(test_base_audio_list, test_enhanced_audio_list, sequence_lengths=test_sequence_lengths_list)
129 |         sdr_enhanced, sir_enhanced, sar_enhanced = sdr_batch_eval(test_base_audio_list, test_other_audio_list, test_enhanced_audio_list, sequence_lengths=test_sequence_lengths_list)
130 |         l2_enhanced = l2_batch_eval(test_base_audio_list, test_enhanced_audio_list, sequence_lengths=test_sequence_lengths_list)
131 | 
132 |         if mix_eval:
133 |             snr_mixed = snr_batch_eval(test_base_audio_list, test_mixed_audio_list, sequence_lengths=test_sequence_lengths_list)
134 |             sdr_mixed, sir_mixed, sar_mixed = sdr_batch_eval(test_base_audio_list, test_other_audio_list, test_mixed_audio_list, sequence_lengths=test_sequence_lengths_list)
135 |             l2_mixed = l2_batch_eval(test_base_audio_list, test_mixed_audio_list, sequence_lengths=test_sequence_lengths_list)
136 |         
137 |         print('done.')
138 | 
139 |         # Print results
140 |         print('')
141 |         print('Experiment number: {:s}'.format(exp_num))
142 |         print('Epoch (step) checkpoint: {:s} ({:s})'.format(*ckp_step.split('_')))
143 |         print('Test set evaluation: {:s}'.format(data_path_test))
144 |         print('Size: {:d}'.format(len(test_base_audio_list)))
145 |         print('Loss per sample: {:.6f}'.format(test_cost / num_examples_test))
146 |         print('')
147 |         print('Enhanced SDR: {:.5f} [{:.5f}]'.format(sdr_enhanced.mean(), sdr_enhanced.std()))
148 |         print('Enhanced SIR: {:.5f} [{:.5f}]'.format(sir_enhanced.mean(), sir_enhanced.std()))
149 |         print('Enhanced SAR: {:.5f} [{:.5f}]'.format(sar_enhanced.mean(), sar_enhanced.std()))
150 |         print('Enhanced L2 (spectrogram): {:.5f} [{:.5f}]'.format(l2_enhanced.mean(), l2_enhanced.std()))
151 |         print('Enhanced SNR: {:.5f} [{:.5f}]'.format(snr_enhanced.mean(), snr_enhanced.std()))
152 |         if mix_eval:
153 |             print('')
154 |             print('Mixed SDR: {:.5f} [{:.5f}]'.format(sdr_mixed.mean(), sdr_mixed.std()))
155 |             print('Mixed SIR: {:.5f} [{:.5f}]'.format(sir_mixed.mean(), sir_mixed.std()))
156 |             print('Mixed SAR: {:.5f} [{:.5f}]'.format(sar_mixed.mean(), sar_mixed.std()))
157 |             print('Mixed L2 (spectrogram): {:.5f} [{:.5f}]'.format(l2_mixed.mean(), l2_mixed.std()))
158 |             print('Mixed SNR: {:.5f} [{:.5f}]'.format(snr_mixed.mean(), snr_mixed.std()))
159 | 
160 |         # Save audio samples: target, mixed and enhanced samples are saved in three
161 |         # subdirectories of "save_samples_dir".
162 |         if save_rec_dir:
163 |             print('\nSaving audio samples...')
164 |             target_dir = os.path.join(data_path, save_rec_dir, 'target')
165 |             mixed_dir = os.path.join(data_path, save_rec_dir, 'mixed')
166 |             enhanced_dir = os.path.join(data_path, save_rec_dir, 'enhanced')
167 | 
168 |             if not os.path.exists(save_rec_dir):
169 |                 os.makedirs(save_rec_dir)
170 |             if not os.path.exists(target_dir):
171 |                 os.makedirs(target_dir)
172 |             if not os.path.exists(mixed_dir):
173 |                 os.makedirs(mixed_dir)
174 |             if not os.path.exists(enhanced_dir):
175 |                 os.makedirs(enhanced_dir)
176 | 
177 |             for base, mixed, enhanced, filename, seq_len in zip(test_base_audio_list, test_mixed_audio_list, test_enhanced_audio_list, test_mixed_paths_list, test_sequence_lengths_list):
178 |                 if tfrecord_mode == 'fixed':
179 |                     filename = filename.decode()
180 |                 else:
181 |                     filename = ''.join([chr(x) for x in np.trim_zeros(filename)])
182 |                 speaker_dir, filename = os.path.split(filename)
183 |                 num_wav_samples = seq_len * 160
184 |                 filename = os.path.basename(speaker_dir) + '_' + filename
185 |                 wavfile.write(os.path.join(target_dir, filename), 16000, base[:num_wav_samples].astype(np.int16))
186 |                 wavfile.write(os.path.join(mixed_dir, filename), 16000, mixed[:num_wav_samples].astype(np.int16))
187 |                 wavfile.write(os.path.join(enhanced_dir, filename), 16000, enhanced[:num_wav_samples].astype(np.int16))
188 | 
189 |             print('done.')
190 | 
191 | 
192 |         # Save outputs: each element is saved in dir "s(n_speaker)/(output_dir)" where
193 |         # (n_speaker) is the id of speaker to whom base belongs and (output_dir)
194 |         # is the value in variable "save_out_dir".
195 |         if save_out_dir:
196 |             print('\nSaving outputs...')
197 |             for output, filename, seq_len in zip(test_output_list, test_base_paths_list, test_sequence_lengths_list):
198 |                 if tfrecord_mode == 'fixed':
199 |                     filename = filename.decode()
200 |                 else:
201 |                     filename = ''.join([chr(x) for x in np.trim_zeros(filename)])
202 |                 dirname, basename = os.path.split(filename)
203 |                 basename = os.path.splitext(basename)[0] + '.npy'
204 |                 dirname = os.path.basename(dirname.replace('/audio', ''))
205 |                 dirname = os.path.join(data_path, dirname, save_out_dir)
206 |                 if not os.path.exists(dirname):
207 |                     os.makedirs(dirname)
208 |            
209 |                 np.save(os.path.join(dirname, basename), output[:seq_len])
210 |                 
211 |             print('done.')
212 | 


--------------------------------------------------------------------------------
/training.py:
--------------------------------------------------------------------------------
  1 | from __future__ import division
  2 | 
  3 | import os
  4 | import sys
  5 | import numpy as np
  6 | import tensorflow as tf
  7 | from time import time
  8 | 
  9 | from dataset_reader import DataManager
 10 | from enhancement_model import VL2M, VL2MRef, AudioVisualConcatMask, AudioVisualConcatMaskRef
 11 | from eval_metrics import snr_batch_eval, sdr_batch_eval, l2_batch_eval
 12 | 
 13 | # Avoid printing TF log messages
 14 | os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'
 15 | 
 16 | # TF constants definition
 17 | TF_INTRA_OP_PT = int(os.getenv('TF_INTRA_OP', 0))
 18 | TF_INTER_OP_PT = int(os.getenv('TF_INTER_OP', 0))
 19 | 
 20 | 
 21 | class Configuration(object):
 22 |     def __init__(self, lr, us, ld, kp, bs, opt, video_dim, audio_dim, a_samples, n_epochs, n_hidden, n_layer, reg, mask_thresh):
 23 |         self.learning_rate = lr
 24 |         self.updating_step = us
 25 |         self.learning_decay = ld
 26 |         self.keep_prob = kp
 27 |         self.batch_size = bs 
 28 |         self.optimizer_choice = opt
 29 |         self.num_epochs = n_epochs
 30 | 
 31 |         self.video_feat_dim = video_dim
 32 |         self.audio_feat_dim = audio_dim
 33 |         self.num_audio_samples = a_samples
 34 | 
 35 |         self.n_hidden = n_hidden
 36 |         self.num_layers = n_layer
 37 |         self.reg = reg
 38 |         self.mask_thresh = mask_thresh
 39 | 
 40 | 
 41 | def train(model_selection, data_path, data_path_train, data_path_val, config, exp_num='0', tfrecord_mode='fixed'):
 42 |     """
 43 |     Train the audio-visual speech enhancement model.
 44 |     """
 45 |     data_path_train = os.path.join(data_path, data_path_train)
 46 |     data_path_val = os.path.join(data_path, data_path_val)
 47 | 
 48 |     checkpoints_dir = os.path.join(data_path, 'logs', 'checkpoints', 'exp_' + exp_num)
 49 |     tensorboard_dir = os.path.join(data_path, 'logs', 'tensorboard', 'exp_' + exp_num)
 50 |     traininglog_dir = os.path.join(data_path, 'logs', 'training_logs')
 51 | 
 52 |     # Training Graph
 53 |     with tf.Graph().as_default():
 54 | 
 55 |         # Create the DataManager that reads TFRecords.
 56 |         with tf.name_scope('train_batch'):
 57 |             train_data_manager = DataManager(config.audio_feat_dim, config.video_feat_dim, config.num_audio_samples, buffer_size=4000, mode=tfrecord_mode)
 58 |             train_dataset, num_examples_train = train_data_manager.get_dataset(data_path_train)
 59 |             train_dataset, train_it = train_data_manager.get_iterator(train_dataset, batch_size=config.batch_size,
 60 |                                                                       n_epochs=config.num_epochs, train=True)
 61 |             next_train_batch = train_it.get_next()
 62 | 
 63 |         with tf.name_scope('validation_batch'):
 64 |             val_data_manager = DataManager(config.audio_feat_dim, config.video_feat_dim, config.num_audio_samples, mode=tfrecord_mode)
 65 |             val_dataset, num_examples_val = val_data_manager.get_dataset(data_path_val)
 66 |             val_dataset, val_it = val_data_manager.get_iterator(val_dataset, batch_size=num_examples_val,
 67 |                                                                 n_epochs=1, train=False)
 68 |             next_val_batch = val_it.get_next()
 69 |     
 70 |         # Placeholders.
 71 |         with tf.name_scope('placeholder'):
 72 |             input_video = tf.placeholder(tf.float64, shape=[None, None, config.video_feat_dim], name='input_video')
 73 |             sequence_lengths = tf.placeholder(tf.int32, shape=[None], name='sequence_lengths')
 74 |             tbm = tf.placeholder(tf.float64, shape=[None, None, config.audio_feat_dim], name='tbm')
 75 |             input_mixed_specs = tf.placeholder(tf.float64, shape=[None, None, config.audio_feat_dim], name='input_mixed_specs')
 76 |             mixed_sources = tf.placeholder(tf.float32, shape=[None, None], name='mixed_sources')
 77 |             target_sources = tf.placeholder(tf.float32, shape=[None, None], name='target_sources')
 78 |             keep_prob = tf.placeholder(tf.float64, name='keep_prob')
 79 |         
 80 |         # Graph building and definition.
 81 |         print('Building model:') 
 82 |         with tf.variable_scope('model'):
 83 |             if model_selection == 'vl2m':
 84 |                 model = VL2M(input_video, sequence_lengths, tbm, mixed_sources, target_sources, keep_prob, config)
 85 |             elif model_selection == 'vl2m_ref':
 86 |                 model = VL2MRef(input_video, sequence_lengths, tbm, mixed_sources, target_sources, input_mixed_specs, keep_prob, config)
 87 |             elif model_selection == 'av_concat_mask':
 88 |                 model = AudioVisualConcatMask(input_video, sequence_lengths, mixed_sources, target_sources, input_mixed_specs, keep_prob, config)
 89 |             elif model_selection == 'av_concat_mask_ref':
 90 |                 model = AudioVisualConcatMaskRef(input_video, sequence_lengths, tbm, mixed_sources, target_sources, input_mixed_specs, keep_prob, config)
 91 |             else:
 92 |                 print('Model selection must be "vl2m", "vl2m_ref", "av_concat_mask" or "av_concat_mask_ref". Closing...')
 93 |                 sys.exit()
 94 |         
 95 |             model.create_graph()
 96 |         print('done.')
 97 | 
 98 |         # The inizializer operation.
 99 |         init_op = tf.group(train_it.initializer, val_it.initializer, tf.global_variables_initializer())
100 |         # save and restore all the variables.
101 |         saver = tf.train.Saver(max_to_keep=10)
102 | 
103 |         # create log directories if not exist.
104 |         if not os.path.exists(checkpoints_dir):
105 |             os.makedirs(checkpoints_dir)
106 |         if not os.path.exists(tensorboard_dir):
107 |             os.makedirs(tensorboard_dir)
108 |         if not os.path.exists(traininglog_dir):
109 |             os.makedirs(traininglog_dir)
110 | 
111 |         trainingLogFile = open(os.path.join(traininglog_dir, 'TrainingExperiment_' + str(exp_num) + '.txt'), 'a')
112 | 
113 |         # Start session
114 |         with tf.Session(config=tf.ConfigProto(allow_soft_placement=True,
115 |                                               log_device_placement=False,
116 |                                               intra_op_parallelism_threads=TF_INTRA_OP_PT,
117 |                                               inter_op_parallelism_threads=TF_INTER_OP_PT)) as sess:
118 | 
119 |             # FileWriter to save TensorBoard summary
120 |             tb_writer = tf.summary.FileWriter(tensorboard_dir, graph=sess.graph)
121 | 
122 |             # Initialize the variables and the batch iterator.
123 |             sess.run(init_op)
124 |         
125 |             n_steps_epoch = int(num_examples_train / config.batch_size)
126 |             n_steps = int(config.num_epochs * n_steps_epoch)
127 |              
128 |             # Fetch validation samples batch.
129 |             val_length, val_tbm, val_video_feature, val_mixed_specs, val_base_audio, val_other_audio, \
130 |                val_mixed_audio, val_base_paths, val_other_paths, val_mixed_paths = sess.run(next_val_batch)
131 |             
132 |             # restore variables
133 |             last_checkpoint = tf.train.latest_checkpoint(checkpoints_dir)
134 |             if last_checkpoint:
135 |                 last_checkpoint_path = saver.restore(sess, last_checkpoint)
136 |                 print('\nVariables restored. Global step: {:d}'.format(sess.run(model.global_step)))
137 |             else:
138 |                 trainingLogFile.write('+-- EXPERIMENT NUMBER - {:s} --+\n'.format(exp_num))
139 |                 trainingLogFile.write('## optimizer: {:s}\n'.format(config.optimizer_choice))
140 |                 trainingLogFile.write('## number of hidden layers (other): {:d}\n'.format(config.num_layers))
141 |                 trainingLogFile.write('## number of hidden units: {:d}\n'.format(config.n_hidden))
142 |                 trainingLogFile.write('## initial learning rate: {:.6f}\n'.format(config.learning_rate))
143 |                 if config.optimizer_choice != 'adam':
144 |                     trainingLogFile.write('## learning rate update steps: {:d}\n'.format(config.updating_step))
145 |                     trainingLogFile.write('## learning rate decay: {:.6f}\n'.format(config.learning_decay))
146 |                 trainingLogFile.write('## regularization: {:.6f}\n'.format(config.reg))
147 |                 trainingLogFile.write('## dropout keep probability (no dropout if 1): {:.6f}\n'.format(config.keep_prob))
148 |                 trainingLogFile.write('## training size: {:d}\n'.format(num_examples_train))
149 |                 trainingLogFile.write('## validation size: {:d}\n'.format(num_examples_val))
150 |                 trainingLogFile.write('## batch size: {:d}\n'.format(config.batch_size))
151 |                 trainingLogFile.write('## approx number of steps: {:d}\n'.format(n_steps))
152 |                 trainingLogFile.write('## approx number of steps per epoch: {:d}\n'.format(n_steps_epoch))
153 |                 trainingLogFile.write('\nEpoch\tLR\tTrain[Cost|L2-SPEC|SNR|SDR|SIR|SAR]\tVal[Cost|L2-SPEC|SNR|SDR|SIR|SAR]\n')
154 |             
155 |             # Print training details.
156 |             print('')
157 |             print('+--- EXPERIMENT NUMBER - {:s} ---+'.format(exp_num))
158 |             print('## optimizer: {:s}'.format(config.optimizer_choice))
159 |             print('## number of hidden layers (other): {:d}'.format(config.num_layers))
160 |             print('## number of hidden units: {:d}'.format(config.n_hidden))
161 |             print('## initial learning rate: {:.6f}'.format(config.learning_rate))
162 |             if config.optimizer_choice != 'adam':
163 |                 print('## learning rate update steps: {:d}'.format(config.updating_step))
164 |                 print('## learning rate decay: {:.6f}'.format(config.learning_decay))
165 |             print('## regularization: {:.6f}'.format(config.reg))
166 |             print('## dropout keep probability (no dropout if 1): {:.6f}'.format(config.keep_prob))
167 |             print('## training size: {:d}'.format(num_examples_train))
168 |             print('## validation size: {:d}'.format(num_examples_val))
169 |             print('## batch size: {:d}'.format(config.batch_size))
170 |             print('## approx number of steps: {:d}'.format(n_steps))
171 |             print('## approx number of steps per epoch: {:d}'.format(n_steps_epoch))
172 |             print('')
173 | 
174 | 
175 |             try:
176 |                 step = sess.run(model.global_step)
177 |                 epoch_counter = int(step / n_steps_epoch)
178 |                 epoch_cost = 0
179 |                 epoch_snr = []
180 |                 epoch_sdr = []
181 |                 epoch_sar = []
182 |                 epoch_sir = []
183 |                 epoch_l2_spec = []
184 | 
185 |                 epoch_start_time = time()
186 |                 
187 |                 while True:
188 |                     step += 1
189 |                     if (step - 1) % n_steps_epoch == 0:
190 |                         print('-> Epoch {:d}'.format(epoch_counter))
191 |                     
192 |                     # Fetch training batch.
193 |                     length_batch, tbm_batch, video_feature_batch, mixed_specs_batch, base_audio_batch, other_audio_batch, \
194 |                         mixed_audio_batch, base_paths_batch, other_paths_batch, mixed_paths_batch = sess.run(next_train_batch)
195 |                     
196 |                     _, cost, lr = sess.run(fetches=[model.train_op, model.loss, model.learning_rate],
197 |                                            feed_dict={
198 |                                             input_video: video_feature_batch,
199 |                                             sequence_lengths: length_batch,
200 |                                             tbm: tbm_batch,
201 |                                             input_mixed_specs: mixed_specs_batch,
202 |                                             mixed_sources: mixed_audio_batch,
203 |                                             target_sources: base_audio_batch,
204 |                                             keep_prob: config.keep_prob})
205 |                     
206 |                     # Enhanced sources w/o dropout
207 |                     enhanced_audio_batch = sess.run(fetches=model.enhanced_sources,
208 |                                                   feed_dict={
209 |                                                     input_video: video_feature_batch,
210 |                                                     sequence_lengths: length_batch,
211 |                                                     tbm: tbm_batch,
212 |                                                     input_mixed_specs: mixed_specs_batch,
213 |                                                     mixed_sources: mixed_audio_batch,
214 |                                                     target_sources: base_audio_batch,
215 |                                                     keep_prob: 1.0})
216 |                   
217 |                     if np.isnan(cost):
218 |                         print('GOT INSTABILITY: cost is NaN. Leaving...')
219 |                         sys.exit()
220 |                    
221 |                     if np.isinf(cost):
222 |                         print('GOT INSTABILITY: cost is inf. Leaving...')
223 |                         sys.exit()
224 | 
225 |                     snr_batch = snr_batch_eval(base_audio_batch, enhanced_audio_batch, sequence_lengths=length_batch)
226 |                     sdr_batch, sir_batch, sar_batch = sdr_batch_eval(base_audio_batch, other_audio_batch, enhanced_audio_batch, sequence_lengths=length_batch)
227 |                     l2_spec_batch = l2_batch_eval(base_audio_batch, enhanced_audio_batch, sequence_lengths=length_batch)
228 | 
229 |                     epoch_cost += cost
230 |                     epoch_snr = np.append(epoch_snr, snr_batch)
231 |                     epoch_sdr = np.append(epoch_sdr, sdr_batch)
232 |                     epoch_sir = np.append(epoch_sir, sir_batch)
233 |                     epoch_sar = np.append(epoch_sar, sar_batch)
234 |                     epoch_l2_spec = np.append(epoch_l2_spec, l2_spec_batch)
235 |                     
236 |                     #Print loss value fairly often.
237 |                     if step % (n_steps_epoch // 16) == 0 or step == 1:
238 |                         print('Step[{:7d}] Cost[{:3.5f}] LR[{:.6f}] L2-SPEC[{:3.5f}] SNR[{:3.5f}] SDR[{:3.5}] SIR[{:3.5}] SAR[{:3.5}]' \
239 |                             .format(step, cost / config.batch_size, lr, l2_spec_batch.mean(), snr_batch.mean(), sdr_batch.mean(), sir_batch.mean(), sar_batch.mean()))
240 |                     
241 |                     if step % 500 == 0 or step % n_steps_epoch == 0:
242 |                         # Save model variables.
243 |                         save_path = saver.save(sess, os.path.join(checkpoints_dir, 'model_epoch_' + str(epoch_counter)
244 |                                                                   + '_' + str(step) + '.ckpt'))
245 |                         print('Model saved.')
246 |                     
247 |                     # Print and write reports at the end of each epoch.
248 |                     if (step % n_steps_epoch == 0) and (step != 0):
249 |                         print('Completed epoch {:d} at step {:d} --> Train cost: {:.6f} - L2-SPEC: {:.6f} - SNR: {:.6f} - SDR: {:.6f} - SIR: {:.6f} - SAR: {:.6f}' \
250 |                               .format(epoch_counter, step, epoch_cost / num_examples_train, epoch_l2_spec.mean(),
251 |                                       epoch_snr.mean(), epoch_sdr.mean(), epoch_sir.mean(), epoch_sar.mean()))
252 |                         print('Epoch training time (seconds) = {:.6f}'.format(time() - epoch_start_time))
253 |                         
254 |                         # Compute validation loss and summaries.
255 |                         val_cost, val_enhanced_audio, summaries = sess.run(fetches=[model.loss, model.enhanced_sources, model.summaries],
256 |                                                                            feed_dict={
257 |                                                                                input_video: val_video_feature,
258 |                                                                                sequence_lengths: val_length,
259 |                                                                                tbm: val_tbm,
260 |                                                                                input_mixed_specs: val_mixed_specs,
261 |                                                                                mixed_sources: val_mixed_audio,
262 |                                                                                target_sources: val_base_audio,
263 |                                                                                keep_prob: 1.0})
264 | 
265 |                         val_snr = snr_batch_eval(val_base_audio, val_enhanced_audio, sequence_lengths=val_length)
266 |                         val_sdr, val_sir, val_sar = sdr_batch_eval(val_base_audio, val_other_audio, val_enhanced_audio, sequence_lengths=val_length)
267 |                         val_l2_spec = l2_batch_eval(val_base_audio, val_enhanced_audio, sequence_lengths=val_length)
268 | 
269 |                         print('Validation cost: {:.6f} - L2-SPEC: {:.6f} - SNR: {:.6f} - SDR: {:.6f} - SIR: {:.6f} - SAR: {:.6f}' \
270 |                               .format(val_cost / num_examples_val, val_l2_spec.mean(), val_snr.mean(), val_sdr.mean(), val_sir.mean(), val_sar.mean()))
271 |                         
272 |                         # Write Tensorboard summaries.
273 |                         tb_summary = tf.Summary()
274 |                         tb_summary.value.add(tag='Training loss', simple_value=epoch_cost / num_examples_train)
275 |                         tb_summary.value.add(tag='Training L2 spectrogram', simple_value=epoch_l2_spec.mean())
276 |                         tb_summary.value.add(tag='Training SNR', simple_value=epoch_snr.mean())
277 |                         tb_summary.value.add(tag='Training SDR', simple_value=epoch_sdr.mean())
278 |                         tb_summary.value.add(tag='Training SIR', simple_value=epoch_sdr.mean())
279 |                         tb_summary.value.add(tag='Training SAR', simple_value=epoch_sdr.mean())
280 |                         tb_summary.value.add(tag='Validation loss', simple_value=val_cost / num_examples_val)
281 |                         tb_summary.value.add(tag='Validation L2 spectrogram', simple_value=val_l2_spec.mean())
282 |                         tb_summary.value.add(tag='Validation SNR', simple_value=val_snr.mean())
283 |                         tb_summary.value.add(tag='Validation SDR', simple_value=val_sdr.mean())
284 |                         tb_summary.value.add(tag='Validation SIR', simple_value=val_sir.mean())
285 |                         tb_summary.value.add(tag='Validation SAR', simple_value=val_sar.mean())
286 |                         
287 |                         tb_writer.add_summary(tb_summary, epoch_counter)
288 | 
289 |                         tb_writer.add_summary(summaries, epoch_counter)
290 |                         tb_writer.flush()
291 |                         
292 |                         # Write trainining log file.
293 |                         trainingLogFile.write('{:d}\t[{:.6f}][{:.5f}|{:.5f}|{:.5f}|{:.5f}|{:.5f}|{:.5f}]\t[{:.5f}|{:.5f}|{:.5f}|{:.5f}|{:.5f}|{:.5f}]\n' \
294 |                                               .format(epoch_counter, lr, epoch_cost / num_examples_train, epoch_l2_spec.mean(), epoch_snr.mean(), epoch_sdr.mean(), epoch_sir.mean(), epoch_sar.mean(),
295 |                                                       val_cost / num_examples_val, val_l2_spec.mean(), val_snr.mean(), val_sdr.mean(), val_sir.mean(), val_sar.mean()))
296 |                        
297 |                         trainingLogFile.flush()
298 | 
299 |                         print('')
300 |                         epoch_counter += 1
301 |                         epoch_cost = 0
302 |                         epoch_snr = []
303 |                         epoch_sdr = []
304 |                         epoch_sar = []
305 |                         epoch_sir = []
306 |                         epoch_l2_spec = []
307 |                         epoch_start_time = time()
308 |             except tf.errors.OutOfRangeError:
309 |                print('Done training for %d epochs, %d steps: validation cost = %.5f' % (config.num_epochs, step, val_cost / num_examples_val))


--------------------------------------------------------------------------------