├── .gitignore ├── LICENSE ├── README.md ├── _config.yml ├── audio_attack.py ├── count_files.sh ├── download_checkpoint.sh ├── download_dataset.sh ├── evaluate_attack.py ├── evaluate_attack.sh ├── evaluate_realdata_accuracy.py ├── extract_times.sh ├── machine_deception2017.ipynb ├── run_attack.sh ├── speech_commands ├── BUILD ├── README.md ├── Untitled.ipynb ├── accuracy_utils.cc ├── accuracy_utils.h ├── accuracy_utils_test.cc ├── freeze.py ├── freeze_test.py ├── generate_streaming_test_wav.py ├── generate_streaming_test_wav_test.py ├── input_data.py ├── input_data_test.py ├── label_wav.cc ├── label_wav.py ├── label_wav_test.py ├── models.py ├── models_test.py ├── output.txt ├── recognize_commands.cc ├── recognize_commands.h ├── recognize_commands_test.cc ├── test_streaming_accuracy.cc └── train.py ├── time_log.txt ├── time_result.txt └── times_all.txt /.gitignore: -------------------------------------------------------------------------------- 1 | *.*.swp 2 | __pycache__/ 3 | *.py[cod] 4 | 5 | # ipython checkpoinits 6 | .ipynb_checkpoints/ 7 | 8 | # checkpoints 9 | ckpts/ 10 | *.ckpt* 11 | *.pbtxt 12 | 13 | # frozen graph 14 | frozen_graph/ 15 | 16 | # dataset 17 | *.wav 18 | data/* 19 | speech_dataset/ 20 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 UCLA Networked & Embedded Systems Laboratory 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Adversarial Speech Commands 2 | 3 | 4 | ### Setup 5 | 6 | Download datset 7 | ``` 8 | bash download_dataset.sh 9 | ``` 10 | 11 | Download model: 12 | ``` 13 | bash download_checkpoint.sh 14 | ``` 15 | ### Attack Experiment 16 | To run: 17 | ``` 18 | ./run_attack.sh dataset_dir ckpts_dir limit max_iters test_size 19 | ``` 20 | -------------------------------------------------------------------------------- /_config.yml: -------------------------------------------------------------------------------- 1 | theme: jekyll-theme-tactile -------------------------------------------------------------------------------- /audio_attack.py: -------------------------------------------------------------------------------- 1 | """ 2 | Author: Moustafa Alzantot (malzantot@ucla.edu) 3 | All rights reserved. 4 | """ 5 | import os, sys 6 | import numpy as np 7 | import sys 8 | import time 9 | import tensorflow as tf 10 | from speech_commands import label_wav 11 | 12 | def load_graph(filename): 13 | with tf.gfile.FastGFile(filename, 'rb') as f: 14 | graph_def = tf.GraphDef() 15 | graph_def.ParseFromString(f.read()) 16 | tf.import_graph_def(graph_def, name='') 17 | 18 | def load_labels(filename): 19 | return [line.rstrip() for line in tf.gfile.FastGFile(filename)] 20 | 21 | 22 | def print_output(output_preds, labels): 23 | top_k = output_pred.argsort()[-5:][::-1] 24 | for node_id in top_k: 25 | human_string = labels[node_id] 26 | score = output_pred[node_id] 27 | print('%s %d score = %0.5f' %(human_string, node_id, score)) 28 | print('----------------------') 29 | 30 | ################## GenAttack again ? 31 | # TODO(malzantot): any thoughts about byte ordering ? 32 | header_len = 44 33 | data_max = 32767 34 | data_min = -32768 35 | mutation_p = 0.0005 36 | 37 | def gen_population_member(x_orig, eps_limit): 38 | new_bytearray = bytearray(x_orig) 39 | # step = 2 40 | # if bps == 8: 41 | step = 2 42 | for i in range(header_len, len(x_orig), step): 43 | if np.random.random() < mutation_p: 44 | # if np.random.random() < 0.5: 45 | # new_bytearray[i] = min(255, new_bytearray[i]+1) 46 | # else: 47 | # new_bytearray[i] = max(0, new_bytearray[i]-1) 48 | int_x = int.from_bytes(x_orig[i:i+2], byteorder='little', signed=True) 49 | new_int_x = min(data_max, max(data_min, int_x + np.random.choice(range(-eps_limit, eps_limit)))) 50 | new_bytes = int(new_int_x).to_bytes(2, byteorder='little', signed=True) 51 | new_bytearray[i] = new_bytes[0] 52 | new_bytearray[i+1] = new_bytes[1] 53 | return bytes(new_bytearray) 54 | 55 | def crossover(x1, x2): 56 | ba1 = bytearray(x1) 57 | ba2 = bytearray(x2) 58 | step = 2 59 | # if bps == 8: 60 | # step = 1 61 | for i in range(header_len, len(x1), step): 62 | if np.random.random() < 0.5: 63 | ba2[i] = ba1[i] 64 | return bytes(ba2) 65 | 66 | # def refine(x_new, x_orig, pbs=16, limit=10): 67 | # ba_new = bytearray(x_new) 68 | # ba_orig = bytearray(x_orig) 69 | # step = 2 70 | # if pbs == 8: 71 | # step = 1 72 | # for i in range(header_len, len(x_new), step): 73 | # # if np.random.random() < 0.5: 74 | # ba_new[i] = min(ba_orig[i]+limit, max(ba_orig[i]-limit, ba_new[i])) 75 | # ba_new[i] = min(255, max(0, ba_new[i])) 76 | # return bytes(ba_new) 77 | 78 | def mutation(x, eps_limit): 79 | ba = bytearray(x) 80 | step = 2 81 | #if pbs == 8: 82 | # step = 1 83 | for i in range(header_len, len(x), step): 84 | #if np.random.random() < 0.05: 85 | # ba[i] = max(0, min(255, np.random.choice(list(range(ba[i]-4, ba[i]+4))))) 86 | #elif np.random.random() < 0.10: 87 | #ba[i] = max(0, min(255, ba[i] + np.random.choice([-1, 1]))) 88 | if np.random.random() < mutation_p: 89 | int_x = int.from_bytes(ba[i:i+2], byteorder='big', signed=True) 90 | new_int_x = min(data_max, max(data_min, int_x + np.random.choice(range(-eps_limit, eps_limit)))) 91 | new_bytes = int(new_int_x).to_bytes(2, byteorder='big', signed=True) 92 | ba[i] = new_bytes[0] 93 | ba[i+1] = new_bytes[1] 94 | return bytes(ba) 95 | 96 | def score(sess, x, target, input_tensor, output_tensor): 97 | output_preds, = sess.run(output_tensor, 98 | feed_dict={input_tensor: x}) 99 | return output_preds 100 | 101 | def generate_attack(x_orig, target, limit, sess, input_node, 102 | output_node, max_iters, eps_limit=256, verbose=False): 103 | pop_size = 20 104 | elite_size = 2 105 | temp = 0.01 106 | initial_pop = [gen_population_member(x_orig, eps_limit) for _ in range(pop_size)] 107 | for idx in range(max_iters): 108 | pop_scores = np.array([score(sess, x, target, input_node, output_node) for x in initial_pop]) 109 | target_scores = pop_scores[:, target] 110 | pop_ranks = list(reversed(np.argsort(target_scores))) 111 | elite_set = [initial_pop[x] for x in pop_ranks[:elite_size]] 112 | 113 | top_attack = initial_pop[pop_ranks[0]] 114 | top_pred = np.argmax(pop_scores[pop_ranks[0],:]) 115 | if verbose: 116 | if top_pred == target: 117 | print("*** SUCCESS ****") 118 | if top_pred == target: 119 | return top_attack 120 | 121 | scores_logits = np.exp(target_scores /temp) 122 | pop_probs = scores_logits / np.sum(scores_logits) 123 | child_set = [crossover( 124 | initial_pop[np.random.choice(pop_size, p=pop_probs)], 125 | initial_pop[np.random.choice(pop_size, p=pop_probs)]) 126 | for _ in range(pop_size - elite_size)] 127 | initial_pop = elite_set + [mutation(child, eps_limit) for child in child_set] 128 | return top_attack 129 | 130 | def save_audiofile(output, filename): 131 | with open(filename, 'wb') as fh: 132 | fh.write(output) 133 | 134 | def load_audiofile(filename): 135 | with open(filename, 'rb') as fh: 136 | return fh.read() 137 | 138 | flags = tf.flags 139 | flags.DEFINE_string("data_dir", "", "Data dir") 140 | flags.DEFINE_string("output_dir", "", "Data dir") 141 | flags.DEFINE_string("target_label", "", "Target classification label") 142 | flags.DEFINE_integer("limit", 4, "Noise limit") 143 | flags.DEFINE_string("graph_path", "", "Path to frozen graph file.") 144 | flags.DEFINE_string("labels_path", "", "Path to labels file.") 145 | flags.DEFINE_boolean("verbose", False, "") 146 | flags.DEFINE_integer("max_iters", 200, "Maxmimum number of iterations") 147 | FLAGS = flags.FLAGS 148 | 149 | if __name__ == '__main__': 150 | data_dir = FLAGS.data_dir 151 | output_dir = FLAGS.output_dir 152 | target_label = FLAGS.target_label 153 | eps_limit = FLAGS.limit 154 | graph_path = FLAGS.graph_path 155 | labels_path = FLAGS.labels_path 156 | max_iters = FLAGS.max_iters 157 | verbose = FLAGS.verbose 158 | input_node_name = 'wav_data:0' 159 | output_node_name = 'labels_softmax:0' 160 | 161 | labels = load_labels(labels_path) 162 | 163 | wav_files_list =\ 164 | [f for f in os.listdir(data_dir) if f.endswith(".wav")] 165 | 166 | target_idx = [idx for idx in range(len(labels)) if labels[idx]==target_label] 167 | if len(target_idx) == 0: 168 | print("Target label not found.") 169 | sys.exit(1) 170 | target_idx = target_idx[0] 171 | 172 | load_graph(graph_path) 173 | with tf.Session() as sess: 174 | output_node = sess.graph.get_tensor_by_name(output_node_name) 175 | for input_file in wav_files_list: 176 | start_time = time.time() 177 | x_orig = load_audiofile(data_dir+'/'+input_file) 178 | #TODO(malzantot) fix 179 | # x_pbs = 1 180 | num_channels = int(x_orig[22]) + int(x_orig[23]*256) 181 | sample_rate = int(x_orig[24]) + int(x_orig[25]*256) + int(x_orig[26]*2**16) + int(x_orig[27]*2**24) 182 | pbs = int(x_orig[34]) 183 | byte_rate = int(x_orig[28]) + int(x_orig[29]*256) + int(x_orig[30]*2**16) + int(x_orig[31]*2**24) 184 | chunk_id = chr(int(x_orig[0])) + chr(int(x_orig[1])) + chr(int(x_orig[2])) + chr(int(x_orig[3])) 185 | # if chunk_id == 'RIFF': 186 | # # chunk_id='RIFF' -> little endian data form. 'RIFX'-> big endian form. 187 | # header_len += 1 188 | assert chunk_id == 'RIFF', 'ONLY RIIF format is supported' 189 | 190 | if verbose: 191 | print("chunk id = %s" %chunk_id) 192 | print("bps = %d - num channels = %d - Sample rate = %d ." 193 | %(pbs, num_channels, sample_rate)) 194 | print("byte rate = %d" %(byte_rate)) 195 | 196 | assert pbs == 16, "Only PBS=16 is supported now" 197 | attack_output = generate_attack(x_orig, target_idx, eps_limit, 198 | sess, input_node_name, output_node, max_iters, pbs, verbose) 199 | save_audiofile(attack_output, output_dir+'/'+input_file) 200 | end_time = time.time() 201 | print("Attack done (%d iterations) in %0.4f seconds" %(max_iters, (end_time-start_time))) 202 | 203 | 204 | 205 | 206 | 207 | -------------------------------------------------------------------------------- /count_files.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | exp_data_dir="output/data" 3 | exp_result_dir="output/result" 4 | count_data=`find $exp_data_dir -name *.wav | wc -l` 5 | count_result=`find $exp_result_dir -name *.wav | wc -l` 6 | echo "Number of input files = $count_data." 7 | echo "Number of output files = $count_result." 8 | -------------------------------------------------------------------------------- /download_checkpoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | test -d ckpts || mkdir ckpts 4 | cd ckpts 5 | wget http://download.tensorflow.org/models/speech_commands_v0.01.zip 6 | unzip speech_commands_v0.01.zip 7 | rm -f speech_commands_v0.01.zip 8 | -------------------------------------------------------------------------------- /download_dataset.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | test -d data/ || mkdir data/ 3 | cd data/ 4 | test -f speech_commands_v0.01.tar.gz || wget http://download.tensorflow.org/data/speech_commands_v0.01.tar.gz 5 | tar -xf speech_commands_v0.01.tar.gz 6 | rm speech_commands_v0.01.tar.gz 7 | -------------------------------------------------------------------------------- /evaluate_attack.py: -------------------------------------------------------------------------------- 1 | """ 2 | Author: Moustafa Alzantot (malzantot@ucla.edu) 3 | 4 | """ 5 | 6 | import numpy as np 7 | import tensorflow as tf 8 | from speech_commands import label_wav 9 | import os, sys 10 | import csv 11 | flags = tf.flags 12 | flags.DEFINE_string('output_dir', '', 'output data directory') 13 | flags.DEFINE_string('labels_file', '', 'Labels file.') 14 | flags.DEFINE_string('graph_file', '', '') 15 | flags.DEFINE_string('output_file', 'eval_output.csv', 'CSV file of evaluation results') 16 | FLAGS = flags.FLAGS 17 | 18 | def load_graph(filename): 19 | with tf.gfile.FastGFile(filename, 'rb') as f: 20 | graph_def = tf.GraphDef() 21 | graph_def.ParseFromString(f.read()) 22 | tf.import_graph_def(graph_def, name='') 23 | 24 | def load_labels(filename): 25 | return [line.rstrip() for line in tf.gfile.FastGFile(filename)] 26 | 27 | 28 | def load_audiofile(filename): 29 | with open(filename, 'rb') as fh: 30 | return fh.read() 31 | 32 | if __name__ == '__main__': 33 | output_dir = FLAGS.output_dir 34 | labels_file = FLAGS.labels_file 35 | graph_file = FLAGS.graph_file 36 | output_file = FLAGS.output_file 37 | labels = load_labels(labels_file) 38 | n_labels = len(labels) 39 | result_mat = np.zeros((n_labels, n_labels)) 40 | input_node_name = 'wav_data:0' 41 | output_node_name = 'labels_softmax:0' 42 | load_graph(graph_file) 43 | 44 | ## Header of output file 45 | output_fh = open(output_file, 'w') 46 | fieldnames = ['filename', 'original', 'target', 'predicted'] 47 | for label in labels: 48 | fieldnames.append(label) 49 | csv_writer = csv.DictWriter(output_fh, fieldnames=fieldnames) 50 | print(fieldnames) 51 | csv_writer.writeheader() 52 | with tf.Session() as sess: 53 | output_node = sess.graph.get_tensor_by_name(output_node_name) 54 | for src_idx, src_label in enumerate(labels): 55 | for target_idx, target_label in enumerate(labels): 56 | case_dir = format("%s/%s/%s" %(output_dir, target_label, src_label)) 57 | if os.path.exists(case_dir): 58 | wav_files =[format('%s/%s' %(case_dir, f)) for f in os.listdir(case_dir) if f.endswith('.wav')] 59 | for wav_filename in wav_files: 60 | wav_data = load_audiofile(wav_filename) 61 | preds = sess.run(output_node, feed_dict = { 62 | input_node_name: wav_data 63 | }) 64 | wav_pred = np.argmax(preds[0]) 65 | if wav_pred == target_idx: 66 | result_mat[src_idx][wav_pred] += 1 67 | row_dict = dict() 68 | row_dict['filename'] = wav_filename 69 | row_dict['original'] = src_label 70 | row_dict['target'] = target_label 71 | row_dict['predicted'] = labels[wav_pred] 72 | for i in range(preds[0].shape[0]): 73 | row_dict[labels[i]] = preds[0][i] 74 | csv_writer.writerow(row_dict) 75 | 76 | print(result_mat) 77 | print(np.sum(result_mat)) 78 | 79 | 80 | -------------------------------------------------------------------------------- /evaluate_attack.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [ $# -lt 2 ] 3 | then 4 | echo "Usage: $(basename $0) output_dir ckpts_dir" 5 | exit 1 6 | fi 7 | output_dir=$1 8 | result_dir="$output_dir/result" 9 | labels_file="$2/conv_actions_labels.txt" 10 | graph_file="$2/conv_actions_frozen.pb" 11 | 12 | python3 evaluate_attack.py --output_dir=$result_dir --labels_file=$labels_file --graph_file=$graph_file 13 | -------------------------------------------------------------------------------- /evaluate_realdata_accuracy.py: -------------------------------------------------------------------------------- 1 | """ 2 | Author: Moustafa Alzantot (malzantot@ucla.edu) 3 | 4 | """ 5 | 6 | import numpy as np 7 | import tensorflow as tf 8 | from speech_commands import label_wav 9 | import os, sys 10 | import csv 11 | flags = tf.flags 12 | flags.DEFINE_string('output_dir', '', 'output data directory') 13 | flags.DEFINE_string('labels_file', '', 'Labels file.') 14 | flags.DEFINE_string('graph_file', '', '') 15 | flags.DEFINE_string('output_file', 'realdata_eval_output.csv', 'CSV file of evaluation results') 16 | FLAGS = flags.FLAGS 17 | 18 | def load_graph(filename): 19 | with tf.gfile.FastGFile(filename, 'rb') as f: 20 | graph_def = tf.GraphDef() 21 | graph_def.ParseFromString(f.read()) 22 | tf.import_graph_def(graph_def, name='') 23 | 24 | def load_labels(filename): 25 | return [line.rstrip() for line in tf.gfile.FastGFile(filename)] 26 | 27 | 28 | def load_audiofile(filename): 29 | with open(filename, 'rb') as fh: 30 | return fh.read() 31 | 32 | if __name__ == '__main__': 33 | output_dir = FLAGS.output_dir 34 | labels_file = FLAGS.labels_file 35 | graph_file = FLAGS.graph_file 36 | output_file = FLAGS.output_file 37 | labels = load_labels(labels_file) 38 | n_labels = len(labels) 39 | result_mat = np.zeros((1, n_labels)) 40 | input_node_name = 'wav_data:0' 41 | output_node_name = 'labels_softmax:0' 42 | load_graph(graph_file) 43 | 44 | ## Header of output file 45 | output_fh = open(output_file, 'w') 46 | fieldnames = ['filename', 'original', 'predicted'] 47 | for label in labels: 48 | fieldnames.append(label) 49 | csv_writer = csv.DictWriter(output_fh, fieldnames=fieldnames) 50 | print(fieldnames) 51 | csv_writer.writeheader() 52 | with tf.Session() as sess: 53 | output_node = sess.graph.get_tensor_by_name(output_node_name) 54 | for label_idx, label_name in enumerate(labels): 55 | case_dir = format("%s/%s" %(output_dir, label_name)) 56 | if os.path.exists(case_dir): 57 | wav_files =[format('%s/%s' %(case_dir, f)) for f in os.listdir(case_dir) if f.endswith('.wav')] 58 | for wav_filename in wav_files: 59 | wav_data = load_audiofile(wav_filename) 60 | preds = sess.run(output_node, feed_dict = { 61 | input_node_name: wav_data 62 | }) 63 | wav_pred = np.argmax(preds[0]) 64 | if wav_pred == label_idx: 65 | result_mat[0][wav_pred] += 1 66 | row_dict = dict() 67 | row_dict['filename'] = wav_filename 68 | row_dict['original'] = label_name 69 | row_dict['predicted'] = labels[wav_pred] 70 | for i in range(preds[0].shape[0]): 71 | row_dict[labels[i]] = preds[0][i] 72 | csv_writer.writerow(row_dict) 73 | 74 | print(result_mat) 75 | print(np.sum(result_mat)) 76 | 77 | 78 | -------------------------------------------------------------------------------- /extract_times.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | cat $1 | grep Attack | sed -r 's/^.*in ([0-9][0-9]*.[0-9][0-9][0-9][0-9]) seconds/\1/g' 3 | -------------------------------------------------------------------------------- /run_attack.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ $# -lt 5 ] 4 | then 5 | echo "Usage: $(basename $0) dataset_dir ckpts_dir limit max_iters test_size." 6 | exit 1 7 | fi 8 | dataset_dir=$1 9 | ckpts_dir=$2 10 | limit=$3 11 | max_iters=$4 12 | test_size=$5 13 | 14 | 15 | frozen_graph="$ckpts_dir/conv_actions_frozen.pb" 16 | labels_file="$ckpts_dir/conv_actions_labels.txt" 17 | 18 | if [ ! -d $ckpts_dir ]; then 19 | echo "Checkpoints dir does not exist." 20 | exit 1 21 | fi 22 | 23 | if [ ! -f $frozen_graph ]; then 24 | echo "Frozen graph does not exist." 25 | exit 1 26 | fi 27 | 28 | if [ ! -f $labels_file ]; then 29 | echo "Labels file does not exist." 30 | exit 1 31 | fi 32 | labels_list=`cat $labels_file` 33 | sample_size=100 34 | 35 | 36 | # remove output dir if exists 37 | if [ -d output ] ; then 38 | echo "Output directory already exists ! " 39 | exit 1 40 | fi 41 | mkdir output 42 | 43 | # Copy data files to output dir 44 | for label in `ls $dataset_dir` 45 | do 46 | #TODO(malzantot) : see why this line ignores the labels with __ 47 | if [ -d "$dataset_dir/$label" ] && [[ $labels_list == *"$label"* ]]; then 48 | mkdir -p "output/data/$label" 49 | find "$dataset_dir/$label/" -name "*.wav" | sort -R \ 50 | | head -n$test_size | xargs -L1 cp -t "output/data/$label" 51 | fi 52 | done 53 | 54 | mkdir -p "output/result/" 55 | for target_label in `ls output/data/` 56 | do 57 | for source_label in `ls output/data` 58 | do 59 | if [ $source_label == $target_label ]; then 60 | continue 61 | fi 62 | echo "Running attack: $source_label --> $target_label" 63 | output_dir="output/result/$target_label/$source_label" 64 | mkdir -p $output_dir 65 | python audio_attack.py \ 66 | --data_dir="output/data/$source_label" \ 67 | --output_dir=$output_dir \ 68 | --target_label=$target_label \ 69 | --labels_path=$labels_file \ 70 | --graph_path=$frozen_graph \ 71 | --limit=$limit \ 72 | --max_iters=$max_iters 73 | done 74 | done 75 | -------------------------------------------------------------------------------- /speech_commands/BUILD: -------------------------------------------------------------------------------- 1 | package( 2 | default_visibility = [ 3 | "//visibility:public", 4 | ], 5 | ) 6 | 7 | licenses(["notice"]) # Apache 2.0 8 | 9 | exports_files([ 10 | "LICENSE", 11 | ]) 12 | 13 | load("//tensorflow:tensorflow.bzl", "tf_cc_binary") 14 | load("//tensorflow:tensorflow.bzl", "tf_cc_test") 15 | load("//tensorflow:tensorflow.bzl", "tf_py_test") 16 | 17 | py_library( 18 | name = "models", 19 | srcs = [ 20 | "models.py", 21 | ], 22 | srcs_version = "PY2AND3", 23 | deps = [ 24 | "//tensorflow:tensorflow_py", 25 | "//third_party/py/numpy", 26 | "@six_archive//:six", 27 | ], 28 | ) 29 | 30 | tf_py_test( 31 | name = "models_test", 32 | size = "small", 33 | srcs = ["models_test.py"], 34 | additional_deps = [ 35 | ":models", 36 | "//tensorflow/python:client_testlib", 37 | ], 38 | ) 39 | 40 | py_library( 41 | name = "input_data", 42 | srcs = [ 43 | "input_data.py", 44 | ], 45 | srcs_version = "PY2AND3", 46 | deps = [ 47 | "//tensorflow:tensorflow_py", 48 | "//third_party/py/numpy", 49 | "@six_archive//:six", 50 | ], 51 | ) 52 | 53 | tf_py_test( 54 | name = "input_data_test", 55 | size = "small", 56 | srcs = ["input_data_test.py"], 57 | additional_deps = [ 58 | ":input_data", 59 | "//tensorflow/python:client_testlib", 60 | ], 61 | ) 62 | 63 | py_binary( 64 | name = "train", 65 | srcs = [ 66 | "train.py", 67 | ], 68 | srcs_version = "PY2AND3", 69 | deps = [ 70 | ":input_data", 71 | ":models", 72 | "//tensorflow:tensorflow_py", 73 | "//third_party/py/numpy", 74 | "@six_archive//:six", 75 | ], 76 | ) 77 | 78 | py_binary( 79 | name = "freeze", 80 | srcs = [ 81 | "freeze.py", 82 | ], 83 | srcs_version = "PY2AND3", 84 | deps = [ 85 | ":input_data", 86 | ":models", 87 | "//tensorflow:tensorflow_py", 88 | "//third_party/py/numpy", 89 | "@six_archive//:six", 90 | ], 91 | ) 92 | 93 | tf_py_test( 94 | name = "freeze_test", 95 | size = "small", 96 | srcs = ["freeze_test.py"], 97 | additional_deps = [ 98 | ":freeze", 99 | "//tensorflow/python:client_testlib", 100 | ], 101 | ) 102 | 103 | py_binary( 104 | name = "generate_streaming_test_wav", 105 | srcs = [ 106 | "generate_streaming_test_wav.py", 107 | ], 108 | srcs_version = "PY2AND3", 109 | deps = [ 110 | ":input_data", 111 | ":models", 112 | "//tensorflow:tensorflow_py", 113 | "//third_party/py/numpy", 114 | "@six_archive//:six", 115 | ], 116 | ) 117 | 118 | tf_py_test( 119 | name = "generate_streaming_test_wav_test", 120 | size = "small", 121 | srcs = ["generate_streaming_test_wav_test.py"], 122 | additional_deps = [ 123 | ":generate_streaming_test_wav", 124 | "//tensorflow/python:client_testlib", 125 | ], 126 | ) 127 | 128 | tf_cc_binary( 129 | name = "label_wav_cc", 130 | srcs = [ 131 | "label_wav.cc", 132 | ], 133 | deps = [ 134 | "//tensorflow/core:core_cpu", 135 | "//tensorflow/core:framework", 136 | "//tensorflow/core:framework_internal", 137 | "//tensorflow/core:lib", 138 | "//tensorflow/core:protos_all_cc", 139 | "//tensorflow/core:tensorflow", 140 | ], 141 | ) 142 | 143 | py_binary( 144 | name = "label_wav", 145 | srcs = [ 146 | "label_wav.py", 147 | ], 148 | srcs_version = "PY2AND3", 149 | deps = [ 150 | "//tensorflow:tensorflow_py", 151 | ], 152 | ) 153 | 154 | tf_py_test( 155 | name = "label_wav_test", 156 | size = "medium", 157 | srcs = ["label_wav_test.py"], 158 | additional_deps = [ 159 | ":label_wav", 160 | "//tensorflow/python:client_testlib", 161 | ], 162 | ) 163 | 164 | cc_library( 165 | name = "recognize_commands", 166 | srcs = [ 167 | "recognize_commands.cc", 168 | ], 169 | hdrs = [ 170 | "recognize_commands.h", 171 | ], 172 | deps = [ 173 | "//tensorflow/core:core_cpu", 174 | "//tensorflow/core:framework", 175 | "//tensorflow/core:framework_internal", 176 | "//tensorflow/core:lib", 177 | "//tensorflow/core:protos_all_cc", 178 | "//tensorflow/core:tensorflow", 179 | ], 180 | ) 181 | 182 | tf_cc_test( 183 | name = "recognize_commands_test", 184 | size = "medium", 185 | srcs = [ 186 | "recognize_commands_test.cc", 187 | ], 188 | deps = [ 189 | ":recognize_commands", 190 | "//tensorflow/core:lib", 191 | "//tensorflow/core:lib_internal", 192 | "//tensorflow/core:test", 193 | "//tensorflow/core:test_main", 194 | "//tensorflow/core:testlib", 195 | ], 196 | ) 197 | 198 | cc_library( 199 | name = "accuracy_utils", 200 | srcs = [ 201 | "accuracy_utils.cc", 202 | ], 203 | hdrs = [ 204 | "accuracy_utils.h", 205 | ], 206 | deps = [ 207 | "//tensorflow/core:core_cpu", 208 | "//tensorflow/core:framework", 209 | "//tensorflow/core:framework_internal", 210 | "//tensorflow/core:lib", 211 | "//tensorflow/core:protos_all_cc", 212 | "//tensorflow/core:tensorflow", 213 | ], 214 | ) 215 | 216 | tf_cc_test( 217 | name = "accuracy_utils_test", 218 | size = "medium", 219 | srcs = [ 220 | "accuracy_utils_test.cc", 221 | ], 222 | deps = [ 223 | ":accuracy_utils", 224 | "//tensorflow/core:lib", 225 | "//tensorflow/core:lib_internal", 226 | "//tensorflow/core:test", 227 | "//tensorflow/core:test_main", 228 | "//tensorflow/core:testlib", 229 | ], 230 | ) 231 | 232 | tf_cc_binary( 233 | name = "test_streaming_accuracy", 234 | srcs = [ 235 | "test_streaming_accuracy.cc", 236 | ], 237 | deps = [ 238 | ":accuracy_utils", 239 | ":recognize_commands", 240 | "//tensorflow/core:core_cpu", 241 | "//tensorflow/core:framework", 242 | "//tensorflow/core:framework_internal", 243 | "//tensorflow/core:lib", 244 | "//tensorflow/core:lib_internal", 245 | "//tensorflow/core:protos_all_cc", 246 | ], 247 | ) 248 | 249 | filegroup( 250 | name = "all_files", 251 | srcs = glob( 252 | ["**/*"], 253 | exclude = [ 254 | "**/METADATA", 255 | "**/OWNERS", 256 | ], 257 | ), 258 | visibility = ["//tensorflow:__subpackages__"], 259 | ) 260 | -------------------------------------------------------------------------------- /speech_commands/README.md: -------------------------------------------------------------------------------- 1 | # Speech Commands Example 2 | 3 | This is a basic speech recognition example. For more information, see the 4 | tutorial at https://www.tensorflow.org/versions/master/tutorials/audio_recognition. 5 | -------------------------------------------------------------------------------- /speech_commands/Untitled.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 1, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import tensorflow as tf" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": 10, 15 | "metadata": { 16 | "collapsed": true 17 | }, 18 | "outputs": [], 19 | "source": [ 20 | "def load_graph(filename):\n", 21 | " with tf.gfile.FastGFile(filename, 'rb') as f:\n", 22 | " graph_def = tf.GraphDef()\n", 23 | " graph_def.ParseFromString(f.read())\n", 24 | " #print(graph_def)\n", 25 | " return graph_def\n", 26 | " " 27 | ] 28 | }, 29 | { 30 | "cell_type": "code", 31 | "execution_count": 11, 32 | "metadata": {}, 33 | "outputs": [], 34 | "source": [ 35 | "graph_def = load_graph('../graph/my_frozen_graph.pb')" 36 | ] 37 | }, 38 | { 39 | "cell_type": "code", 40 | "execution_count": 14, 41 | "metadata": {}, 42 | "outputs": [ 43 | { 44 | "ename": "ValueError", 45 | "evalue": "No op named DecodeWav in defined operations.", 46 | "output_type": "error", 47 | "traceback": [ 48 | "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", 49 | "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)", 50 | "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mimport_graph_def\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgraph_def\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", 51 | "\u001b[0;32m/home/malzantot/anaconda3/lib/python3.6/site-packages/tensorflow/python/util/deprecation.py\u001b[0m in \u001b[0;36mnew_func\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 314\u001b[0m \u001b[0;34m'in a future version'\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mdate\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;34m'after %s'\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mdate\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 315\u001b[0m instructions)\n\u001b[0;32m--> 316\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 317\u001b[0m return tf_decorator.make_decorator(func, new_func, 'deprecated',\n\u001b[1;32m 318\u001b[0m _add_deprecated_arg_notice_to_docstring(\n", 52 | "\u001b[0;32m/home/malzantot/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/importer.py\u001b[0m in \u001b[0;36mimport_graph_def\u001b[0;34m(graph_def, input_map, return_elements, name, op_dict, producer_op_list)\u001b[0m\n\u001b[1;32m 284\u001b[0m \u001b[0;31m# Set any default attr values that aren't present.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 285\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mnode\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mop\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mop_dict\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 286\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'No op named %s in defined operations.'\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mnode\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mop\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 287\u001b[0m \u001b[0mop_def\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mop_dict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnode\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mop\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 288\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mattr_def\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mop_def\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mattr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", 53 | "\u001b[0;31mValueError\u001b[0m: No op named DecodeWav in defined operations." 54 | ] 55 | } 56 | ], 57 | "source": [ 58 | "tf.import_graph_def(graph_def)" 59 | ] 60 | }, 61 | { 62 | "cell_type": "code", 63 | "execution_count": null, 64 | "metadata": { 65 | "collapsed": true 66 | }, 67 | "outputs": [], 68 | "source": [] 69 | } 70 | ], 71 | "metadata": { 72 | "kernelspec": { 73 | "display_name": "Python 3", 74 | "language": "python", 75 | "name": "python3" 76 | }, 77 | "language_info": { 78 | "codemirror_mode": { 79 | "name": "ipython", 80 | "version": 3 81 | }, 82 | "file_extension": ".py", 83 | "mimetype": "text/x-python", 84 | "name": "python", 85 | "nbconvert_exporter": "python", 86 | "pygments_lexer": "ipython3", 87 | "version": "3.6.1" 88 | } 89 | }, 90 | "nbformat": 4, 91 | "nbformat_minor": 2 92 | } 93 | -------------------------------------------------------------------------------- /speech_commands/accuracy_utils.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "tensorflow/examples/speech_commands/accuracy_utils.h" 17 | 18 | #include 19 | #include 20 | #include 21 | 22 | #include "tensorflow/core/lib/io/path.h" 23 | #include "tensorflow/core/lib/strings/str_util.h" 24 | 25 | namespace tensorflow { 26 | 27 | Status ReadGroundTruthFile(const string& file_name, 28 | std::vector>* result) { 29 | std::ifstream file(file_name); 30 | if (!file) { 31 | return tensorflow::errors::NotFound("Ground truth file '", file_name, 32 | "' not found."); 33 | } 34 | result->clear(); 35 | string line; 36 | while (std::getline(file, line)) { 37 | std::vector pieces = tensorflow::str_util::Split(line, ','); 38 | if (pieces.size() != 2) { 39 | continue; 40 | } 41 | float timestamp; 42 | if (!tensorflow::strings::safe_strtof(pieces[1].c_str(), ×tamp)) { 43 | return tensorflow::errors::InvalidArgument( 44 | "Wrong number format at line: ", line); 45 | } 46 | string label = pieces[0]; 47 | auto timestamp_int64 = static_cast(timestamp); 48 | result->push_back({label, timestamp_int64}); 49 | } 50 | std::sort(result->begin(), result->end(), 51 | [](const std::pair& left, 52 | const std::pair& right) { 53 | return left.second < right.second; 54 | }); 55 | return Status::OK(); 56 | } 57 | 58 | void CalculateAccuracyStats( 59 | const std::vector>& ground_truth_list, 60 | const std::vector>& found_words, 61 | int64 up_to_time_ms, int64 time_tolerance_ms, 62 | StreamingAccuracyStats* stats) { 63 | int64 latest_possible_time; 64 | if (up_to_time_ms == -1) { 65 | latest_possible_time = std::numeric_limits::max(); 66 | } else { 67 | latest_possible_time = up_to_time_ms + time_tolerance_ms; 68 | } 69 | stats->how_many_ground_truth_words = 0; 70 | for (const std::pair& ground_truth : ground_truth_list) { 71 | const int64 ground_truth_time = ground_truth.second; 72 | if (ground_truth_time > latest_possible_time) { 73 | break; 74 | } 75 | ++stats->how_many_ground_truth_words; 76 | } 77 | 78 | stats->how_many_false_positives = 0; 79 | stats->how_many_correct_words = 0; 80 | stats->how_many_wrong_words = 0; 81 | std::unordered_set has_ground_truth_been_matched; 82 | for (const std::pair& found_word : found_words) { 83 | const string& found_label = found_word.first; 84 | const int64 found_time = found_word.second; 85 | const int64 earliest_time = found_time - time_tolerance_ms; 86 | const int64 latest_time = found_time + time_tolerance_ms; 87 | bool has_match_been_found = false; 88 | for (const std::pair& ground_truth : ground_truth_list) { 89 | const int64 ground_truth_time = ground_truth.second; 90 | if ((ground_truth_time > latest_time) || 91 | (ground_truth_time > latest_possible_time)) { 92 | break; 93 | } 94 | if (ground_truth_time < earliest_time) { 95 | continue; 96 | } 97 | const string& ground_truth_label = ground_truth.first; 98 | if ((ground_truth_label == found_label) && 99 | (has_ground_truth_been_matched.count(ground_truth_time) == 0)) { 100 | ++stats->how_many_correct_words; 101 | } else { 102 | ++stats->how_many_wrong_words; 103 | } 104 | has_ground_truth_been_matched.insert(ground_truth_time); 105 | has_match_been_found = true; 106 | break; 107 | } 108 | if (!has_match_been_found) { 109 | ++stats->how_many_false_positives; 110 | } 111 | } 112 | stats->how_many_ground_truth_matched = has_ground_truth_been_matched.size(); 113 | } 114 | 115 | void PrintAccuracyStats(const StreamingAccuracyStats& stats) { 116 | if (stats.how_many_ground_truth_words == 0) { 117 | LOG(INFO) << "No ground truth yet, " << stats.how_many_false_positives 118 | << " false positives"; 119 | } else { 120 | float any_match_percentage = 121 | (stats.how_many_ground_truth_matched * 100.0f) / 122 | stats.how_many_ground_truth_words; 123 | float correct_match_percentage = (stats.how_many_correct_words * 100.0f) / 124 | stats.how_many_ground_truth_words; 125 | float wrong_match_percentage = (stats.how_many_wrong_words * 100.0f) / 126 | stats.how_many_ground_truth_words; 127 | float false_positive_percentage = 128 | (stats.how_many_false_positives * 100.0f) / 129 | stats.how_many_ground_truth_words; 130 | 131 | LOG(INFO) << std::setprecision(1) << std::fixed << any_match_percentage 132 | << "% matched, " << correct_match_percentage << "% correctly, " 133 | << wrong_match_percentage << "% wrongly, " 134 | << false_positive_percentage << "% false positives "; 135 | } 136 | } 137 | 138 | } // namespace tensorflow 139 | -------------------------------------------------------------------------------- /speech_commands/accuracy_utils.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef THIRD_PARTY_TENSORFLOW_EXAMPLES_SPEECH_COMMANDS_ACCURACY_UTILS_H_ 17 | #define THIRD_PARTY_TENSORFLOW_EXAMPLES_SPEECH_COMMANDS_ACCURACY_UTILS_H_ 18 | 19 | #include 20 | 21 | #include "tensorflow/core/framework/tensor.h" 22 | #include "tensorflow/core/platform/types.h" 23 | 24 | namespace tensorflow { 25 | 26 | struct StreamingAccuracyStats { 27 | StreamingAccuracyStats() 28 | : how_many_ground_truth_words(0), 29 | how_many_ground_truth_matched(0), 30 | how_many_false_positives(0), 31 | how_many_correct_words(0), 32 | how_many_wrong_words(0) {} 33 | int32 how_many_ground_truth_words; 34 | int32 how_many_ground_truth_matched; 35 | int32 how_many_false_positives; 36 | int32 how_many_correct_words; 37 | int32 how_many_wrong_words; 38 | }; 39 | 40 | // Takes a file name, and loads a list of expected word labels and times from 41 | // it, as comma-separated variables. 42 | Status ReadGroundTruthFile(const string& file_name, 43 | std::vector>* result); 44 | 45 | // Given ground truth labels and corresponding predictions found by a model, 46 | // figure out how many were correct. Takes a time limit, so that only 47 | // predictions up to a point in time are considered, in case we're evaluating 48 | // accuracy when the model has only been run on part of the stream. 49 | void CalculateAccuracyStats( 50 | const std::vector>& ground_truth_list, 51 | const std::vector>& found_words, 52 | int64 up_to_time_ms, int64 time_tolerance_ms, 53 | StreamingAccuracyStats* stats); 54 | 55 | // Writes a human-readable description of the statistics to stdout. 56 | void PrintAccuracyStats(const StreamingAccuracyStats& stats); 57 | 58 | } // namespace tensorflow 59 | 60 | #endif // THIRD_PARTY_TENSORFLOW_EXAMPLES_SPEECH_COMMANDS_ACCURACY_UTILS_H_ 61 | -------------------------------------------------------------------------------- /speech_commands/accuracy_utils_test.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "tensorflow/examples/speech_commands/accuracy_utils.h" 17 | 18 | #include "tensorflow/core/framework/tensor_testutil.h" 19 | #include "tensorflow/core/lib/core/status_test_util.h" 20 | #include "tensorflow/core/lib/io/path.h" 21 | #include "tensorflow/core/lib/strings/str_util.h" 22 | #include "tensorflow/core/platform/env.h" 23 | #include "tensorflow/core/platform/test.h" 24 | 25 | namespace tensorflow { 26 | 27 | TEST(AccuracyUtilsTest, ReadGroundTruthFile) { 28 | string file_name = tensorflow::io::JoinPath(tensorflow::testing::TmpDir(), 29 | "ground_truth.txt"); 30 | string file_data = "a,10\nb,12\n"; 31 | TF_ASSERT_OK(WriteStringToFile(Env::Default(), file_name, file_data)); 32 | 33 | std::vector> ground_truth; 34 | TF_ASSERT_OK(ReadGroundTruthFile(file_name, &ground_truth)); 35 | ASSERT_EQ(2, ground_truth.size()); 36 | EXPECT_EQ("a", ground_truth[0].first); 37 | EXPECT_EQ(10, ground_truth[0].second); 38 | EXPECT_EQ("b", ground_truth[1].first); 39 | EXPECT_EQ(12, ground_truth[1].second); 40 | } 41 | 42 | TEST(AccuracyUtilsTest, CalculateAccuracyStats) { 43 | StreamingAccuracyStats stats; 44 | CalculateAccuracyStats({{"a", 1000}, {"b", 9000}}, 45 | {{"a", 1200}, {"b", 5000}, {"a", 8700}}, 10000, 500, 46 | &stats); 47 | EXPECT_EQ(2, stats.how_many_ground_truth_words); 48 | EXPECT_EQ(2, stats.how_many_ground_truth_matched); 49 | EXPECT_EQ(1, stats.how_many_false_positives); 50 | EXPECT_EQ(1, stats.how_many_correct_words); 51 | EXPECT_EQ(1, stats.how_many_wrong_words); 52 | } 53 | 54 | TEST(AccuracyUtilsTest, PrintAccuracyStats) { 55 | StreamingAccuracyStats stats; 56 | PrintAccuracyStats(stats); 57 | } 58 | 59 | } // namespace tensorflow 60 | -------------------------------------------------------------------------------- /speech_commands/freeze.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | r"""Converts a trained checkpoint into a frozen model for mobile inference. 16 | 17 | Once you've trained a model using the `train.py` script, you can use this tool 18 | to convert it into a binary GraphDef file that can be loaded into the Android, 19 | iOS, or Raspberry Pi example code. Here's an example of how to run it: 20 | 21 | bazel run tensorflow/examples/speech_commands/freeze -- \ 22 | --sample_rate=16000 --dct_coefficient_count=40 --window_size_ms=20 \ 23 | --window_stride_ms=10 --clip_duration_ms=1000 \ 24 | --model_architecture=conv \ 25 | --start_checkpoint=/tmp/speech_commands_train/conv.ckpt-1300 \ 26 | --output_file=/tmp/my_frozen_graph.pb 27 | 28 | One thing to watch out for is that you need to pass in the same arguments for 29 | `sample_rate` and other command line variables here as you did for the training 30 | script. 31 | 32 | The resulting graph has an input for WAV-encoded data named 'wav_data', one for 33 | raw PCM data (as floats in the range -1.0 to 1.0) called 'decoded_sample_data', 34 | and the output is called 'labels_softmax'. 35 | 36 | """ 37 | from __future__ import absolute_import 38 | from __future__ import division 39 | from __future__ import print_function 40 | 41 | import argparse 42 | import os.path 43 | import sys 44 | 45 | import tensorflow as tf 46 | 47 | from tensorflow.contrib.framework.python.ops import audio_ops as contrib_audio 48 | import input_data 49 | import models 50 | from tensorflow.python.framework import graph_util 51 | 52 | FLAGS = None 53 | 54 | 55 | def create_inference_graph(wanted_words, sample_rate, clip_duration_ms, 56 | clip_stride_ms, window_size_ms, window_stride_ms, 57 | dct_coefficient_count, model_architecture): 58 | """Creates an audio model with the nodes needed for inference. 59 | 60 | Uses the supplied arguments to create a model, and inserts the input and 61 | output nodes that are needed to use the graph for inference. 62 | 63 | Args: 64 | wanted_words: Comma-separated list of the words we're trying to recognize. 65 | sample_rate: How many samples per second are in the input audio files. 66 | clip_duration_ms: How many samples to analyze for the audio pattern. 67 | clip_stride_ms: How often to run recognition. Useful for models with cache. 68 | window_size_ms: Time slice duration to estimate frequencies from. 69 | window_stride_ms: How far apart time slices should be. 70 | dct_coefficient_count: Number of frequency bands to analyze. 71 | model_architecture: Name of the kind of model to generate. 72 | """ 73 | 74 | words_list = input_data.prepare_words_list(wanted_words.split(',')) 75 | model_settings = models.prepare_model_settings( 76 | len(words_list), sample_rate, clip_duration_ms, window_size_ms, 77 | window_stride_ms, dct_coefficient_count) 78 | runtime_settings = {'clip_stride_ms': clip_stride_ms} 79 | 80 | wav_data_placeholder = tf.placeholder(tf.string, [], name='wav_data') 81 | decoded_sample_data = contrib_audio.decode_wav( 82 | wav_data_placeholder, 83 | desired_channels=1, 84 | desired_samples=model_settings['desired_samples'], 85 | name='decoded_sample_data') 86 | spectrogram = contrib_audio.audio_spectrogram( 87 | decoded_sample_data.audio, 88 | window_size=model_settings['window_size_samples'], 89 | stride=model_settings['window_stride_samples'], 90 | magnitude_squared=True) 91 | fingerprint_input = contrib_audio.mfcc( 92 | spectrogram, 93 | decoded_sample_data.sample_rate, 94 | dct_coefficient_count=dct_coefficient_count) 95 | fingerprint_frequency_size = model_settings['dct_coefficient_count'] 96 | fingerprint_time_size = model_settings['spectrogram_length'] 97 | reshaped_input = tf.reshape(fingerprint_input, [ 98 | -1, fingerprint_time_size * fingerprint_frequency_size 99 | ]) 100 | 101 | logits = models.create_model( 102 | reshaped_input, model_settings, model_architecture, is_training=False, 103 | runtime_settings=runtime_settings) 104 | 105 | # Create an output to use for inference. 106 | tf.nn.softmax(logits, name='labels_softmax') 107 | 108 | 109 | def main(_): 110 | 111 | # Create the model and load its weights. 112 | sess = tf.InteractiveSession() 113 | create_inference_graph(FLAGS.wanted_words, FLAGS.sample_rate, 114 | FLAGS.clip_duration_ms, FLAGS.clip_stride_ms, 115 | FLAGS.window_size_ms, FLAGS.window_stride_ms, 116 | FLAGS.dct_coefficient_count, FLAGS.model_architecture) 117 | models.load_variables_from_checkpoint(sess, FLAGS.start_checkpoint) 118 | 119 | # Turn all the variables into inline constants inside the graph and save it. 120 | frozen_graph_def = graph_util.convert_variables_to_constants( 121 | sess, sess.graph_def, ['labels_softmax']) 122 | tf.train.write_graph( 123 | frozen_graph_def, 124 | os.path.dirname(FLAGS.output_file), 125 | os.path.basename(FLAGS.output_file), 126 | as_text=False) 127 | tf.logging.info('Saved frozen graph to %s', FLAGS.output_file) 128 | 129 | 130 | if __name__ == '__main__': 131 | parser = argparse.ArgumentParser() 132 | parser.add_argument( 133 | '--sample_rate', 134 | type=int, 135 | default=16000, 136 | help='Expected sample rate of the wavs',) 137 | parser.add_argument( 138 | '--clip_duration_ms', 139 | type=int, 140 | default=1000, 141 | help='Expected duration in milliseconds of the wavs',) 142 | parser.add_argument( 143 | '--clip_stride_ms', 144 | type=int, 145 | default=30, 146 | help='How often to run recognition. Useful for models with cache.',) 147 | parser.add_argument( 148 | '--window_size_ms', 149 | type=float, 150 | default=30.0, 151 | help='How long each spectrogram timeslice is',) 152 | parser.add_argument( 153 | '--window_stride_ms', 154 | type=float, 155 | default=10.0, 156 | help='How long the stride is between spectrogram timeslices',) 157 | parser.add_argument( 158 | '--dct_coefficient_count', 159 | type=int, 160 | default=40, 161 | help='How many bins to use for the MFCC fingerprint',) 162 | parser.add_argument( 163 | '--start_checkpoint', 164 | type=str, 165 | default='', 166 | help='If specified, restore this pretrained model before any training.') 167 | parser.add_argument( 168 | '--model_architecture', 169 | type=str, 170 | default='conv', 171 | help='What model architecture to use') 172 | parser.add_argument( 173 | '--wanted_words', 174 | type=str, 175 | default='yes,no,up,down,left,right,on,off,stop,go', 176 | help='Words to use (others will be added to an unknown label)',) 177 | parser.add_argument( 178 | '--output_file', type=str, help='Where to save the frozen graph.') 179 | FLAGS, unparsed = parser.parse_known_args() 180 | tf.app.run(main=main, argv=[sys.argv[0]] + unparsed) 181 | -------------------------------------------------------------------------------- /speech_commands/freeze_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Tests for data input for speech commands.""" 16 | 17 | from __future__ import absolute_import 18 | from __future__ import division 19 | from __future__ import print_function 20 | 21 | from tensorflow.examples.speech_commands import freeze 22 | from tensorflow.python.platform import test 23 | 24 | 25 | class FreezeTest(test.TestCase): 26 | 27 | def testCreateInferenceGraph(self): 28 | with self.test_session() as sess: 29 | freeze.create_inference_graph('a,b,c,d', 16000, 1000.0, 30.0, 30.0, 10.0, 30 | 40, 'conv') 31 | self.assertIsNotNone(sess.graph.get_tensor_by_name('wav_data:0')) 32 | self.assertIsNotNone( 33 | sess.graph.get_tensor_by_name('decoded_sample_data:0')) 34 | self.assertIsNotNone(sess.graph.get_tensor_by_name('labels_softmax:0')) 35 | 36 | 37 | if __name__ == '__main__': 38 | test.main() 39 | -------------------------------------------------------------------------------- /speech_commands/generate_streaming_test_wav.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | r"""Saves out a .wav file with synthesized conversational data and labels. 16 | 17 | The best way to estimate the real-world performance of an audio recognition 18 | model is by running it against a continuous stream of data, the way that it 19 | would be used in an application. Training evaluations are only run against 20 | discrete individual samples, so the results aren't as realistic. 21 | 22 | To make it easy to run evaluations against audio streams, this script uses 23 | samples from the testing partition of the data set, mixes them in at random 24 | positions together with background noise, and saves out the result as one long 25 | audio file. 26 | 27 | Here's an example of generating a test file: 28 | 29 | bazel run tensorflow/examples/speech_commands:generate_streaming_test_wav -- \ 30 | --data_dir=/tmp/my_wavs --background_dir=/tmp/my_backgrounds \ 31 | --background_volume=0.1 --test_duration_seconds=600 \ 32 | --output_audio_file=/tmp/streaming_test.wav \ 33 | --output_labels_file=/tmp/streaming_test_labels.txt 34 | 35 | Once you've created a streaming audio file, you can then use the 36 | test_streaming_accuracy tool to calculate accuracy metrics for a model. 37 | """ 38 | from __future__ import absolute_import 39 | from __future__ import division 40 | from __future__ import print_function 41 | 42 | import argparse 43 | import math 44 | import sys 45 | 46 | import numpy as np 47 | import tensorflow as tf 48 | 49 | import input_data 50 | import models 51 | 52 | FLAGS = None 53 | 54 | 55 | def mix_in_audio_sample(track_data, track_offset, sample_data, sample_offset, 56 | clip_duration, sample_volume, ramp_in, ramp_out): 57 | """Mixes the sample data into the main track at the specified offset. 58 | 59 | Args: 60 | track_data: Numpy array holding main audio data. Modified in-place. 61 | track_offset: Where to mix the sample into the main track. 62 | sample_data: Numpy array of audio data to mix into the main track. 63 | sample_offset: Where to start in the audio sample. 64 | clip_duration: How long the sample segment is. 65 | sample_volume: Loudness to mix the sample in at. 66 | ramp_in: Length in samples of volume increase stage. 67 | ramp_out: Length in samples of volume decrease stage. 68 | """ 69 | ramp_out_index = clip_duration - ramp_out 70 | track_end = min(track_offset + clip_duration, track_data.shape[0]) 71 | track_end = min(track_end, 72 | track_offset + (sample_data.shape[0] - sample_offset)) 73 | sample_range = track_end - track_offset 74 | for i in range(sample_range): 75 | if i < ramp_in: 76 | envelope_scale = i / ramp_in 77 | elif i > ramp_out_index: 78 | envelope_scale = (clip_duration - i) / ramp_out 79 | else: 80 | envelope_scale = 1 81 | sample_input = sample_data[sample_offset + i] 82 | track_data[track_offset 83 | + i] += sample_input * envelope_scale * sample_volume 84 | 85 | 86 | def main(_): 87 | words_list = input_data.prepare_words_list(FLAGS.wanted_words.split(',')) 88 | model_settings = models.prepare_model_settings( 89 | len(words_list), FLAGS.sample_rate, FLAGS.clip_duration_ms, 90 | FLAGS.window_size_ms, FLAGS.window_stride_ms, FLAGS.dct_coefficient_count) 91 | audio_processor = input_data.AudioProcessor( 92 | '', FLAGS.data_dir, FLAGS.silence_percentage, 10, 93 | FLAGS.wanted_words.split(','), FLAGS.validation_percentage, 94 | FLAGS.testing_percentage, model_settings) 95 | 96 | output_audio_sample_count = FLAGS.sample_rate * FLAGS.test_duration_seconds 97 | output_audio = np.zeros((output_audio_sample_count,), dtype=np.float32) 98 | 99 | # Set up background audio. 100 | background_crossover_ms = 500 101 | background_segment_duration_ms = ( 102 | FLAGS.clip_duration_ms + background_crossover_ms) 103 | background_segment_duration_samples = int( 104 | (background_segment_duration_ms * FLAGS.sample_rate) / 1000) 105 | background_segment_stride_samples = int( 106 | (FLAGS.clip_duration_ms * FLAGS.sample_rate) / 1000) 107 | background_ramp_samples = int( 108 | ((background_crossover_ms / 2) * FLAGS.sample_rate) / 1000) 109 | 110 | # Mix the background audio into the main track. 111 | how_many_backgrounds = int( 112 | math.ceil(output_audio_sample_count / background_segment_stride_samples)) 113 | for i in range(how_many_backgrounds): 114 | output_offset = int(i * background_segment_stride_samples) 115 | background_index = np.random.randint(len(audio_processor.background_data)) 116 | background_samples = audio_processor.background_data[background_index] 117 | background_offset = np.random.randint( 118 | 0, len(background_samples) - model_settings['desired_samples']) 119 | background_volume = np.random.uniform(0, FLAGS.background_volume) 120 | mix_in_audio_sample(output_audio, output_offset, background_samples, 121 | background_offset, background_segment_duration_samples, 122 | background_volume, background_ramp_samples, 123 | background_ramp_samples) 124 | 125 | # Mix the words into the main track, noting their labels and positions. 126 | output_labels = [] 127 | word_stride_ms = FLAGS.clip_duration_ms + FLAGS.word_gap_ms 128 | word_stride_samples = int((word_stride_ms * FLAGS.sample_rate) / 1000) 129 | clip_duration_samples = int( 130 | (FLAGS.clip_duration_ms * FLAGS.sample_rate) / 1000) 131 | word_gap_samples = int((FLAGS.word_gap_ms * FLAGS.sample_rate) / 1000) 132 | how_many_words = int( 133 | math.floor(output_audio_sample_count / word_stride_samples)) 134 | all_test_data, all_test_labels = audio_processor.get_unprocessed_data( 135 | -1, model_settings, 'testing') 136 | for i in range(how_many_words): 137 | output_offset = ( 138 | int(i * word_stride_samples) + np.random.randint(word_gap_samples)) 139 | output_offset_ms = (output_offset * 1000) / FLAGS.sample_rate 140 | is_unknown = np.random.randint(100) < FLAGS.unknown_percentage 141 | if is_unknown: 142 | wanted_label = input_data.UNKNOWN_WORD_LABEL 143 | else: 144 | wanted_label = words_list[2 + np.random.randint(len(words_list) - 2)] 145 | test_data_start = np.random.randint(len(all_test_data)) 146 | found_sample_data = None 147 | index_lookup = np.arange(len(all_test_data), dtype=np.int32) 148 | np.random.shuffle(index_lookup) 149 | for test_data_offset in range(len(all_test_data)): 150 | test_data_index = index_lookup[( 151 | test_data_start + test_data_offset) % len(all_test_data)] 152 | current_label = all_test_labels[test_data_index] 153 | if current_label == wanted_label: 154 | found_sample_data = all_test_data[test_data_index] 155 | break 156 | mix_in_audio_sample(output_audio, output_offset, found_sample_data, 0, 157 | clip_duration_samples, 1.0, 500, 500) 158 | output_labels.append({'label': wanted_label, 'time': output_offset_ms}) 159 | 160 | input_data.save_wav_file(FLAGS.output_audio_file, output_audio, 161 | FLAGS.sample_rate) 162 | tf.logging.info('Saved streaming test wav to %s', FLAGS.output_audio_file) 163 | 164 | with open(FLAGS.output_labels_file, 'w') as f: 165 | for output_label in output_labels: 166 | f.write('%s, %f\n' % (output_label['label'], output_label['time'])) 167 | tf.logging.info('Saved streaming test labels to %s', FLAGS.output_labels_file) 168 | 169 | 170 | if __name__ == '__main__': 171 | parser = argparse.ArgumentParser() 172 | parser.add_argument( 173 | '--data_url', 174 | type=str, 175 | # pylint: disable=line-too-long 176 | default='http://download.tensorflow.org/data/speech_commands_v0.01.tar.gz', 177 | # pylint: enable=line-too-long 178 | help='Location of speech training data') 179 | parser.add_argument( 180 | '--data_dir', 181 | type=str, 182 | default='/tmp/speech_dataset', 183 | help="""\ 184 | Where to download the speech training data to. 185 | """) 186 | parser.add_argument( 187 | '--background_dir', 188 | type=str, 189 | default='', 190 | help="""\ 191 | Path to a directory of .wav files to mix in as background noise during training. 192 | """) 193 | parser.add_argument( 194 | '--background_volume', 195 | type=float, 196 | default=0.1, 197 | help="""\ 198 | How loud the background noise should be, between 0 and 1. 199 | """) 200 | parser.add_argument( 201 | '--background_frequency', 202 | type=float, 203 | default=0.8, 204 | help="""\ 205 | How many of the training samples have background noise mixed in. 206 | """) 207 | parser.add_argument( 208 | '--silence_percentage', 209 | type=float, 210 | default=10.0, 211 | help="""\ 212 | How much of the training data should be silence. 213 | """) 214 | parser.add_argument( 215 | '--testing_percentage', 216 | type=int, 217 | default=10, 218 | help='What percentage of wavs to use as a test set.') 219 | parser.add_argument( 220 | '--validation_percentage', 221 | type=int, 222 | default=10, 223 | help='What percentage of wavs to use as a validation set.') 224 | parser.add_argument( 225 | '--sample_rate', 226 | type=int, 227 | default=16000, 228 | help='Expected sample rate of the wavs.',) 229 | parser.add_argument( 230 | '--clip_duration_ms', 231 | type=int, 232 | default=1000, 233 | help='Expected duration in milliseconds of the wavs.',) 234 | parser.add_argument( 235 | '--window_size_ms', 236 | type=float, 237 | default=30.0, 238 | help='How long each spectrogram timeslice is',) 239 | parser.add_argument( 240 | '--window_stride_ms', 241 | type=float, 242 | default=10.0, 243 | help='How long the stride is between spectrogram timeslices',) 244 | parser.add_argument( 245 | '--dct_coefficient_count', 246 | type=int, 247 | default=40, 248 | help='How many bins to use for the MFCC fingerprint',) 249 | parser.add_argument( 250 | '--wanted_words', 251 | type=str, 252 | default='yes,no,up,down,left,right,on,off,stop,go', 253 | help='Words to use (others will be added to an unknown label)',) 254 | parser.add_argument( 255 | '--output_audio_file', 256 | type=str, 257 | default='/tmp/speech_commands_train/streaming_test.wav', 258 | help='File to save the generated test audio to.') 259 | parser.add_argument( 260 | '--output_labels_file', 261 | type=str, 262 | default='/tmp/speech_commands_train/streaming_test_labels.txt', 263 | help='File to save the generated test labels to.') 264 | parser.add_argument( 265 | '--test_duration_seconds', 266 | type=int, 267 | default=600, 268 | help='How long the generated test audio file should be.',) 269 | parser.add_argument( 270 | '--word_gap_ms', 271 | type=int, 272 | default=2000, 273 | help='How long the average gap should be between words.',) 274 | parser.add_argument( 275 | '--unknown_percentage', 276 | type=int, 277 | default=30, 278 | help='What percentage of words should be unknown.') 279 | 280 | FLAGS, unparsed = parser.parse_known_args() 281 | tf.app.run(main=main, argv=[sys.argv[0]] + unparsed) 282 | -------------------------------------------------------------------------------- /speech_commands/generate_streaming_test_wav_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Tests for test file generation for speech commands.""" 16 | 17 | from __future__ import absolute_import 18 | from __future__ import division 19 | from __future__ import print_function 20 | 21 | import numpy as np 22 | 23 | from tensorflow.examples.speech_commands import generate_streaming_test_wav 24 | from tensorflow.python.platform import test 25 | 26 | 27 | class GenerateStreamingTestWavTest(test.TestCase): 28 | 29 | def testMixInAudioSample(self): 30 | track_data = np.zeros([10000]) 31 | sample_data = np.ones([1000]) 32 | generate_streaming_test_wav.mix_in_audio_sample( 33 | track_data, 2000, sample_data, 0, 1000, 1.0, 100, 100) 34 | self.assertNear(1.0, track_data[2500], 0.0001) 35 | self.assertNear(0.0, track_data[3500], 0.0001) 36 | 37 | 38 | if __name__ == "__main__": 39 | test.main() 40 | -------------------------------------------------------------------------------- /speech_commands/input_data.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Model definitions for simple speech recognition. 16 | 17 | """ 18 | from __future__ import absolute_import 19 | from __future__ import division 20 | from __future__ import print_function 21 | 22 | import hashlib 23 | import math 24 | import os.path 25 | import random 26 | import re 27 | import sys 28 | import tarfile 29 | 30 | import numpy as np 31 | from six.moves import urllib 32 | from six.moves import xrange # pylint: disable=redefined-builtin 33 | import tensorflow as tf 34 | 35 | from tensorflow.contrib.framework.python.ops import audio_ops as contrib_audio 36 | from tensorflow.python.ops import io_ops 37 | from tensorflow.python.platform import gfile 38 | from tensorflow.python.util import compat 39 | 40 | MAX_NUM_WAVS_PER_CLASS = 2**27 - 1 # ~134M 41 | SILENCE_LABEL = '_silence_' 42 | SILENCE_INDEX = 0 43 | UNKNOWN_WORD_LABEL = '_unknown_' 44 | UNKNOWN_WORD_INDEX = 1 45 | BACKGROUND_NOISE_DIR_NAME = '_background_noise_' 46 | RANDOM_SEED = 59185 47 | 48 | 49 | def prepare_words_list(wanted_words): 50 | """Prepends common tokens to the custom word list. 51 | 52 | Args: 53 | wanted_words: List of strings containing the custom words. 54 | 55 | Returns: 56 | List with the standard silence and unknown tokens added. 57 | """ 58 | return [SILENCE_LABEL, UNKNOWN_WORD_LABEL] + wanted_words 59 | 60 | 61 | def which_set(filename, validation_percentage, testing_percentage): 62 | """Determines which data partition the file should belong to. 63 | 64 | We want to keep files in the same training, validation, or testing sets even 65 | if new ones are added over time. This makes it less likely that testing 66 | samples will accidentally be reused in training when long runs are restarted 67 | for example. To keep this stability, a hash of the filename is taken and used 68 | to determine which set it should belong to. This determination only depends on 69 | the name and the set proportions, so it won't change as other files are added. 70 | 71 | It's also useful to associate particular files as related (for example words 72 | spoken by the same person), so anything after '_nohash_' in a filename is 73 | ignored for set determination. This ensures that 'bobby_nohash_0.wav' and 74 | 'bobby_nohash_1.wav' are always in the same set, for example. 75 | 76 | Args: 77 | filename: File path of the data sample. 78 | validation_percentage: How much of the data set to use for validation. 79 | testing_percentage: How much of the data set to use for testing. 80 | 81 | Returns: 82 | String, one of 'training', 'validation', or 'testing'. 83 | """ 84 | base_name = os.path.basename(filename) 85 | # We want to ignore anything after '_nohash_' in the file name when 86 | # deciding which set to put a wav in, so the data set creator has a way of 87 | # grouping wavs that are close variations of each other. 88 | hash_name = re.sub(r'_nohash_.*$', '', base_name) 89 | # This looks a bit magical, but we need to decide whether this file should 90 | # go into the training, testing, or validation sets, and we want to keep 91 | # existing files in the same set even if more files are subsequently 92 | # added. 93 | # To do that, we need a stable way of deciding based on just the file name 94 | # itself, so we do a hash of that and then use that to generate a 95 | # probability value that we use to assign it. 96 | hash_name_hashed = hashlib.sha1(compat.as_bytes(hash_name)).hexdigest() 97 | percentage_hash = ((int(hash_name_hashed, 16) % 98 | (MAX_NUM_WAVS_PER_CLASS + 1)) * 99 | (100.0 / MAX_NUM_WAVS_PER_CLASS)) 100 | if percentage_hash < validation_percentage: 101 | result = 'validation' 102 | elif percentage_hash < (testing_percentage + validation_percentage): 103 | result = 'testing' 104 | else: 105 | result = 'training' 106 | return result 107 | 108 | 109 | def load_wav_file(filename): 110 | """Loads an audio file and returns a float PCM-encoded array of samples. 111 | 112 | Args: 113 | filename: Path to the .wav file to load. 114 | 115 | Returns: 116 | Numpy array holding the sample data as floats between -1.0 and 1.0. 117 | """ 118 | with tf.Session(graph=tf.Graph()) as sess: 119 | wav_filename_placeholder = tf.placeholder(tf.string, []) 120 | wav_loader = io_ops.read_file(wav_filename_placeholder) 121 | wav_decoder = contrib_audio.decode_wav(wav_loader, desired_channels=1) 122 | return sess.run( 123 | wav_decoder, 124 | feed_dict={wav_filename_placeholder: filename}).audio.flatten() 125 | 126 | 127 | def save_wav_file(filename, wav_data, sample_rate): 128 | """Saves audio sample data to a .wav audio file. 129 | 130 | Args: 131 | filename: Path to save the file to. 132 | wav_data: 2D array of float PCM-encoded audio data. 133 | sample_rate: Samples per second to encode in the file. 134 | """ 135 | with tf.Session(graph=tf.Graph()) as sess: 136 | wav_filename_placeholder = tf.placeholder(tf.string, []) 137 | sample_rate_placeholder = tf.placeholder(tf.int32, []) 138 | wav_data_placeholder = tf.placeholder(tf.float32, [None, 1]) 139 | wav_encoder = contrib_audio.encode_wav(wav_data_placeholder, 140 | sample_rate_placeholder) 141 | wav_saver = io_ops.write_file(wav_filename_placeholder, wav_encoder) 142 | sess.run( 143 | wav_saver, 144 | feed_dict={ 145 | wav_filename_placeholder: filename, 146 | sample_rate_placeholder: sample_rate, 147 | wav_data_placeholder: np.reshape(wav_data, (-1, 1)) 148 | }) 149 | 150 | 151 | class AudioProcessor(object): 152 | """Handles loading, partitioning, and preparing audio training data.""" 153 | 154 | def __init__(self, data_url, data_dir, silence_percentage, unknown_percentage, 155 | wanted_words, validation_percentage, testing_percentage, 156 | model_settings): 157 | self.data_dir = data_dir 158 | self.maybe_download_and_extract_dataset(data_url, data_dir) 159 | self.prepare_data_index(silence_percentage, unknown_percentage, 160 | wanted_words, validation_percentage, 161 | testing_percentage) 162 | self.prepare_background_data() 163 | self.prepare_processing_graph(model_settings) 164 | 165 | def maybe_download_and_extract_dataset(self, data_url, dest_directory): 166 | """Download and extract data set tar file. 167 | 168 | If the data set we're using doesn't already exist, this function 169 | downloads it from the TensorFlow.org website and unpacks it into a 170 | directory. 171 | If the data_url is none, don't download anything and expect the data 172 | directory to contain the correct files already. 173 | 174 | Args: 175 | data_url: Web location of the tar file containing the data set. 176 | dest_directory: File path to extract data to. 177 | """ 178 | if not data_url: 179 | return 180 | if not os.path.exists(dest_directory): 181 | os.makedirs(dest_directory) 182 | filename = data_url.split('/')[-1] 183 | filepath = os.path.join(dest_directory, filename) 184 | if not os.path.exists(filepath): 185 | 186 | def _progress(count, block_size, total_size): 187 | sys.stdout.write( 188 | '\r>> Downloading %s %.1f%%' % 189 | (filename, float(count * block_size) / float(total_size) * 100.0)) 190 | sys.stdout.flush() 191 | 192 | try: 193 | filepath, _ = urllib.request.urlretrieve(data_url, filepath, _progress) 194 | except: 195 | tf.logging.error('Failed to download URL: %s to folder: %s', data_url, 196 | filepath) 197 | tf.logging.error('Please make sure you have enough free space and' 198 | ' an internet connection') 199 | raise 200 | print() 201 | statinfo = os.stat(filepath) 202 | tf.logging.info('Successfully downloaded %s (%d bytes)', filename, 203 | statinfo.st_size) 204 | tarfile.open(filepath, 'r:gz').extractall(dest_directory) 205 | 206 | def prepare_data_index(self, silence_percentage, unknown_percentage, 207 | wanted_words, validation_percentage, 208 | testing_percentage): 209 | """Prepares a list of the samples organized by set and label. 210 | 211 | The training loop needs a list of all the available data, organized by 212 | which partition it should belong to, and with ground truth labels attached. 213 | This function analyzes the folders below the `data_dir`, figures out the 214 | right 215 | labels for each file based on the name of the subdirectory it belongs to, 216 | and uses a stable hash to assign it to a data set partition. 217 | 218 | Args: 219 | silence_percentage: How much of the resulting data should be background. 220 | unknown_percentage: How much should be audio outside the wanted classes. 221 | wanted_words: Labels of the classes we want to be able to recognize. 222 | validation_percentage: How much of the data set to use for validation. 223 | testing_percentage: How much of the data set to use for testing. 224 | 225 | Returns: 226 | Dictionary containing a list of file information for each set partition, 227 | and a lookup map for each class to determine its numeric index. 228 | 229 | Raises: 230 | Exception: If expected files are not found. 231 | """ 232 | # Make sure the shuffling and picking of unknowns is deterministic. 233 | random.seed(RANDOM_SEED) 234 | wanted_words_index = {} 235 | for index, wanted_word in enumerate(wanted_words): 236 | wanted_words_index[wanted_word] = index + 2 237 | self.data_index = {'validation': [], 'testing': [], 'training': []} 238 | unknown_index = {'validation': [], 'testing': [], 'training': []} 239 | all_words = {} 240 | # Look through all the subfolders to find audio samples 241 | search_path = os.path.join(self.data_dir, '*', '*.wav') 242 | for wav_path in gfile.Glob(search_path): 243 | word = re.search('.*/([^/]+)/.*.wav', wav_path).group(1).lower() 244 | # Treat the '_background_noise_' folder as a special case, since we expect 245 | # it to contain long audio samples we mix in to improve training. 246 | if word == BACKGROUND_NOISE_DIR_NAME: 247 | continue 248 | all_words[word] = True 249 | set_index = which_set(wav_path, validation_percentage, testing_percentage) 250 | # If it's a known class, store its detail, otherwise add it to the list 251 | # we'll use to train the unknown label. 252 | if word in wanted_words_index: 253 | self.data_index[set_index].append({'label': word, 'file': wav_path}) 254 | else: 255 | unknown_index[set_index].append({'label': word, 'file': wav_path}) 256 | if not all_words: 257 | raise Exception('No .wavs found at ' + search_path) 258 | for index, wanted_word in enumerate(wanted_words): 259 | if wanted_word not in all_words: 260 | raise Exception('Expected to find ' + wanted_word + 261 | ' in labels but only found ' + 262 | ', '.join(all_words.keys())) 263 | # We need an arbitrary file to load as the input for the silence samples. 264 | # It's multiplied by zero later, so the content doesn't matter. 265 | silence_wav_path = self.data_index['training'][0]['file'] 266 | for set_index in ['validation', 'testing', 'training']: 267 | set_size = len(self.data_index[set_index]) 268 | silence_size = int(math.ceil(set_size * silence_percentage / 100)) 269 | for _ in range(silence_size): 270 | self.data_index[set_index].append({ 271 | 'label': SILENCE_LABEL, 272 | 'file': silence_wav_path 273 | }) 274 | # Pick some unknowns to add to each partition of the data set. 275 | random.shuffle(unknown_index[set_index]) 276 | unknown_size = int(math.ceil(set_size * unknown_percentage / 100)) 277 | self.data_index[set_index].extend(unknown_index[set_index][:unknown_size]) 278 | # Make sure the ordering is random. 279 | for set_index in ['validation', 'testing', 'training']: 280 | random.shuffle(self.data_index[set_index]) 281 | # Prepare the rest of the result data structure. 282 | self.words_list = prepare_words_list(wanted_words) 283 | self.word_to_index = {} 284 | for word in all_words: 285 | if word in wanted_words_index: 286 | self.word_to_index[word] = wanted_words_index[word] 287 | else: 288 | self.word_to_index[word] = UNKNOWN_WORD_INDEX 289 | self.word_to_index[SILENCE_LABEL] = SILENCE_INDEX 290 | 291 | def prepare_background_data(self): 292 | """Searches a folder for background noise audio, and loads it into memory. 293 | 294 | It's expected that the background audio samples will be in a subdirectory 295 | named '_background_noise_' inside the 'data_dir' folder, as .wavs that match 296 | the sample rate of the training data, but can be much longer in duration. 297 | 298 | If the '_background_noise_' folder doesn't exist at all, this isn't an 299 | error, it's just taken to mean that no background noise augmentation should 300 | be used. If the folder does exist, but it's empty, that's treated as an 301 | error. 302 | 303 | Returns: 304 | List of raw PCM-encoded audio samples of background noise. 305 | 306 | Raises: 307 | Exception: If files aren't found in the folder. 308 | """ 309 | self.background_data = [] 310 | background_dir = os.path.join(self.data_dir, BACKGROUND_NOISE_DIR_NAME) 311 | if not os.path.exists(background_dir): 312 | return self.background_data 313 | with tf.Session(graph=tf.Graph()) as sess: 314 | wav_filename_placeholder = tf.placeholder(tf.string, []) 315 | wav_loader = io_ops.read_file(wav_filename_placeholder) 316 | wav_decoder = contrib_audio.decode_wav(wav_loader, desired_channels=1) 317 | search_path = os.path.join(self.data_dir, BACKGROUND_NOISE_DIR_NAME, 318 | '*.wav') 319 | for wav_path in gfile.Glob(search_path): 320 | wav_data = sess.run( 321 | wav_decoder, 322 | feed_dict={wav_filename_placeholder: wav_path}).audio.flatten() 323 | self.background_data.append(wav_data) 324 | if not self.background_data: 325 | raise Exception('No background wav files were found in ' + search_path) 326 | 327 | def prepare_processing_graph(self, model_settings): 328 | """Builds a TensorFlow graph to apply the input distortions. 329 | 330 | Creates a graph that loads a WAVE file, decodes it, scales the volume, 331 | shifts it in time, adds in background noise, calculates a spectrogram, and 332 | then builds an MFCC fingerprint from that. 333 | 334 | This must be called with an active TensorFlow session running, and it 335 | creates multiple placeholder inputs, and one output: 336 | 337 | - wav_filename_placeholder_: Filename of the WAV to load. 338 | - foreground_volume_placeholder_: How loud the main clip should be. 339 | - time_shift_padding_placeholder_: Where to pad the clip. 340 | - time_shift_offset_placeholder_: How much to move the clip in time. 341 | - background_data_placeholder_: PCM sample data for background noise. 342 | - background_volume_placeholder_: Loudness of mixed-in background. 343 | - mfcc_: Output 2D fingerprint of processed audio. 344 | 345 | Args: 346 | model_settings: Information about the current model being trained. 347 | """ 348 | desired_samples = model_settings['desired_samples'] 349 | self.wav_filename_placeholder_ = tf.placeholder(tf.string, []) 350 | wav_loader = io_ops.read_file(self.wav_filename_placeholder_) 351 | wav_decoder = contrib_audio.decode_wav( 352 | wav_loader, desired_channels=1, desired_samples=desired_samples) 353 | # Allow the audio sample's volume to be adjusted. 354 | self.foreground_volume_placeholder_ = tf.placeholder(tf.float32, []) 355 | scaled_foreground = tf.multiply(wav_decoder.audio, 356 | self.foreground_volume_placeholder_) 357 | # Shift the sample's start position, and pad any gaps with zeros. 358 | self.time_shift_padding_placeholder_ = tf.placeholder(tf.int32, [2, 2]) 359 | self.time_shift_offset_placeholder_ = tf.placeholder(tf.int32, [2]) 360 | padded_foreground = tf.pad( 361 | scaled_foreground, 362 | self.time_shift_padding_placeholder_, 363 | mode='CONSTANT') 364 | sliced_foreground = tf.slice(padded_foreground, 365 | self.time_shift_offset_placeholder_, 366 | [desired_samples, -1]) 367 | # Mix in background noise. 368 | self.background_data_placeholder_ = tf.placeholder(tf.float32, 369 | [desired_samples, 1]) 370 | self.background_volume_placeholder_ = tf.placeholder(tf.float32, []) 371 | background_mul = tf.multiply(self.background_data_placeholder_, 372 | self.background_volume_placeholder_) 373 | background_add = tf.add(background_mul, sliced_foreground) 374 | background_clamp = tf.clip_by_value(background_add, -1.0, 1.0) 375 | # Run the spectrogram and MFCC ops to get a 2D 'fingerprint' of the audio. 376 | spectrogram = contrib_audio.audio_spectrogram( 377 | background_clamp, 378 | window_size=model_settings['window_size_samples'], 379 | stride=model_settings['window_stride_samples'], 380 | magnitude_squared=True) 381 | self.mfcc_ = contrib_audio.mfcc( 382 | spectrogram, 383 | wav_decoder.sample_rate, 384 | dct_coefficient_count=model_settings['dct_coefficient_count']) 385 | 386 | def set_size(self, mode): 387 | """Calculates the number of samples in the dataset partition. 388 | 389 | Args: 390 | mode: Which partition, must be 'training', 'validation', or 'testing'. 391 | 392 | Returns: 393 | Number of samples in the partition. 394 | """ 395 | return len(self.data_index[mode]) 396 | 397 | def get_data(self, how_many, offset, model_settings, background_frequency, 398 | background_volume_range, time_shift, mode, sess): 399 | """Gather samples from the data set, applying transformations as needed. 400 | 401 | When the mode is 'training', a random selection of samples will be returned, 402 | otherwise the first N clips in the partition will be used. This ensures that 403 | validation always uses the same samples, reducing noise in the metrics. 404 | 405 | Args: 406 | how_many: Desired number of samples to return. -1 means the entire 407 | contents of this partition. 408 | offset: Where to start when fetching deterministically. 409 | model_settings: Information about the current model being trained. 410 | background_frequency: How many clips will have background noise, 0.0 to 411 | 1.0. 412 | background_volume_range: How loud the background noise will be. 413 | time_shift: How much to randomly shift the clips by in time. 414 | mode: Which partition to use, must be 'training', 'validation', or 415 | 'testing'. 416 | sess: TensorFlow session that was active when processor was created. 417 | 418 | Returns: 419 | List of sample data for the transformed samples, and list of labels in 420 | one-hot form. 421 | """ 422 | # Pick one of the partitions to choose samples from. 423 | candidates = self.data_index[mode] 424 | if how_many == -1: 425 | sample_count = len(candidates) 426 | else: 427 | sample_count = max(0, min(how_many, len(candidates) - offset)) 428 | # Data and labels will be populated and returned. 429 | data = np.zeros((sample_count, model_settings['fingerprint_size'])) 430 | labels = np.zeros((sample_count, model_settings['label_count'])) 431 | desired_samples = model_settings['desired_samples'] 432 | use_background = self.background_data and (mode == 'training') 433 | pick_deterministically = (mode != 'training') 434 | # Use the processing graph we created earlier to repeatedly to generate the 435 | # final output sample data we'll use in training. 436 | for i in xrange(offset, offset + sample_count): 437 | # Pick which audio sample to use. 438 | if how_many == -1 or pick_deterministically: 439 | sample_index = i 440 | else: 441 | sample_index = np.random.randint(len(candidates)) 442 | sample = candidates[sample_index] 443 | # If we're time shifting, set up the offset for this sample. 444 | if time_shift > 0: 445 | time_shift_amount = np.random.randint(-time_shift, time_shift) 446 | else: 447 | time_shift_amount = 0 448 | if time_shift_amount > 0: 449 | time_shift_padding = [[time_shift_amount, 0], [0, 0]] 450 | time_shift_offset = [0, 0] 451 | else: 452 | time_shift_padding = [[0, -time_shift_amount], [0, 0]] 453 | time_shift_offset = [-time_shift_amount, 0] 454 | input_dict = { 455 | self.wav_filename_placeholder_: sample['file'], 456 | self.time_shift_padding_placeholder_: time_shift_padding, 457 | self.time_shift_offset_placeholder_: time_shift_offset, 458 | } 459 | # Choose a section of background noise to mix in. 460 | if use_background: 461 | background_index = np.random.randint(len(self.background_data)) 462 | background_samples = self.background_data[background_index] 463 | background_offset = np.random.randint( 464 | 0, len(background_samples) - model_settings['desired_samples']) 465 | background_clipped = background_samples[background_offset:( 466 | background_offset + desired_samples)] 467 | background_reshaped = background_clipped.reshape([desired_samples, 1]) 468 | if np.random.uniform(0, 1) < background_frequency: 469 | background_volume = np.random.uniform(0, background_volume_range) 470 | else: 471 | background_volume = 0 472 | else: 473 | background_reshaped = np.zeros([desired_samples, 1]) 474 | background_volume = 0 475 | input_dict[self.background_data_placeholder_] = background_reshaped 476 | input_dict[self.background_volume_placeholder_] = background_volume 477 | # If we want silence, mute out the main sample but leave the background. 478 | if sample['label'] == SILENCE_LABEL: 479 | input_dict[self.foreground_volume_placeholder_] = 0 480 | else: 481 | input_dict[self.foreground_volume_placeholder_] = 1 482 | # Run the graph to produce the output audio. 483 | data[i - offset, :] = sess.run(self.mfcc_, feed_dict=input_dict).flatten() 484 | label_index = self.word_to_index[sample['label']] 485 | labels[i - offset, label_index] = 1 486 | return data, labels 487 | 488 | def get_unprocessed_data(self, how_many, model_settings, mode): 489 | """Retrieve sample data for the given partition, with no transformations. 490 | 491 | Args: 492 | how_many: Desired number of samples to return. -1 means the entire 493 | contents of this partition. 494 | model_settings: Information about the current model being trained. 495 | mode: Which partition to use, must be 'training', 'validation', or 496 | 'testing'. 497 | 498 | Returns: 499 | List of sample data for the samples, and list of labels in one-hot form. 500 | """ 501 | candidates = self.data_index[mode] 502 | if how_many == -1: 503 | sample_count = len(candidates) 504 | else: 505 | sample_count = how_many 506 | desired_samples = model_settings['desired_samples'] 507 | words_list = self.words_list 508 | data = np.zeros((sample_count, desired_samples)) 509 | labels = [] 510 | with tf.Session(graph=tf.Graph()) as sess: 511 | wav_filename_placeholder = tf.placeholder(tf.string, []) 512 | wav_loader = io_ops.read_file(wav_filename_placeholder) 513 | wav_decoder = contrib_audio.decode_wav( 514 | wav_loader, desired_channels=1, desired_samples=desired_samples) 515 | foreground_volume_placeholder = tf.placeholder(tf.float32, []) 516 | scaled_foreground = tf.multiply(wav_decoder.audio, 517 | foreground_volume_placeholder) 518 | for i in range(sample_count): 519 | if how_many == -1: 520 | sample_index = i 521 | else: 522 | sample_index = np.random.randint(len(candidates)) 523 | sample = candidates[sample_index] 524 | input_dict = {wav_filename_placeholder: sample['file']} 525 | if sample['label'] == SILENCE_LABEL: 526 | input_dict[foreground_volume_placeholder] = 0 527 | else: 528 | input_dict[foreground_volume_placeholder] = 1 529 | data[i, :] = sess.run(scaled_foreground, feed_dict=input_dict).flatten() 530 | label_index = self.word_to_index[sample['label']] 531 | labels.append(words_list[label_index]) 532 | return data, labels 533 | -------------------------------------------------------------------------------- /speech_commands/input_data_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Tests for data input for speech commands.""" 16 | 17 | from __future__ import absolute_import 18 | from __future__ import division 19 | from __future__ import print_function 20 | 21 | import os 22 | 23 | import numpy as np 24 | import tensorflow as tf 25 | 26 | from tensorflow.contrib.framework.python.ops import audio_ops as contrib_audio 27 | from tensorflow.examples.speech_commands import input_data 28 | from tensorflow.python.platform import test 29 | 30 | 31 | class InputDataTest(test.TestCase): 32 | 33 | def _getWavData(self): 34 | with self.test_session() as sess: 35 | sample_data = tf.zeros([1000, 2]) 36 | wav_encoder = contrib_audio.encode_wav(sample_data, 16000) 37 | wav_data = sess.run(wav_encoder) 38 | return wav_data 39 | 40 | def _saveTestWavFile(self, filename, wav_data): 41 | with open(filename, "wb") as f: 42 | f.write(wav_data) 43 | 44 | def _saveWavFolders(self, root_dir, labels, how_many): 45 | wav_data = self._getWavData() 46 | for label in labels: 47 | dir_name = os.path.join(root_dir, label) 48 | os.mkdir(dir_name) 49 | for i in range(how_many): 50 | file_path = os.path.join(dir_name, "some_audio_%d.wav" % i) 51 | self._saveTestWavFile(file_path, wav_data) 52 | 53 | def _model_settings(self): 54 | return { 55 | "desired_samples": 160, 56 | "fingerprint_size": 40, 57 | "label_count": 4, 58 | "window_size_samples": 100, 59 | "window_stride_samples": 100, 60 | "dct_coefficient_count": 40, 61 | } 62 | 63 | def testPrepareWordsList(self): 64 | words_list = ["a", "b"] 65 | self.assertGreater( 66 | len(input_data.prepare_words_list(words_list)), len(words_list)) 67 | 68 | def testWhichSet(self): 69 | self.assertEqual( 70 | input_data.which_set("foo.wav", 10, 10), 71 | input_data.which_set("foo.wav", 10, 10)) 72 | self.assertEqual( 73 | input_data.which_set("foo_nohash_0.wav", 10, 10), 74 | input_data.which_set("foo_nohash_1.wav", 10, 10)) 75 | 76 | def testPrepareDataIndex(self): 77 | tmp_dir = self.get_temp_dir() 78 | self._saveWavFolders(tmp_dir, ["a", "b", "c"], 100) 79 | audio_processor = input_data.AudioProcessor("", tmp_dir, 10, 10, ["a", "b"], 80 | 10, 10, self._model_settings()) 81 | self.assertLess(0, audio_processor.set_size("training")) 82 | self.assertTrue("training" in audio_processor.data_index) 83 | self.assertTrue("validation" in audio_processor.data_index) 84 | self.assertTrue("testing" in audio_processor.data_index) 85 | self.assertEquals(input_data.UNKNOWN_WORD_INDEX, 86 | audio_processor.word_to_index["c"]) 87 | 88 | def testPrepareDataIndexEmpty(self): 89 | tmp_dir = self.get_temp_dir() 90 | self._saveWavFolders(tmp_dir, ["a", "b", "c"], 0) 91 | with self.assertRaises(Exception) as e: 92 | _ = input_data.AudioProcessor("", tmp_dir, 10, 10, ["a", "b"], 10, 10, 93 | self._model_settings()) 94 | self.assertTrue("No .wavs found" in str(e.exception)) 95 | 96 | def testPrepareDataIndexMissing(self): 97 | tmp_dir = self.get_temp_dir() 98 | self._saveWavFolders(tmp_dir, ["a", "b", "c"], 100) 99 | with self.assertRaises(Exception) as e: 100 | _ = input_data.AudioProcessor("", tmp_dir, 10, 10, ["a", "b", "d"], 10, 101 | 10, self._model_settings()) 102 | self.assertTrue("Expected to find" in str(e.exception)) 103 | 104 | def testPrepareBackgroundData(self): 105 | tmp_dir = self.get_temp_dir() 106 | background_dir = os.path.join(tmp_dir, "_background_noise_") 107 | os.mkdir(background_dir) 108 | wav_data = self._getWavData() 109 | for i in range(10): 110 | file_path = os.path.join(background_dir, "background_audio_%d.wav" % i) 111 | self._saveTestWavFile(file_path, wav_data) 112 | self._saveWavFolders(tmp_dir, ["a", "b", "c"], 100) 113 | audio_processor = input_data.AudioProcessor("", tmp_dir, 10, 10, ["a", "b"], 114 | 10, 10, self._model_settings()) 115 | self.assertEqual(10, len(audio_processor.background_data)) 116 | 117 | def testLoadWavFile(self): 118 | tmp_dir = self.get_temp_dir() 119 | file_path = os.path.join(tmp_dir, "load_test.wav") 120 | wav_data = self._getWavData() 121 | self._saveTestWavFile(file_path, wav_data) 122 | sample_data = input_data.load_wav_file(file_path) 123 | self.assertIsNotNone(sample_data) 124 | 125 | def testSaveWavFile(self): 126 | tmp_dir = self.get_temp_dir() 127 | file_path = os.path.join(tmp_dir, "load_test.wav") 128 | save_data = np.zeros([16000, 1]) 129 | input_data.save_wav_file(file_path, save_data, 16000) 130 | loaded_data = input_data.load_wav_file(file_path) 131 | self.assertIsNotNone(loaded_data) 132 | self.assertEqual(16000, len(loaded_data)) 133 | 134 | def testPrepareProcessingGraph(self): 135 | tmp_dir = self.get_temp_dir() 136 | wav_dir = os.path.join(tmp_dir, "wavs") 137 | os.mkdir(wav_dir) 138 | self._saveWavFolders(wav_dir, ["a", "b", "c"], 100) 139 | background_dir = os.path.join(wav_dir, "_background_noise_") 140 | os.mkdir(background_dir) 141 | wav_data = self._getWavData() 142 | for i in range(10): 143 | file_path = os.path.join(background_dir, "background_audio_%d.wav" % i) 144 | self._saveTestWavFile(file_path, wav_data) 145 | model_settings = { 146 | "desired_samples": 160, 147 | "fingerprint_size": 40, 148 | "label_count": 4, 149 | "window_size_samples": 100, 150 | "window_stride_samples": 100, 151 | "dct_coefficient_count": 40, 152 | } 153 | audio_processor = input_data.AudioProcessor("", wav_dir, 10, 10, ["a", "b"], 154 | 10, 10, model_settings) 155 | self.assertIsNotNone(audio_processor.wav_filename_placeholder_) 156 | self.assertIsNotNone(audio_processor.foreground_volume_placeholder_) 157 | self.assertIsNotNone(audio_processor.time_shift_padding_placeholder_) 158 | self.assertIsNotNone(audio_processor.time_shift_offset_placeholder_) 159 | self.assertIsNotNone(audio_processor.background_data_placeholder_) 160 | self.assertIsNotNone(audio_processor.background_volume_placeholder_) 161 | self.assertIsNotNone(audio_processor.mfcc_) 162 | 163 | def testGetData(self): 164 | tmp_dir = self.get_temp_dir() 165 | wav_dir = os.path.join(tmp_dir, "wavs") 166 | os.mkdir(wav_dir) 167 | self._saveWavFolders(wav_dir, ["a", "b", "c"], 100) 168 | background_dir = os.path.join(wav_dir, "_background_noise_") 169 | os.mkdir(background_dir) 170 | wav_data = self._getWavData() 171 | for i in range(10): 172 | file_path = os.path.join(background_dir, "background_audio_%d.wav" % i) 173 | self._saveTestWavFile(file_path, wav_data) 174 | model_settings = { 175 | "desired_samples": 160, 176 | "fingerprint_size": 40, 177 | "label_count": 4, 178 | "window_size_samples": 100, 179 | "window_stride_samples": 100, 180 | "dct_coefficient_count": 40, 181 | } 182 | audio_processor = input_data.AudioProcessor("", wav_dir, 10, 10, ["a", "b"], 183 | 10, 10, model_settings) 184 | with self.test_session() as sess: 185 | result_data, result_labels = audio_processor.get_data( 186 | 10, 0, model_settings, 0.3, 0.1, 100, "training", sess) 187 | self.assertEqual(10, len(result_data)) 188 | self.assertEqual(10, len(result_labels)) 189 | 190 | def testGetUnprocessedData(self): 191 | tmp_dir = self.get_temp_dir() 192 | wav_dir = os.path.join(tmp_dir, "wavs") 193 | os.mkdir(wav_dir) 194 | self._saveWavFolders(wav_dir, ["a", "b", "c"], 100) 195 | model_settings = { 196 | "desired_samples": 160, 197 | "fingerprint_size": 40, 198 | "label_count": 4, 199 | "window_size_samples": 100, 200 | "window_stride_samples": 100, 201 | "dct_coefficient_count": 40, 202 | } 203 | audio_processor = input_data.AudioProcessor("", wav_dir, 10, 10, ["a", "b"], 204 | 10, 10, model_settings) 205 | result_data, result_labels = audio_processor.get_unprocessed_data( 206 | 10, model_settings, "training") 207 | self.assertEqual(10, len(result_data)) 208 | self.assertEqual(10, len(result_labels)) 209 | 210 | 211 | if __name__ == "__main__": 212 | test.main() 213 | -------------------------------------------------------------------------------- /speech_commands/label_wav.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include 17 | #include 18 | 19 | #include "tensorflow/core/framework/tensor.h" 20 | #include "tensorflow/core/lib/io/path.h" 21 | #include "tensorflow/core/platform/init_main.h" 22 | #include "tensorflow/core/platform/logging.h" 23 | #include "tensorflow/core/platform/types.h" 24 | #include "tensorflow/core/public/session.h" 25 | #include "tensorflow/core/util/command_line_flags.h" 26 | 27 | // These are all common classes it's handy to reference with no namespace. 28 | using tensorflow::Flag; 29 | using tensorflow::Status; 30 | using tensorflow::Tensor; 31 | using tensorflow::int32; 32 | using tensorflow::string; 33 | 34 | namespace { 35 | 36 | // Reads a model graph definition from disk, and creates a session object you 37 | // can use to run it. 38 | Status LoadGraph(const string& graph_file_name, 39 | std::unique_ptr* session) { 40 | tensorflow::GraphDef graph_def; 41 | Status load_graph_status = 42 | ReadBinaryProto(tensorflow::Env::Default(), graph_file_name, &graph_def); 43 | if (!load_graph_status.ok()) { 44 | return tensorflow::errors::NotFound("Failed to load compute graph at '", 45 | graph_file_name, "'"); 46 | } 47 | session->reset(tensorflow::NewSession(tensorflow::SessionOptions())); 48 | Status session_create_status = (*session)->Create(graph_def); 49 | if (!session_create_status.ok()) { 50 | return session_create_status; 51 | } 52 | return Status::OK(); 53 | } 54 | 55 | // Takes a file name, and loads a list of labels from it, one per line, and 56 | // returns a vector of the strings. 57 | Status ReadLabelsFile(const string& file_name, std::vector* result) { 58 | std::ifstream file(file_name); 59 | if (!file) { 60 | return tensorflow::errors::NotFound("Labels file ", file_name, 61 | " not found."); 62 | } 63 | result->clear(); 64 | string line; 65 | while (std::getline(file, line)) { 66 | result->push_back(line); 67 | } 68 | return Status::OK(); 69 | } 70 | 71 | // Analyzes the output of the graph to retrieve the highest scores and 72 | // their positions in the tensor. 73 | void GetTopLabels(const std::vector& outputs, int how_many_labels, 74 | Tensor* out_indices, Tensor* out_scores) { 75 | const Tensor& unsorted_scores_tensor = outputs[0]; 76 | auto unsorted_scores_flat = unsorted_scores_tensor.flat(); 77 | std::vector> scores; 78 | scores.reserve(unsorted_scores_flat.size()); 79 | for (int i = 0; i < unsorted_scores_flat.size(); ++i) { 80 | scores.push_back(std::pair({i, unsorted_scores_flat(i)})); 81 | } 82 | std::sort(scores.begin(), scores.end(), 83 | [](const std::pair& left, 84 | const std::pair& right) { 85 | return left.second > right.second; 86 | }); 87 | scores.resize(how_many_labels); 88 | Tensor sorted_indices(tensorflow::DT_INT32, {how_many_labels}); 89 | Tensor sorted_scores(tensorflow::DT_FLOAT, {how_many_labels}); 90 | for (int i = 0; i < scores.size(); ++i) { 91 | sorted_indices.flat()(i) = scores[i].first; 92 | sorted_scores.flat()(i) = scores[i].second; 93 | } 94 | *out_indices = sorted_indices; 95 | *out_scores = sorted_scores; 96 | } 97 | 98 | } // namespace 99 | 100 | int main(int argc, char* argv[]) { 101 | string wav = ""; 102 | string graph = ""; 103 | string labels = ""; 104 | string input_name = "wav_data"; 105 | string output_name = "labels_softmax"; 106 | int32 how_many_labels = 3; 107 | std::vector flag_list = { 108 | Flag("wav", &wav, "audio file to be identified"), 109 | Flag("graph", &graph, "model to be executed"), 110 | Flag("labels", &labels, "path to file containing labels"), 111 | Flag("input_name", &input_name, "name of input node in model"), 112 | Flag("output_name", &output_name, "name of output node in model"), 113 | Flag("how_many_labels", &how_many_labels, "number of results to show"), 114 | }; 115 | string usage = tensorflow::Flags::Usage(argv[0], flag_list); 116 | const bool parse_result = tensorflow::Flags::Parse(&argc, argv, flag_list); 117 | if (!parse_result) { 118 | LOG(ERROR) << usage; 119 | return -1; 120 | } 121 | 122 | // We need to call this to set up global state for TensorFlow. 123 | tensorflow::port::InitMain(argv[0], &argc, &argv); 124 | if (argc > 1) { 125 | LOG(ERROR) << "Unknown argument " << argv[1] << "\n" << usage; 126 | return -1; 127 | } 128 | 129 | // First we load and initialize the model. 130 | std::unique_ptr session; 131 | Status load_graph_status = LoadGraph(graph, &session); 132 | if (!load_graph_status.ok()) { 133 | LOG(ERROR) << load_graph_status; 134 | return -1; 135 | } 136 | 137 | std::vector labels_list; 138 | Status read_labels_status = ReadLabelsFile(labels, &labels_list); 139 | if (!read_labels_status.ok()) { 140 | LOG(ERROR) << read_labels_status; 141 | return -1; 142 | } 143 | 144 | string wav_string; 145 | Status read_wav_status = tensorflow::ReadFileToString( 146 | tensorflow::Env::Default(), wav, &wav_string); 147 | if (!read_wav_status.ok()) { 148 | LOG(ERROR) << read_wav_status; 149 | return -1; 150 | } 151 | Tensor wav_tensor(tensorflow::DT_STRING, tensorflow::TensorShape({})); 152 | wav_tensor.scalar()() = wav_string; 153 | 154 | // Actually run the audio through the model. 155 | std::vector outputs; 156 | Status run_status = 157 | session->Run({{input_name, wav_tensor}}, {output_name}, {}, &outputs); 158 | if (!run_status.ok()) { 159 | LOG(ERROR) << "Running model failed: " << run_status; 160 | return -1; 161 | } 162 | 163 | Tensor indices; 164 | Tensor scores; 165 | GetTopLabels(outputs, how_many_labels, &indices, &scores); 166 | tensorflow::TTypes::Flat scores_flat = scores.flat(); 167 | tensorflow::TTypes::Flat indices_flat = indices.flat(); 168 | for (int pos = 0; pos < how_many_labels; ++pos) { 169 | const int label_index = indices_flat(pos); 170 | const float score = scores_flat(pos); 171 | LOG(INFO) << labels_list[label_index] << " (" << label_index 172 | << "): " << score; 173 | } 174 | 175 | return 0; 176 | } 177 | -------------------------------------------------------------------------------- /speech_commands/label_wav.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | r"""Runs a trained audio graph against a WAVE file and reports the results. 16 | 17 | The model, labels and .wav file specified in the arguments will be loaded, and 18 | then the predictions from running the model against the audio data will be 19 | printed to the console. This is a useful script for sanity checking trained 20 | models, and as an example of how to use an audio model from Python. 21 | 22 | Here's an example of running it: 23 | 24 | python tensorflow/examples/speech_commands/label_wav.py \ 25 | --graph=/tmp/my_frozen_graph.pb \ 26 | --labels=/tmp/speech_commands_train/conv_labels.txt \ 27 | --wav=/tmp/speech_dataset/left/a5d485dc_nohash_0.wav 28 | 29 | """ 30 | from __future__ import absolute_import 31 | from __future__ import division 32 | from __future__ import print_function 33 | 34 | import argparse 35 | import sys 36 | 37 | import tensorflow as tf 38 | 39 | # pylint: disable=unused-import 40 | from tensorflow.contrib.framework.python.ops import audio_ops as contrib_audio 41 | # pylint: enable=unused-import 42 | 43 | FLAGS = None 44 | 45 | 46 | def load_graph(filename): 47 | """Unpersists graph from file as default graph.""" 48 | with tf.gfile.FastGFile(filename, 'rb') as f: 49 | graph_def = tf.GraphDef() 50 | graph_def.ParseFromString(f.read()) 51 | tf.import_graph_def(graph_def, name='') 52 | 53 | 54 | def load_labels(filename): 55 | """Read in labels, one label per line.""" 56 | return [line.rstrip() for line in tf.gfile.GFile(filename)] 57 | 58 | 59 | def run_graph(wav_data, labels, input_layer_name, output_layer_name, 60 | num_top_predictions): 61 | """Runs the audio data through the graph and prints predictions.""" 62 | with tf.Session() as sess: 63 | # Feed the audio data as input to the graph. 64 | # predictions will contain a two-dimensional array, where one 65 | # dimension represents the input image count, and the other has 66 | # predictions per class 67 | softmax_tensor = sess.graph.get_tensor_by_name(output_layer_name) 68 | 69 | predictions, = sess.run(softmax_tensor, {input_layer_name: wav_data}) 70 | # sess.run(grads_op, feed_dict={input_layer_na 71 | 72 | # Sort to show labels in order of confidence 73 | top_k = predictions.argsort()[-num_top_predictions:][::-1] 74 | for node_id in top_k: 75 | human_string = labels[node_id] 76 | score = predictions[node_id] 77 | print('%s (score = %.5f)' % (human_string, score)) 78 | 79 | return 0 80 | 81 | 82 | def label_wav(wav, labels, graph, input_name, output_name, how_many_labels): 83 | """Loads the model and labels, and runs the inference to print predictions.""" 84 | if not wav or not tf.gfile.Exists(wav): 85 | tf.logging.fatal('Audio file does not exist %s', wav) 86 | 87 | if not labels or not tf.gfile.Exists(labels): 88 | tf.logging.fatal('Labels file does not exist %s', labels) 89 | 90 | if not graph or not tf.gfile.Exists(graph): 91 | tf.logging.fatal('Graph file does not exist %s', graph) 92 | 93 | labels_list = load_labels(labels) 94 | 95 | # load graph, which is stored in the default session 96 | load_graph(graph) 97 | 98 | with open(wav, 'rb') as wav_file: 99 | wav_data = wav_file.read() 100 | 101 | run_graph(wav_data, labels_list, input_name, output_name, how_many_labels) 102 | 103 | 104 | def main(_): 105 | """Entry point for script, converts flags to arguments.""" 106 | label_wav(FLAGS.wav, FLAGS.labels, FLAGS.graph, FLAGS.input_name, 107 | FLAGS.output_name, FLAGS.how_many_labels) 108 | 109 | 110 | if __name__ == '__main__': 111 | parser = argparse.ArgumentParser() 112 | parser.add_argument( 113 | '--wav', type=str, default='', help='Audio file to be identified.') 114 | parser.add_argument( 115 | '--graph', type=str, default='', help='Model to use for identification.') 116 | parser.add_argument( 117 | '--labels', type=str, default='', help='Path to file containing labels.') 118 | parser.add_argument( 119 | '--input_name', 120 | type=str, 121 | default='wav_data:0', 122 | help='Name of WAVE data input node in model.') 123 | parser.add_argument( 124 | '--output_name', 125 | type=str, 126 | default='labels_softmax:0', 127 | help='Name of node outputting a prediction in the model.') 128 | parser.add_argument( 129 | '--how_many_labels', 130 | type=int, 131 | default=3, 132 | help='Number of results to show.') 133 | 134 | FLAGS, unparsed = parser.parse_known_args() 135 | tf.app.run(main=main, argv=[sys.argv[0]] + unparsed) 136 | -------------------------------------------------------------------------------- /speech_commands/label_wav_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Tests for WAVE file labeling tool.""" 16 | 17 | from __future__ import absolute_import 18 | from __future__ import division 19 | from __future__ import print_function 20 | 21 | import os 22 | 23 | import tensorflow as tf 24 | 25 | from tensorflow.contrib.framework.python.ops import audio_ops as contrib_audio 26 | from tensorflow.examples.speech_commands import label_wav 27 | from tensorflow.python.platform import test 28 | 29 | 30 | class LabelWavTest(test.TestCase): 31 | 32 | def _getWavData(self): 33 | with self.test_session() as sess: 34 | sample_data = tf.zeros([1000, 2]) 35 | wav_encoder = contrib_audio.encode_wav(sample_data, 16000) 36 | wav_data = sess.run(wav_encoder) 37 | return wav_data 38 | 39 | def _saveTestWavFile(self, filename, wav_data): 40 | with open(filename, "wb") as f: 41 | f.write(wav_data) 42 | 43 | def testLabelWav(self): 44 | tmp_dir = self.get_temp_dir() 45 | wav_data = self._getWavData() 46 | wav_filename = os.path.join(tmp_dir, "wav_file.wav") 47 | self._saveTestWavFile(wav_filename, wav_data) 48 | input_name = "test_input" 49 | output_name = "test_output" 50 | graph_filename = os.path.join(tmp_dir, "test_graph.pb") 51 | with tf.Session() as sess: 52 | tf.placeholder(tf.string, name=input_name) 53 | tf.zeros([1, 3], name=output_name) 54 | with open(graph_filename, "wb") as f: 55 | f.write(sess.graph.as_graph_def().SerializeToString()) 56 | labels_filename = os.path.join(tmp_dir, "test_labels.txt") 57 | with open(labels_filename, "w") as f: 58 | f.write("a\nb\nc\n") 59 | label_wav.label_wav(wav_filename, labels_filename, graph_filename, 60 | input_name + ":0", output_name + ":0", 3) 61 | 62 | 63 | if __name__ == "__main__": 64 | test.main() 65 | -------------------------------------------------------------------------------- /speech_commands/models.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Model definitions for simple speech recognition. 16 | 17 | """ 18 | from __future__ import absolute_import 19 | from __future__ import division 20 | from __future__ import print_function 21 | 22 | import math 23 | 24 | import tensorflow as tf 25 | 26 | 27 | def prepare_model_settings(label_count, sample_rate, clip_duration_ms, 28 | window_size_ms, window_stride_ms, 29 | dct_coefficient_count): 30 | """Calculates common settings needed for all models. 31 | 32 | Args: 33 | label_count: How many classes are to be recognized. 34 | sample_rate: Number of audio samples per second. 35 | clip_duration_ms: Length of each audio clip to be analyzed. 36 | window_size_ms: Duration of frequency analysis window. 37 | window_stride_ms: How far to move in time between frequency windows. 38 | dct_coefficient_count: Number of frequency bins to use for analysis. 39 | 40 | Returns: 41 | Dictionary containing common settings. 42 | """ 43 | desired_samples = int(sample_rate * clip_duration_ms / 1000) 44 | window_size_samples = int(sample_rate * window_size_ms / 1000) 45 | window_stride_samples = int(sample_rate * window_stride_ms / 1000) 46 | length_minus_window = (desired_samples - window_size_samples) 47 | if length_minus_window < 0: 48 | spectrogram_length = 0 49 | else: 50 | spectrogram_length = 1 + int(length_minus_window / window_stride_samples) 51 | fingerprint_size = dct_coefficient_count * spectrogram_length 52 | return { 53 | 'desired_samples': desired_samples, 54 | 'window_size_samples': window_size_samples, 55 | 'window_stride_samples': window_stride_samples, 56 | 'spectrogram_length': spectrogram_length, 57 | 'dct_coefficient_count': dct_coefficient_count, 58 | 'fingerprint_size': fingerprint_size, 59 | 'label_count': label_count, 60 | 'sample_rate': sample_rate, 61 | } 62 | 63 | 64 | def create_model(fingerprint_input, model_settings, model_architecture, 65 | is_training, runtime_settings=None): 66 | """Builds a model of the requested architecture compatible with the settings. 67 | 68 | There are many possible ways of deriving predictions from a spectrogram 69 | input, so this function provides an abstract interface for creating different 70 | kinds of models in a black-box way. You need to pass in a TensorFlow node as 71 | the 'fingerprint' input, and this should output a batch of 1D features that 72 | describe the audio. Typically this will be derived from a spectrogram that's 73 | been run through an MFCC, but in theory it can be any feature vector of the 74 | size specified in model_settings['fingerprint_size']. 75 | 76 | The function will build the graph it needs in the current TensorFlow graph, 77 | and return the tensorflow output that will contain the 'logits' input to the 78 | softmax prediction process. If training flag is on, it will also return a 79 | placeholder node that can be used to control the dropout amount. 80 | 81 | See the implementations below for the possible model architectures that can be 82 | requested. 83 | 84 | Args: 85 | fingerprint_input: TensorFlow node that will output audio feature vectors. 86 | model_settings: Dictionary of information about the model. 87 | model_architecture: String specifying which kind of model to create. 88 | is_training: Whether the model is going to be used for training. 89 | runtime_settings: Dictionary of information about the runtime. 90 | 91 | Returns: 92 | TensorFlow node outputting logits results, and optionally a dropout 93 | placeholder. 94 | 95 | Raises: 96 | Exception: If the architecture type isn't recognized. 97 | """ 98 | print(fingerprint_input) 99 | if model_architecture == 'single_fc': 100 | return create_single_fc_model(fingerprint_input, model_settings, 101 | is_training) 102 | elif model_architecture == 'conv': 103 | return create_conv_model(fingerprint_input, model_settings, is_training) 104 | elif model_architecture == 'low_latency_conv': 105 | return create_low_latency_conv_model(fingerprint_input, model_settings, 106 | is_training) 107 | elif model_architecture == 'low_latency_svdf': 108 | return create_low_latency_svdf_model(fingerprint_input, model_settings, 109 | is_training, runtime_settings) 110 | else: 111 | raise Exception('model_architecture argument "' + model_architecture + 112 | '" not recognized, should be one of "single_fc", "conv",' + 113 | ' "low_latency_conv, or "low_latency_svdf"') 114 | 115 | 116 | def load_variables_from_checkpoint(sess, start_checkpoint): 117 | """Utility function to centralize checkpoint restoration. 118 | 119 | Args: 120 | sess: TensorFlow session. 121 | start_checkpoint: Path to saved checkpoint on disk. 122 | """ 123 | saver = tf.train.Saver(tf.global_variables()) 124 | saver.restore(sess, start_checkpoint) 125 | 126 | 127 | def create_single_fc_model(fingerprint_input, model_settings, is_training): 128 | """Builds a model with a single hidden fully-connected layer. 129 | 130 | This is a very simple model with just one matmul and bias layer. As you'd 131 | expect, it doesn't produce very accurate results, but it is very fast and 132 | simple, so it's useful for sanity testing. 133 | 134 | Here's the layout of the graph: 135 | 136 | (fingerprint_input) 137 | v 138 | [MatMul]<-(weights) 139 | v 140 | [BiasAdd]<-(bias) 141 | v 142 | 143 | Args: 144 | fingerprint_input: TensorFlow node that will output audio feature vectors. 145 | model_settings: Dictionary of information about the model. 146 | is_training: Whether the model is going to be used for training. 147 | 148 | Returns: 149 | TensorFlow node outputting logits results, and optionally a dropout 150 | placeholder. 151 | """ 152 | if is_training: 153 | dropout_prob = tf.placeholder(tf.float32, name='dropout_prob') 154 | fingerprint_size = model_settings['fingerprint_size'] 155 | label_count = model_settings['label_count'] 156 | weights = tf.Variable( 157 | tf.truncated_normal([fingerprint_size, label_count], stddev=0.001)) 158 | bias = tf.Variable(tf.zeros([label_count])) 159 | logits = tf.matmul(fingerprint_input, weights) + bias 160 | if is_training: 161 | return logits, dropout_prob 162 | else: 163 | return logits 164 | 165 | 166 | def create_conv_model(fingerprint_input, model_settings, is_training): 167 | """Builds a standard convolutional model. 168 | 169 | This is roughly the network labeled as 'cnn-trad-fpool3' in the 170 | 'Convolutional Neural Networks for Small-footprint Keyword Spotting' paper: 171 | http://www.isca-speech.org/archive/interspeech_2015/papers/i15_1478.pdf 172 | 173 | Here's the layout of the graph: 174 | 175 | (fingerprint_input) 176 | v 177 | [Conv2D]<-(weights) 178 | v 179 | [BiasAdd]<-(bias) 180 | v 181 | [Relu] 182 | v 183 | [MaxPool] 184 | v 185 | [Conv2D]<-(weights) 186 | v 187 | [BiasAdd]<-(bias) 188 | v 189 | [Relu] 190 | v 191 | [MaxPool] 192 | v 193 | [MatMul]<-(weights) 194 | v 195 | [BiasAdd]<-(bias) 196 | v 197 | 198 | This produces fairly good quality results, but can involve a large number of 199 | weight parameters and computations. For a cheaper alternative from the same 200 | paper with slightly less accuracy, see 'low_latency_conv' below. 201 | 202 | During training, dropout nodes are introduced after each relu, controlled by a 203 | placeholder. 204 | 205 | Args: 206 | fingerprint_input: TensorFlow node that will output audio feature vectors. 207 | model_settings: Dictionary of information about the model. 208 | is_training: Whether the model is going to be used for training. 209 | 210 | Returns: 211 | TensorFlow node outputting logits results, and optionally a dropout 212 | placeholder. 213 | """ 214 | if is_training: 215 | dropout_prob = tf.placeholder(tf.float32, name='dropout_prob') 216 | input_frequency_size = model_settings['dct_coefficient_count'] 217 | input_time_size = model_settings['spectrogram_length'] 218 | fingerprint_4d = tf.reshape(fingerprint_input, 219 | [-1, input_time_size, input_frequency_size, 1]) 220 | first_filter_width = 8 221 | first_filter_height = 20 222 | first_filter_count = 64 223 | first_weights = tf.Variable( 224 | tf.truncated_normal( 225 | [first_filter_height, first_filter_width, 1, first_filter_count], 226 | stddev=0.01)) 227 | first_bias = tf.Variable(tf.zeros([first_filter_count])) 228 | first_conv = tf.nn.conv2d(fingerprint_4d, first_weights, [1, 1, 1, 1], 229 | 'SAME') + first_bias 230 | first_relu = tf.nn.relu(first_conv) 231 | if is_training: 232 | first_dropout = tf.nn.dropout(first_relu, dropout_prob) 233 | else: 234 | first_dropout = first_relu 235 | max_pool = tf.nn.max_pool(first_dropout, [1, 2, 2, 1], [1, 2, 2, 1], 'SAME') 236 | second_filter_width = 4 237 | second_filter_height = 10 238 | second_filter_count = 64 239 | second_weights = tf.Variable( 240 | tf.truncated_normal( 241 | [ 242 | second_filter_height, second_filter_width, first_filter_count, 243 | second_filter_count 244 | ], 245 | stddev=0.01)) 246 | second_bias = tf.Variable(tf.zeros([second_filter_count])) 247 | second_conv = tf.nn.conv2d(max_pool, second_weights, [1, 1, 1, 1], 248 | 'SAME') + second_bias 249 | second_relu = tf.nn.relu(second_conv) 250 | if is_training: 251 | second_dropout = tf.nn.dropout(second_relu, dropout_prob) 252 | else: 253 | second_dropout = second_relu 254 | second_conv_shape = second_dropout.get_shape() 255 | second_conv_output_width = second_conv_shape[2] 256 | second_conv_output_height = second_conv_shape[1] 257 | second_conv_element_count = int( 258 | second_conv_output_width * second_conv_output_height * 259 | second_filter_count) 260 | flattened_second_conv = tf.reshape(second_dropout, 261 | [-1, second_conv_element_count]) 262 | label_count = model_settings['label_count'] 263 | final_fc_weights = tf.Variable( 264 | tf.truncated_normal( 265 | [second_conv_element_count, label_count], stddev=0.01)) 266 | final_fc_bias = tf.Variable(tf.zeros([label_count])) 267 | final_fc = tf.matmul(flattened_second_conv, final_fc_weights) + final_fc_bias 268 | if is_training: 269 | return final_fc, dropout_prob 270 | else: 271 | return final_fc 272 | 273 | 274 | def create_low_latency_conv_model(fingerprint_input, model_settings, 275 | is_training): 276 | """Builds a convolutional model with low compute requirements. 277 | 278 | This is roughly the network labeled as 'cnn-one-fstride4' in the 279 | 'Convolutional Neural Networks for Small-footprint Keyword Spotting' paper: 280 | http://www.isca-speech.org/archive/interspeech_2015/papers/i15_1478.pdf 281 | 282 | Here's the layout of the graph: 283 | 284 | (fingerprint_input) 285 | v 286 | [Conv2D]<-(weights) 287 | v 288 | [BiasAdd]<-(bias) 289 | v 290 | [Relu] 291 | v 292 | [MatMul]<-(weights) 293 | v 294 | [BiasAdd]<-(bias) 295 | v 296 | [MatMul]<-(weights) 297 | v 298 | [BiasAdd]<-(bias) 299 | v 300 | [MatMul]<-(weights) 301 | v 302 | [BiasAdd]<-(bias) 303 | v 304 | 305 | This produces slightly lower quality results than the 'conv' model, but needs 306 | fewer weight parameters and computations. 307 | 308 | During training, dropout nodes are introduced after the relu, controlled by a 309 | placeholder. 310 | 311 | Args: 312 | fingerprint_input: TensorFlow node that will output audio feature vectors. 313 | model_settings: Dictionary of information about the model. 314 | is_training: Whether the model is going to be used for training. 315 | 316 | Returns: 317 | TensorFlow node outputting logits results, and optionally a dropout 318 | placeholder. 319 | """ 320 | if is_training: 321 | dropout_prob = tf.placeholder(tf.float32, name='dropout_prob') 322 | input_frequency_size = model_settings['dct_coefficient_count'] 323 | input_time_size = model_settings['spectrogram_length'] 324 | fingerprint_4d = tf.reshape(fingerprint_input, 325 | [-1, input_time_size, input_frequency_size, 1]) 326 | first_filter_width = 8 327 | first_filter_height = input_time_size 328 | first_filter_count = 186 329 | first_filter_stride_x = 1 330 | first_filter_stride_y = 4 331 | first_weights = tf.Variable( 332 | tf.truncated_normal( 333 | [first_filter_height, first_filter_width, 1, first_filter_count], 334 | stddev=0.01)) 335 | first_bias = tf.Variable(tf.zeros([first_filter_count])) 336 | first_conv = tf.nn.conv2d(fingerprint_4d, first_weights, [ 337 | 1, first_filter_stride_y, first_filter_stride_x, 1 338 | ], 'VALID') + first_bias 339 | first_relu = tf.nn.relu(first_conv) 340 | if is_training: 341 | first_dropout = tf.nn.dropout(first_relu, dropout_prob) 342 | else: 343 | first_dropout = first_relu 344 | first_conv_output_width = math.floor( 345 | (input_frequency_size - first_filter_width + first_filter_stride_x) / 346 | first_filter_stride_x) 347 | first_conv_output_height = math.floor( 348 | (input_time_size - first_filter_height + first_filter_stride_y) / 349 | first_filter_stride_y) 350 | first_conv_element_count = int( 351 | first_conv_output_width * first_conv_output_height * first_filter_count) 352 | flattened_first_conv = tf.reshape(first_dropout, 353 | [-1, first_conv_element_count]) 354 | first_fc_output_channels = 128 355 | first_fc_weights = tf.Variable( 356 | tf.truncated_normal( 357 | [first_conv_element_count, first_fc_output_channels], stddev=0.01)) 358 | first_fc_bias = tf.Variable(tf.zeros([first_fc_output_channels])) 359 | first_fc = tf.matmul(flattened_first_conv, first_fc_weights) + first_fc_bias 360 | if is_training: 361 | second_fc_input = tf.nn.dropout(first_fc, dropout_prob) 362 | else: 363 | second_fc_input = first_fc 364 | second_fc_output_channels = 128 365 | second_fc_weights = tf.Variable( 366 | tf.truncated_normal( 367 | [first_fc_output_channels, second_fc_output_channels], stddev=0.01)) 368 | second_fc_bias = tf.Variable(tf.zeros([second_fc_output_channels])) 369 | second_fc = tf.matmul(second_fc_input, second_fc_weights) + second_fc_bias 370 | if is_training: 371 | final_fc_input = tf.nn.dropout(second_fc, dropout_prob) 372 | else: 373 | final_fc_input = second_fc 374 | label_count = model_settings['label_count'] 375 | final_fc_weights = tf.Variable( 376 | tf.truncated_normal( 377 | [second_fc_output_channels, label_count], stddev=0.01)) 378 | final_fc_bias = tf.Variable(tf.zeros([label_count])) 379 | final_fc = tf.matmul(final_fc_input, final_fc_weights) + final_fc_bias 380 | if is_training: 381 | return final_fc, dropout_prob 382 | else: 383 | return final_fc 384 | 385 | 386 | def create_low_latency_svdf_model(fingerprint_input, model_settings, 387 | is_training, runtime_settings): 388 | """Builds an SVDF model with low compute requirements. 389 | 390 | This is based in the topology presented in the 'Compressing Deep Neural 391 | Networks using a Rank-Constrained Topology' paper: 392 | https://static.googleusercontent.com/media/research.google.com/en//pubs/archive/43813.pdf 393 | 394 | Here's the layout of the graph: 395 | 396 | (fingerprint_input) 397 | v 398 | [SVDF]<-(weights) 399 | v 400 | [BiasAdd]<-(bias) 401 | v 402 | [Relu] 403 | v 404 | [MatMul]<-(weights) 405 | v 406 | [BiasAdd]<-(bias) 407 | v 408 | [MatMul]<-(weights) 409 | v 410 | [BiasAdd]<-(bias) 411 | v 412 | [MatMul]<-(weights) 413 | v 414 | [BiasAdd]<-(bias) 415 | v 416 | 417 | This model produces lower recognition accuracy than the 'conv' model above, 418 | but requires fewer weight parameters and, significantly fewer computations. 419 | 420 | During training, dropout nodes are introduced after the relu, controlled by a 421 | placeholder. 422 | 423 | Args: 424 | fingerprint_input: TensorFlow node that will output audio feature vectors. 425 | The node is expected to produce a 2D Tensor of shape: 426 | [batch, model_settings['dct_coefficient_count'] * 427 | model_settings['spectrogram_length']] 428 | with the features corresponding to the same time slot arranged contiguously, 429 | and the oldest slot at index [:, 0], and newest at [:, -1]. 430 | model_settings: Dictionary of information about the model. 431 | is_training: Whether the model is going to be used for training. 432 | runtime_settings: Dictionary of information about the runtime. 433 | 434 | Returns: 435 | TensorFlow node outputting logits results, and optionally a dropout 436 | placeholder. 437 | 438 | Raises: 439 | ValueError: If the inputs tensor is incorrectly shaped. 440 | """ 441 | if is_training: 442 | dropout_prob = tf.placeholder(tf.float32, name='dropout_prob') 443 | 444 | input_frequency_size = model_settings['dct_coefficient_count'] 445 | input_time_size = model_settings['spectrogram_length'] 446 | 447 | # Validation. 448 | input_shape = fingerprint_input.get_shape() 449 | if len(input_shape) != 2: 450 | raise ValueError('Inputs to `SVDF` should have rank == 2.') 451 | if input_shape[-1].value is None: 452 | raise ValueError('The last dimension of the inputs to `SVDF` ' 453 | 'should be defined. Found `None`.') 454 | if input_shape[-1].value % input_frequency_size != 0: 455 | raise ValueError('Inputs feature dimension %d must be a multiple of ' 456 | 'frame size %d', fingerprint_input.shape[-1].value, 457 | input_frequency_size) 458 | 459 | # Set number of units (i.e. nodes) and rank. 460 | rank = 2 461 | num_units = 1280 462 | # Number of filters: pairs of feature and time filters. 463 | num_filters = rank * num_units 464 | # Create the runtime memory: [num_filters, batch, input_time_size] 465 | batch = 1 466 | memory = tf.Variable(tf.zeros([num_filters, batch, input_time_size]), 467 | trainable=False, name='runtime-memory') 468 | # Determine the number of new frames in the input, such that we only operate 469 | # on those. For training we do not use the memory, and thus use all frames 470 | # provided in the input. 471 | # new_fingerprint_input: [batch, num_new_frames*input_frequency_size] 472 | if is_training: 473 | num_new_frames = input_time_size 474 | else: 475 | window_stride_ms = int(model_settings['window_stride_samples'] * 1000 / 476 | model_settings['sample_rate']) 477 | num_new_frames = tf.cond( 478 | tf.equal(tf.count_nonzero(memory), 0), 479 | lambda: input_time_size, 480 | lambda: int(runtime_settings['clip_stride_ms'] / window_stride_ms)) 481 | new_fingerprint_input = fingerprint_input[ 482 | :, -num_new_frames*input_frequency_size:] 483 | # Expand to add input channels dimension. 484 | new_fingerprint_input = tf.expand_dims(new_fingerprint_input, 2) 485 | 486 | # Create the frequency filters. 487 | weights_frequency = tf.Variable( 488 | tf.truncated_normal([input_frequency_size, num_filters], stddev=0.01)) 489 | # Expand to add input channels dimensions. 490 | # weights_frequency: [input_frequency_size, 1, num_filters] 491 | weights_frequency = tf.expand_dims(weights_frequency, 1) 492 | # Convolve the 1D feature filters sliding over the time dimension. 493 | # activations_time: [batch, num_new_frames, num_filters] 494 | activations_time = tf.nn.conv1d( 495 | new_fingerprint_input, weights_frequency, input_frequency_size, 'VALID') 496 | # Rearrange such that we can perform the batched matmul. 497 | # activations_time: [num_filters, batch, num_new_frames] 498 | activations_time = tf.transpose(activations_time, perm=[2, 0, 1]) 499 | 500 | # Runtime memory optimization. 501 | if not is_training: 502 | # We need to drop the activations corresponding to the oldest frames, and 503 | # then add those corresponding to the new frames. 504 | new_memory = memory[:, :, num_new_frames:] 505 | new_memory = tf.concat([new_memory, activations_time], 2) 506 | tf.assign(memory, new_memory) 507 | activations_time = new_memory 508 | 509 | # Create the time filters. 510 | weights_time = tf.Variable( 511 | tf.truncated_normal([num_filters, input_time_size], stddev=0.01)) 512 | # Apply the time filter on the outputs of the feature filters. 513 | # weights_time: [num_filters, input_time_size, 1] 514 | # outputs: [num_filters, batch, 1] 515 | weights_time = tf.expand_dims(weights_time, 2) 516 | outputs = tf.matmul(activations_time, weights_time) 517 | # Split num_units and rank into separate dimensions (the remaining 518 | # dimension is the input_shape[0] -i.e. batch size). This also squeezes 519 | # the last dimension, since it's not used. 520 | # [num_filters, batch, 1] => [num_units, rank, batch] 521 | outputs = tf.reshape(outputs, [num_units, rank, -1]) 522 | # Sum the rank outputs per unit => [num_units, batch]. 523 | units_output = tf.reduce_sum(outputs, axis=1) 524 | # Transpose to shape [batch, num_units] 525 | units_output = tf.transpose(units_output) 526 | 527 | # Appy bias. 528 | bias = tf.Variable(tf.zeros([num_units])) 529 | first_bias = tf.nn.bias_add(units_output, bias) 530 | 531 | # Relu. 532 | first_relu = tf.nn.relu(first_bias) 533 | 534 | if is_training: 535 | first_dropout = tf.nn.dropout(first_relu, dropout_prob) 536 | else: 537 | first_dropout = first_relu 538 | 539 | first_fc_output_channels = 256 540 | first_fc_weights = tf.Variable( 541 | tf.truncated_normal([num_units, first_fc_output_channels], stddev=0.01)) 542 | first_fc_bias = tf.Variable(tf.zeros([first_fc_output_channels])) 543 | first_fc = tf.matmul(first_dropout, first_fc_weights) + first_fc_bias 544 | if is_training: 545 | second_fc_input = tf.nn.dropout(first_fc, dropout_prob) 546 | else: 547 | second_fc_input = first_fc 548 | second_fc_output_channels = 256 549 | second_fc_weights = tf.Variable( 550 | tf.truncated_normal( 551 | [first_fc_output_channels, second_fc_output_channels], stddev=0.01)) 552 | second_fc_bias = tf.Variable(tf.zeros([second_fc_output_channels])) 553 | second_fc = tf.matmul(second_fc_input, second_fc_weights) + second_fc_bias 554 | if is_training: 555 | final_fc_input = tf.nn.dropout(second_fc, dropout_prob) 556 | else: 557 | final_fc_input = second_fc 558 | label_count = model_settings['label_count'] 559 | final_fc_weights = tf.Variable( 560 | tf.truncated_normal( 561 | [second_fc_output_channels, label_count], stddev=0.01)) 562 | final_fc_bias = tf.Variable(tf.zeros([label_count])) 563 | final_fc = tf.matmul(final_fc_input, final_fc_weights) + final_fc_bias 564 | if is_training: 565 | return final_fc, dropout_prob 566 | else: 567 | return final_fc 568 | -------------------------------------------------------------------------------- /speech_commands/models_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | """Tests for speech commands models.""" 16 | 17 | from __future__ import absolute_import 18 | from __future__ import division 19 | from __future__ import print_function 20 | 21 | import tensorflow as tf 22 | 23 | from tensorflow.examples.speech_commands import models 24 | from tensorflow.python.platform import test 25 | 26 | 27 | class ModelsTest(test.TestCase): 28 | 29 | def testPrepareModelSettings(self): 30 | self.assertIsNotNone( 31 | models.prepare_model_settings(10, 16000, 1000, 20, 10, 40)) 32 | 33 | def testCreateModelConvTraining(self): 34 | model_settings = models.prepare_model_settings(10, 16000, 1000, 20, 10, 40) 35 | with self.test_session() as sess: 36 | fingerprint_input = tf.zeros([1, model_settings["fingerprint_size"]]) 37 | logits, dropout_prob = models.create_model(fingerprint_input, 38 | model_settings, "conv", True) 39 | self.assertIsNotNone(logits) 40 | self.assertIsNotNone(dropout_prob) 41 | self.assertIsNotNone(sess.graph.get_tensor_by_name(logits.name)) 42 | self.assertIsNotNone(sess.graph.get_tensor_by_name(dropout_prob.name)) 43 | 44 | def testCreateModelConvInference(self): 45 | model_settings = models.prepare_model_settings(10, 16000, 1000, 20, 10, 40) 46 | with self.test_session() as sess: 47 | fingerprint_input = tf.zeros([1, model_settings["fingerprint_size"]]) 48 | logits = models.create_model(fingerprint_input, model_settings, "conv", 49 | False) 50 | self.assertIsNotNone(logits) 51 | self.assertIsNotNone(sess.graph.get_tensor_by_name(logits.name)) 52 | 53 | def testCreateModelLowLatencyConvTraining(self): 54 | model_settings = models.prepare_model_settings(10, 16000, 1000, 20, 10, 40) 55 | with self.test_session() as sess: 56 | fingerprint_input = tf.zeros([1, model_settings["fingerprint_size"]]) 57 | logits, dropout_prob = models.create_model( 58 | fingerprint_input, model_settings, "low_latency_conv", True) 59 | self.assertIsNotNone(logits) 60 | self.assertIsNotNone(dropout_prob) 61 | self.assertIsNotNone(sess.graph.get_tensor_by_name(logits.name)) 62 | self.assertIsNotNone(sess.graph.get_tensor_by_name(dropout_prob.name)) 63 | 64 | def testCreateModelFullyConnectedTraining(self): 65 | model_settings = models.prepare_model_settings(10, 16000, 1000, 20, 10, 40) 66 | with self.test_session() as sess: 67 | fingerprint_input = tf.zeros([1, model_settings["fingerprint_size"]]) 68 | logits, dropout_prob = models.create_model( 69 | fingerprint_input, model_settings, "single_fc", True) 70 | self.assertIsNotNone(logits) 71 | self.assertIsNotNone(dropout_prob) 72 | self.assertIsNotNone(sess.graph.get_tensor_by_name(logits.name)) 73 | self.assertIsNotNone(sess.graph.get_tensor_by_name(dropout_prob.name)) 74 | 75 | def testCreateModelBadArchitecture(self): 76 | model_settings = models.prepare_model_settings(10, 16000, 1000, 20, 10, 40) 77 | with self.test_session(): 78 | fingerprint_input = tf.zeros([1, model_settings["fingerprint_size"]]) 79 | with self.assertRaises(Exception) as e: 80 | models.create_model(fingerprint_input, model_settings, 81 | "bad_architecture", True) 82 | self.assertTrue("not recognized" in str(e.exception)) 83 | 84 | 85 | if __name__ == "__main__": 86 | test.main() 87 | -------------------------------------------------------------------------------- /speech_commands/output.txt: -------------------------------------------------------------------------------- 1 | yes (score = 0.32898) 2 | left (score = 0.17696) 3 | stop (score = 0.12051) 4 | _unknown_ (score = 0.11003) 5 | no (score = 0.09265) 6 | go (score = 0.07126) 7 | down (score = 0.05835) 8 | off (score = 0.01777) 9 | right (score = 0.01647) 10 | up (score = 0.00544) 11 | on (score = 0.00147) 12 | _silence_ (score = 0.00010) 13 | -------------------------------------------------------------------------------- /speech_commands/recognize_commands.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "tensorflow/examples/speech_commands/recognize_commands.h" 17 | 18 | namespace tensorflow { 19 | 20 | RecognizeCommands::RecognizeCommands(const std::vector& labels, 21 | int32 average_window_duration_ms, 22 | float detection_threshold, 23 | int32 suppression_ms, int32 minimum_count) 24 | : labels_(labels), 25 | average_window_duration_ms_(average_window_duration_ms), 26 | detection_threshold_(detection_threshold), 27 | suppression_ms_(suppression_ms), 28 | minimum_count_(minimum_count) { 29 | labels_count_ = labels.size(); 30 | previous_top_label_ = "_silence_"; 31 | previous_top_label_time_ = std::numeric_limits::min(); 32 | } 33 | 34 | Status RecognizeCommands::ProcessLatestResults(const Tensor& latest_results, 35 | const int64 current_time_ms, 36 | string* found_command, 37 | float* score, 38 | bool* is_new_command) { 39 | if (latest_results.NumElements() != labels_count_) { 40 | return errors::InvalidArgument( 41 | "The results for recognition should contain ", labels_count_, 42 | " elements, but there are ", latest_results.NumElements()); 43 | } 44 | 45 | if ((!previous_results_.empty()) && 46 | (current_time_ms < previous_results_.front().first)) { 47 | return errors::InvalidArgument( 48 | "Results must be fed in increasing time order, but received a " 49 | "timestamp of ", 50 | current_time_ms, " that was earlier than the previous one of ", 51 | previous_results_.front().first); 52 | } 53 | 54 | // Add the latest results to the head of the queue. 55 | previous_results_.push_back({current_time_ms, latest_results}); 56 | 57 | // Prune any earlier results that are too old for the averaging window. 58 | const int64 time_limit = current_time_ms - average_window_duration_ms_; 59 | while (previous_results_.front().first < time_limit) { 60 | previous_results_.pop_front(); 61 | } 62 | 63 | // If there are too few results, assume the result will be unreliable and 64 | // bail. 65 | const int64 how_many_results = previous_results_.size(); 66 | const int64 earliest_time = previous_results_.front().first; 67 | const int64 samples_duration = current_time_ms - earliest_time; 68 | if ((how_many_results < minimum_count_) || 69 | (samples_duration < (average_window_duration_ms_ / 4))) { 70 | *found_command = previous_top_label_; 71 | *score = 0.0f; 72 | *is_new_command = false; 73 | return Status::OK(); 74 | } 75 | 76 | // Calculate the average score across all the results in the window. 77 | std::vector average_scores(labels_count_); 78 | for (const auto& previous_result : previous_results_) { 79 | const Tensor& scores_tensor = previous_result.second; 80 | auto scores_flat = scores_tensor.flat(); 81 | for (int i = 0; i < scores_flat.size(); ++i) { 82 | average_scores[i] += scores_flat(i) / how_many_results; 83 | } 84 | } 85 | 86 | // Sort the averaged results in descending score order. 87 | std::vector> sorted_average_scores; 88 | sorted_average_scores.reserve(labels_count_); 89 | for (int i = 0; i < labels_count_; ++i) { 90 | sorted_average_scores.push_back( 91 | std::pair({i, average_scores[i]})); 92 | } 93 | std::sort(sorted_average_scores.begin(), sorted_average_scores.end(), 94 | [](const std::pair& left, 95 | const std::pair& right) { 96 | return left.second > right.second; 97 | }); 98 | 99 | // See if the latest top score is enough to trigger a detection. 100 | const int current_top_index = sorted_average_scores[0].first; 101 | const string current_top_label = labels_[current_top_index]; 102 | const float current_top_score = sorted_average_scores[0].second; 103 | // If we've recently had another label trigger, assume one that occurs too 104 | // soon afterwards is a bad result. 105 | int64 time_since_last_top; 106 | if ((previous_top_label_ == "_silence_") || 107 | (previous_top_label_time_ == std::numeric_limits::min())) { 108 | time_since_last_top = std::numeric_limits::max(); 109 | } else { 110 | time_since_last_top = current_time_ms - previous_top_label_time_; 111 | } 112 | if ((current_top_score > detection_threshold_) && 113 | (current_top_label != previous_top_label_) && 114 | (time_since_last_top > suppression_ms_)) { 115 | previous_top_label_ = current_top_label; 116 | previous_top_label_time_ = current_time_ms; 117 | *is_new_command = true; 118 | } else { 119 | *is_new_command = false; 120 | } 121 | *found_command = current_top_label; 122 | *score = current_top_score; 123 | 124 | return Status::OK(); 125 | } 126 | 127 | } // namespace tensorflow 128 | -------------------------------------------------------------------------------- /speech_commands/recognize_commands.h: -------------------------------------------------------------------------------- 1 | /* Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #ifndef THIRD_PARTY_TENSORFLOW_EXAMPLES_SPEECH_COMMANDS_RECOGNIZE_COMMANDS_H_ 17 | #define THIRD_PARTY_TENSORFLOW_EXAMPLES_SPEECH_COMMANDS_RECOGNIZE_COMMANDS_H_ 18 | 19 | #include 20 | #include 21 | #include 22 | 23 | #include "tensorflow/core/framework/tensor.h" 24 | #include "tensorflow/core/platform/types.h" 25 | 26 | namespace tensorflow { 27 | 28 | // This class is designed to apply a very primitive decoding model on top of the 29 | // instantaneous results from running an audio recognition model on a single 30 | // window of samples. It applies smoothing over time so that noisy individual 31 | // label scores are averaged, increasing the confidence that apparent matches 32 | // are real. 33 | // To use it, you should create a class object with the configuration you 34 | // want, and then feed results from running a TensorFlow model into the 35 | // processing method. The timestamp for each subsequent call should be 36 | // increasing from the previous, since the class is designed to process a stream 37 | // of data over time. 38 | class RecognizeCommands { 39 | public: 40 | // labels should be a list of the strings associated with each one-hot score. 41 | // The window duration controls the smoothing. Longer durations will give a 42 | // higher confidence that the results are correct, but may miss some commands. 43 | // The detection threshold has a similar effect, with high values increasing 44 | // the precision at the cost of recall. The minimum count controls how many 45 | // results need to be in the averaging window before it's seen as a reliable 46 | // average. This prevents erroneous results when the averaging window is 47 | // initially being populated for example. The suppression argument disables 48 | // further recognitions for a set time after one has been triggered, which can 49 | // help reduce spurious recognitions. 50 | explicit RecognizeCommands(const std::vector& labels, 51 | int32 average_window_duration_ms = 1000, 52 | float detection_threshold = 0.2, 53 | int32 suppression_ms = 500, 54 | int32 minimum_count = 3); 55 | 56 | // Call this with the results of running a model on sample data. 57 | Status ProcessLatestResults(const Tensor& latest_results, 58 | const int64 current_time_ms, 59 | string* found_command, float* score, 60 | bool* is_new_command); 61 | 62 | private: 63 | // Configuration 64 | std::vector labels_; 65 | int32 average_window_duration_ms_; 66 | float detection_threshold_; 67 | int32 suppression_ms_; 68 | int32 minimum_count_; 69 | 70 | // Working variables 71 | std::deque> previous_results_; 72 | string previous_top_label_; 73 | int64 labels_count_; 74 | int64 previous_top_label_time_; 75 | }; 76 | 77 | } // namespace tensorflow 78 | 79 | #endif // THIRD_PARTY_TENSORFLOW_EXAMPLES_SPEECH_COMMANDS_RECOGNIZE_COMMANDS_H_ 80 | -------------------------------------------------------------------------------- /speech_commands/recognize_commands_test.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | 16 | #include "tensorflow/examples/speech_commands/recognize_commands.h" 17 | 18 | #include "tensorflow/core/framework/tensor_testutil.h" 19 | #include "tensorflow/core/lib/core/status_test_util.h" 20 | #include "tensorflow/core/platform/env.h" 21 | #include "tensorflow/core/platform/test.h" 22 | 23 | namespace tensorflow { 24 | 25 | TEST(RecognizeCommandsTest, Basic) { 26 | RecognizeCommands recognize_commands({"_silence_", "a", "b"}); 27 | 28 | Tensor results(DT_FLOAT, {3}); 29 | test::FillValues(&results, {1.0f, 0.0f, 0.0f}); 30 | 31 | string found_command; 32 | float score; 33 | bool is_new_command; 34 | TF_EXPECT_OK(recognize_commands.ProcessLatestResults( 35 | results, 0, &found_command, &score, &is_new_command)); 36 | } 37 | 38 | TEST(RecognizeCommandsTest, FindCommands) { 39 | RecognizeCommands recognize_commands({"_silence_", "a", "b"}, 1000, 0.2f); 40 | 41 | Tensor results(DT_FLOAT, {3}); 42 | 43 | test::FillValues(&results, {0.0f, 1.0f, 0.0f}); 44 | bool has_found_new_command = false; 45 | string new_command; 46 | for (int i = 0; i < 10; ++i) { 47 | string found_command; 48 | float score; 49 | bool is_new_command; 50 | int64 current_time_ms = 0 + (i * 100); 51 | TF_EXPECT_OK(recognize_commands.ProcessLatestResults( 52 | results, current_time_ms, &found_command, &score, &is_new_command)); 53 | if (is_new_command) { 54 | EXPECT_FALSE(has_found_new_command); 55 | has_found_new_command = true; 56 | new_command = found_command; 57 | } 58 | } 59 | EXPECT_TRUE(has_found_new_command); 60 | EXPECT_EQ("a", new_command); 61 | 62 | test::FillValues(&results, {0.0f, 0.0f, 1.0f}); 63 | has_found_new_command = false; 64 | new_command = ""; 65 | for (int i = 0; i < 10; ++i) { 66 | string found_command; 67 | float score; 68 | bool is_new_command; 69 | int64 current_time_ms = 1000 + (i * 100); 70 | TF_EXPECT_OK(recognize_commands.ProcessLatestResults( 71 | results, current_time_ms, &found_command, &score, &is_new_command)); 72 | if (is_new_command) { 73 | EXPECT_FALSE(has_found_new_command); 74 | has_found_new_command = true; 75 | new_command = found_command; 76 | } 77 | } 78 | EXPECT_TRUE(has_found_new_command); 79 | EXPECT_EQ("b", new_command); 80 | } 81 | 82 | TEST(RecognizeCommandsTest, BadInputLength) { 83 | RecognizeCommands recognize_commands({"_silence_", "a", "b"}, 1000, 0.2f); 84 | 85 | Tensor bad_results(DT_FLOAT, {2}); 86 | test::FillValues(&bad_results, {1.0f, 0.0f}); 87 | 88 | string found_command; 89 | float score; 90 | bool is_new_command; 91 | EXPECT_FALSE(recognize_commands 92 | .ProcessLatestResults(bad_results, 0, &found_command, &score, 93 | &is_new_command) 94 | .ok()); 95 | } 96 | 97 | TEST(RecognizeCommandsTest, BadInputTimes) { 98 | RecognizeCommands recognize_commands({"_silence_", "a", "b"}, 1000, 0.2f); 99 | 100 | Tensor results(DT_FLOAT, {3}); 101 | test::FillValues(&results, {1.0f, 0.0f, 0.0f}); 102 | 103 | string found_command; 104 | float score; 105 | bool is_new_command; 106 | TF_EXPECT_OK(recognize_commands.ProcessLatestResults( 107 | results, 100, &found_command, &score, &is_new_command)); 108 | EXPECT_FALSE(recognize_commands 109 | .ProcessLatestResults(results, 0, &found_command, &score, 110 | &is_new_command) 111 | .ok()); 112 | } 113 | 114 | } // namespace tensorflow 115 | -------------------------------------------------------------------------------- /speech_commands/test_streaming_accuracy.cc: -------------------------------------------------------------------------------- 1 | /* Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | ==============================================================================*/ 15 | /* 16 | 17 | Tool to create accuracy statistics from running an audio recognition model on a 18 | continuous stream of samples. 19 | 20 | This is designed to be an environment for running experiments on new models and 21 | settings to understand the effects they will have in a real application. You 22 | need to supply it with a long audio file containing sounds you want to recognize 23 | and a text file listing the labels of each sound along with the time they occur. 24 | With this information, and a frozen model, the tool will process the audio 25 | stream, apply the model, and keep track of how many mistakes and successes the 26 | model achieved. 27 | 28 | The matched percentage is the number of sounds that were correctly classified, 29 | as a percentage of the total number of sounds listed in the ground truth file. 30 | A correct classification is when the right label is chosen within a short time 31 | of the expected ground truth, where the time tolerance is controlled by the 32 | 'time_tolerance_ms' command line flag. 33 | 34 | The wrong percentage is how many sounds triggered a detection (the classifier 35 | figured out it wasn't silence or background noise), but the detected class was 36 | wrong. This is also a percentage of the total number of ground truth sounds. 37 | 38 | The false positive percentage is how many sounds were detected when there was 39 | only silence or background noise. This is also expressed as a percentage of the 40 | total number of ground truth sounds, though since it can be large it may go 41 | above 100%. 42 | 43 | The easiest way to get an audio file and labels to test with is by using the 44 | 'generate_streaming_test_wav' script. This will synthesize a test file with 45 | randomly placed sounds and background noise, and output a text file with the 46 | ground truth. 47 | 48 | If you want to test natural data, you need to use a .wav with the same sample 49 | rate as your model (often 16,000 samples per second), and note down where the 50 | sounds occur in time. Save this information out as a comma-separated text file, 51 | where the first column is the label and the second is the time in seconds from 52 | the start of the file that it occurs. 53 | 54 | Here's an example of how to run the tool: 55 | 56 | bazel run tensorflow/examples/speech_commands:test_streaming_accuracy -- \ 57 | --wav=/tmp/streaming_test_bg.wav \ 58 | --graph=/tmp/conv_frozen.pb \ 59 | --labels=/tmp/speech_commands_train/conv_labels.txt \ 60 | --ground_truth=/tmp/streaming_test_labels.txt --verbose \ 61 | --clip_duration_ms=1000 --detection_threshold=0.70 --average_window_ms=500 \ 62 | --suppression_ms=500 --time_tolerance_ms=1500 63 | 64 | */ 65 | 66 | #include 67 | #include 68 | #include 69 | #include 70 | 71 | #include "tensorflow/core/framework/tensor.h" 72 | #include "tensorflow/core/lib/io/path.h" 73 | #include "tensorflow/core/lib/strings/numbers.h" 74 | #include "tensorflow/core/lib/strings/str_util.h" 75 | #include "tensorflow/core/lib/wav/wav_io.h" 76 | #include "tensorflow/core/platform/init_main.h" 77 | #include "tensorflow/core/platform/logging.h" 78 | #include "tensorflow/core/platform/types.h" 79 | #include "tensorflow/core/public/session.h" 80 | #include "tensorflow/core/util/command_line_flags.h" 81 | #include "tensorflow/examples/speech_commands/accuracy_utils.h" 82 | #include "tensorflow/examples/speech_commands/recognize_commands.h" 83 | 84 | // These are all common classes it's handy to reference with no namespace. 85 | using tensorflow::Flag; 86 | using tensorflow::Status; 87 | using tensorflow::Tensor; 88 | using tensorflow::int32; 89 | using tensorflow::int64; 90 | using tensorflow::string; 91 | using tensorflow::uint16; 92 | using tensorflow::uint32; 93 | 94 | namespace { 95 | 96 | // Reads a model graph definition from disk, and creates a session object you 97 | // can use to run it. 98 | Status LoadGraph(const string& graph_file_name, 99 | std::unique_ptr* session) { 100 | tensorflow::GraphDef graph_def; 101 | Status load_graph_status = 102 | ReadBinaryProto(tensorflow::Env::Default(), graph_file_name, &graph_def); 103 | if (!load_graph_status.ok()) { 104 | return tensorflow::errors::NotFound("Failed to load compute graph at '", 105 | graph_file_name, "'"); 106 | } 107 | session->reset(tensorflow::NewSession(tensorflow::SessionOptions())); 108 | Status session_create_status = (*session)->Create(graph_def); 109 | if (!session_create_status.ok()) { 110 | return session_create_status; 111 | } 112 | return Status::OK(); 113 | } 114 | 115 | // Takes a file name, and loads a list of labels from it, one per line, and 116 | // returns a vector of the strings. 117 | Status ReadLabelsFile(const string& file_name, std::vector* result) { 118 | std::ifstream file(file_name); 119 | if (!file) { 120 | return tensorflow::errors::NotFound("Labels file '", file_name, 121 | "' not found."); 122 | } 123 | result->clear(); 124 | string line; 125 | while (std::getline(file, line)) { 126 | result->push_back(line); 127 | } 128 | return Status::OK(); 129 | } 130 | 131 | } // namespace 132 | 133 | int main(int argc, char* argv[]) { 134 | string wav = ""; 135 | string graph = ""; 136 | string labels = ""; 137 | string ground_truth = ""; 138 | string input_data_name = "decoded_sample_data:0"; 139 | string input_rate_name = "decoded_sample_data:1"; 140 | string output_name = "labels_softmax"; 141 | int32 clip_duration_ms = 1000; 142 | int32 clip_stride_ms = 30; 143 | int32 average_window_ms = 500; 144 | int32 time_tolerance_ms = 750; 145 | int32 suppression_ms = 1500; 146 | float detection_threshold = 0.7f; 147 | bool verbose = false; 148 | std::vector flag_list = { 149 | Flag("wav", &wav, "audio file to be identified"), 150 | Flag("graph", &graph, "model to be executed"), 151 | Flag("labels", &labels, "path to file containing labels"), 152 | Flag("ground_truth", &ground_truth, 153 | "path to file containing correct times and labels of words in the " 154 | "audio as , lines"), 155 | Flag("input_data_name", &input_data_name, 156 | "name of input data node in model"), 157 | Flag("input_rate_name", &input_rate_name, 158 | "name of input sample rate node in model"), 159 | Flag("output_name", &output_name, "name of output node in model"), 160 | Flag("clip_duration_ms", &clip_duration_ms, 161 | "length of recognition window"), 162 | Flag("average_window_ms", &average_window_ms, 163 | "length of window to smooth results over"), 164 | Flag("time_tolerance_ms", &time_tolerance_ms, 165 | "maximum gap allowed between a recognition and ground truth"), 166 | Flag("suppression_ms", &suppression_ms, 167 | "how long to ignore others for after a recognition"), 168 | Flag("clip_stride_ms", &clip_stride_ms, "how often to run recognition"), 169 | Flag("detection_threshold", &detection_threshold, 170 | "what score is required to trigger detection of a word"), 171 | Flag("verbose", &verbose, "whether to log extra debugging information"), 172 | }; 173 | string usage = tensorflow::Flags::Usage(argv[0], flag_list); 174 | const bool parse_result = tensorflow::Flags::Parse(&argc, argv, flag_list); 175 | if (!parse_result) { 176 | LOG(ERROR) << usage; 177 | return -1; 178 | } 179 | 180 | // We need to call this to set up global state for TensorFlow. 181 | tensorflow::port::InitMain(argv[0], &argc, &argv); 182 | if (argc > 1) { 183 | LOG(ERROR) << "Unknown argument " << argv[1] << "\n" << usage; 184 | return -1; 185 | } 186 | 187 | // First we load and initialize the model. 188 | std::unique_ptr session; 189 | Status load_graph_status = LoadGraph(graph, &session); 190 | if (!load_graph_status.ok()) { 191 | LOG(ERROR) << load_graph_status; 192 | return -1; 193 | } 194 | 195 | std::vector labels_list; 196 | Status read_labels_status = ReadLabelsFile(labels, &labels_list); 197 | if (!read_labels_status.ok()) { 198 | LOG(ERROR) << read_labels_status; 199 | return -1; 200 | } 201 | 202 | std::vector> ground_truth_list; 203 | Status read_ground_truth_status = 204 | tensorflow::ReadGroundTruthFile(ground_truth, &ground_truth_list); 205 | if (!read_ground_truth_status.ok()) { 206 | LOG(ERROR) << read_ground_truth_status; 207 | return -1; 208 | } 209 | 210 | string wav_string; 211 | Status read_wav_status = tensorflow::ReadFileToString( 212 | tensorflow::Env::Default(), wav, &wav_string); 213 | if (!read_wav_status.ok()) { 214 | LOG(ERROR) << read_wav_status; 215 | return -1; 216 | } 217 | std::vector audio_data; 218 | uint32 sample_count; 219 | uint16 channel_count; 220 | uint32 sample_rate; 221 | Status decode_wav_status = tensorflow::wav::DecodeLin16WaveAsFloatVector( 222 | wav_string, &audio_data, &sample_count, &channel_count, &sample_rate); 223 | if (!decode_wav_status.ok()) { 224 | LOG(ERROR) << decode_wav_status; 225 | return -1; 226 | } 227 | if (channel_count != 1) { 228 | LOG(ERROR) << "Only mono .wav files can be used, but input has " 229 | << channel_count << " channels."; 230 | return -1; 231 | } 232 | 233 | const int64 clip_duration_samples = (clip_duration_ms * sample_rate) / 1000; 234 | const int64 clip_stride_samples = (clip_stride_ms * sample_rate) / 1000; 235 | Tensor audio_data_tensor(tensorflow::DT_FLOAT, 236 | tensorflow::TensorShape({clip_duration_samples, 1})); 237 | 238 | Tensor sample_rate_tensor(tensorflow::DT_INT32, tensorflow::TensorShape({})); 239 | sample_rate_tensor.scalar()() = sample_rate; 240 | 241 | tensorflow::RecognizeCommands recognize_commands( 242 | labels_list, average_window_ms, detection_threshold, suppression_ms); 243 | 244 | std::vector> all_found_words; 245 | tensorflow::StreamingAccuracyStats previous_stats; 246 | 247 | const int64 audio_data_end = (sample_count - clip_duration_ms); 248 | for (int64 audio_data_offset = 0; audio_data_offset < audio_data_end; 249 | audio_data_offset += clip_stride_samples) { 250 | const float* input_start = &(audio_data[audio_data_offset]); 251 | const float* input_end = input_start + clip_duration_samples; 252 | std::copy(input_start, input_end, audio_data_tensor.flat().data()); 253 | 254 | // Actually run the audio through the model. 255 | std::vector outputs; 256 | Status run_status = session->Run({{input_data_name, audio_data_tensor}, 257 | {input_rate_name, sample_rate_tensor}}, 258 | {output_name}, {}, &outputs); 259 | if (!run_status.ok()) { 260 | LOG(ERROR) << "Running model failed: " << run_status; 261 | return -1; 262 | } 263 | 264 | const int64 current_time_ms = (audio_data_offset * 1000) / sample_rate; 265 | string found_command; 266 | float score; 267 | bool is_new_command; 268 | Status recognize_status = recognize_commands.ProcessLatestResults( 269 | outputs[0], current_time_ms, &found_command, &score, &is_new_command); 270 | if (!recognize_status.ok()) { 271 | LOG(ERROR) << "Recognition processing failed: " << recognize_status; 272 | return -1; 273 | } 274 | 275 | if (is_new_command && (found_command != "_silence_")) { 276 | all_found_words.push_back({found_command, current_time_ms}); 277 | if (verbose) { 278 | tensorflow::StreamingAccuracyStats stats; 279 | tensorflow::CalculateAccuracyStats(ground_truth_list, all_found_words, 280 | current_time_ms, time_tolerance_ms, 281 | &stats); 282 | int32 false_positive_delta = stats.how_many_false_positives - 283 | previous_stats.how_many_false_positives; 284 | int32 correct_delta = stats.how_many_correct_words - 285 | previous_stats.how_many_correct_words; 286 | int32 wrong_delta = 287 | stats.how_many_wrong_words - previous_stats.how_many_wrong_words; 288 | string recognition_state; 289 | if (false_positive_delta == 1) { 290 | recognition_state = " (False Positive)"; 291 | } else if (correct_delta == 1) { 292 | recognition_state = " (Correct)"; 293 | } else if (wrong_delta == 1) { 294 | recognition_state = " (Wrong)"; 295 | } else { 296 | LOG(ERROR) << "Unexpected state in statistics"; 297 | } 298 | LOG(INFO) << current_time_ms << "ms: " << found_command << ": " << score 299 | << recognition_state; 300 | previous_stats = stats; 301 | tensorflow::PrintAccuracyStats(stats); 302 | } 303 | } 304 | } 305 | 306 | tensorflow::StreamingAccuracyStats stats; 307 | tensorflow::CalculateAccuracyStats(ground_truth_list, all_found_words, -1, 308 | time_tolerance_ms, &stats); 309 | tensorflow::PrintAccuracyStats(stats); 310 | 311 | return 0; 312 | } 313 | -------------------------------------------------------------------------------- /speech_commands/train.py: -------------------------------------------------------------------------------- 1 | # Copyright 2017 The TensorFlow Authors. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # ============================================================================== 15 | r"""Simple speech recognition to spot a limited number of keywords. 16 | 17 | This is a self-contained example script that will train a very basic audio 18 | recognition model in TensorFlow. It downloads the necessary training data and 19 | runs with reasonable defaults to train within a few hours even only using a CPU. 20 | For more information, please see 21 | https://www.tensorflow.org/tutorials/audio_recognition. 22 | 23 | It is intended as an introduction to using neural networks for audio 24 | recognition, and is not a full speech recognition system. For more advanced 25 | speech systems, I recommend looking into Kaldi. This network uses a keyword 26 | detection style to spot discrete words from a small vocabulary, consisting of 27 | "yes", "no", "up", "down", "left", "right", "on", "off", "stop", and "go". 28 | 29 | To run the training process, use: 30 | 31 | bazel run tensorflow/examples/speech_commands:train 32 | 33 | This will write out checkpoints to /tmp/speech_commands_train/, and will 34 | download over 1GB of open source training data, so you'll need enough free space 35 | and a good internet connection. The default data is a collection of thousands of 36 | one-second .wav files, each containing one spoken word. This data set is 37 | collected from https://aiyprojects.withgoogle.com/open_speech_recording, please 38 | consider contributing to help improve this and other models! 39 | 40 | As training progresses, it will print out its accuracy metrics, which should 41 | rise above 90% by the end. Once it's complete, you can run the freeze script to 42 | get a binary GraphDef that you can easily deploy on mobile applications. 43 | 44 | If you want to train on your own data, you'll need to create .wavs with your 45 | recordings, all at a consistent length, and then arrange them into subfolders 46 | organized by label. For example, here's a possible file structure: 47 | 48 | my_wavs > 49 | up > 50 | audio_0.wav 51 | audio_1.wav 52 | down > 53 | audio_2.wav 54 | audio_3.wav 55 | other> 56 | audio_4.wav 57 | audio_5.wav 58 | 59 | You'll also need to tell the script what labels to look for, using the 60 | `--wanted_words` argument. In this case, 'up,down' might be what you want, and 61 | the audio in the 'other' folder would be used to train an 'unknown' category. 62 | 63 | To pull this all together, you'd run: 64 | 65 | bazel run tensorflow/examples/speech_commands:train -- \ 66 | --data_dir=my_wavs --wanted_words=up,down 67 | 68 | """ 69 | from __future__ import absolute_import 70 | from __future__ import division 71 | from __future__ import print_function 72 | 73 | import argparse 74 | import os.path 75 | import sys 76 | 77 | import numpy as np 78 | from six.moves import xrange # pylint: disable=redefined-builtin 79 | import tensorflow as tf 80 | 81 | import input_data 82 | import models 83 | from tensorflow.python.platform import gfile 84 | 85 | FLAGS = None 86 | 87 | 88 | def main(_): 89 | # We want to see all the logging messages for this tutorial. 90 | tf.logging.set_verbosity(tf.logging.INFO) 91 | 92 | # Start a new TensorFlow session. 93 | sess = tf.InteractiveSession() 94 | 95 | # Begin by making sure we have the training data we need. If you already have 96 | # training data of your own, use `--data_url= ` on the command line to avoid 97 | # downloading. 98 | model_settings = models.prepare_model_settings( 99 | len(input_data.prepare_words_list(FLAGS.wanted_words.split(','))), 100 | FLAGS.sample_rate, FLAGS.clip_duration_ms, FLAGS.window_size_ms, 101 | FLAGS.window_stride_ms, FLAGS.dct_coefficient_count) 102 | audio_processor = input_data.AudioProcessor( 103 | FLAGS.data_url, FLAGS.data_dir, FLAGS.silence_percentage, 104 | FLAGS.unknown_percentage, 105 | FLAGS.wanted_words.split(','), FLAGS.validation_percentage, 106 | FLAGS.testing_percentage, model_settings) 107 | fingerprint_size = model_settings['fingerprint_size'] 108 | label_count = model_settings['label_count'] 109 | time_shift_samples = int((FLAGS.time_shift_ms * FLAGS.sample_rate) / 1000) 110 | # Figure out the learning rates for each training phase. Since it's often 111 | # effective to have high learning rates at the start of training, followed by 112 | # lower levels towards the end, the number of steps and learning rates can be 113 | # specified as comma-separated lists to define the rate at each stage. For 114 | # example --how_many_training_steps=10000,3000 --learning_rate=0.001,0.0001 115 | # will run 13,000 training loops in total, with a rate of 0.001 for the first 116 | # 10,000, and 0.0001 for the final 3,000. 117 | training_steps_list = list(map(int, FLAGS.how_many_training_steps.split(','))) 118 | learning_rates_list = list(map(float, FLAGS.learning_rate.split(','))) 119 | if len(training_steps_list) != len(learning_rates_list): 120 | raise Exception( 121 | '--how_many_training_steps and --learning_rate must be equal length ' 122 | 'lists, but are %d and %d long instead' % (len(training_steps_list), 123 | len(learning_rates_list))) 124 | 125 | fingerprint_input = tf.placeholder( 126 | tf.float32, [None, fingerprint_size], name='fingerprint_input') 127 | 128 | logits, dropout_prob = models.create_model( 129 | fingerprint_input, 130 | model_settings, 131 | FLAGS.model_architecture, 132 | is_training=True) 133 | 134 | # Define loss and optimizer 135 | ground_truth_input = tf.placeholder( 136 | tf.float32, [None, label_count], name='groundtruth_input') 137 | 138 | # Optionally we can add runtime checks to spot when NaNs or other symptoms of 139 | # numerical errors start occurring during training. 140 | control_dependencies = [] 141 | if FLAGS.check_nans: 142 | checks = tf.add_check_numerics_ops() 143 | control_dependencies = [checks] 144 | 145 | # Create the back propagation and training evaluation machinery in the graph. 146 | with tf.name_scope('cross_entropy'): 147 | cross_entropy_mean = tf.reduce_mean( 148 | tf.nn.softmax_cross_entropy_with_logits( 149 | labels=ground_truth_input, logits=logits)) 150 | tf.summary.scalar('cross_entropy', cross_entropy_mean) 151 | with tf.name_scope('train'), tf.control_dependencies(control_dependencies): 152 | learning_rate_input = tf.placeholder( 153 | tf.float32, [], name='learning_rate_input') 154 | train_step = tf.train.GradientDescentOptimizer( 155 | learning_rate_input).minimize(cross_entropy_mean) 156 | predicted_indices = tf.argmax(logits, 1) 157 | expected_indices = tf.argmax(ground_truth_input, 1) 158 | correct_prediction = tf.equal(predicted_indices, expected_indices) 159 | confusion_matrix = tf.confusion_matrix(expected_indices, predicted_indices) 160 | evaluation_step = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) 161 | tf.summary.scalar('accuracy', evaluation_step) 162 | 163 | global_step = tf.contrib.framework.get_or_create_global_step() 164 | increment_global_step = tf.assign(global_step, global_step + 1) 165 | 166 | saver = tf.train.Saver(tf.global_variables()) 167 | 168 | # Merge all the summaries and write them out to /tmp/retrain_logs (by default) 169 | merged_summaries = tf.summary.merge_all() 170 | train_writer = tf.summary.FileWriter(FLAGS.summaries_dir + '/train', 171 | sess.graph) 172 | validation_writer = tf.summary.FileWriter(FLAGS.summaries_dir + '/validation') 173 | 174 | tf.global_variables_initializer().run() 175 | 176 | start_step = 1 177 | 178 | if FLAGS.start_checkpoint: 179 | models.load_variables_from_checkpoint(sess, FLAGS.start_checkpoint) 180 | start_step = global_step.eval(session=sess) 181 | 182 | tf.logging.info('Training from step: %d ', start_step) 183 | 184 | # Save graph.pbtxt. 185 | tf.train.write_graph(sess.graph_def, FLAGS.train_dir, 186 | FLAGS.model_architecture + '.pbtxt') 187 | 188 | # Save list of words. 189 | with gfile.GFile( 190 | os.path.join(FLAGS.train_dir, FLAGS.model_architecture + '_labels.txt'), 191 | 'w') as f: 192 | f.write('\n'.join(audio_processor.words_list)) 193 | 194 | # Training loop. 195 | training_steps_max = np.sum(training_steps_list) 196 | for training_step in xrange(start_step, training_steps_max + 1): 197 | # Figure out what the current learning rate is. 198 | training_steps_sum = 0 199 | for i in range(len(training_steps_list)): 200 | training_steps_sum += training_steps_list[i] 201 | if training_step <= training_steps_sum: 202 | learning_rate_value = learning_rates_list[i] 203 | break 204 | # Pull the audio samples we'll use for training. 205 | train_fingerprints, train_ground_truth = audio_processor.get_data( 206 | FLAGS.batch_size, 0, model_settings, FLAGS.background_frequency, 207 | FLAGS.background_volume, time_shift_samples, 'training', sess) 208 | # Run the graph with this batch of training data. 209 | train_summary, train_accuracy, cross_entropy_value, _, _ = sess.run( 210 | [ 211 | merged_summaries, evaluation_step, cross_entropy_mean, train_step, 212 | increment_global_step 213 | ], 214 | feed_dict={ 215 | fingerprint_input: train_fingerprints, 216 | ground_truth_input: train_ground_truth, 217 | learning_rate_input: learning_rate_value, 218 | dropout_prob: 0.5 219 | }) 220 | train_writer.add_summary(train_summary, training_step) 221 | tf.logging.info('Step #%d: rate %f, accuracy %.1f%%, cross entropy %f' % 222 | (training_step, learning_rate_value, train_accuracy * 100, 223 | cross_entropy_value)) 224 | is_last_step = (training_step == training_steps_max) 225 | if (training_step % FLAGS.eval_step_interval) == 0 or is_last_step: 226 | set_size = audio_processor.set_size('validation') 227 | total_accuracy = 0 228 | total_conf_matrix = None 229 | for i in xrange(0, set_size, FLAGS.batch_size): 230 | validation_fingerprints, validation_ground_truth = ( 231 | audio_processor.get_data(FLAGS.batch_size, i, model_settings, 0.0, 232 | 0.0, 0, 'validation', sess)) 233 | # Run a validation step and capture training summaries for TensorBoard 234 | # with the `merged` op. 235 | validation_summary, validation_accuracy, conf_matrix = sess.run( 236 | [merged_summaries, evaluation_step, confusion_matrix], 237 | feed_dict={ 238 | fingerprint_input: validation_fingerprints, 239 | ground_truth_input: validation_ground_truth, 240 | dropout_prob: 1.0 241 | }) 242 | validation_writer.add_summary(validation_summary, training_step) 243 | batch_size = min(FLAGS.batch_size, set_size - i) 244 | total_accuracy += (validation_accuracy * batch_size) / set_size 245 | if total_conf_matrix is None: 246 | total_conf_matrix = conf_matrix 247 | else: 248 | total_conf_matrix += conf_matrix 249 | tf.logging.info('Confusion Matrix:\n %s' % (total_conf_matrix)) 250 | tf.logging.info('Step %d: Validation accuracy = %.1f%% (N=%d)' % 251 | (training_step, total_accuracy * 100, set_size)) 252 | 253 | # Save the model checkpoint periodically. 254 | if (training_step % FLAGS.save_step_interval == 0 or 255 | training_step == training_steps_max): 256 | checkpoint_path = os.path.join(FLAGS.train_dir, 257 | FLAGS.model_architecture + '.ckpt') 258 | tf.logging.info('Saving to "%s-%d"', checkpoint_path, training_step) 259 | saver.save(sess, checkpoint_path, global_step=training_step) 260 | 261 | set_size = audio_processor.set_size('testing') 262 | tf.logging.info('set_size=%d', set_size) 263 | total_accuracy = 0 264 | total_conf_matrix = None 265 | for i in xrange(0, set_size, FLAGS.batch_size): 266 | test_fingerprints, test_ground_truth = audio_processor.get_data( 267 | FLAGS.batch_size, i, model_settings, 0.0, 0.0, 0, 'testing', sess) 268 | test_accuracy, conf_matrix = sess.run( 269 | [evaluation_step, confusion_matrix], 270 | feed_dict={ 271 | fingerprint_input: test_fingerprints, 272 | ground_truth_input: test_ground_truth, 273 | dropout_prob: 1.0 274 | }) 275 | batch_size = min(FLAGS.batch_size, set_size - i) 276 | total_accuracy += (test_accuracy * batch_size) / set_size 277 | if total_conf_matrix is None: 278 | total_conf_matrix = conf_matrix 279 | else: 280 | total_conf_matrix += conf_matrix 281 | tf.logging.info('Confusion Matrix:\n %s' % (total_conf_matrix)) 282 | tf.logging.info('Final test accuracy = %.1f%% (N=%d)' % (total_accuracy * 100, 283 | set_size)) 284 | 285 | 286 | if __name__ == '__main__': 287 | parser = argparse.ArgumentParser() 288 | parser.add_argument( 289 | '--data_url', 290 | type=str, 291 | # pylint: disable=line-too-long 292 | default='http://download.tensorflow.org/data/speech_commands_v0.01.tar.gz', 293 | # pylint: enable=line-too-long 294 | help='Location of speech training data archive on the web.') 295 | parser.add_argument( 296 | '--data_dir', 297 | type=str, 298 | default='/tmp/speech_dataset/', 299 | help="""\ 300 | Where to download the speech training data to. 301 | """) 302 | parser.add_argument( 303 | '--background_volume', 304 | type=float, 305 | default=0.1, 306 | help="""\ 307 | How loud the background noise should be, between 0 and 1. 308 | """) 309 | parser.add_argument( 310 | '--background_frequency', 311 | type=float, 312 | default=0.8, 313 | help="""\ 314 | How many of the training samples have background noise mixed in. 315 | """) 316 | parser.add_argument( 317 | '--silence_percentage', 318 | type=float, 319 | default=10.0, 320 | help="""\ 321 | How much of the training data should be silence. 322 | """) 323 | parser.add_argument( 324 | '--unknown_percentage', 325 | type=float, 326 | default=10.0, 327 | help="""\ 328 | How much of the training data should be unknown words. 329 | """) 330 | parser.add_argument( 331 | '--time_shift_ms', 332 | type=float, 333 | default=100.0, 334 | help="""\ 335 | Range to randomly shift the training audio by in time. 336 | """) 337 | parser.add_argument( 338 | '--testing_percentage', 339 | type=int, 340 | default=10, 341 | help='What percentage of wavs to use as a test set.') 342 | parser.add_argument( 343 | '--validation_percentage', 344 | type=int, 345 | default=10, 346 | help='What percentage of wavs to use as a validation set.') 347 | parser.add_argument( 348 | '--sample_rate', 349 | type=int, 350 | default=16000, 351 | help='Expected sample rate of the wavs',) 352 | parser.add_argument( 353 | '--clip_duration_ms', 354 | type=int, 355 | default=1000, 356 | help='Expected duration in milliseconds of the wavs',) 357 | parser.add_argument( 358 | '--window_size_ms', 359 | type=float, 360 | default=30.0, 361 | help='How long each spectrogram timeslice is',) 362 | parser.add_argument( 363 | '--window_stride_ms', 364 | type=float, 365 | default=10.0, 366 | help='How long each spectrogram timeslice is',) 367 | parser.add_argument( 368 | '--dct_coefficient_count', 369 | type=int, 370 | default=40, 371 | help='How many bins to use for the MFCC fingerprint',) 372 | parser.add_argument( 373 | '--how_many_training_steps', 374 | type=str, 375 | default='15000,3000', 376 | help='How many training loops to run',) 377 | parser.add_argument( 378 | '--eval_step_interval', 379 | type=int, 380 | default=400, 381 | help='How often to evaluate the training results.') 382 | parser.add_argument( 383 | '--learning_rate', 384 | type=str, 385 | default='0.001,0.0001', 386 | help='How large a learning rate to use when training.') 387 | parser.add_argument( 388 | '--batch_size', 389 | type=int, 390 | default=100, 391 | help='How many items to train with at once',) 392 | parser.add_argument( 393 | '--summaries_dir', 394 | type=str, 395 | default='/tmp/retrain_logs', 396 | help='Where to save summary logs for TensorBoard.') 397 | parser.add_argument( 398 | '--wanted_words', 399 | type=str, 400 | default='yes,no,up,down,left,right,on,off,stop,go', 401 | help='Words to use (others will be added to an unknown label)',) 402 | parser.add_argument( 403 | '--train_dir', 404 | type=str, 405 | default='/tmp/speech_commands_train', 406 | help='Directory to write event logs and checkpoint.') 407 | parser.add_argument( 408 | '--save_step_interval', 409 | type=int, 410 | default=100, 411 | help='Save model checkpoint every save_steps.') 412 | parser.add_argument( 413 | '--start_checkpoint', 414 | type=str, 415 | default='', 416 | help='If specified, restore this pretrained model before any training.') 417 | parser.add_argument( 418 | '--model_architecture', 419 | type=str, 420 | default='conv', 421 | help='What model architecture to use') 422 | parser.add_argument( 423 | '--check_nans', 424 | type=bool, 425 | default=False, 426 | help='Whether to check for invalid numbers during processing') 427 | 428 | FLAGS, unparsed = parser.parse_known_args() 429 | tf.app.run(main=main, argv=[sys.argv[0]] + unparsed) 430 | -------------------------------------------------------------------------------- /time_result.txt: -------------------------------------------------------------------------------- 1 | 17.1096 2 | 39.0555 3 | 0.2303 4 | 54.4187 5 | 54.0748 6 | 77.5612 7 | 63.0076 8 | 31.7504 9 | 79.1837 10 | 19.8636 11 | 18.7639 12 | 17.6010 13 | 1.2691 14 | 25.1455 15 | 49.4147 16 | 43.9083 17 | 26.1632 18 | 46.9988 19 | 47.8827 20 | 104.4578 21 | 63.3241 22 | 111.3988 23 | 118.0704 24 | 130.0131 25 | 140.6477 26 | 10.7368 27 | 65.8320 28 | 23.8368 29 | 63.1834 30 | 139.8245 31 | 53.5005 32 | 145.5692 33 | 104.2643 34 | 54.1706 35 | 32.0414 36 | 12.3187 37 | 18.6994 38 | 141.3865 39 | 96.6535 40 | 38.1732 41 | 67.9692 42 | 4.1075 43 | 4.3812 44 | 25.3969 45 | 47.2932 46 | 127.2838 47 | 21.5544 48 | 36.5270 49 | 9.9807 50 | 20.3957 51 | 14.0885 52 | 2.4214 53 | 22.1025 54 | 21.1632 55 | 2.0416 56 | 9.6773 57 | 2.6388 58 | 9.2343 59 | 10.7344 60 | 19.3604 61 | 14.4956 62 | 53.6991 63 | 28.1472 64 | 59.6000 65 | 28.4227 66 | 28.8335 67 | 36.4441 68 | 34.6203 69 | 107.9486 70 | 140.0864 71 | 39.0158 72 | 59.5250 73 | 22.3414 74 | 57.1576 75 | 37.3969 76 | 12.4797 77 | 35.7610 78 | 15.2712 79 | 102.0578 80 | 75.5051 81 | 3.0644 82 | 43.1590 83 | 14.3475 84 | 52.3304 85 | 4.6207 86 | 58.7237 87 | 151.3307 88 | 1.5041 89 | 98.5836 90 | 15.8842 91 | 113.5173 92 | 0.2115 93 | 1.8735 94 | 51.9216 95 | 117.4684 96 | 64.3236 97 | 6.8612 98 | 3.7313 99 | 16.5268 100 | 9.0296 101 | 140.8570 102 | 17.1080 103 | 82.3550 104 | 111.1049 105 | 1.6779 106 | 44.8342 107 | 101.1979 108 | 45.9162 109 | 8.1783 110 | 30.5023 111 | 40.2973 112 | 132.6416 113 | 24.5404 114 | 18.0608 115 | 128.2360 116 | 84.8774 117 | 22.1785 118 | 59.7239 119 | 115.0997 120 | 59.9550 121 | 142.6363 122 | 33.1882 123 | 5.2092 124 | 67.0225 125 | 28.3692 126 | 32.3948 127 | 16.6806 128 | 22.9328 129 | 28.0641 130 | 12.4817 131 | 25.1258 132 | 7.6407 133 | 26.3536 134 | 88.2617 135 | 90.3930 136 | 39.5848 137 | 14.7916 138 | 27.2379 139 | 124.5612 140 | 22.0597 141 | 21.0099 142 | 152.6868 143 | 104.4420 144 | 3.7650 145 | 144.7107 146 | 142.1570 147 | 8.7764 148 | 45.3974 149 | 68.3778 150 | 33.3649 151 | 72.8438 152 | 119.2222 153 | 46.9398 154 | 92.9798 155 | 11.9401 156 | 107.9348 157 | 137.2207 158 | 45.4746 159 | 55.5396 160 | 25.9474 161 | 22.7856 162 | 104.6727 163 | 0.4280 164 | 141.5980 165 | 60.9580 166 | 13.6134 167 | 38.0896 168 | 140.5806 169 | 141.3959 170 | 55.5611 171 | 94.9445 172 | 113.2595 173 | 26.0266 174 | 85.7888 175 | 25.9084 176 | 75.6535 177 | 143.2161 178 | 66.9020 179 | 11.6767 180 | 51.9222 181 | 6.3700 182 | 4.8182 183 | 41.1372 184 | 15.0717 185 | 142.0483 186 | 2.3188 187 | 102.3454 188 | 12.0595 189 | 2.2885 190 | 3.9850 191 | 1.4721 192 | 58.5363 193 | 2.2289 194 | 18.2779 195 | 13.2342 196 | 2.7723 197 | 4.7821 198 | 87.9677 199 | 41.3738 200 | 1.6938 201 | 43.1196 202 | 39.3235 203 | 121.2568 204 | 129.0257 205 | 73.8579 206 | 9.6050 207 | 12.3974 208 | 30.9931 209 | 45.4138 210 | 149.6114 211 | 69.5298 212 | 146.2447 213 | 149.8379 214 | 39.9152 215 | 27.5357 216 | 11.5517 217 | 16.1533 218 | 156.0176 219 | 89.0116 220 | 37.1579 221 | 15.5182 222 | 0.2422 223 | 6.1164 224 | 0.9955 225 | 13.0656 226 | 14.2669 227 | 0.7722 228 | 3.5672 229 | 4.9995 230 | 1.9984 231 | 27.3361 232 | 5.4742 233 | 0.2315 234 | 107.3712 235 | 4.5407 236 | 1.6593 237 | 3.7531 238 | 1.5647 239 | 1.2946 240 | 10.5330 241 | 7.9323 242 | 69.2195 243 | 31.5967 244 | 1.5865 245 | 35.6940 246 | 26.0001 247 | 48.9257 248 | 115.0865 249 | 45.7829 250 | 145.6835 251 | 22.8389 252 | 6.1591 253 | 25.3286 254 | 40.4820 255 | 32.9603 256 | 14.9118 257 | 38.8699 258 | 1.5337 259 | 77.0691 260 | 79.5108 261 | 14.1318 262 | 28.2024 263 | 91.9487 264 | 125.5210 265 | 3.7003 266 | 83.8904 267 | 142.4914 268 | 4.3942 269 | 128.1913 270 | 57.1212 271 | 153.2654 272 | 7.2292 273 | 19.8411 274 | 140.5946 275 | 146.6334 276 | 95.8393 277 | 19.9194 278 | 19.2653 279 | 31.4515 280 | 51.3485 281 | 107.0890 282 | 13.1097 283 | 143.8200 284 | 55.2034 285 | 3.0232 286 | 46.4632 287 | 63.7542 288 | 48.1166 289 | 15.9506 290 | 4.1087 291 | 37.8803 292 | 103.4466 293 | 20.2329 294 | 20.0738 295 | 141.8863 296 | 45.2509 297 | 100.8771 298 | 103.2023 299 | 132.3928 300 | 16.9719 301 | 147.7448 302 | 32.6029 303 | 9.9600 304 | 30.9625 305 | 15.2826 306 | 33.3585 307 | 0.9337 308 | 21.2703 309 | 71.6814 310 | 3.5273 311 | 14.6250 312 | 6.8132 313 | 33.7229 314 | 95.4753 315 | 38.6951 316 | 55.4762 317 | 3.4655 318 | 47.4560 319 | 96.3297 320 | 23.3100 321 | 2.1579 322 | 145.9147 323 | 24.0080 324 | 7.3919 325 | 72.6559 326 | 33.7183 327 | 1.0749 328 | 51.1608 329 | 57.4460 330 | 17.7215 331 | 10.4715 332 | 25.1300 333 | 3.6404 334 | 39.5573 335 | 2.9960 336 | 47.6390 337 | 3.2534 338 | 4.1710 339 | 7.1255 340 | 23.3236 341 | 132.2683 342 | 96.5359 343 | 64.0586 344 | 83.4121 345 | 59.9480 346 | 4.4465 347 | 30.0102 348 | 142.2167 349 | 117.6148 350 | 55.4599 351 | 76.4847 352 | 30.6262 353 | 61.7659 354 | 108.5204 355 | 42.9351 356 | 67.4590 357 | 57.3712 358 | 122.7133 359 | 4.7110 360 | 41.7364 361 | 150.9233 362 | 63.2468 363 | 143.8464 364 | 150.5185 365 | 149.4570 366 | 16.5760 367 | 147.3039 368 | 52.6083 369 | 87.9962 370 | 44.1981 371 | 136.4359 372 | 63.2872 373 | 7.0741 374 | 25.5209 375 | 46.0869 376 | 146.4649 377 | 121.1806 378 | 0.2581 379 | 5.0885 380 | 14.9027 381 | 6.6357 382 | 147.5159 383 | 96.1629 384 | 25.9300 385 | 148.3261 386 | 38.5867 387 | 104.8021 388 | 93.3055 389 | 44.7566 390 | 6.0622 391 | 17.4973 392 | 8.4585 393 | 33.3551 394 | 6.2553 395 | 140.9516 396 | 111.1198 397 | 1.5739 398 | 7.9270 399 | 55.7398 400 | 52.1438 401 | 27.7700 402 | 141.7181 403 | 3.2689 404 | 14.8923 405 | 17.7984 406 | 67.8990 407 | 26.7585 408 | 15.1333 409 | 2.0239 410 | 18.7653 411 | 23.9754 412 | 146.4471 413 | 3.1881 414 | 83.2783 415 | 6.0950 416 | 12.0518 417 | 37.4007 418 | 11.6148 419 | 21.1086 420 | 11.7136 421 | 58.4304 422 | 40.5205 423 | 90.5347 424 | 122.2655 425 | 71.4611 426 | 101.7932 427 | 54.3718 428 | 44.4985 429 | 28.5206 430 | 125.4118 431 | 8.8295 432 | 2.5677 433 | 4.8627 434 | 94.3955 435 | 29.6911 436 | 144.1469 437 | 60.1331 438 | 8.7518 439 | 38.1831 440 | 139.8536 441 | 12.7389 442 | 58.2671 443 | 63.8071 444 | 77.7928 445 | 1.3947 446 | 143.7328 447 | 150.0125 448 | 6.7044 449 | 147.3384 450 | 55.1741 451 | 140.8260 452 | 63.8825 453 | 29.4988 454 | 127.8428 455 | 144.5503 456 | 108.4069 457 | 95.6995 458 | 4.3538 459 | 30.0901 460 | 48.9157 461 | 60.0629 462 | 2.1020 463 | 69.0856 464 | 4.0033 465 | 2.2775 466 | 27.6643 467 | 29.0688 468 | 10.2222 469 | 9.3587 470 | 11.4346 471 | 26.4220 472 | 34.8860 473 | 1.7292 474 | 3.2445 475 | 143.3448 476 | 21.7173 477 | 15.1987 478 | 12.7297 479 | 7.5748 480 | 35.5754 481 | 66.6907 482 | 19.6206 483 | 4.0369 484 | 24.6870 485 | 5.9831 486 | 8.1583 487 | 3.1553 488 | 6.9379 489 | 52.4070 490 | 142.4070 491 | 14.8066 492 | 8.4760 493 | 5.1401 494 | 30.6774 495 | 132.3111 496 | 33.7201 497 | 91.9216 498 | 98.4644 499 | 29.2372 500 | 12.4705 501 | 12.4484 502 | 2.9627 503 | 18.3470 504 | 1.7783 505 | 23.7982 506 | 66.4465 507 | 15.6623 508 | 51.9362 509 | 37.5236 510 | 5.4357 511 | 38.0248 512 | 58.6278 513 | 30.7442 514 | 66.8056 515 | 11.7328 516 | 18.1108 517 | 13.4062 518 | 10.7651 519 | 9.0382 520 | 5.1968 521 | 18.7952 522 | 34.3902 523 | 114.6318 524 | 13.3443 525 | 5.2589 526 | 2.0753 527 | 9.9824 528 | 8.2662 529 | 20.5553 530 | 20.1276 531 | 24.9730 532 | 1.5298 533 | 15.6700 534 | 23.5283 535 | 13.3786 536 | 4.8698 537 | 7.6980 538 | 44.4621 539 | 2.2232 540 | 11.7662 541 | 44.8576 542 | 27.3300 543 | 37.9605 544 | 57.3414 545 | 72.8665 546 | 8.4792 547 | 95.3389 548 | 5.0071 549 | 47.8150 550 | 5.3275 551 | 64.4713 552 | 49.1333 553 | 0.9413 554 | 69.7198 555 | 4.1351 556 | 77.1503 557 | 21.0366 558 | 0.7080 559 | 43.7014 560 | 18.3289 561 | 32.4469 562 | 38.7677 563 | 1.2750 564 | 23.9355 565 | 67.9118 566 | 39.9812 567 | 68.4388 568 | 0.9761 569 | 59.7021 570 | 12.3019 571 | 16.3947 572 | 21.8528 573 | 21.9012 574 | 4.8562 575 | 5.2517 576 | 25.4449 577 | 2.0241 578 | 9.4925 579 | 2.8569 580 | 74.7665 581 | 22.1205 582 | 39.0770 583 | 34.2501 584 | 120.1350 585 | 49.7909 586 | 7.4766 587 | 17.6554 588 | 13.6021 589 | 27.2241 590 | 83.6433 591 | 19.7983 592 | 58.6933 593 | 96.1600 594 | 8.6126 595 | 15.3416 596 | 1.1754 597 | 15.7059 598 | 57.7343 599 | 33.5711 600 | 10.3982 601 | 18.6022 602 | 85.9385 603 | 144.7059 604 | 101.2870 605 | 110.9166 606 | 78.3741 607 | 55.8076 608 | 125.5177 609 | 98.4499 610 | 149.9822 611 | 41.3793 612 | 56.5305 613 | 22.0352 614 | 60.9451 615 | 59.7054 616 | 151.3827 617 | 77.9570 618 | 18.1555 619 | 74.1782 620 | 83.9479 621 | 51.8614 622 | 80.3917 623 | 77.6519 624 | 123.8033 625 | 12.1771 626 | 128.4930 627 | 140.4089 628 | 6.6044 629 | 150.9374 630 | 67.6072 631 | 151.1906 632 | 4.3192 633 | 12.1641 634 | 144.9452 635 | 146.0959 636 | 112.3672 637 | 76.9044 638 | 16.9232 639 | 28.6580 640 | 75.2863 641 | 154.6649 642 | 13.5605 643 | 138.6283 644 | 65.0556 645 | 1.4583 646 | 42.5708 647 | 53.2723 648 | 49.7485 649 | 14.8588 650 | 43.3948 651 | 41.5978 652 | 107.7749 653 | 43.7103 654 | 13.2020 655 | 89.0451 656 | 91.8523 657 | 19.8838 658 | 50.7845 659 | 103.9201 660 | 74.1808 661 | 143.6265 662 | 41.8073 663 | 12.1310 664 | 72.8054 665 | 24.0094 666 | 17.4527 667 | 18.4139 668 | 25.1389 669 | 106.8169 670 | 21.6182 671 | 33.5434 672 | 5.4527 673 | 19.7684 674 | 137.7228 675 | 140.6601 676 | 55.0007 677 | 43.9263 678 | 49.6277 679 | 86.0147 680 | 28.4745 681 | 42.4853 682 | 142.3135 683 | 31.5111 684 | 9.6352 685 | 126.3432 686 | 142.4141 687 | 10.4100 688 | 48.0022 689 | 100.6718 690 | 30.3048 691 | 34.4961 692 | 51.6274 693 | 48.6921 694 | 65.1139 695 | 11.8738 696 | 72.1664 697 | 18.3769 698 | 16.6625 699 | 22.5695 700 | 29.9502 701 | 51.2120 702 | 141.9136 703 | 1.5149 704 | 39.9864 705 | 92.1795 706 | 14.5541 707 | 0.1801 708 | 44.9290 709 | 53.4642 710 | 47.1930 711 | 55.6262 712 | 19.0544 713 | 59.3665 714 | 27.3389 715 | 30.9024 716 | 3.8689 717 | 20.9152 718 | 38.9112 719 | 13.7533 720 | 38.5681 721 | 151.0894 722 | 17.3267 723 | 150.4311 724 | 147.8600 725 | 144.9902 726 | 41.0863 727 | 106.8840 728 | 129.6686 729 | 96.2321 730 | 35.0959 731 | 60.7306 732 | 57.2399 733 | 86.6861 734 | 31.6996 735 | 148.3573 736 | 9.0268 737 | 151.8997 738 | 141.3672 739 | 24.3420 740 | 44.6812 741 | 37.3945 742 | 44.0438 743 | 11.9986 744 | 90.3960 745 | 145.9337 746 | 79.0785 747 | 31.7183 748 | 61.5429 749 | 5.5149 750 | 47.5937 751 | 25.0235 752 | 3.5497 753 | 55.8791 754 | 27.8336 755 | 146.7599 756 | 56.5417 757 | 5.3526 758 | 63.2123 759 | 150.2706 760 | 8.8183 761 | 44.1393 762 | 0.7066 763 | 142.1068 764 | 125.9834 765 | 141.2701 766 | 27.5882 767 | 83.9858 768 | 20.7918 769 | 50.6885 770 | 142.6050 771 | 32.4192 772 | 139.1043 773 | 146.9559 774 | 145.7502 775 | 40.7959 776 | 35.0644 777 | 15.5158 778 | 150.5584 779 | 85.6581 780 | 30.8206 781 | 141.8985 782 | 22.0611 783 | 41.6101 784 | 34.0402 785 | 145.2438 786 | 140.1989 787 | 146.5875 788 | 38.3080 789 | 55.8910 790 | 83.3613 791 | 128.9602 792 | 138.8247 793 | 23.4989 794 | 144.0651 795 | 13.1372 796 | 62.0643 797 | 139.3685 798 | 50.5160 799 | 148.7936 800 | 85.3464 801 | 22.0965 802 | 3.1745 803 | 5.4460 804 | 119.0749 805 | 6.9578 806 | 80.0772 807 | 25.0002 808 | 0.9636 809 | 13.5902 810 | 6.7209 811 | 143.0810 812 | 4.4560 813 | 17.7075 814 | 126.0789 815 | 138.1064 816 | 42.1143 817 | 106.7471 818 | 0.5263 819 | 48.3873 820 | 8.6537 821 | 142.8807 822 | 28.0726 823 | 149.2267 824 | 90.5596 825 | 12.2326 826 | 85.7347 827 | 127.1331 828 | 141.8880 829 | 10.1679 830 | 25.9020 831 | 54.4204 832 | 150.8773 833 | 13.4139 834 | 28.1293 835 | 146.0211 836 | 103.0666 837 | 27.1047 838 | 135.5302 839 | 140.8957 840 | 34.2354 841 | 46.7026 842 | 5.6246 843 | 1.2394 844 | 39.5896 845 | 2.9152 846 | 89.0284 847 | 2.6782 848 | 0.1849 849 | 33.0538 850 | 32.2335 851 | 6.7232 852 | 14.5632 853 | 5.1802 854 | 21.2170 855 | 66.8996 856 | 11.0308 857 | 23.3087 858 | 39.8408 859 | 147.1329 860 | 56.9672 861 | 1.6467 862 | 26.5775 863 | 58.0623 864 | 1.9223 865 | 51.5964 866 | 2.5583 867 | 5.6757 868 | 69.4168 869 | 7.6999 870 | 18.8917 871 | 9.1198 872 | 3.5231 873 | 1.3200 874 | 84.4414 875 | 3.2160 876 | 54.9005 877 | 15.2203 878 | 4.3122 879 | 52.1346 880 | 4.9700 881 | 67.8958 882 | 116.3300 883 | 149.5362 884 | 119.8346 885 | 116.5688 886 | 50.9231 887 | 138.2909 888 | 148.0986 889 | 151.6178 890 | 154.9061 891 | 132.5468 892 | 145.7125 893 | 81.4851 894 | 148.5240 895 | 153.2625 896 | 150.1754 897 | 146.9768 898 | 144.5661 899 | 3.4992 900 | 154.6605 901 | 144.2214 902 | 36.0226 903 | 61.2129 904 | 143.8522 905 | 143.8852 906 | 12.8648 907 | 40.6772 908 | 120.1016 909 | 136.8846 910 | 12.6999 911 | 15.3145 912 | 36.8832 913 | 38.1197 914 | 34.1063 915 | 135.9841 916 | 3.1742 917 | 139.9168 918 | 147.2079 919 | 41.8369 920 | 41.1079 921 | 17.5262 922 | 87.1278 923 | 58.3992 924 | 144.6201 925 | 141.8935 926 | 131.1161 927 | 142.3176 928 | 52.2428 929 | 95.1946 930 | 40.1206 931 | 28.6512 932 | 22.7446 933 | 25.9199 934 | 73.0159 935 | 143.8759 936 | 61.8326 937 | 31.9469 938 | 49.7819 939 | 142.5554 940 | 143.1478 941 | 80.8857 942 | 18.8913 943 | 149.6819 944 | 130.0616 945 | 144.2053 946 | 37.9950 947 | 108.2496 948 | 50.9321 949 | 141.9669 950 | 142.7897 951 | 82.0019 952 | 142.6549 953 | 143.4207 954 | 138.4140 955 | 86.1678 956 | 25.7086 957 | 19.8353 958 | 148.1319 959 | 141.9024 960 | 27.3549 961 | 143.6305 962 | 12.9590 963 | 9.4954 964 | 79.0655 965 | 144.9587 966 | 147.8024 967 | 145.5515 968 | 59.6286 969 | 102.9614 970 | 91.9663 971 | 111.7728 972 | 148.8711 973 | 23.2045 974 | 146.3006 975 | 12.4373 976 | 72.4344 977 | 153.3374 978 | 37.1486 979 | 82.8149 980 | 117.1137 981 | 7.7672 982 | 18.6728 983 | 0.4185 984 | 3.0757 985 | 4.9902 986 | 10.4049 987 | 18.2257 988 | 26.6538 989 | 38.0646 990 | 15.6097 991 | 9.0641 992 | 34.0219 993 | 1.3115 994 | 2.3585 995 | 18.2872 996 | 0.1549 997 | 4.2552 998 | 9.8866 999 | 26.7948 1000 | 9.4280 1001 | 141.9159 1002 | 10.6172 1003 | 96.2572 1004 | 64.0065 1005 | 20.1214 1006 | 27.8145 1007 | 77.1913 1008 | 75.4473 1009 | 7.5224 1010 | 46.5719 1011 | 47.0895 1012 | 127.9266 1013 | 47.4834 1014 | 3.4339 1015 | 138.4519 1016 | 17.5149 1017 | 30.3741 1018 | 107.8037 1019 | 92.1980 1020 | 67.6365 1021 | 84.4091 1022 | 29.9779 1023 | 6.5377 1024 | 121.3248 1025 | 19.7484 1026 | 86.4577 1027 | 16.2209 1028 | 21.7224 1029 | 144.0781 1030 | 93.9521 1031 | 24.0023 1032 | 50.0025 1033 | 31.5387 1034 | 141.3488 1035 | 146.2117 1036 | 33.0418 1037 | 37.9579 1038 | 79.9477 1039 | 142.6778 1040 | 85.0276 1041 | 8.1025 1042 | 57.5765 1043 | 77.7877 1044 | 1.0664 1045 | 75.7921 1046 | 37.8876 1047 | 16.7196 1048 | 82.9060 1049 | 29.4008 1050 | 21.6070 1051 | 44.0643 1052 | 104.7741 1053 | 1.0341 1054 | 153.5153 1055 | 16.4715 1056 | 128.6295 1057 | 37.8427 1058 | 16.2386 1059 | 116.7780 1060 | 13.1252 1061 | 150.1977 1062 | 140.1425 1063 | 143.4744 1064 | 138.6088 1065 | 117.9313 1066 | 15.1949 1067 | 147.8191 1068 | 150.1406 1069 | 152.5213 1070 | 148.5606 1071 | 120.3815 1072 | 101.6994 1073 | 108.0462 1074 | 151.0699 1075 | 97.6984 1076 | 145.2680 1077 | 143.7045 1078 | 153.3860 1079 | 26.6638 1080 | 149.2815 1081 | 153.6747 1082 | 103.3783 1083 | 100.7035 1084 | 147.9552 1085 | 142.5047 1086 | 23.3588 1087 | 143.8756 1088 | 125.8544 1089 | 96.1533 1090 | 119.3090 1091 | 34.5497 1092 | 39.1673 1093 | 8.2357 1094 | 146.3898 1095 | 32.3843 1096 | 25.1152 1097 | 140.8602 1098 | 42.9664 1099 | 13.7021 1100 | 17.9977 1101 | 16.1988 1102 | 125.5690 1103 | 33.9564 1104 | 11.0900 1105 | 141.2521 1106 | 47.9270 1107 | 131.8727 1108 | 1.5695 1109 | 12.7683 1110 | 2.3022 1111 | 16.9789 1112 | 3.6957 1113 | 47.1326 1114 | 6.3820 1115 | 135.8437 1116 | 29.8538 1117 | 1.1025 1118 | 5.5144 1119 | 49.9136 1120 | 22.1694 1121 | 15.9020 1122 | 1.1736 1123 | 74.5145 1124 | 43.2777 1125 | 10.6859 1126 | 12.0831 1127 | 2.9403 1128 | 6.7998 1129 | 31.1538 1130 | 24.2770 1131 | 23.9145 1132 | 14.4536 1133 | 53.1574 1134 | 19.4903 1135 | 28.0690 1136 | 5.1912 1137 | 7.8724 1138 | 48.4442 1139 | 21.6882 1140 | 3.6645 1141 | 7.7970 1142 | 29.0136 1143 | 7.1001 1144 | 16.9026 1145 | 4.2772 1146 | 57.8197 1147 | 11.3797 1148 | 8.1096 1149 | 13.1978 1150 | 28.2567 1151 | 36.8104 1152 | 148.2991 1153 | 2.9969 1154 | 121.2050 1155 | 5.8962 1156 | 10.3021 1157 | 41.5583 1158 | 24.2508 1159 | 6.4557 1160 | 34.5014 1161 | 14.1632 1162 | 36.3909 1163 | 45.4848 1164 | 1.0218 1165 | 46.5858 1166 | 98.1196 1167 | 50.4710 1168 | 124.7846 1169 | 67.0420 1170 | 142.3538 1171 | 24.4617 1172 | 7.4959 1173 | 5.2956 1174 | 63.8885 1175 | 45.0866 1176 | 71.5217 1177 | 29.4042 1178 | 6.9017 1179 | 42.2293 1180 | 87.7802 1181 | 8.1095 1182 | 26.0009 1183 | 29.0530 1184 | 122.7702 1185 | 8.8356 1186 | 44.3811 1187 | 73.6743 1188 | 3.2400 1189 | 138.9872 1190 | 30.5971 1191 | 13.3746 1192 | 26.7628 1193 | 12.6320 1194 | 70.0857 1195 | 88.9373 1196 | 117.8024 1197 | 41.3063 1198 | 10.8782 1199 | 15.8254 1200 | 23.0094 1201 | 93.3139 1202 | 14.6468 1203 | 5.0136 1204 | 35.9575 1205 | 4.9769 1206 | 89.8853 1207 | 2.6855 1208 | 15.7678 1209 | 146.1412 1210 | 141.0738 1211 | 15.9090 1212 | 12.4702 1213 | 6.9317 1214 | 40.3367 1215 | 77.2104 1216 | 38.1316 1217 | 77.3013 1218 | 143.0473 1219 | 52.9546 1220 | 49.5927 1221 | 2.3838 1222 | 5.6280 1223 | 27.8872 1224 | 1.6614 1225 | 12.6656 1226 | 147.6655 1227 | 32.7131 1228 | 55.9840 1229 | 54.6586 1230 | 6.9620 1231 | 45.5310 1232 | 46.9498 1233 | 1.4071 1234 | 50.0019 1235 | 2.4585 1236 | 136.9974 1237 | 19.5314 1238 | 5.4838 1239 | 19.6584 1240 | 6.0322 1241 | 60.2580 1242 | 92.8173 1243 | 150.0480 1244 | 50.2541 1245 | 30.8502 1246 | 0.1986 1247 | 59.2752 1248 | 67.6662 1249 | 43.1574 1250 | 73.2865 1251 | 87.2126 1252 | 12.1274 1253 | 58.0323 1254 | 92.2628 1255 | 26.4851 1256 | 28.5567 1257 | 52.8539 1258 | 148.5543 1259 | 33.5886 1260 | 68.6984 1261 | 7.2304 1262 | 1.7435 1263 | 10.1392 1264 | 114.3316 1265 | 147.3608 1266 | 6.8467 1267 | 89.0745 1268 | 1.5979 1269 | 2.8048 1270 | 2.1066 1271 | 6.8813 1272 | 17.5840 1273 | 1.3798 1274 | 1.4737 1275 | 61.9141 1276 | 1.8053 1277 | 15.7952 1278 | 1.0043 1279 | 2.0691 1280 | 1.9541 1281 | 6.5830 1282 | 3.2151 1283 | 1.4682 1284 | 1.1682 1285 | 49.0029 1286 | 31.6571 1287 | 1.9975 1288 | 2.5342 1289 | 1.2890 1290 | 5.0352 1291 | 3.7746 1292 | 2.3723 1293 | 1.9035 1294 | 2.5865 1295 | 71.8912 1296 | 47.5370 1297 | 1.5175 1298 | 17.1616 1299 | 50.7755 1300 | 2.3455 1301 | 31.6073 1302 | 1.5593 1303 | 56.6222 1304 | 19.7239 1305 | 64.3135 1306 | 5.0330 1307 | 4.5392 1308 | 5.8300 1309 | 9.8461 1310 | 144.3152 1311 | 22.3977 1312 | 34.2903 1313 | 95.6929 1314 | 27.3205 1315 | 22.4798 1316 | 5.4246 1317 | 2.1301 1318 | 146.4437 1319 | 11.5815 1320 | 12.9020 1321 | 44.5370 1322 | 1.4954 1323 | 1.1921 1324 | 1.1971 1325 | 3.4643 1326 | 71.3254 1327 | 45.3073 1328 | 6.6428 1329 | 6.2290 1330 | 12.3436 1331 | 18.9231 1332 | 111.5401 1333 | 2.5140 1334 | 134.4635 1335 | 1.3034 1336 | 2.6878 1337 | 58.4474 1338 | 2.0014 1339 | 20.6146 1340 | 6.9701 1341 | 2.5544 1342 | 1.7414 1343 | 6.8676 1344 | 3.7364 1345 | 9.9601 1346 | 1.4228 1347 | 0.7986 1348 | 68.7062 1349 | 4.2352 1350 | 39.7632 1351 | 4.9039 1352 | 2.6384 1353 | 1.7238 1354 | 16.5449 1355 | 3.2118 1356 | 77.7410 1357 | 0.7149 1358 | 2.2215 1359 | 3.4871 1360 | 16.9186 1361 | 6.3581 1362 | 12.2351 1363 | 6.4425 1364 | 31.3323 1365 | 3.9175 1366 | 136.4674 1367 | 94.3649 1368 | 1.2507 1369 | 64.0629 1370 | 7.5092 1371 | 91.7179 1372 | 1.9226 1373 | 29.1497 1374 | 153.4242 1375 | 162.9178 1376 | 68.6516 1377 | 77.2356 1378 | 3.4762 1379 | 33.5209 1380 | 10.5186 1381 | 159.0460 1382 | 1.3124 1383 | 145.7876 1384 | 84.6861 1385 | 1.9853 1386 | 70.5194 1387 | 108.0294 1388 | 110.1561 1389 | 12.4828 1390 | 7.8623 1391 | 39.1507 1392 | 149.0763 1393 | 13.1176 1394 | 7.5276 1395 | 144.4734 1396 | 61.1234 1397 | 48.5215 1398 | 94.4014 1399 | 105.1019 1400 | 54.7977 1401 | 1.5839 1402 | 7.0890 1403 | 3.0772 1404 | 1.3005 1405 | 2.3548 1406 | 1.1885 1407 | 1.3593 1408 | 28.1583 1409 | 7.7162 1410 | 3.0127 1411 | 1.5932 1412 | 7.8343 1413 | 0.7195 1414 | 4.1274 1415 | 1.7481 1416 | 6.0950 1417 | 2.0057 1418 | 1.3225 1419 | 2.7322 1420 | 7.7285 1421 | 22.6858 1422 | 82.5494 1423 | 9.5144 1424 | 146.2253 1425 | 68.3413 1426 | 0.9906 1427 | 29.0707 1428 | 145.0697 1429 | 140.9270 1430 | 36.0953 1431 | 121.7394 1432 | 82.5619 1433 | 54.5553 1434 | 150.0277 1435 | 145.5178 1436 | 98.1598 1437 | 148.8080 1438 | 107.2903 1439 | 2.3870 1440 | 62.3112 1441 | 80.8176 1442 | 19.3569 1443 | 49.6527 1444 | 148.5704 1445 | 143.5712 1446 | 7.7839 1447 | 145.6212 1448 | 2.3299 1449 | 64.2635 1450 | 56.5707 1451 | 23.8738 1452 | 47.2975 1453 | 11.6004 1454 | 22.4655 1455 | 144.1556 1456 | 6.0273 1457 | 63.0311 1458 | 31.9970 1459 | 11.8061 1460 | 15.9963 1461 | 1.3330 1462 | 11.1150 1463 | 3.1999 1464 | 1.7262 1465 | 142.6462 1466 | 37.9274 1467 | 13.1500 1468 | 21.7567 1469 | 0.7097 1470 | 5.5332 1471 | 7.2206 1472 | 2.7214 1473 | 24.3450 1474 | 24.2617 1475 | 148.3215 1476 | 40.6807 1477 | 0.8519 1478 | 78.2972 1479 | 106.9494 1480 | 1.9386 1481 | 40.3823 1482 | 4.4920 1483 | 113.5813 1484 | 22.5141 1485 | 33.5167 1486 | 14.0950 1487 | 1.1748 1488 | 1.5863 1489 | 47.9489 1490 | 146.7349 1491 | 7.3906 1492 | 52.5785 1493 | 106.5986 1494 | 39.6605 1495 | 48.2657 1496 | 4.1434 1497 | 3.7716 1498 | 150.9261 1499 | 38.2816 1500 | 23.5153 1501 | 150.8626 1502 | 1.3794 1503 | 5.0530 1504 | 11.5590 1505 | 73.2166 1506 | 49.0748 1507 | 49.4428 1508 | 18.2139 1509 | 12.9137 1510 | 22.6345 1511 | 69.4785 1512 | 143.8219 1513 | 1.6173 1514 | 143.3243 1515 | 7.7833 1516 | 21.0161 1517 | 124.7362 1518 | 36.2614 1519 | 23.6049 1520 | 37.2941 1521 | 8.0202 1522 | 1.1477 1523 | 7.2889 1524 | 6.1576 1525 | 32.5899 1526 | 11.9789 1527 | 0.1542 1528 | 53.3717 1529 | 0.6930 1530 | 28.3528 1531 | 3.1343 1532 | 10.3389 1533 | 1.6351 1534 | 5.7535 1535 | 5.1988 1536 | 77.2337 1537 | 2.0463 1538 | 5.0844 1539 | 7.8953 1540 | 13.1080 1541 | 9.0846 1542 | 6.2899 1543 | 14.4154 1544 | 28.9888 1545 | 0.2298 1546 | 61.9460 1547 | 50.0943 1548 | 0.9952 1549 | 96.6784 1550 | 6.3715 1551 | 137.6811 1552 | 2.7729 1553 | 3.7996 1554 | 150.8308 1555 | 145.8467 1556 | 108.0466 1557 | 43.0569 1558 | 0.8110 1559 | 16.8933 1560 | 22.4444 1561 | 132.2878 1562 | 2.6383 1563 | 146.3800 1564 | 28.9107 1565 | 2.7085 1566 | 56.7494 1567 | 79.9938 1568 | 112.4639 1569 | 14.0905 1570 | 7.9769 1571 | 56.1781 1572 | 55.2313 1573 | 23.5154 1574 | 3.1848 1575 | 146.2881 1576 | 33.9138 1577 | 61.9245 1578 | 71.6925 1579 | 149.0499 1580 | 25.5506 1581 | 24.5929 1582 | 10.5735 1583 | 2.1162 1584 | 50.5927 1585 | 5.7357 1586 | 1.3879 1587 | 1.3737 1588 | 0.4727 1589 | 35.3152 1590 | 61.4927 1591 | 0.6831 1592 | 4.4998 1593 | 4.8402 1594 | 27.4945 1595 | 4.2635 1596 | 11.3031 1597 | 18.1592 1598 | 29.2245 1599 | 12.5939 1600 | 5.9888 1601 | 42.0683 1602 | 146.1147 1603 | 143.8956 1604 | 140.3559 1605 | 82.5626 1606 | 0.5665 1607 | 96.3849 1608 | 148.0078 1609 | 137.1482 1610 | 81.4215 1611 | 121.3276 1612 | 116.9001 1613 | 51.6229 1614 | 144.8925 1615 | 149.7968 1616 | 123.3901 1617 | 144.2994 1618 | 143.2107 1619 | 13.5891 1620 | 100.2990 1621 | 146.4800 1622 | 144.6126 1623 | 90.0628 1624 | 143.6756 1625 | 147.9001 1626 | 13.0277 1627 | 138.7175 1628 | 33.2223 1629 | 43.7210 1630 | 25.3796 1631 | 144.7211 1632 | 38.4023 1633 | 34.8214 1634 | 42.0941 1635 | 49.5687 1636 | 142.3841 1637 | 73.0631 1638 | 1.2512 1639 | 45.6843 1640 | 8.4848 1641 | 22.8933 1642 | 155.9038 1643 | 56.4788 1644 | 94.1385 1645 | 142.0361 1646 | 84.8821 1647 | 148.1673 1648 | 159.6954 1649 | 92.2826 1650 | 4.9755 1651 | 76.9543 1652 | 10.2029 1653 | 56.1356 1654 | 8.8089 1655 | 153.4941 1656 | 100.1896 1657 | 3.0961 1658 | 37.0511 1659 | 36.0836 1660 | 106.6724 1661 | 6.8166 1662 | 27.6118 1663 | 33.0030 1664 | 66.2813 1665 | 86.3931 1666 | 1.2828 1667 | 35.0983 1668 | 2.3576 1669 | 1.1599 1670 | 92.8588 1671 | 8.1146 1672 | 55.6568 1673 | 51.7482 1674 | 6.2179 1675 | 0.1702 1676 | 4.1623 1677 | 4.4518 1678 | 36.4852 1679 | 6.2103 1680 | 7.9715 1681 | 52.1352 1682 | 148.6854 1683 | 1.4627 1684 | 16.8152 1685 | 17.4257 1686 | 138.9600 1687 | 52.2322 1688 | 11.3161 1689 | 2.1329 1690 | 25.0257 1691 | 31.1639 1692 | 74.3789 1693 | 9.0732 1694 | 143.9985 1695 | 12.6459 1696 | 8.2297 1697 | 4.2714 1698 | 6.4151 1699 | 21.9242 1700 | 12.8160 1701 | 98.4336 1702 | 79.3572 1703 | 153.6987 1704 | 160.1868 1705 | 122.9365 1706 | 143.8784 1707 | 103.1421 1708 | 49.4274 1709 | 67.9417 1710 | 143.4089 1711 | 32.0760 1712 | 18.4778 1713 | 44.8192 1714 | 135.0280 1715 | 79.9561 1716 | 148.2030 1717 | 151.9446 1718 | 24.1681 1719 | 131.1499 1720 | 149.5982 1721 | 99.9083 1722 | 140.9586 1723 | 148.6705 1724 | 99.9511 1725 | 21.4849 1726 | 146.8159 1727 | 144.3414 1728 | 7.7592 1729 | 148.7992 1730 | 86.2872 1731 | 146.6617 1732 | 96.9765 1733 | 10.7014 1734 | 141.3413 1735 | 152.9148 1736 | 113.3306 1737 | 143.7137 1738 | 7.8103 1739 | 80.0260 1740 | 90.5116 1741 | 148.1215 1742 | 4.7483 1743 | 88.5062 1744 | 73.2545 1745 | 3.0702 1746 | 49.0062 1747 | 81.3213 1748 | 56.0541 1749 | 15.9592 1750 | 37.3886 1751 | 39.7740 1752 | 117.4870 1753 | 16.8163 1754 | 10.3658 1755 | 139.4571 1756 | 113.7419 1757 | 33.2954 1758 | 53.3366 1759 | 113.4529 1760 | 98.1170 1761 | 146.6356 1762 | 52.1958 1763 | 20.2207 1764 | 28.6115 1765 | 13.6368 1766 | 46.8805 1767 | 10.5133 1768 | 18.2373 1769 | 93.2120 1770 | 147.6334 1771 | 16.9781 1772 | 16.5081 1773 | 9.8947 1774 | 96.0890 1775 | 146.5288 1776 | 66.8956 1777 | 152.0530 1778 | 110.9011 1779 | 144.2596 1780 | 19.6199 1781 | 93.9691 1782 | 140.8471 1783 | 66.2752 1784 | 12.5218 1785 | 84.8649 1786 | 149.5376 1787 | 26.3644 1788 | 132.2238 1789 | 114.9379 1790 | 27.5455 1791 | 115.2197 1792 | 62.0676 1793 | 78.7513 1794 | 142.1670 1795 | 22.2805 1796 | 32.5588 1797 | 42.6557 1798 | 21.3827 1799 | 38.4069 1800 | 16.2365 1801 | -------------------------------------------------------------------------------- /times_all.txt: -------------------------------------------------------------------------------- 1 | 7.1096 2 | 9.0555 3 | 0.2303 4 | 4.4187 5 | 4.0748 6 | 7.5612 7 | 3.0076 8 | 1.7504 9 | 9.1837 10 | 9.8636 11 | 8.7639 12 | 7.6010 13 | 1.2691 14 | 5.1455 15 | 9.4147 16 | 3.9083 17 | 6.1632 18 | 6.9988 19 | 7.8827 20 | 4.4578 21 | 3.3241 22 | 1.3988 23 | 8.0704 24 | 0.0131 25 | 0.6477 26 | 0.7368 27 | 5.8320 28 | 3.8368 29 | 3.1834 30 | 9.8245 31 | 3.5005 32 | 5.5692 33 | 4.2643 34 | 4.1706 35 | 2.0414 36 | 2.3187 37 | 8.6994 38 | 1.3865 39 | 6.6535 40 | 8.1732 41 | 7.9692 42 | 4.1075 43 | 4.3812 44 | 5.3969 45 | 7.2932 46 | 7.2838 47 | 1.5544 48 | 6.5270 49 | 9.9807 50 | 0.3957 51 | 4.0885 52 | 2.4214 53 | 2.1025 54 | 1.1632 55 | 2.0416 56 | 9.6773 57 | 2.6388 58 | 9.2343 59 | 0.7344 60 | 9.3604 61 | 4.4956 62 | 3.6991 63 | 8.1472 64 | 9.6000 65 | 8.4227 66 | 8.8335 67 | 6.4441 68 | 4.6203 69 | 7.9486 70 | 0.0864 71 | 9.0158 72 | 9.5250 73 | 2.3414 74 | 7.1576 75 | 7.3969 76 | 2.4797 77 | 5.7610 78 | 5.2712 79 | 2.0578 80 | 5.5051 81 | 3.0644 82 | 3.1590 83 | 4.3475 84 | 2.3304 85 | 4.6207 86 | 8.7237 87 | 1.3307 88 | 1.5041 89 | 8.5836 90 | 5.8842 91 | 3.5173 92 | 0.2115 93 | 1.8735 94 | 1.9216 95 | 7.4684 96 | 4.3236 97 | 6.8612 98 | 3.7313 99 | 6.5268 100 | 9.0296 101 | 0.8570 102 | 7.1080 103 | 2.3550 104 | 1.1049 105 | 1.6779 106 | 4.8342 107 | 1.1979 108 | 5.9162 109 | 8.1783 110 | 0.5023 111 | 0.2973 112 | 2.6416 113 | 4.5404 114 | 8.0608 115 | 8.2360 116 | 4.8774 117 | 2.1785 118 | 9.7239 119 | 5.0997 120 | 9.9550 121 | 2.6363 122 | 3.1882 123 | 5.2092 124 | 7.0225 125 | 8.3692 126 | 2.3948 127 | 6.6806 128 | 2.9328 129 | 8.0641 130 | 2.4817 131 | 5.1258 132 | 7.6407 133 | 6.3536 134 | 8.2617 135 | 0.3930 136 | 9.5848 137 | 4.7916 138 | 7.2379 139 | 4.5612 140 | 2.0597 141 | 1.0099 142 | 2.6868 143 | 4.4420 144 | 3.7650 145 | 4.7107 146 | 2.1570 147 | 8.7764 148 | 5.3974 149 | 8.3778 150 | 3.3649 151 | 2.8438 152 | 9.2222 153 | 6.9398 154 | 2.9798 155 | 1.9401 156 | 7.9348 157 | 7.2207 158 | 5.4746 159 | 5.5396 160 | 5.9474 161 | 2.7856 162 | 4.6727 163 | 0.4280 164 | 1.5980 165 | 0.9580 166 | 3.6134 167 | 8.0896 168 | 0.5806 169 | 1.3959 170 | 5.5611 171 | 4.9445 172 | 3.2595 173 | 6.0266 174 | 5.7888 175 | 5.9084 176 | 5.6535 177 | 3.2161 178 | 6.9020 179 | 1.6767 180 | 1.9222 181 | 6.3700 182 | 4.8182 183 | 1.1372 184 | 5.0717 185 | 2.0483 186 | 2.3188 187 | 2.3454 188 | 2.0595 189 | 2.2885 190 | 3.9850 191 | 1.4721 192 | 8.5363 193 | 2.2289 194 | 8.2779 195 | 3.2342 196 | 2.7723 197 | 4.7821 198 | 7.9677 199 | 1.3738 200 | 1.6938 201 | 3.1196 202 | 9.3235 203 | 1.2568 204 | 9.0257 205 | 3.8579 206 | 9.6050 207 | 2.3974 208 | 0.9931 209 | 5.4138 210 | 9.6114 211 | 9.5298 212 | 6.2447 213 | 9.8379 214 | 9.9152 215 | 7.5357 216 | 1.5517 217 | 6.1533 218 | 6.0176 219 | 9.0116 220 | 7.1579 221 | 5.5182 222 | 0.2422 223 | 6.1164 224 | 0.9955 225 | 3.0656 226 | 4.2669 227 | 0.7722 228 | 3.5672 229 | 4.9995 230 | 1.9984 231 | 7.3361 232 | 5.4742 233 | 0.2315 234 | 7.3712 235 | 4.5407 236 | 1.6593 237 | 3.7531 238 | 1.5647 239 | 1.2946 240 | 0.5330 241 | 7.9323 242 | 9.2195 243 | 1.5967 244 | 1.5865 245 | 5.6940 246 | 6.0001 247 | 8.9257 248 | 5.0865 249 | 5.7829 250 | 5.6835 251 | 2.8389 252 | 6.1591 253 | 5.3286 254 | 0.4820 255 | 2.9603 256 | 4.9118 257 | 8.8699 258 | 1.5337 259 | 7.0691 260 | 9.5108 261 | 4.1318 262 | 8.2024 263 | 1.9487 264 | 5.5210 265 | 3.7003 266 | 3.8904 267 | 2.4914 268 | 4.3942 269 | 8.1913 270 | 7.1212 271 | 3.2654 272 | 7.2292 273 | 9.8411 274 | 0.5946 275 | 6.6334 276 | 5.8393 277 | 9.9194 278 | 9.2653 279 | 1.4515 280 | 1.3485 281 | 7.0890 282 | 3.1097 283 | 3.8200 284 | 5.2034 285 | 3.0232 286 | 6.4632 287 | 3.7542 288 | 8.1166 289 | 5.9506 290 | 4.1087 291 | 7.8803 292 | 3.4466 293 | 0.2329 294 | 0.0738 295 | 1.8863 296 | 5.2509 297 | 0.8771 298 | 3.2023 299 | 2.3928 300 | 6.9719 301 | 7.7448 302 | 2.6029 303 | 9.9600 304 | 0.9625 305 | 5.2826 306 | 3.3585 307 | 0.9337 308 | 1.2703 309 | 1.6814 310 | 3.5273 311 | 4.6250 312 | 6.8132 313 | 3.7229 314 | 5.4753 315 | 8.6951 316 | 5.4762 317 | 3.4655 318 | 7.4560 319 | 6.3297 320 | 3.3100 321 | 2.1579 322 | 5.9147 323 | 4.0080 324 | 7.3919 325 | 2.6559 326 | 3.7183 327 | 1.0749 328 | 1.1608 329 | 7.4460 330 | 7.7215 331 | 0.4715 332 | 5.1300 333 | 3.6404 334 | 9.5573 335 | 2.9960 336 | 7.6390 337 | 3.2534 338 | 4.1710 339 | 7.1255 340 | 3.3236 341 | 2.2683 342 | 6.5359 343 | 4.0586 344 | 3.4121 345 | 9.9480 346 | 4.4465 347 | 0.0102 348 | 2.2167 349 | 7.6148 350 | 5.4599 351 | 6.4847 352 | 0.6262 353 | 1.7659 354 | 8.5204 355 | 2.9351 356 | 7.4590 357 | 7.3712 358 | 2.7133 359 | 4.7110 360 | 1.7364 361 | 0.9233 362 | 3.2468 363 | 3.8464 364 | 0.5185 365 | 9.4570 366 | 6.5760 367 | 7.3039 368 | 2.6083 369 | 7.9962 370 | 4.1981 371 | 6.4359 372 | 3.2872 373 | 7.0741 374 | 5.5209 375 | 6.0869 376 | 6.4649 377 | 1.1806 378 | 0.2581 379 | 5.0885 380 | 4.9027 381 | 6.6357 382 | 7.5159 383 | 6.1629 384 | 5.9300 385 | 8.3261 386 | 8.5867 387 | 4.8021 388 | 3.3055 389 | 4.7566 390 | 6.0622 391 | 7.4973 392 | 8.4585 393 | 3.3551 394 | 6.2553 395 | 0.9516 396 | 1.1198 397 | 1.5739 398 | 7.9270 399 | 5.7398 400 | 2.1438 401 | 7.7700 402 | 1.7181 403 | 3.2689 404 | 4.8923 405 | 7.7984 406 | 7.8990 407 | 6.7585 408 | 5.1333 409 | 2.0239 410 | 8.7653 411 | 3.9754 412 | 6.4471 413 | 3.1881 414 | 3.2783 415 | 6.0950 416 | 2.0518 417 | 7.4007 418 | 1.6148 419 | 1.1086 420 | 1.7136 421 | 8.4304 422 | 0.5205 423 | 0.5347 424 | 2.2655 425 | 1.4611 426 | 1.7932 427 | 4.3718 428 | 4.4985 429 | 8.5206 430 | 5.4118 431 | 8.8295 432 | 2.5677 433 | 4.8627 434 | 4.3955 435 | 9.6911 436 | 4.1469 437 | 0.1331 438 | 8.7518 439 | 8.1831 440 | 9.8536 441 | 2.7389 442 | 8.2671 443 | 3.8071 444 | 7.7928 445 | 1.3947 446 | 3.7328 447 | 0.0125 448 | 6.7044 449 | 7.3384 450 | 5.1741 451 | 0.8260 452 | 3.8825 453 | 9.4988 454 | 7.8428 455 | 4.5503 456 | 8.4069 457 | 5.6995 458 | 4.3538 459 | 0.0901 460 | 8.9157 461 | 0.0629 462 | 2.1020 463 | 9.0856 464 | 4.0033 465 | 2.2775 466 | 7.6643 467 | 9.0688 468 | 0.2222 469 | 9.3587 470 | 1.4346 471 | 6.4220 472 | 4.8860 473 | 1.7292 474 | 3.2445 475 | 3.3448 476 | 1.7173 477 | 5.1987 478 | 2.7297 479 | 7.5748 480 | 5.5754 481 | 6.6907 482 | 9.6206 483 | 4.0369 484 | 4.6870 485 | 5.9831 486 | 8.1583 487 | 3.1553 488 | 6.9379 489 | 2.4070 490 | 2.4070 491 | 4.8066 492 | 8.4760 493 | 5.1401 494 | 0.6774 495 | 2.3111 496 | 3.7201 497 | 1.9216 498 | 8.4644 499 | 9.2372 500 | 2.4705 501 | 2.4484 502 | 2.9627 503 | 8.3470 504 | 1.7783 505 | 3.7982 506 | 6.4465 507 | 5.6623 508 | 1.9362 509 | 7.5236 510 | 5.4357 511 | 8.0248 512 | 8.6278 513 | 0.7442 514 | 6.8056 515 | 1.7328 516 | 8.1108 517 | 3.4062 518 | 0.7651 519 | 9.0382 520 | 5.1968 521 | 8.7952 522 | 4.3902 523 | 4.6318 524 | 3.3443 525 | 5.2589 526 | 2.0753 527 | 9.9824 528 | 8.2662 529 | 0.5553 530 | 0.1276 531 | 4.9730 532 | 1.5298 533 | 5.6700 534 | 3.5283 535 | 3.3786 536 | 4.8698 537 | 7.6980 538 | 4.4621 539 | 2.2232 540 | 1.7662 541 | 4.8576 542 | 7.3300 543 | 7.9605 544 | 7.3414 545 | 2.8665 546 | 8.4792 547 | 5.3389 548 | 5.0071 549 | 7.8150 550 | 5.3275 551 | 4.4713 552 | 9.1333 553 | 0.9413 554 | 9.7198 555 | 4.1351 556 | 7.1503 557 | 1.0366 558 | 0.7080 559 | 3.7014 560 | 8.3289 561 | 2.4469 562 | 8.7677 563 | 1.2750 564 | 3.9355 565 | 7.9118 566 | 9.9812 567 | 8.4388 568 | 0.9761 569 | 9.7021 570 | 2.3019 571 | 6.3947 572 | 1.8528 573 | 1.9012 574 | 4.8562 575 | 5.2517 576 | 5.4449 577 | 2.0241 578 | 9.4925 579 | 2.8569 580 | 4.7665 581 | 2.1205 582 | 9.0770 583 | 4.2501 584 | 0.1350 585 | 9.7909 586 | 7.4766 587 | 7.6554 588 | 3.6021 589 | 7.2241 590 | 3.6433 591 | 9.7983 592 | 8.6933 593 | 6.1600 594 | 8.6126 595 | 5.3416 596 | 1.1754 597 | 5.7059 598 | 7.7343 599 | 3.5711 600 | 0.3982 601 | 8.6022 602 | 5.9385 603 | 4.7059 604 | 1.2870 605 | 0.9166 606 | 8.3741 607 | 5.8076 608 | 5.5177 609 | 8.4499 610 | 9.9822 611 | 1.3793 612 | 6.5305 613 | 2.0352 614 | 0.9451 615 | 9.7054 616 | 1.3827 617 | 7.9570 618 | 8.1555 619 | 4.1782 620 | 3.9479 621 | 1.8614 622 | 0.3917 623 | 7.6519 624 | 3.8033 625 | 2.1771 626 | 8.4930 627 | 0.4089 628 | 6.6044 629 | 0.9374 630 | 7.6072 631 | 1.1906 632 | 4.3192 633 | 2.1641 634 | 4.9452 635 | 6.0959 636 | 2.3672 637 | 6.9044 638 | 6.9232 639 | 8.6580 640 | 5.2863 641 | 4.6649 642 | 3.5605 643 | 8.6283 644 | 5.0556 645 | 1.4583 646 | 2.5708 647 | 3.2723 648 | 9.7485 649 | 4.8588 650 | 3.3948 651 | 1.5978 652 | 7.7749 653 | 3.7103 654 | 3.2020 655 | 9.0451 656 | 1.8523 657 | 9.8838 658 | 0.7845 659 | 3.9201 660 | 4.1808 661 | 3.6265 662 | 1.8073 663 | 2.1310 664 | 2.8054 665 | 4.0094 666 | 7.4527 667 | 8.4139 668 | 5.1389 669 | 6.8169 670 | 1.6182 671 | 3.5434 672 | 5.4527 673 | 9.7684 674 | 7.7228 675 | 0.6601 676 | 5.0007 677 | 3.9263 678 | 9.6277 679 | 6.0147 680 | 8.4745 681 | 2.4853 682 | 2.3135 683 | 1.5111 684 | 9.6352 685 | 6.3432 686 | 2.4141 687 | 0.4100 688 | 8.0022 689 | 0.6718 690 | 0.3048 691 | 4.4961 692 | 1.6274 693 | 8.6921 694 | 5.1139 695 | 1.8738 696 | 2.1664 697 | 8.3769 698 | 6.6625 699 | 2.5695 700 | 9.9502 701 | 1.2120 702 | 1.9136 703 | 1.5149 704 | 9.9864 705 | 2.1795 706 | 4.5541 707 | 0.1801 708 | 4.9290 709 | 3.4642 710 | 7.1930 711 | 5.6262 712 | 9.0544 713 | 9.3665 714 | 7.3389 715 | 0.9024 716 | 3.8689 717 | 0.9152 718 | 8.9112 719 | 3.7533 720 | 8.5681 721 | 1.0894 722 | 7.3267 723 | 0.4311 724 | 7.8600 725 | 4.9902 726 | 1.0863 727 | 6.8840 728 | 9.6686 729 | 6.2321 730 | 5.0959 731 | 0.7306 732 | 7.2399 733 | 6.6861 734 | 1.6996 735 | 8.3573 736 | 9.0268 737 | 1.8997 738 | 1.3672 739 | 4.3420 740 | 4.6812 741 | 7.3945 742 | 4.0438 743 | 1.9986 744 | 0.3960 745 | 5.9337 746 | 9.0785 747 | 1.7183 748 | 1.5429 749 | 5.5149 750 | 7.5937 751 | 5.0235 752 | 3.5497 753 | 5.8791 754 | 7.8336 755 | 6.7599 756 | 6.5417 757 | 5.3526 758 | 3.2123 759 | 0.2706 760 | 8.8183 761 | 4.1393 762 | 0.7066 763 | 2.1068 764 | 5.9834 765 | 1.2701 766 | 7.5882 767 | 3.9858 768 | 0.7918 769 | 0.6885 770 | 2.6050 771 | 2.4192 772 | 9.1043 773 | 6.9559 774 | 5.7502 775 | 0.7959 776 | 5.0644 777 | 5.5158 778 | 0.5584 779 | 5.6581 780 | 0.8206 781 | 1.8985 782 | 2.0611 783 | 1.6101 784 | 4.0402 785 | 5.2438 786 | 0.1989 787 | 6.5875 788 | 8.3080 789 | 5.8910 790 | 3.3613 791 | 8.9602 792 | 8.8247 793 | 3.4989 794 | 4.0651 795 | 3.1372 796 | 2.0643 797 | 9.3685 798 | 0.5160 799 | 8.7936 800 | 5.3464 801 | 2.0965 802 | 3.1745 803 | 5.4460 804 | 9.0749 805 | 6.9578 806 | 0.0772 807 | 5.0002 808 | 0.9636 809 | 3.5902 810 | 6.7209 811 | 3.0810 812 | 4.4560 813 | 7.7075 814 | 6.0789 815 | 8.1064 816 | 2.1143 817 | 6.7471 818 | 0.5263 819 | 8.3873 820 | 8.6537 821 | 2.8807 822 | 8.0726 823 | 9.2267 824 | 0.5596 825 | 2.2326 826 | 5.7347 827 | 7.1331 828 | 1.8880 829 | 0.1679 830 | 5.9020 831 | 4.4204 832 | 0.8773 833 | 3.4139 834 | 8.1293 835 | 6.0211 836 | 3.0666 837 | 7.1047 838 | 5.5302 839 | 0.8957 840 | 4.2354 841 | 6.7026 842 | 5.6246 843 | 1.2394 844 | 9.5896 845 | 2.9152 846 | 9.0284 847 | 2.6782 848 | 0.1849 849 | 3.0538 850 | 2.2335 851 | 6.7232 852 | 4.5632 853 | 5.1802 854 | 1.2170 855 | 6.8996 856 | 1.0308 857 | 3.3087 858 | 9.8408 859 | 7.1329 860 | 6.9672 861 | 1.6467 862 | 6.5775 863 | 8.0623 864 | 1.9223 865 | 1.5964 866 | 2.5583 867 | 5.6757 868 | 9.4168 869 | 7.6999 870 | 8.8917 871 | 9.1198 872 | 3.5231 873 | 1.3200 874 | 4.4414 875 | 3.2160 876 | 4.9005 877 | 5.2203 878 | 4.3122 879 | 2.1346 880 | 4.9700 881 | 7.8958 882 | 6.3300 883 | 9.5362 884 | 9.8346 885 | 6.5688 886 | 0.9231 887 | 8.2909 888 | 8.0986 889 | 1.6178 890 | 4.9061 891 | 2.5468 892 | 5.7125 893 | 1.4851 894 | 8.5240 895 | 3.2625 896 | 0.1754 897 | 6.9768 898 | 4.5661 899 | 3.4992 900 | 4.6605 901 | 4.2214 902 | 6.0226 903 | 1.2129 904 | 3.8522 905 | 3.8852 906 | 2.8648 907 | 0.6772 908 | 0.1016 909 | 6.8846 910 | 2.6999 911 | 5.3145 912 | 6.8832 913 | 8.1197 914 | 4.1063 915 | 5.9841 916 | 3.1742 917 | 9.9168 918 | 7.2079 919 | 1.8369 920 | 1.1079 921 | 7.5262 922 | 7.1278 923 | 8.3992 924 | 4.6201 925 | 1.8935 926 | 1.1161 927 | 2.3176 928 | 2.2428 929 | 5.1946 930 | 0.1206 931 | 8.6512 932 | 2.7446 933 | 5.9199 934 | 3.0159 935 | 3.8759 936 | 1.8326 937 | 1.9469 938 | 9.7819 939 | 2.5554 940 | 3.1478 941 | 0.8857 942 | 8.8913 943 | 9.6819 944 | 0.0616 945 | 4.2053 946 | 7.9950 947 | 8.2496 948 | 0.9321 949 | 1.9669 950 | 2.7897 951 | 2.0019 952 | 2.6549 953 | 3.4207 954 | 8.4140 955 | 6.1678 956 | 5.7086 957 | 9.8353 958 | 8.1319 959 | 1.9024 960 | 7.3549 961 | --------------------------------------------------------------------------------