├── figures └── architecture.png ├── requirements.txt ├── configs └── example_run.json ├── losses.py ├── README.md ├── pretrain_eeg_module.py ├── .gitignore ├── inference.py ├── mel_processing.py ├── s4_block ├── s4_model.py └── s4d.py ├── filelists └── n400 │ ├── filelist_test_unseen_both_text.txt │ ├── filelist_test_unseen_audio_text.txt │ └── filelist_test_unseen_subject_text.txt ├── commons.py ├── data_utils.py ├── EEGModule.py ├── utils.py ├── transforms.py ├── attentions.py ├── models.py ├── modules.py └── train.py /figures/architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lee-jhwn/fesde/HEAD/figures/architecture.png -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | librosa==0.9.0 2 | Cython==0.29.21 3 | numpy==1.24.3 4 | scipy==1.10.1 5 | torch==1.13.1 6 | torchvision==0.14.1 7 | tensorboard==2.12.0 8 | lightning==2.0.6 9 | einops==0.6.1 10 | wandb==0.16.1 11 | matplotlib==3.7.2 -------------------------------------------------------------------------------- /configs/example_run.json: -------------------------------------------------------------------------------- 1 | { 2 | "train": { 3 | "log_interval": 200, 4 | "eval_interval": 1000, 5 | "seed": 1234, 6 | "epochs": 20000, 7 | "learning_rate": 2e-4, 8 | "betas": [0.8, 0.99], 9 | "eps": 1e-9, 10 | "batch_size": 64, 11 | "lr_decay": 0.999875, 12 | "segment_size": 8192, 13 | "init_lr_ratio": 1, 14 | "warmup_epochs": 0, 15 | "c_mel": 45, 16 | "c_kl": 1.0, 17 | "pretrained_audio":"", 18 | "pretrained_eeg":"", 19 | "freeze_modules": [] 20 | }, 21 | "data": { 22 | "data_root_dir":"dataset_dir", 23 | "training_files":"filelists/n400/filelist_train_text.txt", 24 | "validation_files_both":"filelists/n400/filelist_test_unseen_both_text.txt", 25 | "validation_files_audio":"filelists/n400/filelist_test_unseen_audio_text.txt", 26 | "validation_files_subject":"filelists/n400/filelist_test_unseen_subject_text.txt", 27 | "max_wav_value": 32768.0, 28 | "sampling_rate": 22050, 29 | "filter_length": 1024, 30 | "hop_length": 256, 31 | "win_length": 1024, 32 | "n_mel_channels": 80, 33 | "mel_fmin": 0.0, 34 | "mel_fmax": null 35 | }, 36 | "model": { 37 | "inter_channels": 192, 38 | "hidden_channels": 192, 39 | "filter_channels": 768, 40 | "n_heads": 2, 41 | "n_layers": 6, 42 | "kernel_size": 3, 43 | "p_dropout": 0.1, 44 | "resblock": "1", 45 | "resblock_kernel_sizes": [3,7,11], 46 | "resblock_dilation_sizes": [[1,3,5], [1,3,5], [1,3,5]], 47 | "upsample_rates": [8,8,2,2], 48 | "upsample_initial_channel": 512, 49 | "upsample_kernel_sizes": [16,16,4,4], 50 | "n_layers_q": 3, 51 | "use_spectral_norm": false, 52 | "eeg_module" : { 53 | "n_layers_cnn": 2, 54 | "use_s4":true, 55 | "n_layers_s4":4, 56 | "in_channels":128 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /losses.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch.nn import functional as F 3 | 4 | import commons 5 | 6 | 7 | def feature_loss(fmap_r, fmap_g): 8 | loss = 0 9 | for dr, dg in zip(fmap_r, fmap_g): 10 | for rl, gl in zip(dr, dg): 11 | rl = rl.float().detach() 12 | gl = gl.float() 13 | loss += torch.mean(torch.abs(rl - gl)) 14 | 15 | return loss * 2 16 | 17 | 18 | def discriminator_loss(disc_real_outputs, disc_generated_outputs): 19 | loss = 0 20 | r_losses = [] 21 | g_losses = [] 22 | for dr, dg in zip(disc_real_outputs, disc_generated_outputs): 23 | dr = dr.float() 24 | dg = dg.float() 25 | r_loss = torch.mean((1-dr)**2) 26 | g_loss = torch.mean(dg**2) 27 | loss += (r_loss + g_loss) 28 | r_losses.append(r_loss.item()) 29 | g_losses.append(g_loss.item()) 30 | 31 | return loss, r_losses, g_losses 32 | 33 | 34 | def generator_loss(disc_outputs): 35 | loss = 0 36 | gen_losses = [] 37 | for dg in disc_outputs: 38 | dg = dg.float() 39 | l = torch.mean((1-dg)**2) 40 | gen_losses.append(l) 41 | loss += l 42 | 43 | return loss, gen_losses 44 | 45 | def eeg_loss(eeg, eeg_hat, eeg_lengths): 46 | cossim_loss_fn = torch.nn.CosineSimilarity(dim=2) 47 | 48 | mask_from_lengths = torch.zeros(eeg.size(), device=eeg.device) 49 | 50 | for item_i in range(mask_from_lengths.size(0)): 51 | mask_from_lengths[item_i,:,:eeg_lengths[item_i]] = 1 52 | 53 | masked_eeg = mask_from_lengths * eeg 54 | masked_eeg_hat = mask_from_lengths * eeg_hat 55 | cossim_loss = cossim_loss_fn(masked_eeg, masked_eeg_hat) 56 | 57 | cossim_loss = 1 - torch.sum(cossim_loss) / (eeg.size(0) * eeg.size(1)) # divide by batch size * channel size 58 | 59 | 60 | return cossim_loss 61 | 62 | 63 | def kl_loss(z_p, logs_q, m_p, logs_p, z_mask): 64 | """ 65 | z_p, logs_q: [b, h, t_t] 66 | m_p, logs_p: [b, h, t_t] 67 | """ 68 | shorter_len = min(z_p.size(2), m_p.size(2)) 69 | z_p = z_p[:,:,:shorter_len] 70 | m_p = m_p[:, :, :shorter_len] 71 | logs_p = logs_p[:,:,:shorter_len] 72 | logs_q = logs_q[:,:,:shorter_len] 73 | z_mask = z_mask[:,:,:shorter_len] 74 | 75 | z_p = z_p.float() 76 | logs_q = logs_q.float() 77 | m_p = m_p.float() 78 | logs_p = logs_p.float() 79 | z_mask = z_mask.float() 80 | 81 | kl = logs_p - logs_q - 0.5 82 | kl += 0.5 * ((z_p - m_p)**2) * torch.exp(-2. * logs_p) 83 | kl = torch.sum(kl * z_mask) 84 | l = kl / torch.sum(z_mask) 85 | return l 86 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Toward Fully-End-to-End Listened Speech Decoding from EEG Signals 2 | ### Jihwan Lee, Aditya Kommineni, Tiantian Feng, Kleanthis Avramidis, Xuan Shi, Sudarsana Kadiri, and Shrikanth Narayanan 3 | 4 | ### Code implementation for [paper](https://www.isca-archive.org/interspeech_2024/lee24c_interspeech.html "Toward Fully-End-to-End Listened Speech Decoding from EEG Signals") accepted at Interspeech 2024 5 | 6 | ### Update (January, 2025) 7 | An improved subsequent version of the work is available [here](https://github.com/lee-jhwn/icassp25-fesde-phoneme "Enhancing Listened Speech Decoding from EEG via Parallel Phoneme Sequence Prediction"). 8 | 9 | ## Abstract 10 | Speech decoding from EEG signals is a challenging task, where brain activity is modeled to estimate useful characteristics of acoustic stimuli. We propose **FESDE**, a novel framework for **F**ully-**E**nd-to-end **S**peech **D**ecoding from **E**EG signals. Our approach aims to directly reconstruct listened speech waveforms given EEG signals, and it does not require any intermediate acoustic feature processing step. The proposed method consists of an EEG module and a speech module along with the connector. The EEG module learns to better represent noisy EEG signals, while the speech module generates speech waveforms from model representations. The connector learns to bridge the distributions of the latent spaces of EEG and speech. The proposed framework is both simple and efficient, by allowing single-step inference, and outperforms prior works on objective metrics. A fine-grained phoneme analysis is conducted to unveil model characteristics of speech decoding. 11 | 12 | ![overall_architecture](figures/architecture.png) 13 | 14 | 15 | ## Environment 16 | - Recommended: `Python >=3.8` 17 | - Install required python packages. 18 | - Refer to `requirements.txt` 19 | - e.g.) `pip install -r requirements.txt` 20 | 21 | 22 | 23 | ## Training Exmaple 24 | ### Pre-training the EEG module only 25 | ```sh 26 | python3 pretrain_eeg_module.py -c configs/config.json -m 27 | ``` 28 | ### Training the whole network 29 | ```sh 30 | python3 train.py -c configs/config.json -m 31 | ``` 32 | 33 | ## Inference Exmaple 34 | ```sh 35 | python3 inference.py --run_name --checkpoint_idx 36 | ``` 37 | 38 | 39 | 40 | 41 | ## References 42 | #### We adopt some of the backbone code from the following repos: 43 | - Speech Module: https://github.com/jaywalnut310/vits 44 | - EEG Module: https://github.com/Uncertain-Quark/s4_eeg 45 | 46 | ## Contact 47 | - Jihwan Lee 48 | -------------------------------------------------------------------------------- /pretrain_eeg_module.py: -------------------------------------------------------------------------------- 1 | import matplotlib.pyplot as plt 2 | import torch 3 | import numpy as np 4 | from torch.utils.data import DataLoader 5 | 6 | import utils 7 | from data_utils import EEGAudioLoader, EEGAudioCollate 8 | 9 | from EEGModule import EEGModule 10 | # from eeg_enc.engine.trainer import trainer 11 | 12 | from tqdm import tqdm 13 | import wandb 14 | from losses import eeg_loss 15 | import os 16 | 17 | USE_WANDB=False 18 | 19 | def main(): 20 | hps = utils.get_hparams() 21 | 22 | 23 | if USE_WANDB: 24 | wandb.login() 25 | wandb.init(project='fesde', name='pretrain_eeg_module') 26 | 27 | train_dataset = EEGAudioLoader(hps.data.training_files, hps.data) 28 | train_loader = DataLoader(train_dataset, num_workers=8, shuffle=True, 29 | sampler=None, 30 | batch_size=32, 31 | pin_memory=True, 32 | collate_fn=EEGAudioCollate()) 33 | 34 | val_dataset = EEGAudioLoader(hps.data.validation_files_both, hps.data) 35 | val_loader = DataLoader(val_dataset, num_workers=8, shuffle=False, 36 | sampler=None, 37 | batch_size=32, 38 | pin_memory=True, 39 | collate_fn=EEGAudioCollate()) 40 | 41 | eeg_module = EEGModule( 42 | n_layers_cnn=hps.model.eeg_module.n_layers_cnn, 43 | use_s4=hps.model.eeg_module.use_s4, 44 | n_layers_s4=hps.model.eeg_module.n_layers_s4, 45 | embedding_size=hps.model.inter_channels, 46 | is_mask=False, 47 | in_channels=hps.model.eeg_module.in_channels 48 | ) 49 | 50 | eeg_module.to('cuda') 51 | optim = torch.optim.AdamW(eeg_module.parameters(), 10*hps.train.learning_rate, betas=hps.train.betas, eps=hps.train.eps) 52 | 53 | 54 | for epoch in range(1,300): 55 | eeg_module.train() 56 | 57 | for batch_idx, (x, x_lengths, spec, spec_lengths, y, y_lengths) in enumerate(train_loader): 58 | x, x_lengths = x.cuda(0, non_blocking=True), x_lengths.cuda(0, non_blocking=True) 59 | spec, spec_lengths = spec.cuda(0, non_blocking=True), spec_lengths.cuda(0, non_blocking=True) 60 | y, y_lengths = y.cuda(0, non_blocking=True), y_lengths.cuda(0, non_blocking=True) 61 | 62 | x, x_mask_output, mid_output, ecog_decoder_output = eeg_module(x) 63 | 64 | 65 | loss_eeg_enc = eeg_loss(x, ecog_decoder_output, x_lengths) 66 | 67 | optim.zero_grad() 68 | loss_eeg_enc.backward() 69 | optim.step() 70 | if USE_WANDB: 71 | wandb.log({'train_loss': loss_eeg_enc}) 72 | 73 | eeg_module.eval() 74 | val_loss_total = [] 75 | for batch_idx, (x, x_lengths, spec, spec_lengths, y, y_lengths) in enumerate(val_loader): 76 | x, x_lengths = x.cuda(0, non_blocking=True), x_lengths.cuda(0, non_blocking=True) 77 | spec, spec_lengths = spec.cuda(0, non_blocking=True), spec_lengths.cuda(0, non_blocking=True) 78 | y, y_lengths = y.cuda(0, non_blocking=True), y_lengths.cuda(0, non_blocking=True) 79 | 80 | x, x_mask_output, mid_output, eeg_decoder_output = eeg_module(x) 81 | 82 | 83 | loss_eeg_module = eeg_loss(x, eeg_decoder_output, x_lengths) 84 | 85 | val_loss_total.append(loss_eeg_module.cpu().item()) 86 | 87 | val_loss_total = np.mean(val_loss_total) 88 | 89 | if USE_WANDB: 90 | wandb.log({'val_loss': val_loss_total}) 91 | 92 | utils.save_checkpoint(eeg_module, optim, 10*hps.train.learning_rate, epoch, os.path.join(hps.model_dir, "pretrained_E_{}.pth".format(epoch))) 93 | 94 | eeg_module.train() 95 | 96 | 97 | 98 | if __name__=='__main__': 99 | main() 100 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # poetry 98 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 99 | # This is especially recommended for binary packages to ensure reproducibility, and is more 100 | # commonly ignored for libraries. 101 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 102 | #poetry.lock 103 | 104 | # pdm 105 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 106 | #pdm.lock 107 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 108 | # in version control. 109 | # https://pdm.fming.dev/latest/usage/project/#working-with-version-control 110 | .pdm.toml 111 | .pdm-python 112 | .pdm-build/ 113 | 114 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 115 | __pypackages__/ 116 | 117 | # Celery stuff 118 | celerybeat-schedule 119 | celerybeat.pid 120 | 121 | # SageMath parsed files 122 | *.sage.py 123 | 124 | # Environments 125 | .env 126 | .venv 127 | env/ 128 | venv/ 129 | ENV/ 130 | env.bak/ 131 | venv.bak/ 132 | 133 | # Spyder project settings 134 | .spyderproject 135 | .spyproject 136 | 137 | # Rope project settings 138 | .ropeproject 139 | 140 | # mkdocs documentation 141 | /site 142 | 143 | # mypy 144 | .mypy_cache/ 145 | .dmypy.json 146 | dmypy.json 147 | 148 | # Pyre type checker 149 | .pyre/ 150 | 151 | # pytype static type analyzer 152 | .pytype/ 153 | 154 | # Cython debug symbols 155 | cython_debug/ 156 | 157 | # PyCharm 158 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 159 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 160 | # and can be added to the global gitignore or merged into this file. For a more nuclear 161 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 162 | #.idea/ 163 | -------------------------------------------------------------------------------- /inference.py: -------------------------------------------------------------------------------- 1 | import os 2 | import utils 3 | import argparse 4 | import torch 5 | from data_utils import ( 6 | EEGAudioLoader, 7 | EEGAudioCollate 8 | ) 9 | from torch.utils.data import DataLoader 10 | from EEGModule import EEGModule 11 | from models import SpeechDecoder 12 | from scipy.io.wavfile import write 13 | 14 | torch.manual_seed(777) 15 | 16 | OUTPUT_DIR = './logs' 17 | 18 | def synthesize(eeg_enc, audio_gen, eval_loader, suffix, hps, args): 19 | 20 | eeg_enc.eval() 21 | audio_gen.eval() 22 | 23 | with torch.no_grad(): 24 | for batch_idx, (x, x_lengths, spec, spec_lengths, y, y_lengths) in enumerate(eval_loader): 25 | x, x_lengths = x.cuda(0), x_lengths.cuda(0) 26 | spec, spec_lengths = spec.cuda(0), spec_lengths.cuda(0) 27 | y, y_lengths = y.cuda(0), y_lengths.cuda(0) 28 | 29 | x, x_mask_output, mid_output, eeg_decoder_output = eeg_enc(x) 30 | 31 | 32 | mid_output_lengths = x_lengths.clone() * mid_output.size(2) / x.size(2) 33 | mid_output_lengths = mid_output_lengths.long() 34 | 35 | y_hat, _, mask, *_ = audio_gen.infer(mid_output, mid_output_lengths, max_len=1000, noise_scale=1) 36 | 37 | os.makedirs(os.path.join(OUTPUT_DIR, args.run_name, 'synthesized', str(args.checkpoint_idx), suffix), exist_ok=True) 38 | 39 | write(os.path.join(OUTPUT_DIR, args.run_name, 'synthesized', str(args.checkpoint_idx), suffix, f'{batch_idx}_gt.wav'), hps.data.sampling_rate, y[0,:,:y_hat.size(2)][0].cpu().float().numpy()) 40 | write(os.path.join(OUTPUT_DIR, args.run_name, 'synthesized', str(args.checkpoint_idx), suffix, f'{batch_idx}_syn.wav'), hps.data.sampling_rate, y_hat[0,:,:y_hat.size(2)][0].cpu().float().numpy()) 41 | 42 | 43 | return 44 | 45 | 46 | def run(args): 47 | 48 | config_dir = os.path.join('./logs', args.run_name, 'config.json') 49 | hps = utils.get_hparams_from_file(config_dir) 50 | print(hps) 51 | 52 | 53 | collate_fn = EEGAudioCollate() 54 | 55 | eval_loader_both = DataLoader(EEGAudioLoader(hps.data.validation_files_both, hps.data), num_workers=1, shuffle=False, 56 | batch_size=1, pin_memory=True, 57 | drop_last=False, collate_fn=collate_fn) 58 | eval_loader_audio = DataLoader(EEGAudioLoader(hps.data.validation_files_audio, hps.data), num_workers=1, shuffle=False, 59 | batch_size=1, pin_memory=True, 60 | drop_last=False, collate_fn=collate_fn) 61 | eval_loader_subject = DataLoader(EEGAudioLoader(hps.data.validation_files_subject, hps.data), num_workers=1, shuffle=False, 62 | batch_size=1, pin_memory=True, 63 | drop_last=False, collate_fn=collate_fn) 64 | 65 | eval_loaders = [eval_loader_both, eval_loader_audio, eval_loader_subject] 66 | 67 | 68 | eeg_module = EEGModule( 69 | n_layers_cnn=hps.model.eeg_module.n_layers_cnn, 70 | use_s4=hps.model.eeg_module.use_s4, 71 | n_layers_s4=hps.model.eeg_module.n_layers_s4, 72 | embedding_size=hps.model.inter_channels, 73 | is_mask=False, 74 | in_channels=hps.model.eeg_module.in_channels 75 | ).cuda() 76 | 77 | net_g = SpeechDecoder( 78 | hps.data.filter_length // 2 + 1, 79 | hps.train.segment_size // hps.data.hop_length, 80 | **hps.model).cuda() 81 | 82 | utils.load_checkpoint(os.path.join('./logs', args.run_name, f"G_{args.checkpoint_idx}.pth"), net_g, None) 83 | utils.load_checkpoint(os.path.join('./logs', args.run_name, f"E_{args.checkpoint_idx}.pth"), eeg_module, None) 84 | 85 | 86 | net_g.eval() 87 | eeg_module.eval() 88 | 89 | for val_type, eval_loader in list(zip(['both', 'audio', 'subject'], eval_loaders)): 90 | synthesize(eeg_module, net_g, eval_loader, val_type, hps, args) 91 | 92 | return 93 | 94 | 95 | 96 | if __name__=="__main__": 97 | parser = argparse.ArgumentParser() 98 | parser.add_argument('--run_name', type=str, default="fesde") 99 | parser.add_argument('--checkpoint_idx', default=0) 100 | args = parser.parse_args() 101 | 102 | run(args) 103 | -------------------------------------------------------------------------------- /mel_processing.py: -------------------------------------------------------------------------------- 1 | import math 2 | import os 3 | import random 4 | import torch 5 | from torch import nn 6 | import torch.nn.functional as F 7 | import torch.utils.data 8 | import numpy as np 9 | import librosa 10 | import librosa.util as librosa_util 11 | from librosa.util import normalize, pad_center, tiny 12 | from scipy.signal import get_window 13 | from scipy.io.wavfile import read 14 | from librosa.filters import mel as librosa_mel_fn 15 | 16 | MAX_WAV_VALUE = 32768.0 17 | 18 | 19 | def dynamic_range_compression_torch(x, C=1, clip_val=1e-5): 20 | """ 21 | PARAMS 22 | ------ 23 | C: compression factor 24 | """ 25 | return torch.log(torch.clamp(x, min=clip_val) * C) 26 | 27 | 28 | def dynamic_range_decompression_torch(x, C=1): 29 | """ 30 | PARAMS 31 | ------ 32 | C: compression factor used to compress 33 | """ 34 | return torch.exp(x) / C 35 | 36 | 37 | def spectral_normalize_torch(magnitudes): 38 | output = dynamic_range_compression_torch(magnitudes) 39 | return output 40 | 41 | 42 | def spectral_de_normalize_torch(magnitudes): 43 | output = dynamic_range_decompression_torch(magnitudes) 44 | return output 45 | 46 | 47 | mel_basis = {} 48 | hann_window = {} 49 | 50 | 51 | def spectrogram_torch(y, n_fft, sampling_rate, hop_size, win_size, center=False): 52 | if torch.min(y) < -1.: 53 | print('min value is ', torch.min(y)) 54 | if torch.max(y) > 1.: 55 | print('max value is ', torch.max(y)) 56 | 57 | global hann_window 58 | dtype_device = str(y.dtype) + '_' + str(y.device) 59 | wnsize_dtype_device = str(win_size) + '_' + dtype_device 60 | if wnsize_dtype_device not in hann_window: 61 | hann_window[wnsize_dtype_device] = torch.hann_window(win_size).to(dtype=y.dtype, device=y.device) 62 | 63 | y = torch.nn.functional.pad(y.unsqueeze(1), (int((n_fft-hop_size)/2), int((n_fft-hop_size)/2)), mode='reflect') 64 | y = y.squeeze(1) 65 | 66 | spec = torch.stft(y, n_fft, hop_length=hop_size, win_length=win_size, window=hann_window[wnsize_dtype_device], 67 | center=center, pad_mode='reflect', normalized=False, onesided=True) 68 | 69 | spec = torch.sqrt(spec.pow(2).sum(-1) + 1e-6) 70 | return spec 71 | 72 | 73 | def spec_to_mel_torch(spec, n_fft, num_mels, sampling_rate, fmin, fmax): 74 | global mel_basis 75 | dtype_device = str(spec.dtype) + '_' + str(spec.device) 76 | fmax_dtype_device = str(fmax) + '_' + dtype_device 77 | if fmax_dtype_device not in mel_basis: 78 | mel = librosa_mel_fn(sr=sampling_rate, n_fft=n_fft, n_mels=num_mels, fmin=fmin, fmax=fmax) 79 | mel_basis[fmax_dtype_device] = torch.from_numpy(mel).to(dtype=spec.dtype, device=spec.device) 80 | spec = torch.matmul(mel_basis[fmax_dtype_device], spec) 81 | spec = spectral_normalize_torch(spec) 82 | return spec 83 | 84 | 85 | def mel_spectrogram_torch(y, n_fft, num_mels, sampling_rate, hop_size, win_size, fmin, fmax, center=False): 86 | if torch.min(y) < -1.: 87 | print('min value is ', torch.min(y)) 88 | if torch.max(y) > 1.: 89 | print('max value is ', torch.max(y)) 90 | 91 | global mel_basis, hann_window 92 | dtype_device = str(y.dtype) + '_' + str(y.device) 93 | fmax_dtype_device = str(fmax) + '_' + dtype_device 94 | wnsize_dtype_device = str(win_size) + '_' + dtype_device 95 | if fmax_dtype_device not in mel_basis: 96 | mel = librosa_mel_fn(sr=sampling_rate, n_fft=n_fft, n_mels=num_mels, fmin=fmin, fmax=fmax) 97 | mel_basis[fmax_dtype_device] = torch.from_numpy(mel).to(dtype=y.dtype, device=y.device) 98 | if wnsize_dtype_device not in hann_window: 99 | hann_window[wnsize_dtype_device] = torch.hann_window(win_size).to(dtype=y.dtype, device=y.device) 100 | 101 | y = torch.nn.functional.pad(y.unsqueeze(1), (int((n_fft-hop_size)/2), int((n_fft-hop_size)/2)), mode='reflect') 102 | y = y.squeeze(1) 103 | 104 | spec = torch.stft(y, n_fft, hop_length=hop_size, win_length=win_size, window=hann_window[wnsize_dtype_device], 105 | center=center, pad_mode='reflect', normalized=False, onesided=True) 106 | 107 | spec = torch.sqrt(spec.pow(2).sum(-1) + 1e-6) 108 | 109 | spec = torch.matmul(mel_basis[fmax_dtype_device], spec) 110 | spec = spectral_normalize_torch(spec) 111 | 112 | return spec 113 | -------------------------------------------------------------------------------- /s4_block/s4_model.py: -------------------------------------------------------------------------------- 1 | ''' 2 | Train an S4 model on sequential CIFAR10 / sequential MNIST with PyTorch for demonstration purposes. 3 | This code borrows heavily from https://github.com/kuangliu/pytorch-cifar. 4 | 5 | This file only depends on the standalone S4 layer 6 | available in /models/s4/ 7 | 8 | * Train standard sequential CIFAR: 9 | python -m example 10 | * Train sequential CIFAR grayscale: 11 | python -m example --grayscale 12 | * Train MNIST: 13 | python -m example --dataset mnist --d_model 256 --weight_decay 0.0 14 | 15 | The `S4Model` class defined in this file provides a simple backbone to train S4 models. 16 | This backbone is a good starting point for many problems, although some tasks (especially generation) 17 | may require using other backbones. 18 | 19 | The default CIFAR10 model trained by this file should get 20 | 89+% accuracy on the CIFAR10 test set in 80 epochs. 21 | 22 | Each epoch takes approximately 7m20s on a T4 GPU (will be much faster on V100 / A100). 23 | ''' 24 | import torch 25 | import torch.nn as nn 26 | import torch.optim as optim 27 | import torch.backends.cudnn as cudnn 28 | 29 | import torchvision 30 | import torchvision.transforms as transforms 31 | 32 | import os 33 | import argparse 34 | 35 | from .s4 import S4Block as S4 # Can use full version instead of minimal S4D standalone below 36 | from .s4d import S4D 37 | from tqdm.auto import tqdm 38 | 39 | device = 'cuda' if torch.cuda.is_available() else 'cpu' 40 | 41 | # Dropout broke in PyTorch 1.11 42 | if tuple(map(int, torch.__version__.split('.')[:2])) == (1, 11): 43 | print("WARNING: Dropout is bugged in PyTorch 1.11. Results may be worse.") 44 | dropout_fn = nn.Dropout 45 | if tuple(map(int, torch.__version__.split('.')[:2])) >= (1, 12): 46 | dropout_fn = nn.Dropout1d 47 | else: 48 | dropout_fn = nn.Dropout2d 49 | 50 | class S4Model(nn.Module): 51 | 52 | def __init__( 53 | self, 54 | d_input=256, 55 | d_output=256, 56 | d_model=256, 57 | n_layers=4, 58 | dropout=0.2, 59 | prenorm=False, 60 | ): 61 | super().__init__() 62 | 63 | self.prenorm = prenorm 64 | 65 | # Linear encoder (d_input = 1 for grayscale and 3 for RGB) 66 | self.encoder = nn.Linear(d_input, d_model) 67 | 68 | # Stack S4 layers as residual blocks 69 | self.s4_layers = nn.ModuleList() 70 | self.norms = nn.ModuleList() 71 | self.dropouts = nn.ModuleList() 72 | for _ in range(n_layers): 73 | self.s4_layers.append( 74 | S4(d_model, dropout=dropout, transposed=True, lr=0.001) 75 | ) 76 | self.norms.append(nn.LayerNorm(d_model)) 77 | self.dropouts.append(dropout_fn(dropout)) 78 | 79 | # Linear decoder 80 | # self.decoder = nn.Linear(d_model, d_output) 81 | 82 | def forward(self, x): 83 | """ 84 | Input x is shape (B, L, d_input) 85 | """ 86 | x = self.encoder(x) # (B, L, d_model) -> (B, L, d_model) 87 | 88 | x = x.transpose(-1, -2) # (B, L, d_model) -> (B, d_model, L) 89 | for layer, norm, dropout in zip(self.s4_layers, self.norms, self.dropouts): 90 | # Each iteration of this loop will map (B, d_model, L) -> (B, d_model, L) 91 | 92 | z = x 93 | if self.prenorm: 94 | # Prenorm 95 | z = norm(z.transpose(-1, -2)).transpose(-1, -2) 96 | 97 | # Apply S4 block: we ignore the state input and output 98 | z, _ = layer(z) 99 | 100 | # Dropout on the output of the S4 block 101 | z = dropout(z) 102 | 103 | # Residual connection 104 | x = z + x 105 | 106 | if not self.prenorm: 107 | # Postnorm 108 | x = norm(x.transpose(-1, -2)).transpose(-1, -2) 109 | 110 | x = x.transpose(-1, -2) 111 | 112 | # Pooling: average pooling over the sequence length 113 | #x = x.mean(dim=1) 114 | 115 | # Decode the outputs 116 | # x = self.decoder(x) # (B, L, d_model) -> (B, L, d_output) 117 | 118 | return x 119 | 120 | if __name__ == '__main__': 121 | # Model 122 | print('==> Building model..') 123 | model = S4Model( 124 | d_input=304, 125 | d_output=304, 126 | d_model=512, 127 | n_layers=4, 128 | dropout=0.3, 129 | prenorm=False, 130 | ) 131 | 132 | model = model.to(device) 133 | 134 | x = torch.randn(32, 1736, 304) 135 | y = model(x.to(device)) 136 | 137 | print(f'Shape of the output {y.shape}') 138 | 139 | -------------------------------------------------------------------------------- /filelists/n400/filelist_test_unseen_both_text.txt: -------------------------------------------------------------------------------- 1 | sub-23/sub-23-_-(LD5_Eliminated)NPC_live||The woods are where bears live. 2 | sub-23/sub-23-_-NPC_aisle||The bride walked down the aisle. 3 | sub-23/sub-23-_-NPC_animals||The sign said do not feed the animals. 4 | sub-23/sub-23-_-NPC_apple||His favorite fruit was a red crisp apple. 5 | sub-23/sub-23-_-NPC_arm||I have two legs and two arms. 6 | sub-23/sub-23-_-NPC_bag||He puts his lunch in a paper bag. 7 | sub-23/sub-23-_-NPC_bank||My mom puts money in the bank. 8 | sub-23/sub-23-_-NPC_button||On the remote he clicked the button. 9 | sub-23/sub-23-_-NPC_cart||At the store I pushed the cart. 10 | sub-23/sub-23-_-NPC_chores||My mom makes me do household chores. 11 | sub-23/sub-23-_-NPC_clothes||My parents fold my clothes. 12 | sub-23/sub-23-_-NPC_cribs||Babies sleep in cribs. 13 | sub-23/sub-23-_-NPC_cup||I drink from a cup. 14 | sub-23/sub-23-_-NPC_dream||She woke up after a bad dream. 15 | sub-23/sub-23-_-NPC_frog||Ribbit goes the frog. 16 | sub-23/sub-23-_-NPC_fur||Cats have soft fur. 17 | sub-23/sub-23-_-NPC_mouse||The cat chased the mouse. 18 | sub-23/sub-23-_-NPC_one||First place is number one. 19 | sub-23/sub-23-_-NPC_pink||Pigs are colored pink. 20 | sub-23/sub-23-_-NPC_snake||Hiss goes the snake. 21 | sub-23/sub-23-_-NPI_bear(pie)||The woods had a big brown scary pie. 22 | sub-23/sub-23-_-NPI_bite(north)||He had an itchy bug north. 23 | sub-23/sub-23-_-NPI_cages(woman)||Animals are locked in woman. 24 | sub-23/sub-23-_-NPI_card(fly)||I signed a birthday fly. 25 | sub-23/sub-23-_-NPI_cliff(sheep)||He stood at the edge of a sheep. 26 | sub-23/sub-23-_-NPI_cry(fork)||When I am sad I fork. 27 | sub-23/sub-23-_-NPI_dare(brick)||Lets play truth or brick. 28 | sub-23/sub-23-_-NPI_dice(art)||In games I roll the art. 29 | sub-23/sub-23-_-NPI_down(tie)||She told me to calm tie. 30 | sub-23/sub-23-_-NPI_fall(duck)||After summer is duck. 31 | sub-23/sub-23-_-NPI_fork(cleats)||He eats steak with a knife and cleats. 32 | sub-23/sub-23-_-NPI_gel(sign)||He styled his hair with sign. 33 | sub-23/sub-23-_-NPI_night(inch)||It is dark at inch. 34 | sub-23/sub-23-_-NPI_nine(cat)||The number before ten is cat. 35 | sub-23/sub-23-_-NPI_one(cold)||First place is number cold. 36 | sub-23/sub-23-_-NPI_out(ant)||Three strikes and you are ant. 37 | sub-23/sub-23-_-NPI_pink(door)||Pigs are colored door. 38 | sub-23/sub-23-_-NPI_plank(shark)||The pirate said walk the shark. 39 | sub-23/sub-23-_-NPI_popped(square)||The water balloon was so full it square. 40 | sub-23/sub-23-_-NPI_round(kiss)||The ball is not square but kiss. 41 | sub-24/sub-24-_-(LD5_Eliminated)NPC_live||The woods are where bears live. 42 | sub-24/sub-24-_-NPC_aisle||The bride walked down the aisle. 43 | sub-24/sub-24-_-NPC_animals||The sign said do not feed the animals. 44 | sub-24/sub-24-_-NPC_apple||His favorite fruit was a red crisp apple. 45 | sub-24/sub-24-_-NPC_arm||I have two legs and two arms. 46 | sub-24/sub-24-_-NPC_bag||He puts his lunch in a paper bag. 47 | sub-24/sub-24-_-NPC_bank||My mom puts money in the bank. 48 | sub-24/sub-24-_-NPC_button||On the remote he clicked the button. 49 | sub-24/sub-24-_-NPC_cart||At the store I pushed the cart. 50 | sub-24/sub-24-_-NPC_chores||My mom makes me do household chores. 51 | sub-24/sub-24-_-NPC_clothes||My parents fold my clothes. 52 | sub-24/sub-24-_-NPC_cribs||Babies sleep in cribs. 53 | sub-24/sub-24-_-NPC_cup||I drink from a cup. 54 | sub-24/sub-24-_-NPC_dream||She woke up after a bad dream. 55 | sub-24/sub-24-_-NPC_frog||Ribbit goes the frog. 56 | sub-24/sub-24-_-NPC_fur||Cats have soft fur. 57 | sub-24/sub-24-_-NPC_mouse||The cat chased the mouse. 58 | sub-24/sub-24-_-NPC_one||First place is number one. 59 | sub-24/sub-24-_-NPC_pink||Pigs are colored pink. 60 | sub-24/sub-24-_-NPC_snake||Hiss goes the snake. 61 | sub-24/sub-24-_-NPI_bear(pie)||The woods had a big brown scary pie. 62 | sub-24/sub-24-_-NPI_bite(north)||He had an itchy bug north. 63 | sub-24/sub-24-_-NPI_cages(woman)||Animals are locked in woman. 64 | sub-24/sub-24-_-NPI_card(fly)||I signed a birthday fly. 65 | sub-24/sub-24-_-NPI_cliff(sheep)||He stood at the edge of a sheep. 66 | sub-24/sub-24-_-NPI_cry(fork)||When I am sad I fork. 67 | sub-24/sub-24-_-NPI_dare(brick)||Lets play truth or brick. 68 | sub-24/sub-24-_-NPI_dice(art)||In games I roll the art. 69 | sub-24/sub-24-_-NPI_down(tie)||She told me to calm tie. 70 | sub-24/sub-24-_-NPI_fall(duck)||After summer is duck. 71 | sub-24/sub-24-_-NPI_fork(cleats)||He eats steak with a knife and cleats. 72 | sub-24/sub-24-_-NPI_gel(sign)||He styled his hair with sign. 73 | sub-24/sub-24-_-NPI_night(inch)||It is dark at inch. 74 | sub-24/sub-24-_-NPI_nine(cat)||The number before ten is cat. 75 | sub-24/sub-24-_-NPI_one(cold)||First place is number cold. 76 | sub-24/sub-24-_-NPI_out(ant)||Three strikes and you are ant. 77 | sub-24/sub-24-_-NPI_pink(door)||Pigs are colored door. 78 | sub-24/sub-24-_-NPI_plank(shark)||The pirate said walk the shark. 79 | sub-24/sub-24-_-NPI_popped(square)||The water balloon was so full it square. 80 | sub-24/sub-24-_-NPI_round(kiss)||The ball is not square but kiss. -------------------------------------------------------------------------------- /s4_block/s4d.py: -------------------------------------------------------------------------------- 1 | """Minimal version of S4D with extra options and features stripped out, for pedagogical purposes.""" 2 | 3 | import math 4 | import torch 5 | import torch.nn as nn 6 | import torch.nn.functional as F 7 | from einops import rearrange, repeat 8 | 9 | class DropoutNd(nn.Module): 10 | def __init__(self, p: float = 0.5, tie=True, transposed=True): 11 | """ 12 | tie: tie dropout mask across sequence lengths (Dropout1d/2d/3d) 13 | """ 14 | super().__init__() 15 | if p < 0 or p >= 1: 16 | raise ValueError("dropout probability has to be in [0, 1), " "but got {}".format(p)) 17 | self.p = p 18 | self.tie = tie 19 | self.transposed = transposed 20 | self.binomial = torch.distributions.binomial.Binomial(probs=1-self.p) 21 | 22 | def forward(self, X): 23 | """X: (batch, dim, lengths...).""" 24 | if self.training: 25 | if not self.transposed: X = rearrange(X, 'b ... d -> b d ...') 26 | # binomial = torch.distributions.binomial.Binomial(probs=1-self.p) # This is incredibly slow because of CPU -> GPU copying 27 | mask_shape = X.shape[:2] + (1,)*(X.ndim-2) if self.tie else X.shape 28 | # mask = self.binomial.sample(mask_shape) 29 | mask = torch.rand(*mask_shape, device=X.device) < 1.-self.p 30 | X = X * mask * (1.0/(1-self.p)) 31 | if not self.transposed: X = rearrange(X, 'b d ... -> b ... d') 32 | return X 33 | return X 34 | 35 | class S4DKernel(nn.Module): 36 | """Generate convolution kernel from diagonal SSM parameters.""" 37 | 38 | def __init__(self, d_model, N=64, dt_min=0.001, dt_max=0.1, lr=None): 39 | super().__init__() 40 | # Generate dt 41 | H = d_model 42 | log_dt = torch.rand(H) * ( 43 | math.log(dt_max) - math.log(dt_min) 44 | ) + math.log(dt_min) 45 | 46 | C = torch.randn(H, N // 2, dtype=torch.cfloat) 47 | self.C = nn.Parameter(torch.view_as_real(C)) 48 | self.register("log_dt", log_dt, lr) 49 | 50 | log_A_real = torch.log(0.5 * torch.ones(H, N//2)) 51 | A_imag = math.pi * repeat(torch.arange(N//2), 'n -> h n', h=H) 52 | self.register("log_A_real", log_A_real, lr) 53 | self.register("A_imag", A_imag, lr) 54 | 55 | def forward(self, L): 56 | """ 57 | returns: (..., c, L) where c is number of channels (default 1) 58 | """ 59 | 60 | # Materialize parameters 61 | dt = torch.exp(self.log_dt) # (H) 62 | C = torch.view_as_complex(self.C) # (H N) 63 | A = -torch.exp(self.log_A_real) + 1j * self.A_imag # (H N) 64 | 65 | # Vandermonde multiplication 66 | dtA = A * dt.unsqueeze(-1) # (H N) 67 | K = dtA.unsqueeze(-1) * torch.arange(L, device=A.device) # (H N L) 68 | C = C * (torch.exp(dtA)-1.) / A 69 | K = 2 * torch.einsum('hn, hnl -> hl', C, torch.exp(K)).real 70 | 71 | return K 72 | 73 | def register(self, name, tensor, lr=None): 74 | """Register a tensor with a configurable learning rate and 0 weight decay""" 75 | 76 | if lr == 0.0: 77 | self.register_buffer(name, tensor) 78 | else: 79 | self.register_parameter(name, nn.Parameter(tensor)) 80 | 81 | optim = {"weight_decay": 0.0} 82 | if lr is not None: optim["lr"] = lr 83 | setattr(getattr(self, name), "_optim", optim) 84 | 85 | 86 | class S4D(nn.Module): 87 | def __init__(self, d_model, d_state=64, dropout=0.0, transposed=True, **kernel_args): 88 | super().__init__() 89 | 90 | self.h = d_model 91 | self.n = d_state 92 | self.d_output = self.h 93 | self.transposed = transposed 94 | 95 | self.D = nn.Parameter(torch.randn(self.h)) 96 | 97 | # SSM Kernel 98 | self.kernel = S4DKernel(self.h, N=self.n, **kernel_args) 99 | 100 | # Pointwise 101 | self.activation = nn.GELU() 102 | # dropout_fn = nn.Dropout2d # NOTE: bugged in PyTorch 1.11 103 | dropout_fn = DropoutNd 104 | self.dropout = dropout_fn(dropout) if dropout > 0.0 else nn.Identity() 105 | 106 | # position-wise output transform to mix features 107 | self.output_linear = nn.Sequential( 108 | nn.Conv1d(self.h, 2*self.h, kernel_size=1), 109 | nn.GLU(dim=-2), 110 | ) 111 | 112 | def forward(self, u, **kwargs): # absorbs return_output and transformer src mask 113 | """ Input and output shape (B, H, L) """ 114 | if not self.transposed: u = u.transpose(-1, -2) 115 | L = u.size(-1) 116 | 117 | # Compute SSM Kernel 118 | k = self.kernel(L=L) # (H L) 119 | 120 | # Convolution 121 | k_f = torch.fft.rfft(k, n=2*L) # (H L) 122 | u_f = torch.fft.rfft(u, n=2*L) # (B H L) 123 | y = torch.fft.irfft(u_f*k_f, n=2*L)[..., :L] # (B H L) 124 | 125 | # Compute D term in state space equation - essentially a skip connection 126 | y = y + u * self.D.unsqueeze(-1) 127 | 128 | y = self.dropout(self.activation(y)) 129 | y = self.output_linear(y) 130 | if not self.transposed: y = y.transpose(-1, -2) 131 | return y, None # Return a dummy state to satisfy this repo's interface, but this can be modified -------------------------------------------------------------------------------- /commons.py: -------------------------------------------------------------------------------- 1 | import math 2 | import numpy as np 3 | import torch 4 | from torch import nn 5 | from torch.nn import functional as F 6 | 7 | 8 | def init_weights(m, mean=0.0, std=0.01): 9 | classname = m.__class__.__name__ 10 | if classname.find("Conv") != -1: 11 | m.weight.data.normal_(mean, std) 12 | 13 | 14 | def get_padding(kernel_size, dilation=1): 15 | return int((kernel_size*dilation - dilation)/2) 16 | 17 | 18 | def convert_pad_shape(pad_shape): 19 | l = pad_shape[::-1] 20 | pad_shape = [item for sublist in l for item in sublist] 21 | return pad_shape 22 | 23 | 24 | def intersperse(lst, item): 25 | result = [item] * (len(lst) * 2 + 1) 26 | result[1::2] = lst 27 | return result 28 | 29 | 30 | def kl_divergence(m_p, logs_p, m_q, logs_q): 31 | """KL(P||Q)""" 32 | kl = (logs_q - logs_p) - 0.5 33 | kl += 0.5 * (torch.exp(2. * logs_p) + ((m_p - m_q)**2)) * torch.exp(-2. * logs_q) 34 | return kl 35 | 36 | 37 | def rand_gumbel(shape): 38 | """Sample from the Gumbel distribution, protect from overflows.""" 39 | uniform_samples = torch.rand(shape) * 0.99998 + 0.00001 40 | return -torch.log(-torch.log(uniform_samples)) 41 | 42 | 43 | def rand_gumbel_like(x): 44 | g = rand_gumbel(x.size()).to(dtype=x.dtype, device=x.device) 45 | return g 46 | 47 | 48 | def slice_segments(x, ids_str, segment_size=4): 49 | ret = torch.zeros_like(x[:, :, :segment_size]) 50 | for i in range(x.size(0)): 51 | idx_str = ids_str[i] 52 | idx_end = idx_str + segment_size 53 | ret[i] = x[i, :, idx_str:idx_end] 54 | return ret 55 | 56 | 57 | def rand_slice_segments(x, x_lengths=None, segment_size=4): 58 | b, d, t = x.size() 59 | if x_lengths is None: 60 | x_lengths = t 61 | ids_str_max = x_lengths - segment_size + 1 62 | ids_str = (torch.rand([b]).to(device=x.device) * ids_str_max).to(dtype=torch.long) 63 | ret = slice_segments(x, ids_str, segment_size) 64 | return ret, ids_str 65 | 66 | 67 | def get_timing_signal_1d( 68 | length, channels, min_timescale=1.0, max_timescale=1.0e4): 69 | position = torch.arange(length, dtype=torch.float) 70 | num_timescales = channels // 2 71 | log_timescale_increment = ( 72 | math.log(float(max_timescale) / float(min_timescale)) / 73 | (num_timescales - 1)) 74 | inv_timescales = min_timescale * torch.exp( 75 | torch.arange(num_timescales, dtype=torch.float) * -log_timescale_increment) 76 | scaled_time = position.unsqueeze(0) * inv_timescales.unsqueeze(1) 77 | signal = torch.cat([torch.sin(scaled_time), torch.cos(scaled_time)], 0) 78 | signal = F.pad(signal, [0, 0, 0, channels % 2]) 79 | signal = signal.view(1, channels, length) 80 | return signal 81 | 82 | 83 | def add_timing_signal_1d(x, min_timescale=1.0, max_timescale=1.0e4): 84 | b, channels, length = x.size() 85 | signal = get_timing_signal_1d(length, channels, min_timescale, max_timescale) 86 | return x + signal.to(dtype=x.dtype, device=x.device) 87 | 88 | 89 | def cat_timing_signal_1d(x, min_timescale=1.0, max_timescale=1.0e4, axis=1): 90 | b, channels, length = x.size() 91 | signal = get_timing_signal_1d(length, channels, min_timescale, max_timescale) 92 | return torch.cat([x, signal.to(dtype=x.dtype, device=x.device)], axis) 93 | 94 | 95 | def subsequent_mask(length): 96 | mask = torch.tril(torch.ones(length, length)).unsqueeze(0).unsqueeze(0) 97 | return mask 98 | 99 | 100 | @torch.jit.script 101 | def fused_add_tanh_sigmoid_multiply(input_a, input_b, n_channels): 102 | n_channels_int = n_channels[0] 103 | in_act = input_a + input_b 104 | t_act = torch.tanh(in_act[:, :n_channels_int, :]) 105 | s_act = torch.sigmoid(in_act[:, n_channels_int:, :]) 106 | acts = t_act * s_act 107 | return acts 108 | 109 | 110 | def convert_pad_shape(pad_shape): 111 | l = pad_shape[::-1] 112 | pad_shape = [item for sublist in l for item in sublist] 113 | return pad_shape 114 | 115 | 116 | def shift_1d(x): 117 | x = F.pad(x, convert_pad_shape([[0, 0], [0, 0], [1, 0]]))[:, :, :-1] 118 | return x 119 | 120 | 121 | def sequence_mask(length, max_length=None): 122 | if max_length is None: 123 | max_length = length.max() 124 | x = torch.arange(max_length, dtype=length.dtype, device=length.device) 125 | return x.unsqueeze(0) < length.unsqueeze(1) 126 | 127 | 128 | def generate_path(duration, mask): 129 | """ 130 | duration: [b, 1, t_x] 131 | mask: [b, 1, t_y, t_x] 132 | """ 133 | device = duration.device 134 | 135 | b, _, t_y, t_x = mask.shape 136 | cum_duration = torch.cumsum(duration, -1) 137 | 138 | cum_duration_flat = cum_duration.view(b * t_x) 139 | path = sequence_mask(cum_duration_flat, t_y).to(mask.dtype) 140 | path = path.view(b, t_x, t_y) 141 | path = path - F.pad(path, convert_pad_shape([[0, 0], [1, 0], [0, 0]]))[:, :-1] 142 | path = path.unsqueeze(1).transpose(2,3) * mask 143 | return path 144 | 145 | 146 | def clip_grad_value_(parameters, clip_value, norm_type=2): 147 | if isinstance(parameters, torch.Tensor): 148 | parameters = [parameters] 149 | parameters = list(filter(lambda p: p.grad is not None, parameters)) 150 | norm_type = float(norm_type) 151 | if clip_value is not None: 152 | clip_value = float(clip_value) 153 | 154 | total_norm = 0 155 | for p in parameters: 156 | param_norm = p.grad.data.norm(norm_type) 157 | total_norm += param_norm.item() ** norm_type 158 | if clip_value is not None: 159 | p.grad.data.clamp_(min=-clip_value, max=clip_value) 160 | total_norm = total_norm ** (1. / norm_type) 161 | return total_norm 162 | -------------------------------------------------------------------------------- /data_utils.py: -------------------------------------------------------------------------------- 1 | import time 2 | import os 3 | import random 4 | import numpy as np 5 | import torch 6 | import torch.utils.data 7 | 8 | import commons 9 | from mel_processing import spectrogram_torch 10 | from utils import load_wav_to_torch, load_filepaths_and_eeg, get_hparams 11 | 12 | EEG_CHANNEL_LIST = ['A1', 'A2', 'A3', 'A4', 'A5', 'A6', 'A7', 'A8', 'A9', 'A10', 'A11', 'A12', 'A13', 'A14', 'A15', 'A16', 'A17', 'A18', 'A19', 'A20', 'A21', 'A22', 'A23', 'A24', 'A25', 'A26', 'A27', 'A28', 'A29', 'A30', 'A31', 'A32', 'B1', 'B2', 'B3', 'B4', 'B5', 'B6', 'B7', 'B8', 'B9', 'B10', 'B11', 'B12', 'B13', 'B14', 'B15', 'B16', 'B17', 'B18', 'B19', 'B20', 'B21', 'B22', 'B23', 'B24', 'B25', 'B26', 'B27', 'B28', 'B29', 'B30', 'B31', 'B32', 'C1', 'C2', 'C3', 'C4', 'C5', 'C6', 'C7', 'C8', 'C9', 'C10', 'C11', 'C12', 'C13', 'C14', 'C15', 'C16', 'C17', 'C18', 'C19', 'C20', 'C21', 'C22', 'C23', 'C24', 'C25', 'C26', 'C27', 'C28', 'C29', 'C30', 'C31', 'C32', 'D1', 'D2', 'D3', 'D4', 'D5', 'D6', 'D7', 'D8', 'D9', 'D10', 'D11', 'D12', 'D13', 'D14', 'D15', 'D16', 'D17', 'D18', 'D19', 'D20', 'D21', 'D22', 'D23', 'D24', 'D25', 'D26', 'D27', 'D28', 'D29', 'D30', 'D31', 'D32', 'EXG1', 'EXG2', 'EXG3', 'EXG4', 'EXG5', 'EXG6', 'EXG7', 'EXG8', 'Status'] 13 | EEG_CHANNEL_DICT = {k:i for i,k in enumerate(EEG_CHANNEL_LIST)} 14 | 15 | 16 | class EEGAudioLoader(torch.utils.data.Dataset): 17 | """ 18 | loads EEG and audio pairs 19 | """ 20 | def __init__(self, audiopaths_and_eeg, hparams): 21 | self.audiopaths_and_eeg = load_filepaths_and_eeg(audiopaths_and_eeg) 22 | self.data_root_dir = hparams.data_root_dir 23 | self.max_wav_value = hparams.max_wav_value 24 | self.sampling_rate = hparams.sampling_rate 25 | self.filter_length = hparams.filter_length 26 | self.hop_length = hparams.hop_length 27 | self.win_length = hparams.win_length 28 | self.sampling_rate = hparams.sampling_rate 29 | 30 | try: 31 | self.eeg_channels_select = hparams.eeg_channels 32 | self.eeg_channels_select = [EEG_CHANNEL_DICT[ch] for ch in self.eeg_channels_select] 33 | print(f'{len(self.eeg_channels_select)} channels selected:{hparams.eeg_channels}') 34 | except: 35 | self.eeg_channels_select = 'all' 36 | 37 | try: 38 | self.dataset_type = hparams.dataset_type 39 | except: 40 | self.dataset_type = 'n400' 41 | 42 | 43 | random.seed(1234) 44 | random.shuffle(self.audiopaths_and_eeg) 45 | 46 | self._get_audio_length() 47 | 48 | 49 | 50 | def _get_audio_length(self): 51 | 52 | lengths = [] 53 | for audiopath in self.audiopaths_and_eeg: 54 | audiopath = audiopath.split('||')[0] 55 | lengths.append(os.path.getsize(os.path.join(self.data_root_dir,'2022N400_Jan_cp', 'stimuli_22k', audiopath.split('-_-')[-1])+'_22k.wav') // (2 * self.hop_length)) 56 | 57 | self.lengths = lengths 58 | 59 | def get_audio_eeg_pair(self, audiopath_and_eeg): 60 | datapath, text = audiopath_and_eeg.split('||') 61 | audiopath = os.path.join(self.data_root_dir,'2022N400_Jan_cp', 'stimuli_22k', datapath.split('-_-')[-1])+'_22k.wav' 62 | eeg_path = os.path.join(self.data_root_dir, '2022N400_Epoched', datapath)+'.npy' 63 | 64 | eeg_data = self.get_eeg(eeg_path) 65 | spec, wav = self.get_audio(audiopath) 66 | return (eeg_data, spec, wav, text) 67 | 68 | def get_audio(self, filename): 69 | audio, sampling_rate = load_wav_to_torch(filename) 70 | if sampling_rate != self.sampling_rate: 71 | raise ValueError("{} {} SR doesn't match target {} SR".format( 72 | sampling_rate, self.sampling_rate)) 73 | audio_norm = audio / self.max_wav_value 74 | audio_norm = audio_norm.unsqueeze(0) 75 | spec = spectrogram_torch(audio_norm, self.filter_length, 76 | self.sampling_rate, self.hop_length, self.win_length, 77 | center=False) 78 | spec = torch.squeeze(spec, 0) 79 | return spec, audio_norm 80 | 81 | def get_eeg(self, filename): 82 | eeg = np.load(filename) 83 | eeg = torch.FloatTensor(eeg) # T x Ch 84 | 85 | if self.eeg_channels_select == 'all': 86 | eeg = eeg[:128] 87 | else: 88 | eeg = eeg[self.eeg_channels_select] 89 | eeg = torch.nn.functional.normalize(eeg, p=2, dim=1) # time-wise normalization 90 | 91 | return eeg 92 | 93 | 94 | def __getitem__(self, index): 95 | return self.get_audio_eeg_pair(self.audiopaths_and_eeg[index]) 96 | 97 | def __len__(self): 98 | return len(self.audiopaths_and_eeg) 99 | 100 | 101 | class EEGAudioCollate(): 102 | """ Zero-pads model inputs and targets 103 | """ 104 | def __init__(self, return_ids=False): 105 | self.return_ids = return_ids 106 | 107 | def __call__(self, batch): 108 | """Collate's training batch from normalized eeg and audio 109 | PARAMS 110 | ------ 111 | batch: [eeg, spec_normalized, wav_normalized] 112 | """ 113 | # Right zero-pad all one-hot text sequences to max input length 114 | _, ids_sorted_decreasing = torch.sort( 115 | torch.LongTensor([x[0].size(1) for x in batch]), # eeg, spec, wav 116 | dim=0, descending=True) 117 | 118 | max_eeg_len = max([x[0].size(1) for x in batch]) 119 | max_spec_len = max([x[1].size(1) for x in batch]) 120 | max_wav_len = max([x[2].size(1) for x in batch]) 121 | 122 | 123 | eeg_lengths = torch.LongTensor(len(batch)) 124 | spec_lengths = torch.LongTensor(len(batch)) 125 | wav_lengths = torch.LongTensor(len(batch)) 126 | 127 | 128 | eeg_padded = torch.FloatTensor(len(batch), batch[0][0].size(0), max_eeg_len) 129 | spec_padded = torch.FloatTensor(len(batch), batch[0][1].size(0), max_spec_len) 130 | wav_padded = torch.FloatTensor(len(batch), 1, max_wav_len) 131 | 132 | eeg_padded.zero_() 133 | spec_padded.zero_() 134 | wav_padded.zero_() 135 | 136 | for i in range(len(ids_sorted_decreasing)): 137 | row = batch[ids_sorted_decreasing[i]] 138 | 139 | eeg = row[0] 140 | eeg_padded[i,:,:eeg.size(1)] = eeg 141 | eeg_lengths[i] = eeg.size(1) 142 | 143 | spec = row[1] 144 | spec_padded[i, :, :spec.size(1)] = spec 145 | spec_lengths[i] = spec.size(1) 146 | 147 | wav = row[2] 148 | wav_padded[i, :, :wav.size(1)] = wav 149 | wav_lengths[i] = wav.size(1) 150 | 151 | if self.return_ids: 152 | return eeg_padded, eeg_lengths, spec_padded, spec_lengths, wav_padded, wav_lengths, ids_sorted_decreasing 153 | return eeg_padded, eeg_lengths, spec_padded, spec_lengths, wav_padded, wav_lengths 154 | 155 | 156 | 157 | 158 | 159 | if __name__=='__main__': 160 | hps = get_hparams() 161 | temp_dataset = EEGAudioLoader(hps.data.validation_files_subject, hps.data) 162 | -------------------------------------------------------------------------------- /EEGModule.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | 5 | import numpy as np 6 | from s4_block.s4_model import S4Model 7 | 8 | 9 | class block_conv(nn.Module): 10 | def __init__(self, in_channel: int=512, out_channel: int=512, 11 | kernel_size: int=3, stride: int=1, padding: int=1, output_padding: int=1, 12 | norm=nn.GroupNorm, dropout: float=0.3, residual: bool=False, 13 | act_layer = nn.GELU, conv: str='conv'): 14 | '''A single convolution block with normaliation and dropout''' 15 | super().__init__() 16 | 17 | self.kernel_size = kernel_size 18 | self.in_channel = in_channel 19 | self.out_channel = out_channel 20 | self.padding = padding 21 | self.output_padding = output_padding 22 | self.stride = stride 23 | self.dropout = dropout 24 | 25 | if residual is not False: 26 | raise NotImplementedError 27 | 28 | if conv == 'conv': 29 | self.conv_layer = nn.Conv1d(in_channels=self.in_channel, 30 | out_channels=self.out_channel, 31 | kernel_size=self.kernel_size, 32 | stride=self.stride, padding=self.padding) 33 | else: 34 | self.conv_layer = nn.ConvTranspose1d(in_channels=self.in_channel, 35 | out_channels=self.out_channel, 36 | kernel_size=self.kernel_size, 37 | stride=self.stride, padding=self.padding, 38 | output_padding=self.output_padding) 39 | 40 | self.norm_layer = norm(1, out_channel) 41 | 42 | self.dropout_layer = nn.Dropout1d(self.dropout) 43 | 44 | if act_layer is not None: 45 | self.act_layer = act_layer() 46 | else: 47 | self.act_layer = None 48 | 49 | def forward(self, x): 50 | x = self.conv_layer(x) 51 | x = self.dropout_layer(x) 52 | x = self.norm_layer(x) 53 | 54 | if self.act_layer is not None: 55 | x = self.act_layer(x) 56 | return x 57 | 58 | class block_last_decoder(block_conv): 59 | def __init__(self, in_channel: int = 512, out_channel: int = 512, kernel_size: int = 3, stride: int = 1, padding: int = 1, output_padding: int = 1, norm=nn.GroupNorm, dropout: float = 0.3, residual: bool = False, act_layer=nn.GELU, conv: str = 'conv'): 60 | super().__init__(in_channel, out_channel, kernel_size, stride, padding, output_padding, norm, dropout, residual, act_layer, conv) 61 | 62 | def forward(self, x): 63 | x = self.conv_layer(x) 64 | return x 65 | 66 | class conv_encoder_tueg(nn.Module): 67 | def __init__(self, n_layers: int=4, input_channels: int=19, embedding_size: int=512, 68 | kernel_size: int=3, stride: int=2, padding: int=1, output_padding: int=1, 69 | norm=nn.GroupNorm, dropout: float=0.3, residual: bool=False, 70 | act_layer = nn.GELU): 71 | super().__init__() 72 | 73 | self.n_layers = n_layers 74 | 75 | self.block_layers = nn.ModuleList() 76 | 77 | self.block_layers.append(block_conv(input_channels, embedding_size, 78 | kernel_size, stride, padding, output_padding, 79 | norm, dropout, residual, act_layer)) 80 | 81 | for i in range(n_layers - 1): 82 | self.block_layers.append(block_conv(embedding_size, embedding_size, 83 | kernel_size, stride, padding, output_padding, 84 | norm, dropout, residual, act_layer)) 85 | 86 | 87 | def forward(self, x): 88 | for i, block_layer in enumerate(self.block_layers): 89 | x = block_layer(x) 90 | return x 91 | 92 | class conv_decoder_tueg(nn.Module): 93 | def __init__(self, n_layers: int=4, input_channels: int=19, embedding_size: int=512, 94 | kernel_size: int=3, stride: int=2, padding: int=1, output_padding: int=1, 95 | norm=nn.GroupNorm, dropout: float=0.3, residual: bool=False, 96 | act_layer = nn.GELU, is_last_layer=False): 97 | super().__init__() 98 | 99 | self.n_layers = n_layers 100 | 101 | self.block_layers = nn.ModuleList() 102 | 103 | for i in range(n_layers - 1 * is_last_layer): 104 | self.block_layers.append(block_conv(embedding_size, embedding_size, 105 | kernel_size, stride, padding, output_padding, 106 | norm, dropout, residual, act_layer, 'deconv')) 107 | 108 | if is_last_layer: 109 | self.block_layers.append(block_last_decoder(embedding_size, input_channels, 110 | kernel_size, stride, padding, output_padding, 111 | norm, dropout, residual, None, 'deconv')) 112 | 113 | def forward(self, x): 114 | for i, block_layer in enumerate(self.block_layers): 115 | x = block_layer(x) 116 | return x 117 | 118 | 119 | 120 | class EEGModule(nn.Module): 121 | def __init__(self, n_layers_cnn: int=6, 122 | use_s4=False, 123 | n_layers_s4: int=8, 124 | device: str='cuda', embedding_size: int=512, 125 | is_mask: bool=True, in_channels: int=19): 126 | super().__init__() 127 | 128 | self.n_layers_cnn = n_layers_cnn 129 | self.use_s4 = use_s4 130 | self.n_layers_s4 = n_layers_s4 131 | self.device = device 132 | self.is_mask = is_mask 133 | 134 | self.conv_encoder = conv_encoder_tueg(n_layers_cnn, stride=1, padding=3, kernel_size=4, embedding_size=embedding_size, input_channels=in_channels) 135 | self.conv_encoder2 = conv_encoder_tueg(1, stride=3, padding=2, kernel_size=4, embedding_size=embedding_size, input_channels=embedding_size) 136 | 137 | self.deconv_encoder = conv_decoder_tueg(n_layers_cnn, stride=1, padding=3, kernel_size=4, output_padding=0, embedding_size=embedding_size, input_channels=in_channels, is_last_layer=True) 138 | self.deconv_encoder2 = conv_decoder_tueg(1, stride=3, padding=2, kernel_size=4, output_padding=2, embedding_size=embedding_size, input_channels=embedding_size, is_last_layer=False) 139 | 140 | self.s4_model = S4Model(d_input=embedding_size, 141 | d_output=embedding_size, 142 | d_model=embedding_size, 143 | n_layers=n_layers_s4, 144 | dropout=0.3, 145 | prenorm=False) 146 | 147 | 148 | def forward(self, x): 149 | if type(x) is list: 150 | x = torch.cat((x[0], x[1]), dim=1) 151 | x = x.to(self.device) 152 | mask = None 153 | if self.is_mask: 154 | mask, masked_input = self.mask(x) 155 | else: 156 | masked_input = x 157 | 158 | conv_output = self.conv_encoder(masked_input.clone()) 159 | conv_output = self.conv_encoder2(conv_output) 160 | 161 | if self.use_s4: 162 | mid_output = self.s4_model(conv_output.transpose(-1,-2).clone()) 163 | mid_output = mid_output.transpose(1,2) 164 | else: 165 | mid_output = conv_output # not s4 output 166 | 167 | 168 | decoder_out = self.deconv_encoder2(mid_output.clone()) 169 | decoder_out = self.deconv_encoder(decoder_out) 170 | 171 | return x, mask, mid_output, decoder_out[:,:,:x.size(2)] 172 | 173 | -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import glob 3 | import sys 4 | import argparse 5 | import logging 6 | import json 7 | import subprocess 8 | import numpy as np 9 | from scipy.io.wavfile import read 10 | import torch 11 | 12 | MATPLOTLIB_FLAG = False 13 | 14 | logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) 15 | logger = logging 16 | 17 | 18 | def load_checkpoint(checkpoint_path, model, optimizer=None): 19 | assert os.path.isfile(checkpoint_path) 20 | checkpoint_dict = torch.load(checkpoint_path, map_location='cpu') 21 | iteration = checkpoint_dict['iteration'] 22 | learning_rate = checkpoint_dict['learning_rate'] 23 | if optimizer is not None: 24 | optimizer.load_state_dict(checkpoint_dict['optimizer']) 25 | saved_state_dict = checkpoint_dict['model'] 26 | if hasattr(model, 'module'): 27 | state_dict = model.module.state_dict() 28 | else: 29 | state_dict = model.state_dict() 30 | new_state_dict= {} 31 | for k, v in state_dict.items(): 32 | try: 33 | new_state_dict[k] = saved_state_dict[k] 34 | except: 35 | logger.info("%s is not in the checkpoint" % k) 36 | new_state_dict[k] = v 37 | if hasattr(model, 'module'): 38 | model.module.load_state_dict(new_state_dict) 39 | else: 40 | model.load_state_dict(new_state_dict) 41 | logger.info("Loaded checkpoint '{}' (iteration {})" .format( 42 | checkpoint_path, iteration)) 43 | return model, optimizer, learning_rate, iteration 44 | 45 | 46 | def save_checkpoint(model, optimizer, learning_rate, iteration, checkpoint_path): 47 | logger.info("Saving model and optimizer state at iteration {} to {}".format( 48 | iteration, checkpoint_path)) 49 | if hasattr(model, 'module'): 50 | state_dict = model.module.state_dict() 51 | else: 52 | state_dict = model.state_dict() 53 | torch.save({'model': state_dict, 54 | 'iteration': iteration, 55 | 'optimizer': optimizer.state_dict(), 56 | 'learning_rate': learning_rate}, checkpoint_path) 57 | 58 | 59 | def summarize(writer, global_step, scalars={}, histograms={}, images={}, audios={}, audio_sampling_rate=22050): 60 | for k, v in scalars.items(): 61 | writer.add_scalar(k, v, global_step) 62 | for k, v in histograms.items(): 63 | writer.add_histogram(k, v, global_step) 64 | for k, v in images.items(): 65 | writer.add_image(k, v, global_step, dataformats='HWC') 66 | for k, v in audios.items(): 67 | writer.add_audio(k, v, global_step, audio_sampling_rate) 68 | 69 | 70 | def latest_checkpoint_path(dir_path, regex="G_*.pth"): 71 | f_list = glob.glob(os.path.join(dir_path, regex)) 72 | f_list.sort(key=lambda f: int("".join(filter(str.isdigit, f)))) 73 | x = f_list[-1] 74 | print(x) 75 | return x 76 | 77 | 78 | def plot_spectrogram_to_numpy(spectrogram): 79 | global MATPLOTLIB_FLAG 80 | if not MATPLOTLIB_FLAG: 81 | import matplotlib 82 | matplotlib.use("Agg") 83 | MATPLOTLIB_FLAG = True 84 | mpl_logger = logging.getLogger('matplotlib') 85 | mpl_logger.setLevel(logging.WARNING) 86 | import matplotlib.pylab as plt 87 | import numpy as np 88 | 89 | fig, ax = plt.subplots(figsize=(10,2)) 90 | im = ax.imshow(spectrogram, aspect="auto", origin="lower", 91 | interpolation='none') 92 | plt.colorbar(im, ax=ax) 93 | plt.xlabel("Frames") 94 | plt.ylabel("Channels") 95 | plt.tight_layout() 96 | 97 | fig.canvas.draw() 98 | data = np.fromstring(fig.canvas.tostring_rgb(), dtype=np.uint8, sep='') 99 | data = data.reshape(fig.canvas.get_width_height()[::-1] + (3,)) 100 | plt.close() 101 | return data 102 | 103 | 104 | def plot_alignment_to_numpy(alignment, info=None): 105 | global MATPLOTLIB_FLAG 106 | if not MATPLOTLIB_FLAG: 107 | import matplotlib 108 | matplotlib.use("Agg") 109 | MATPLOTLIB_FLAG = True 110 | mpl_logger = logging.getLogger('matplotlib') 111 | mpl_logger.setLevel(logging.WARNING) 112 | import matplotlib.pylab as plt 113 | import numpy as np 114 | 115 | fig, ax = plt.subplots(figsize=(6, 4)) 116 | im = ax.imshow(alignment.transpose(), aspect='auto', origin='lower', 117 | interpolation='none') 118 | fig.colorbar(im, ax=ax) 119 | xlabel = 'Decoder timestep' 120 | if info is not None: 121 | xlabel += '\n\n' + info 122 | plt.xlabel(xlabel) 123 | plt.ylabel('Encoder timestep') 124 | plt.tight_layout() 125 | 126 | fig.canvas.draw() 127 | data = np.fromstring(fig.canvas.tostring_rgb(), dtype=np.uint8, sep='') 128 | data = data.reshape(fig.canvas.get_width_height()[::-1] + (3,)) 129 | plt.close() 130 | return data 131 | 132 | 133 | def load_wav_to_torch(full_path): 134 | sampling_rate, data = read(full_path) 135 | return torch.FloatTensor(data.astype(np.float32)), sampling_rate 136 | 137 | 138 | def load_filepaths_and_text(filename, split="||"): 139 | with open(filename, encoding='utf-8') as f: 140 | filepaths_and_text = [line.strip().split(split) for line in f] 141 | return filepaths_and_text 142 | 143 | def load_filepaths_and_eeg(filename): 144 | with open(filename, encoding='utf-8') as f: 145 | filepaths_and_eeg = [line.strip() for line in f] 146 | return filepaths_and_eeg 147 | 148 | 149 | def get_hparams(init=True): 150 | parser = argparse.ArgumentParser() 151 | parser.add_argument('-c', '--config', type=str, default="./configs/base.json", 152 | help='JSON file for configuration') 153 | parser.add_argument('-m', '--model', type=str, required=True, 154 | help='Model name') 155 | parser.add_argument('-w', '--wandb', type=str, required=False, 156 | help='use wandb', default='n') 157 | parser.add_argument('-p', '--port', type=str, default='7000', 158 | help='port number') 159 | 160 | args = parser.parse_args() 161 | model_dir = os.path.join("./logs", args.model) 162 | 163 | if not os.path.exists(model_dir): 164 | os.makedirs(model_dir) 165 | 166 | config_path = args.config 167 | config_save_path = os.path.join(model_dir, "config.json") 168 | if init: 169 | with open(config_path, "r") as f: 170 | data = f.read() 171 | with open(config_save_path, "w") as f: 172 | f.write(data) 173 | else: 174 | with open(config_save_path, "r") as f: 175 | data = f.read() 176 | config = json.loads(data) 177 | 178 | hparams = HParams(**config) 179 | hparams.model_dir = model_dir 180 | hparams.run_name = args.model 181 | hparams.port = args.port 182 | 183 | if args.wandb.lower() in ['t', 'y', 'yes']: 184 | hparams.wandb = True 185 | else: 186 | hparams.wandb = False 187 | 188 | return hparams 189 | 190 | 191 | def get_hparams_from_dir(model_dir): 192 | config_save_path = os.path.join(model_dir, "config.json") 193 | with open(config_save_path, "r") as f: 194 | data = f.read() 195 | config = json.loads(data) 196 | 197 | hparams =HParams(**config) 198 | hparams.model_dir = model_dir 199 | return hparams 200 | 201 | 202 | def get_hparams_from_file(config_path): 203 | with open(config_path, "r") as f: 204 | data = f.read() 205 | config = json.loads(data) 206 | 207 | hparams =HParams(**config) 208 | return hparams 209 | 210 | 211 | def check_git_hash(model_dir): 212 | source_dir = os.path.dirname(os.path.realpath(__file__)) 213 | if not os.path.exists(os.path.join(source_dir, ".git")): 214 | logger.warn("{} is not a git repository, therefore hash value comparison will be ignored.".format( 215 | source_dir 216 | )) 217 | return 218 | 219 | cur_hash = subprocess.getoutput("git rev-parse HEAD") 220 | 221 | path = os.path.join(model_dir, "githash") 222 | if os.path.exists(path): 223 | saved_hash = open(path).read() 224 | if saved_hash != cur_hash: 225 | logger.warn("git hash values are different. {}(saved) != {}(current)".format( 226 | saved_hash[:8], cur_hash[:8])) 227 | else: 228 | open(path, "w").write(cur_hash) 229 | 230 | 231 | def get_logger(model_dir, filename="train.log"): 232 | global logger 233 | logger = logging.getLogger(os.path.basename(model_dir)) 234 | logger.setLevel(logging.DEBUG) 235 | 236 | formatter = logging.Formatter("%(asctime)s\t%(name)s\t%(levelname)s\t%(message)s") 237 | if not os.path.exists(model_dir): 238 | os.makedirs(model_dir) 239 | h = logging.FileHandler(os.path.join(model_dir, filename)) 240 | h.setLevel(logging.DEBUG) 241 | h.setFormatter(formatter) 242 | logger.addHandler(h) 243 | return logger 244 | 245 | 246 | class HParams(): 247 | def __init__(self, **kwargs): 248 | for k, v in kwargs.items(): 249 | if type(v) == dict: 250 | v = HParams(**v) 251 | self[k] = v 252 | 253 | def keys(self): 254 | return self.__dict__.keys() 255 | 256 | def items(self): 257 | return self.__dict__.items() 258 | 259 | def values(self): 260 | return self.__dict__.values() 261 | 262 | def __len__(self): 263 | return len(self.__dict__) 264 | 265 | def __getitem__(self, key): 266 | return getattr(self, key) 267 | 268 | def __setitem__(self, key, value): 269 | return setattr(self, key, value) 270 | 271 | def __contains__(self, key): 272 | return key in self.__dict__ 273 | 274 | def __repr__(self): 275 | return self.__dict__.__repr__() 276 | -------------------------------------------------------------------------------- /transforms.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch.nn import functional as F 3 | 4 | import numpy as np 5 | 6 | 7 | DEFAULT_MIN_BIN_WIDTH = 1e-3 8 | DEFAULT_MIN_BIN_HEIGHT = 1e-3 9 | DEFAULT_MIN_DERIVATIVE = 1e-3 10 | 11 | 12 | def piecewise_rational_quadratic_transform(inputs, 13 | unnormalized_widths, 14 | unnormalized_heights, 15 | unnormalized_derivatives, 16 | inverse=False, 17 | tails=None, 18 | tail_bound=1., 19 | min_bin_width=DEFAULT_MIN_BIN_WIDTH, 20 | min_bin_height=DEFAULT_MIN_BIN_HEIGHT, 21 | min_derivative=DEFAULT_MIN_DERIVATIVE): 22 | 23 | if tails is None: 24 | spline_fn = rational_quadratic_spline 25 | spline_kwargs = {} 26 | else: 27 | spline_fn = unconstrained_rational_quadratic_spline 28 | spline_kwargs = { 29 | 'tails': tails, 30 | 'tail_bound': tail_bound 31 | } 32 | 33 | outputs, logabsdet = spline_fn( 34 | inputs=inputs, 35 | unnormalized_widths=unnormalized_widths, 36 | unnormalized_heights=unnormalized_heights, 37 | unnormalized_derivatives=unnormalized_derivatives, 38 | inverse=inverse, 39 | min_bin_width=min_bin_width, 40 | min_bin_height=min_bin_height, 41 | min_derivative=min_derivative, 42 | **spline_kwargs 43 | ) 44 | return outputs, logabsdet 45 | 46 | 47 | def searchsorted(bin_locations, inputs, eps=1e-6): 48 | bin_locations[..., -1] += eps 49 | return torch.sum( 50 | inputs[..., None] >= bin_locations, 51 | dim=-1 52 | ) - 1 53 | 54 | 55 | def unconstrained_rational_quadratic_spline(inputs, 56 | unnormalized_widths, 57 | unnormalized_heights, 58 | unnormalized_derivatives, 59 | inverse=False, 60 | tails='linear', 61 | tail_bound=1., 62 | min_bin_width=DEFAULT_MIN_BIN_WIDTH, 63 | min_bin_height=DEFAULT_MIN_BIN_HEIGHT, 64 | min_derivative=DEFAULT_MIN_DERIVATIVE): 65 | inside_interval_mask = (inputs >= -tail_bound) & (inputs <= tail_bound) 66 | outside_interval_mask = ~inside_interval_mask 67 | 68 | outputs = torch.zeros_like(inputs) 69 | logabsdet = torch.zeros_like(inputs) 70 | 71 | if tails == 'linear': 72 | unnormalized_derivatives = F.pad(unnormalized_derivatives, pad=(1, 1)) 73 | constant = np.log(np.exp(1 - min_derivative) - 1) 74 | unnormalized_derivatives[..., 0] = constant 75 | unnormalized_derivatives[..., -1] = constant 76 | 77 | outputs[outside_interval_mask] = inputs[outside_interval_mask] 78 | logabsdet[outside_interval_mask] = 0 79 | else: 80 | raise RuntimeError('{} tails are not implemented.'.format(tails)) 81 | 82 | outputs[inside_interval_mask], logabsdet[inside_interval_mask] = rational_quadratic_spline( 83 | inputs=inputs[inside_interval_mask], 84 | unnormalized_widths=unnormalized_widths[inside_interval_mask, :], 85 | unnormalized_heights=unnormalized_heights[inside_interval_mask, :], 86 | unnormalized_derivatives=unnormalized_derivatives[inside_interval_mask, :], 87 | inverse=inverse, 88 | left=-tail_bound, right=tail_bound, bottom=-tail_bound, top=tail_bound, 89 | min_bin_width=min_bin_width, 90 | min_bin_height=min_bin_height, 91 | min_derivative=min_derivative 92 | ) 93 | 94 | return outputs, logabsdet 95 | 96 | def rational_quadratic_spline(inputs, 97 | unnormalized_widths, 98 | unnormalized_heights, 99 | unnormalized_derivatives, 100 | inverse=False, 101 | left=0., right=1., bottom=0., top=1., 102 | min_bin_width=DEFAULT_MIN_BIN_WIDTH, 103 | min_bin_height=DEFAULT_MIN_BIN_HEIGHT, 104 | min_derivative=DEFAULT_MIN_DERIVATIVE): 105 | if torch.min(inputs) < left or torch.max(inputs) > right: 106 | raise ValueError('Input to a transform is not within its domain') 107 | 108 | num_bins = unnormalized_widths.shape[-1] 109 | 110 | if min_bin_width * num_bins > 1.0: 111 | raise ValueError('Minimal bin width too large for the number of bins') 112 | if min_bin_height * num_bins > 1.0: 113 | raise ValueError('Minimal bin height too large for the number of bins') 114 | 115 | widths = F.softmax(unnormalized_widths, dim=-1) 116 | widths = min_bin_width + (1 - min_bin_width * num_bins) * widths 117 | cumwidths = torch.cumsum(widths, dim=-1) 118 | cumwidths = F.pad(cumwidths, pad=(1, 0), mode='constant', value=0.0) 119 | cumwidths = (right - left) * cumwidths + left 120 | cumwidths[..., 0] = left 121 | cumwidths[..., -1] = right 122 | widths = cumwidths[..., 1:] - cumwidths[..., :-1] 123 | 124 | derivatives = min_derivative + F.softplus(unnormalized_derivatives) 125 | 126 | heights = F.softmax(unnormalized_heights, dim=-1) 127 | heights = min_bin_height + (1 - min_bin_height * num_bins) * heights 128 | cumheights = torch.cumsum(heights, dim=-1) 129 | cumheights = F.pad(cumheights, pad=(1, 0), mode='constant', value=0.0) 130 | cumheights = (top - bottom) * cumheights + bottom 131 | cumheights[..., 0] = bottom 132 | cumheights[..., -1] = top 133 | heights = cumheights[..., 1:] - cumheights[..., :-1] 134 | 135 | if inverse: 136 | bin_idx = searchsorted(cumheights, inputs)[..., None] 137 | else: 138 | bin_idx = searchsorted(cumwidths, inputs)[..., None] 139 | 140 | input_cumwidths = cumwidths.gather(-1, bin_idx)[..., 0] 141 | input_bin_widths = widths.gather(-1, bin_idx)[..., 0] 142 | 143 | input_cumheights = cumheights.gather(-1, bin_idx)[..., 0] 144 | delta = heights / widths 145 | input_delta = delta.gather(-1, bin_idx)[..., 0] 146 | 147 | input_derivatives = derivatives.gather(-1, bin_idx)[..., 0] 148 | input_derivatives_plus_one = derivatives[..., 1:].gather(-1, bin_idx)[..., 0] 149 | 150 | input_heights = heights.gather(-1, bin_idx)[..., 0] 151 | 152 | if inverse: 153 | a = (((inputs - input_cumheights) * (input_derivatives 154 | + input_derivatives_plus_one 155 | - 2 * input_delta) 156 | + input_heights * (input_delta - input_derivatives))) 157 | b = (input_heights * input_derivatives 158 | - (inputs - input_cumheights) * (input_derivatives 159 | + input_derivatives_plus_one 160 | - 2 * input_delta)) 161 | c = - input_delta * (inputs - input_cumheights) 162 | 163 | discriminant = b.pow(2) - 4 * a * c 164 | assert (discriminant >= 0).all() 165 | 166 | root = (2 * c) / (-b - torch.sqrt(discriminant)) 167 | outputs = root * input_bin_widths + input_cumwidths 168 | 169 | theta_one_minus_theta = root * (1 - root) 170 | denominator = input_delta + ((input_derivatives + input_derivatives_plus_one - 2 * input_delta) 171 | * theta_one_minus_theta) 172 | derivative_numerator = input_delta.pow(2) * (input_derivatives_plus_one * root.pow(2) 173 | + 2 * input_delta * theta_one_minus_theta 174 | + input_derivatives * (1 - root).pow(2)) 175 | logabsdet = torch.log(derivative_numerator) - 2 * torch.log(denominator) 176 | 177 | return outputs, -logabsdet 178 | else: 179 | theta = (inputs - input_cumwidths) / input_bin_widths 180 | theta_one_minus_theta = theta * (1 - theta) 181 | 182 | numerator = input_heights * (input_delta * theta.pow(2) 183 | + input_derivatives * theta_one_minus_theta) 184 | denominator = input_delta + ((input_derivatives + input_derivatives_plus_one - 2 * input_delta) 185 | * theta_one_minus_theta) 186 | outputs = input_cumheights + numerator / denominator 187 | 188 | derivative_numerator = input_delta.pow(2) * (input_derivatives_plus_one * theta.pow(2) 189 | + 2 * input_delta * theta_one_minus_theta 190 | + input_derivatives * (1 - theta).pow(2)) 191 | logabsdet = torch.log(derivative_numerator) - 2 * torch.log(denominator) 192 | 193 | return outputs, logabsdet 194 | -------------------------------------------------------------------------------- /attentions.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import math 3 | import numpy as np 4 | import torch 5 | from torch import nn 6 | from torch.nn import functional as F 7 | 8 | import commons 9 | import modules 10 | from modules import LayerNorm 11 | 12 | 13 | class Encoder(nn.Module): 14 | def __init__(self, hidden_channels, filter_channels, n_heads, n_layers, kernel_size=1, p_dropout=0., window_size=4, **kwargs): 15 | super().__init__() 16 | self.hidden_channels = hidden_channels 17 | self.filter_channels = filter_channels 18 | self.n_heads = n_heads 19 | self.n_layers = n_layers 20 | self.kernel_size = kernel_size 21 | self.p_dropout = p_dropout 22 | self.window_size = window_size 23 | 24 | self.drop = nn.Dropout(p_dropout) 25 | self.attn_layers = nn.ModuleList() 26 | self.norm_layers_1 = nn.ModuleList() 27 | self.ffn_layers = nn.ModuleList() 28 | self.norm_layers_2 = nn.ModuleList() 29 | for i in range(self.n_layers): 30 | self.attn_layers.append(MultiHeadAttention(hidden_channels, hidden_channels, n_heads, p_dropout=p_dropout, window_size=window_size)) 31 | self.norm_layers_1.append(LayerNorm(hidden_channels)) 32 | self.ffn_layers.append(FFN(hidden_channels, hidden_channels, filter_channels, kernel_size, p_dropout=p_dropout)) 33 | self.norm_layers_2.append(LayerNorm(hidden_channels)) 34 | 35 | def forward(self, x, x_mask): 36 | attn_mask = x_mask.unsqueeze(2) * x_mask.unsqueeze(-1) 37 | x = x * x_mask 38 | for i in range(self.n_layers): 39 | y = self.attn_layers[i](x, x, attn_mask) 40 | y = self.drop(y) 41 | x = self.norm_layers_1[i](x + y) 42 | 43 | y = self.ffn_layers[i](x, x_mask) 44 | y = self.drop(y) 45 | x = self.norm_layers_2[i](x + y) 46 | x = x * x_mask 47 | return x 48 | 49 | 50 | class Decoder(nn.Module): 51 | def __init__(self, hidden_channels, filter_channels, n_heads, n_layers, kernel_size=1, p_dropout=0., proximal_bias=False, proximal_init=True, **kwargs): 52 | super().__init__() 53 | self.hidden_channels = hidden_channels 54 | self.filter_channels = filter_channels 55 | self.n_heads = n_heads 56 | self.n_layers = n_layers 57 | self.kernel_size = kernel_size 58 | self.p_dropout = p_dropout 59 | self.proximal_bias = proximal_bias 60 | self.proximal_init = proximal_init 61 | 62 | self.drop = nn.Dropout(p_dropout) 63 | self.self_attn_layers = nn.ModuleList() 64 | self.norm_layers_0 = nn.ModuleList() 65 | self.encdec_attn_layers = nn.ModuleList() 66 | self.norm_layers_1 = nn.ModuleList() 67 | self.ffn_layers = nn.ModuleList() 68 | self.norm_layers_2 = nn.ModuleList() 69 | for i in range(self.n_layers): 70 | self.self_attn_layers.append(MultiHeadAttention(hidden_channels, hidden_channels, n_heads, p_dropout=p_dropout, proximal_bias=proximal_bias, proximal_init=proximal_init)) 71 | self.norm_layers_0.append(LayerNorm(hidden_channels)) 72 | self.encdec_attn_layers.append(MultiHeadAttention(hidden_channels, hidden_channels, n_heads, p_dropout=p_dropout)) 73 | self.norm_layers_1.append(LayerNorm(hidden_channels)) 74 | self.ffn_layers.append(FFN(hidden_channels, hidden_channels, filter_channels, kernel_size, p_dropout=p_dropout, causal=True)) 75 | self.norm_layers_2.append(LayerNorm(hidden_channels)) 76 | 77 | def forward(self, x, x_mask, h, h_mask): 78 | """ 79 | x: decoder input 80 | h: encoder output 81 | """ 82 | self_attn_mask = commons.subsequent_mask(x_mask.size(2)).to(device=x.device, dtype=x.dtype) 83 | encdec_attn_mask = h_mask.unsqueeze(2) * x_mask.unsqueeze(-1) 84 | x = x * x_mask 85 | for i in range(self.n_layers): 86 | y = self.self_attn_layers[i](x, x, self_attn_mask) 87 | y = self.drop(y) 88 | x = self.norm_layers_0[i](x + y) 89 | 90 | y = self.encdec_attn_layers[i](x, h, encdec_attn_mask) 91 | y = self.drop(y) 92 | x = self.norm_layers_1[i](x + y) 93 | 94 | y = self.ffn_layers[i](x, x_mask) 95 | y = self.drop(y) 96 | x = self.norm_layers_2[i](x + y) 97 | x = x * x_mask 98 | return x 99 | 100 | 101 | class MultiHeadAttention(nn.Module): 102 | def __init__(self, channels, out_channels, n_heads, p_dropout=0., window_size=None, heads_share=True, block_length=None, proximal_bias=False, proximal_init=False): 103 | super().__init__() 104 | assert channels % n_heads == 0 105 | 106 | self.channels = channels 107 | self.out_channels = out_channels 108 | self.n_heads = n_heads 109 | self.p_dropout = p_dropout 110 | self.window_size = window_size 111 | self.heads_share = heads_share 112 | self.block_length = block_length 113 | self.proximal_bias = proximal_bias 114 | self.proximal_init = proximal_init 115 | self.attn = None 116 | 117 | self.k_channels = channels // n_heads 118 | self.conv_q = nn.Conv1d(channels, channels, 1) 119 | self.conv_k = nn.Conv1d(channels, channels, 1) 120 | self.conv_v = nn.Conv1d(channels, channels, 1) 121 | self.conv_o = nn.Conv1d(channels, out_channels, 1) 122 | self.drop = nn.Dropout(p_dropout) 123 | 124 | if window_size is not None: 125 | n_heads_rel = 1 if heads_share else n_heads 126 | rel_stddev = self.k_channels**-0.5 127 | self.emb_rel_k = nn.Parameter(torch.randn(n_heads_rel, window_size * 2 + 1, self.k_channels) * rel_stddev) 128 | self.emb_rel_v = nn.Parameter(torch.randn(n_heads_rel, window_size * 2 + 1, self.k_channels) * rel_stddev) 129 | 130 | nn.init.xavier_uniform_(self.conv_q.weight) 131 | nn.init.xavier_uniform_(self.conv_k.weight) 132 | nn.init.xavier_uniform_(self.conv_v.weight) 133 | if proximal_init: 134 | with torch.no_grad(): 135 | self.conv_k.weight.copy_(self.conv_q.weight) 136 | self.conv_k.bias.copy_(self.conv_q.bias) 137 | 138 | def forward(self, x, c, attn_mask=None): 139 | q = self.conv_q(x) 140 | k = self.conv_k(c) 141 | v = self.conv_v(c) 142 | 143 | x, self.attn = self.attention(q, k, v, mask=attn_mask) 144 | 145 | x = self.conv_o(x) 146 | return x 147 | 148 | def attention(self, query, key, value, mask=None): 149 | # reshape [b, d, t] -> [b, n_h, t, d_k] 150 | b, d, t_s, t_t = (*key.size(), query.size(2)) 151 | query = query.view(b, self.n_heads, self.k_channels, t_t).transpose(2, 3) 152 | key = key.view(b, self.n_heads, self.k_channels, t_s).transpose(2, 3) 153 | value = value.view(b, self.n_heads, self.k_channels, t_s).transpose(2, 3) 154 | 155 | scores = torch.matmul(query / math.sqrt(self.k_channels), key.transpose(-2, -1)) 156 | if self.window_size is not None: 157 | assert t_s == t_t, "Relative attention is only available for self-attention." 158 | key_relative_embeddings = self._get_relative_embeddings(self.emb_rel_k, t_s) 159 | rel_logits = self._matmul_with_relative_keys(query /math.sqrt(self.k_channels), key_relative_embeddings) 160 | scores_local = self._relative_position_to_absolute_position(rel_logits) 161 | scores = scores + scores_local 162 | if self.proximal_bias: 163 | assert t_s == t_t, "Proximal bias is only available for self-attention." 164 | scores = scores + self._attention_bias_proximal(t_s).to(device=scores.device, dtype=scores.dtype) 165 | if mask is not None: 166 | scores = scores.masked_fill(mask == 0, -1e4) 167 | if self.block_length is not None: 168 | assert t_s == t_t, "Local attention is only available for self-attention." 169 | block_mask = torch.ones_like(scores).triu(-self.block_length).tril(self.block_length) 170 | scores = scores.masked_fill(block_mask == 0, -1e4) 171 | p_attn = F.softmax(scores, dim=-1) # [b, n_h, t_t, t_s] 172 | p_attn = self.drop(p_attn) 173 | output = torch.matmul(p_attn, value) 174 | if self.window_size is not None: 175 | relative_weights = self._absolute_position_to_relative_position(p_attn) 176 | value_relative_embeddings = self._get_relative_embeddings(self.emb_rel_v, t_s) 177 | output = output + self._matmul_with_relative_values(relative_weights, value_relative_embeddings) 178 | output = output.transpose(2, 3).contiguous().view(b, d, t_t) # [b, n_h, t_t, d_k] -> [b, d, t_t] 179 | return output, p_attn 180 | 181 | def _matmul_with_relative_values(self, x, y): 182 | """ 183 | x: [b, h, l, m] 184 | y: [h or 1, m, d] 185 | ret: [b, h, l, d] 186 | """ 187 | ret = torch.matmul(x, y.unsqueeze(0)) 188 | return ret 189 | 190 | def _matmul_with_relative_keys(self, x, y): 191 | """ 192 | x: [b, h, l, d] 193 | y: [h or 1, m, d] 194 | ret: [b, h, l, m] 195 | """ 196 | ret = torch.matmul(x, y.unsqueeze(0).transpose(-2, -1)) 197 | return ret 198 | 199 | def _get_relative_embeddings(self, relative_embeddings, length): 200 | max_relative_position = 2 * self.window_size + 1 201 | # Pad first before slice to avoid using cond ops. 202 | pad_length = max(length - (self.window_size + 1), 0) 203 | slice_start_position = max((self.window_size + 1) - length, 0) 204 | slice_end_position = slice_start_position + 2 * length - 1 205 | if pad_length > 0: 206 | padded_relative_embeddings = F.pad( 207 | relative_embeddings, 208 | commons.convert_pad_shape([[0, 0], [pad_length, pad_length], [0, 0]])) 209 | else: 210 | padded_relative_embeddings = relative_embeddings 211 | used_relative_embeddings = padded_relative_embeddings[:,slice_start_position:slice_end_position] 212 | return used_relative_embeddings 213 | 214 | def _relative_position_to_absolute_position(self, x): 215 | """ 216 | x: [b, h, l, 2*l-1] 217 | ret: [b, h, l, l] 218 | """ 219 | batch, heads, length, _ = x.size() 220 | # Concat columns of pad to shift from relative to absolute indexing. 221 | x = F.pad(x, commons.convert_pad_shape([[0,0],[0,0],[0,0],[0,1]])) 222 | 223 | # Concat extra elements so to add up to shape (len+1, 2*len-1). 224 | x_flat = x.view([batch, heads, length * 2 * length]) 225 | x_flat = F.pad(x_flat, commons.convert_pad_shape([[0,0],[0,0],[0,length-1]])) 226 | 227 | # Reshape and slice out the padded elements. 228 | x_final = x_flat.view([batch, heads, length+1, 2*length-1])[:, :, :length, length-1:] 229 | return x_final 230 | 231 | def _absolute_position_to_relative_position(self, x): 232 | """ 233 | x: [b, h, l, l] 234 | ret: [b, h, l, 2*l-1] 235 | """ 236 | batch, heads, length, _ = x.size() 237 | # padd along column 238 | x = F.pad(x, commons.convert_pad_shape([[0, 0], [0, 0], [0, 0], [0, length-1]])) 239 | x_flat = x.view([batch, heads, length**2 + length*(length -1)]) 240 | # add 0's in the beginning that will skew the elements after reshape 241 | x_flat = F.pad(x_flat, commons.convert_pad_shape([[0, 0], [0, 0], [length, 0]])) 242 | x_final = x_flat.view([batch, heads, length, 2*length])[:,:,:,1:] 243 | return x_final 244 | 245 | def _attention_bias_proximal(self, length): 246 | """Bias for self-attention to encourage attention to close positions. 247 | Args: 248 | length: an integer scalar. 249 | Returns: 250 | a Tensor with shape [1, 1, length, length] 251 | """ 252 | r = torch.arange(length, dtype=torch.float32) 253 | diff = torch.unsqueeze(r, 0) - torch.unsqueeze(r, 1) 254 | return torch.unsqueeze(torch.unsqueeze(-torch.log1p(torch.abs(diff)), 0), 0) 255 | 256 | 257 | class FFN(nn.Module): 258 | def __init__(self, in_channels, out_channels, filter_channels, kernel_size, p_dropout=0., activation=None, causal=False): 259 | super().__init__() 260 | self.in_channels = in_channels 261 | self.out_channels = out_channels 262 | self.filter_channels = filter_channels 263 | self.kernel_size = kernel_size 264 | self.p_dropout = p_dropout 265 | self.activation = activation 266 | self.causal = causal 267 | 268 | if causal: 269 | self.padding = self._causal_padding 270 | else: 271 | self.padding = self._same_padding 272 | 273 | self.conv_1 = nn.Conv1d(in_channels, filter_channels, kernel_size) 274 | self.conv_2 = nn.Conv1d(filter_channels, out_channels, kernel_size) 275 | self.drop = nn.Dropout(p_dropout) 276 | 277 | def forward(self, x, x_mask): 278 | x = self.conv_1(self.padding(x * x_mask)) 279 | if self.activation == "gelu": 280 | x = x * torch.sigmoid(1.702 * x) 281 | else: 282 | x = torch.relu(x) 283 | x = self.drop(x) 284 | x = self.conv_2(self.padding(x * x_mask)) 285 | return x * x_mask 286 | 287 | def _causal_padding(self, x): 288 | if self.kernel_size == 1: 289 | return x 290 | pad_l = self.kernel_size - 1 291 | pad_r = 0 292 | padding = [[0, 0], [0, 0], [pad_l, pad_r]] 293 | x = F.pad(x, commons.convert_pad_shape(padding)) 294 | return x 295 | 296 | def _same_padding(self, x): 297 | if self.kernel_size == 1: 298 | return x 299 | pad_l = (self.kernel_size - 1) // 2 300 | pad_r = self.kernel_size // 2 301 | padding = [[0, 0], [0, 0], [pad_l, pad_r]] 302 | x = F.pad(x, commons.convert_pad_shape(padding)) 303 | return x 304 | -------------------------------------------------------------------------------- /models.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import math 3 | import torch 4 | from torch import nn 5 | from torch.nn import functional as F 6 | 7 | import commons 8 | import modules 9 | import attentions 10 | 11 | from torch.nn import Conv1d, ConvTranspose1d, AvgPool1d, Conv2d 12 | from torch.nn.utils import weight_norm, remove_weight_norm, spectral_norm 13 | from commons import init_weights, get_padding 14 | 15 | 16 | class Connector(nn.Module): 17 | def __init__(self, 18 | out_channels, 19 | hidden_channels, 20 | filter_channels, 21 | n_heads, 22 | n_layers, 23 | kernel_size, 24 | p_dropout): 25 | super().__init__() 26 | self.out_channels = out_channels 27 | self.hidden_channels = hidden_channels 28 | self.filter_channels = filter_channels 29 | self.n_heads = n_heads 30 | self.n_layers = n_layers 31 | self.kernel_size = kernel_size 32 | self.p_dropout = p_dropout 33 | 34 | self.encoder = attentions.Encoder( 35 | hidden_channels, 36 | filter_channels, 37 | n_heads, 38 | n_layers, 39 | kernel_size, 40 | p_dropout) 41 | self.proj= nn.Conv1d(hidden_channels, out_channels * 2 42 | , 1) 43 | 44 | def forward(self, x, x_lengths): 45 | x = x * math.sqrt(self.hidden_channels) # [b, h, t] 46 | x_mask = torch.unsqueeze(commons.sequence_mask(x_lengths, x.size(2)), 1).to(x.dtype) 47 | 48 | x = self.encoder(x * x_mask, x_mask) 49 | stats = self.proj(x) * x_mask 50 | m, logs = torch.split(stats, self.out_channels, dim=1) 51 | return x, m, logs, x_mask 52 | 53 | 54 | 55 | class ResidualCouplingBlock(nn.Module): 56 | def __init__(self, 57 | channels, 58 | hidden_channels, 59 | kernel_size, 60 | dilation_rate, 61 | n_layers, 62 | n_flows=4, 63 | gin_channels=0): 64 | super().__init__() 65 | self.channels = channels 66 | self.hidden_channels = hidden_channels 67 | self.kernel_size = kernel_size 68 | self.dilation_rate = dilation_rate 69 | self.n_layers = n_layers 70 | self.n_flows = n_flows 71 | self.gin_channels = gin_channels 72 | 73 | self.flows = nn.ModuleList() 74 | for i in range(n_flows): 75 | self.flows.append(modules.ResidualCouplingLayer(channels, hidden_channels, kernel_size, dilation_rate, n_layers, gin_channels=gin_channels, mean_only=True)) 76 | self.flows.append(modules.Flip()) 77 | 78 | def forward(self, x, x_mask, g=None, reverse=False): 79 | if not reverse: 80 | for flow in self.flows: 81 | x, _ = flow(x, x_mask, g=g, reverse=reverse) 82 | else: 83 | for flow in reversed(self.flows): 84 | x = flow(x, x_mask, g=g, reverse=reverse) 85 | return x 86 | 87 | 88 | class SpeechEncoder(nn.Module): 89 | def __init__(self, 90 | in_channels, 91 | out_channels, 92 | hidden_channels, 93 | kernel_size, 94 | dilation_rate, 95 | n_layers, 96 | gin_channels=0): 97 | super().__init__() 98 | self.in_channels = in_channels 99 | self.out_channels = out_channels 100 | self.hidden_channels = hidden_channels 101 | self.kernel_size = kernel_size 102 | self.dilation_rate = dilation_rate 103 | self.n_layers = n_layers 104 | self.gin_channels = gin_channels 105 | 106 | self.pre = nn.Conv1d(in_channels, hidden_channels, 1) 107 | self.enc = modules.WN(hidden_channels, kernel_size, dilation_rate, n_layers, gin_channels=gin_channels) 108 | self.proj = nn.Conv1d(hidden_channels, out_channels * 2, 1) 109 | 110 | def forward(self, x, x_lengths, g=None): 111 | x_mask = torch.unsqueeze(commons.sequence_mask(x_lengths, x.size(2)), 1).to(x.dtype) 112 | x = self.pre(x) * x_mask 113 | x = self.enc(x, x_mask, g=g) 114 | stats = self.proj(x) * x_mask 115 | m, logs = torch.split(stats, self.out_channels, dim=1) 116 | z = (m + torch.randn_like(m) * torch.exp(logs)) * x_mask 117 | return z, m, logs, x_mask 118 | 119 | 120 | class Generator(torch.nn.Module): 121 | def __init__(self, initial_channel, resblock, resblock_kernel_sizes, resblock_dilation_sizes, upsample_rates, upsample_initial_channel, upsample_kernel_sizes, gin_channels=0): 122 | super(Generator, self).__init__() 123 | self.num_kernels = len(resblock_kernel_sizes) 124 | self.num_upsamples = len(upsample_rates) 125 | self.conv_pre = Conv1d(initial_channel, upsample_initial_channel, 7, 1, padding=3) 126 | resblock = modules.ResBlock1 if resblock == '1' else modules.ResBlock2 127 | 128 | self.ups = nn.ModuleList() 129 | for i, (u, k) in enumerate(zip(upsample_rates, upsample_kernel_sizes)): 130 | self.ups.append(weight_norm( 131 | ConvTranspose1d(upsample_initial_channel//(2**i), upsample_initial_channel//(2**(i+1)), 132 | k, u, padding=(k-u)//2))) 133 | 134 | self.resblocks = nn.ModuleList() 135 | for i in range(len(self.ups)): 136 | ch = upsample_initial_channel//(2**(i+1)) 137 | for j, (k, d) in enumerate(zip(resblock_kernel_sizes, resblock_dilation_sizes)): 138 | self.resblocks.append(resblock(ch, k, d)) 139 | 140 | self.conv_post = Conv1d(ch, 1, 7, 1, padding=3, bias=False) 141 | self.ups.apply(init_weights) 142 | 143 | if gin_channels != 0: 144 | self.cond = nn.Conv1d(gin_channels, upsample_initial_channel, 1) 145 | 146 | def forward(self, x, g=None): 147 | x = self.conv_pre(x) 148 | if g is not None: 149 | x = x + self.cond(g) 150 | 151 | for i in range(self.num_upsamples): 152 | x = F.leaky_relu(x, modules.LRELU_SLOPE) 153 | x = self.ups[i](x) 154 | xs = None 155 | for j in range(self.num_kernels): 156 | if xs is None: 157 | xs = self.resblocks[i*self.num_kernels+j](x) 158 | else: 159 | xs += self.resblocks[i*self.num_kernels+j](x) 160 | x = xs / self.num_kernels 161 | x = F.leaky_relu(x) 162 | x = self.conv_post(x) 163 | x = torch.tanh(x) 164 | 165 | return x 166 | 167 | def remove_weight_norm(self): 168 | print('Removing weight norm...') 169 | for l in self.ups: 170 | remove_weight_norm(l) 171 | for l in self.resblocks: 172 | l.remove_weight_norm() 173 | 174 | 175 | class DiscriminatorP(torch.nn.Module): 176 | def __init__(self, period, kernel_size=5, stride=3, use_spectral_norm=False): 177 | super(DiscriminatorP, self).__init__() 178 | self.period = period 179 | self.use_spectral_norm = use_spectral_norm 180 | norm_f = weight_norm if use_spectral_norm == False else spectral_norm 181 | self.convs = nn.ModuleList([ 182 | norm_f(Conv2d(1, 32, (kernel_size, 1), (stride, 1), padding=(get_padding(kernel_size, 1), 0))), 183 | norm_f(Conv2d(32, 128, (kernel_size, 1), (stride, 1), padding=(get_padding(kernel_size, 1), 0))), 184 | norm_f(Conv2d(128, 512, (kernel_size, 1), (stride, 1), padding=(get_padding(kernel_size, 1), 0))), 185 | norm_f(Conv2d(512, 1024, (kernel_size, 1), (stride, 1), padding=(get_padding(kernel_size, 1), 0))), 186 | norm_f(Conv2d(1024, 1024, (kernel_size, 1), 1, padding=(get_padding(kernel_size, 1), 0))), 187 | ]) 188 | self.conv_post = norm_f(Conv2d(1024, 1, (3, 1), 1, padding=(1, 0))) 189 | 190 | def forward(self, x): 191 | fmap = [] 192 | 193 | # 1d to 2d 194 | b, c, t = x.shape 195 | if t % self.period != 0: # pad first 196 | n_pad = self.period - (t % self.period) 197 | x = F.pad(x, (0, n_pad), "reflect") 198 | t = t + n_pad 199 | x = x.view(b, c, t // self.period, self.period) 200 | 201 | for l in self.convs: 202 | x = l(x) 203 | x = F.leaky_relu(x, modules.LRELU_SLOPE) 204 | fmap.append(x) 205 | x = self.conv_post(x) 206 | fmap.append(x) 207 | x = torch.flatten(x, 1, -1) 208 | 209 | return x, fmap 210 | 211 | 212 | class DiscriminatorS(torch.nn.Module): 213 | def __init__(self, use_spectral_norm=False): 214 | super(DiscriminatorS, self).__init__() 215 | norm_f = weight_norm if use_spectral_norm == False else spectral_norm 216 | self.convs = nn.ModuleList([ 217 | norm_f(Conv1d(1, 16, 15, 1, padding=7)), 218 | norm_f(Conv1d(16, 64, 41, 4, groups=4, padding=20)), 219 | norm_f(Conv1d(64, 256, 41, 4, groups=16, padding=20)), 220 | norm_f(Conv1d(256, 1024, 41, 4, groups=64, padding=20)), 221 | norm_f(Conv1d(1024, 1024, 41, 4, groups=256, padding=20)), 222 | norm_f(Conv1d(1024, 1024, 5, 1, padding=2)), 223 | ]) 224 | self.conv_post = norm_f(Conv1d(1024, 1, 3, 1, padding=1)) 225 | 226 | def forward(self, x): 227 | fmap = [] 228 | 229 | for l in self.convs: 230 | x = l(x) 231 | x = F.leaky_relu(x, modules.LRELU_SLOPE) 232 | fmap.append(x) 233 | x = self.conv_post(x) 234 | fmap.append(x) 235 | x = torch.flatten(x, 1, -1) 236 | 237 | return x, fmap 238 | 239 | 240 | class MultiPeriodDiscriminator(torch.nn.Module): 241 | def __init__(self, use_spectral_norm=False): 242 | super(MultiPeriodDiscriminator, self).__init__() 243 | periods = [2,3,5,7,11] 244 | 245 | discs = [DiscriminatorS(use_spectral_norm=use_spectral_norm)] 246 | discs = discs + [DiscriminatorP(i, use_spectral_norm=use_spectral_norm) for i in periods] 247 | self.discriminators = nn.ModuleList(discs) 248 | 249 | def forward(self, y, y_hat): 250 | y_d_rs = [] 251 | y_d_gs = [] 252 | fmap_rs = [] 253 | fmap_gs = [] 254 | for i, d in enumerate(self.discriminators): 255 | y_d_r, fmap_r = d(y) 256 | y_d_g, fmap_g = d(y_hat) 257 | y_d_rs.append(y_d_r) 258 | y_d_gs.append(y_d_g) 259 | fmap_rs.append(fmap_r) 260 | fmap_gs.append(fmap_g) 261 | 262 | return y_d_rs, y_d_gs, fmap_rs, fmap_gs 263 | 264 | 265 | 266 | 267 | class SpeechDecoder(nn.Module): 268 | """ 269 | From EEG to Wav for Training 270 | """ 271 | 272 | def __init__(self, 273 | spec_channels, 274 | segment_size, 275 | inter_channels, 276 | hidden_channels, 277 | filter_channels, 278 | n_heads, 279 | n_layers, 280 | kernel_size, 281 | p_dropout, 282 | resblock, 283 | resblock_kernel_sizes, 284 | resblock_dilation_sizes, 285 | upsample_rates, 286 | upsample_initial_channel, 287 | upsample_kernel_sizes, 288 | **kwargs): 289 | 290 | super().__init__() 291 | self.spec_channels = spec_channels 292 | self.inter_channels = inter_channels 293 | self.hidden_channels = hidden_channels 294 | self.filter_channels = filter_channels 295 | self.n_heads = n_heads 296 | self.n_layers = n_layers 297 | self.kernel_size = kernel_size 298 | self.p_dropout = p_dropout 299 | self.resblock = resblock 300 | self.resblock_kernel_sizes = resblock_kernel_sizes 301 | self.resblock_dilation_sizes = resblock_dilation_sizes 302 | self.upsample_rates = upsample_rates 303 | self.upsample_initial_channel = upsample_initial_channel 304 | self.upsample_kernel_sizes = upsample_kernel_sizes 305 | self.segment_size = segment_size 306 | 307 | 308 | self.enc_proj = Connector( 309 | inter_channels, 310 | hidden_channels, 311 | filter_channels, 312 | n_heads, 313 | n_layers, 314 | kernel_size, 315 | p_dropout) 316 | self.dec = Generator(inter_channels, resblock, resblock_kernel_sizes, resblock_dilation_sizes, upsample_rates, upsample_initial_channel, upsample_kernel_sizes) 317 | self.enc_q = SpeechEncoder(spec_channels, inter_channels, hidden_channels, 5, 1, 16) 318 | self.flow = ResidualCouplingBlock(inter_channels, hidden_channels, 5, 1, 4) 319 | 320 | 321 | def forward(self, x, x_lengths, y, y_lengths): 322 | 323 | x, m_p, logs_p, x_mask = self.enc_proj(x, x_lengths) 324 | g=None 325 | z, m_q, logs_q, y_mask = self.enc_q(y, y_lengths, g=g) 326 | z_p = self.flow(z, y_mask, g=g) 327 | 328 | z_slice, ids_slice = commons.rand_slice_segments(z, y_lengths, self.segment_size) 329 | o = self.dec(z_slice, g=g) 330 | return o, None, None, ids_slice, x_mask, y_mask, (z, z_p, m_p, logs_p, m_q, logs_q) 331 | 332 | def infer(self, x, x_lengths, noise_scale=0.667, max_len=None): 333 | x, m_p, logs_p, x_mask = self.enc_proj(x, x_lengths) 334 | g = None 335 | 336 | 337 | y_lengths = x_lengths.clone() 338 | y_mask = torch.unsqueeze(commons.sequence_mask(y_lengths, None), 1).to(x_mask.dtype) 339 | 340 | 341 | m_p=m_p[:,:,:y_mask.size(2)] 342 | logs_p=logs_p[:,:,:y_mask.size(2)] 343 | 344 | z_p = m_p + torch.randn_like(m_p) * torch.exp(logs_p) * noise_scale 345 | z = self.flow(z_p, y_mask, g=g, reverse=True) 346 | o = self.dec((z * y_mask)[:,:,:max_len], g=g) 347 | return o, None, y_mask, (z, z_p, m_p, logs_p) 348 | 349 | -------------------------------------------------------------------------------- /modules.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import math 3 | import numpy as np 4 | import scipy 5 | import torch 6 | from torch import nn 7 | from torch.nn import functional as F 8 | 9 | from torch.nn import Conv1d, ConvTranspose1d, AvgPool1d, Conv2d 10 | from torch.nn.utils import weight_norm, remove_weight_norm 11 | 12 | import commons 13 | from commons import init_weights, get_padding 14 | from transforms import piecewise_rational_quadratic_transform 15 | 16 | 17 | LRELU_SLOPE = 0.1 18 | 19 | 20 | class LayerNorm(nn.Module): 21 | def __init__(self, channels, eps=1e-5): 22 | super().__init__() 23 | self.channels = channels 24 | self.eps = eps 25 | 26 | self.gamma = nn.Parameter(torch.ones(channels)) 27 | self.beta = nn.Parameter(torch.zeros(channels)) 28 | 29 | def forward(self, x): 30 | x = x.transpose(1, -1) 31 | x = F.layer_norm(x, (self.channels,), self.gamma, self.beta, self.eps) 32 | return x.transpose(1, -1) 33 | 34 | 35 | class ConvReluNorm(nn.Module): 36 | def __init__(self, in_channels, hidden_channels, out_channels, kernel_size, n_layers, p_dropout): 37 | super().__init__() 38 | self.in_channels = in_channels 39 | self.hidden_channels = hidden_channels 40 | self.out_channels = out_channels 41 | self.kernel_size = kernel_size 42 | self.n_layers = n_layers 43 | self.p_dropout = p_dropout 44 | assert n_layers > 1, "Number of layers should be larger than 0." 45 | 46 | self.conv_layers = nn.ModuleList() 47 | self.norm_layers = nn.ModuleList() 48 | self.conv_layers.append(nn.Conv1d(in_channels, hidden_channels, kernel_size, padding=kernel_size//2)) 49 | self.norm_layers.append(LayerNorm(hidden_channels)) 50 | self.relu_drop = nn.Sequential( 51 | nn.ReLU(), 52 | nn.Dropout(p_dropout)) 53 | for _ in range(n_layers-1): 54 | self.conv_layers.append(nn.Conv1d(hidden_channels, hidden_channels, kernel_size, padding=kernel_size//2)) 55 | self.norm_layers.append(LayerNorm(hidden_channels)) 56 | self.proj = nn.Conv1d(hidden_channels, out_channels, 1) 57 | self.proj.weight.data.zero_() 58 | self.proj.bias.data.zero_() 59 | 60 | def forward(self, x, x_mask): 61 | x_org = x 62 | for i in range(self.n_layers): 63 | x = self.conv_layers[i](x * x_mask) 64 | x = self.norm_layers[i](x) 65 | x = self.relu_drop(x) 66 | x = x_org + self.proj(x) 67 | return x * x_mask 68 | 69 | 70 | class DDSConv(nn.Module): 71 | """ 72 | Dialted and Depth-Separable Convolution 73 | """ 74 | def __init__(self, channels, kernel_size, n_layers, p_dropout=0.): 75 | super().__init__() 76 | self.channels = channels 77 | self.kernel_size = kernel_size 78 | self.n_layers = n_layers 79 | self.p_dropout = p_dropout 80 | 81 | self.drop = nn.Dropout(p_dropout) 82 | self.convs_sep = nn.ModuleList() 83 | self.convs_1x1 = nn.ModuleList() 84 | self.norms_1 = nn.ModuleList() 85 | self.norms_2 = nn.ModuleList() 86 | for i in range(n_layers): 87 | dilation = kernel_size ** i 88 | padding = (kernel_size * dilation - dilation) // 2 89 | self.convs_sep.append(nn.Conv1d(channels, channels, kernel_size, 90 | groups=channels, dilation=dilation, padding=padding 91 | )) 92 | self.convs_1x1.append(nn.Conv1d(channels, channels, 1)) 93 | self.norms_1.append(LayerNorm(channels)) 94 | self.norms_2.append(LayerNorm(channels)) 95 | 96 | def forward(self, x, x_mask, g=None): 97 | if g is not None: 98 | x = x + g 99 | for i in range(self.n_layers): 100 | y = self.convs_sep[i](x * x_mask) 101 | y = self.norms_1[i](y) 102 | y = F.gelu(y) 103 | y = self.convs_1x1[i](y) 104 | y = self.norms_2[i](y) 105 | y = F.gelu(y) 106 | y = self.drop(y) 107 | x = x + y 108 | return x * x_mask 109 | 110 | 111 | class WN(torch.nn.Module): 112 | def __init__(self, hidden_channels, kernel_size, dilation_rate, n_layers, gin_channels=0, p_dropout=0): 113 | super(WN, self).__init__() 114 | assert(kernel_size % 2 == 1) 115 | self.hidden_channels =hidden_channels 116 | self.kernel_size = kernel_size, 117 | self.dilation_rate = dilation_rate 118 | self.n_layers = n_layers 119 | self.gin_channels = gin_channels 120 | self.p_dropout = p_dropout 121 | 122 | self.in_layers = torch.nn.ModuleList() 123 | self.res_skip_layers = torch.nn.ModuleList() 124 | self.drop = nn.Dropout(p_dropout) 125 | 126 | if gin_channels != 0: 127 | cond_layer = torch.nn.Conv1d(gin_channels, 2*hidden_channels*n_layers, 1) 128 | self.cond_layer = torch.nn.utils.weight_norm(cond_layer, name='weight') 129 | 130 | for i in range(n_layers): 131 | dilation = dilation_rate ** i 132 | padding = int((kernel_size * dilation - dilation) / 2) 133 | in_layer = torch.nn.Conv1d(hidden_channels, 2*hidden_channels, kernel_size, 134 | dilation=dilation, padding=padding) 135 | in_layer = torch.nn.utils.weight_norm(in_layer, name='weight') 136 | self.in_layers.append(in_layer) 137 | 138 | # last one is not necessary 139 | if i < n_layers - 1: 140 | res_skip_channels = 2 * hidden_channels 141 | else: 142 | res_skip_channels = hidden_channels 143 | 144 | res_skip_layer = torch.nn.Conv1d(hidden_channels, res_skip_channels, 1) 145 | res_skip_layer = torch.nn.utils.weight_norm(res_skip_layer, name='weight') 146 | self.res_skip_layers.append(res_skip_layer) 147 | 148 | def forward(self, x, x_mask, g=None, **kwargs): 149 | output = torch.zeros_like(x) 150 | n_channels_tensor = torch.IntTensor([self.hidden_channels]) 151 | 152 | if g is not None: 153 | g = self.cond_layer(g) 154 | 155 | for i in range(self.n_layers): 156 | x_in = self.in_layers[i](x) 157 | if g is not None: 158 | cond_offset = i * 2 * self.hidden_channels 159 | g_l = g[:,cond_offset:cond_offset+2*self.hidden_channels,:] 160 | else: 161 | g_l = torch.zeros_like(x_in) 162 | 163 | acts = commons.fused_add_tanh_sigmoid_multiply( 164 | x_in, 165 | g_l, 166 | n_channels_tensor) 167 | acts = self.drop(acts) 168 | 169 | res_skip_acts = self.res_skip_layers[i](acts) 170 | if i < self.n_layers - 1: 171 | res_acts = res_skip_acts[:,:self.hidden_channels,:] 172 | x = (x + res_acts) * x_mask 173 | output = output + res_skip_acts[:,self.hidden_channels:,:] 174 | else: 175 | output = output + res_skip_acts 176 | return output * x_mask 177 | 178 | def remove_weight_norm(self): 179 | if self.gin_channels != 0: 180 | torch.nn.utils.remove_weight_norm(self.cond_layer) 181 | for l in self.in_layers: 182 | torch.nn.utils.remove_weight_norm(l) 183 | for l in self.res_skip_layers: 184 | torch.nn.utils.remove_weight_norm(l) 185 | 186 | 187 | class ResBlock1(torch.nn.Module): 188 | def __init__(self, channels, kernel_size=3, dilation=(1, 3, 5)): 189 | super(ResBlock1, self).__init__() 190 | self.convs1 = nn.ModuleList([ 191 | weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[0], 192 | padding=get_padding(kernel_size, dilation[0]))), 193 | weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[1], 194 | padding=get_padding(kernel_size, dilation[1]))), 195 | weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[2], 196 | padding=get_padding(kernel_size, dilation[2]))) 197 | ]) 198 | self.convs1.apply(init_weights) 199 | 200 | self.convs2 = nn.ModuleList([ 201 | weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=1, 202 | padding=get_padding(kernel_size, 1))), 203 | weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=1, 204 | padding=get_padding(kernel_size, 1))), 205 | weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=1, 206 | padding=get_padding(kernel_size, 1))) 207 | ]) 208 | self.convs2.apply(init_weights) 209 | 210 | def forward(self, x, x_mask=None): 211 | for c1, c2 in zip(self.convs1, self.convs2): 212 | xt = F.leaky_relu(x, LRELU_SLOPE) 213 | if x_mask is not None: 214 | xt = xt * x_mask 215 | xt = c1(xt) 216 | xt = F.leaky_relu(xt, LRELU_SLOPE) 217 | if x_mask is not None: 218 | xt = xt * x_mask 219 | xt = c2(xt) 220 | x = xt + x 221 | if x_mask is not None: 222 | x = x * x_mask 223 | return x 224 | 225 | def remove_weight_norm(self): 226 | for l in self.convs1: 227 | remove_weight_norm(l) 228 | for l in self.convs2: 229 | remove_weight_norm(l) 230 | 231 | 232 | class ResBlock2(torch.nn.Module): 233 | def __init__(self, channels, kernel_size=3, dilation=(1, 3)): 234 | super(ResBlock2, self).__init__() 235 | self.convs = nn.ModuleList([ 236 | weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[0], 237 | padding=get_padding(kernel_size, dilation[0]))), 238 | weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[1], 239 | padding=get_padding(kernel_size, dilation[1]))) 240 | ]) 241 | self.convs.apply(init_weights) 242 | 243 | def forward(self, x, x_mask=None): 244 | for c in self.convs: 245 | xt = F.leaky_relu(x, LRELU_SLOPE) 246 | if x_mask is not None: 247 | xt = xt * x_mask 248 | xt = c(xt) 249 | x = xt + x 250 | if x_mask is not None: 251 | x = x * x_mask 252 | return x 253 | 254 | def remove_weight_norm(self): 255 | for l in self.convs: 256 | remove_weight_norm(l) 257 | 258 | 259 | class Log(nn.Module): 260 | def forward(self, x, x_mask, reverse=False, **kwargs): 261 | if not reverse: 262 | y = torch.log(torch.clamp_min(x, 1e-5)) * x_mask 263 | logdet = torch.sum(-y, [1, 2]) 264 | return y, logdet 265 | else: 266 | x = torch.exp(x) * x_mask 267 | return x 268 | 269 | 270 | class Flip(nn.Module): 271 | def forward(self, x, *args, reverse=False, **kwargs): 272 | x = torch.flip(x, [1]) 273 | if not reverse: 274 | logdet = torch.zeros(x.size(0)).to(dtype=x.dtype, device=x.device) 275 | return x, logdet 276 | else: 277 | return x 278 | 279 | 280 | class ElementwiseAffine(nn.Module): 281 | def __init__(self, channels): 282 | super().__init__() 283 | self.channels = channels 284 | self.m = nn.Parameter(torch.zeros(channels,1)) 285 | self.logs = nn.Parameter(torch.zeros(channels,1)) 286 | 287 | def forward(self, x, x_mask, reverse=False, **kwargs): 288 | if not reverse: 289 | y = self.m + torch.exp(self.logs) * x 290 | y = y * x_mask 291 | logdet = torch.sum(self.logs * x_mask, [1,2]) 292 | return y, logdet 293 | else: 294 | x = (x - self.m) * torch.exp(-self.logs) * x_mask 295 | return x 296 | 297 | 298 | class ResidualCouplingLayer(nn.Module): 299 | def __init__(self, 300 | channels, 301 | hidden_channels, 302 | kernel_size, 303 | dilation_rate, 304 | n_layers, 305 | p_dropout=0, 306 | gin_channels=0, 307 | mean_only=False): 308 | assert channels % 2 == 0, "channels should be divisible by 2" 309 | super().__init__() 310 | self.channels = channels 311 | self.hidden_channels = hidden_channels 312 | self.kernel_size = kernel_size 313 | self.dilation_rate = dilation_rate 314 | self.n_layers = n_layers 315 | self.half_channels = channels // 2 316 | self.mean_only = mean_only 317 | 318 | self.pre = nn.Conv1d(self.half_channels, hidden_channels, 1) 319 | self.enc = WN(hidden_channels, kernel_size, dilation_rate, n_layers, p_dropout=p_dropout, gin_channels=gin_channels) 320 | self.post = nn.Conv1d(hidden_channels, self.half_channels * (2 - mean_only), 1) 321 | self.post.weight.data.zero_() 322 | self.post.bias.data.zero_() 323 | 324 | def forward(self, x, x_mask, g=None, reverse=False): 325 | x0, x1 = torch.split(x, [self.half_channels]*2, 1) 326 | h = self.pre(x0) * x_mask 327 | h = self.enc(h, x_mask, g=g) 328 | stats = self.post(h) * x_mask 329 | if not self.mean_only: 330 | m, logs = torch.split(stats, [self.half_channels]*2, 1) 331 | else: 332 | m = stats 333 | logs = torch.zeros_like(m) 334 | 335 | if not reverse: 336 | x1 = m + x1 * torch.exp(logs) * x_mask 337 | x = torch.cat([x0, x1], 1) 338 | logdet = torch.sum(logs, [1,2]) 339 | return x, logdet 340 | else: 341 | x1 = (x1 - m) * torch.exp(-logs) * x_mask 342 | x = torch.cat([x0, x1], 1) 343 | return x 344 | 345 | 346 | class ConvFlow(nn.Module): 347 | def __init__(self, in_channels, filter_channels, kernel_size, n_layers, num_bins=10, tail_bound=5.0): 348 | super().__init__() 349 | self.in_channels = in_channels 350 | self.filter_channels = filter_channels 351 | self.kernel_size = kernel_size 352 | self.n_layers = n_layers 353 | self.num_bins = num_bins 354 | self.tail_bound = tail_bound 355 | self.half_channels = in_channels // 2 356 | 357 | self.pre = nn.Conv1d(self.half_channels, filter_channels, 1) 358 | self.convs = DDSConv(filter_channels, kernel_size, n_layers, p_dropout=0.) 359 | self.proj = nn.Conv1d(filter_channels, self.half_channels * (num_bins * 3 - 1), 1) 360 | self.proj.weight.data.zero_() 361 | self.proj.bias.data.zero_() 362 | 363 | def forward(self, x, x_mask, g=None, reverse=False): 364 | x0, x1 = torch.split(x, [self.half_channels]*2, 1) 365 | h = self.pre(x0) 366 | h = self.convs(h, x_mask, g=g) 367 | h = self.proj(h) * x_mask 368 | 369 | b, c, t = x0.shape 370 | h = h.reshape(b, c, -1, t).permute(0, 1, 3, 2) # [b, cx?, t] -> [b, c, t, ?] 371 | 372 | unnormalized_widths = h[..., :self.num_bins] / math.sqrt(self.filter_channels) 373 | unnormalized_heights = h[..., self.num_bins:2*self.num_bins] / math.sqrt(self.filter_channels) 374 | unnormalized_derivatives = h[..., 2 * self.num_bins:] 375 | 376 | x1, logabsdet = piecewise_rational_quadratic_transform(x1, 377 | unnormalized_widths, 378 | unnormalized_heights, 379 | unnormalized_derivatives, 380 | inverse=reverse, 381 | tails='linear', 382 | tail_bound=self.tail_bound 383 | ) 384 | 385 | x = torch.cat([x0, x1], 1) * x_mask 386 | logdet = torch.sum(logabsdet * x_mask, [1,2]) 387 | if not reverse: 388 | return x, logdet 389 | else: 390 | return x 391 | -------------------------------------------------------------------------------- /train.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | import argparse 4 | import itertools 5 | import math 6 | import torch 7 | from torch.nn import functional as F 8 | from torch.utils.data import DataLoader 9 | from torch.utils.tensorboard import SummaryWriter 10 | import torch.distributed as dist 11 | from torch.cuda.amp import autocast, GradScaler 12 | from torch.nn.parallel import DistributedDataParallel as DDP 13 | from scipy.io.wavfile import write 14 | 15 | 16 | import commons 17 | import utils 18 | from data_utils import EEGAudioLoader, EEGAudioCollate 19 | from models import SpeechDecoder, MultiPeriodDiscriminator 20 | from losses import ( 21 | generator_loss, 22 | discriminator_loss, 23 | feature_loss, 24 | kl_loss, 25 | eeg_loss 26 | ) 27 | from mel_processing import mel_spectrogram_torch, spec_to_mel_torch 28 | from EEGModule import EEGModule 29 | 30 | import wandb 31 | 32 | torch.backends.cudnn.benchmark = True 33 | global_step = 0 34 | 35 | 36 | import warnings 37 | warnings.filterwarnings('ignore') 38 | 39 | 40 | 41 | def main(): 42 | """Assume Single Node Multi GPUs Training Only""" 43 | assert torch.cuda.is_available(), "CPU training is not allowed." 44 | 45 | n_gpus = torch.cuda.device_count() 46 | hps = utils.get_hparams() 47 | 48 | os.environ['MASTER_ADDR'] = 'localhost' 49 | os.environ['MASTER_PORT'] = '7000' 50 | 51 | hps = utils.get_hparams() 52 | run(rank=0, n_gpus=1, hps=hps) # using only single GPU 53 | 54 | 55 | def run(rank, n_gpus, hps): 56 | global global_step 57 | 58 | if hps.wandb: 59 | wandb.login() 60 | wandb.init(project='fesde', name=hps.run_name) 61 | 62 | if rank == 0: 63 | logger = utils.get_logger(hps.model_dir) 64 | logger.info(hps) 65 | utils.check_git_hash(hps.model_dir) 66 | writer = SummaryWriter(log_dir=hps.model_dir) 67 | writer_eval = SummaryWriter(log_dir=os.path.join(hps.model_dir, "eval")) 68 | 69 | dist.init_process_group(backend='nccl', init_method='env://', world_size=n_gpus, rank=rank) 70 | torch.manual_seed(hps.train.seed) 71 | torch.cuda.set_device(rank) 72 | 73 | train_dataset = EEGAudioLoader(hps.data.training_files, hps.data) 74 | collate_fn = EEGAudioCollate() 75 | train_loader = DataLoader(train_dataset, num_workers=8, shuffle=True, pin_memory=True, 76 | collate_fn=collate_fn, batch_size=hps.train.batch_size) 77 | if rank == 0: 78 | eval_loader_both = DataLoader(EEGAudioLoader(hps.data.validation_files_both, hps.data), num_workers=8, shuffle=False, 79 | batch_size=hps.train.batch_size, pin_memory=True, 80 | drop_last=False, collate_fn=collate_fn) 81 | eval_loader_audio = DataLoader(EEGAudioLoader(hps.data.validation_files_audio, hps.data), num_workers=8, shuffle=False, 82 | batch_size=hps.train.batch_size, pin_memory=True, 83 | drop_last=False, collate_fn=collate_fn) 84 | eval_loader_subject = DataLoader(EEGAudioLoader(hps.data.validation_files_subject, hps.data), num_workers=8, shuffle=False, 85 | batch_size=hps.train.batch_size, pin_memory=True, 86 | drop_last=False, collate_fn=collate_fn) 87 | 88 | eval_loaders = [eval_loader_both, eval_loader_audio, eval_loader_subject] 89 | 90 | 91 | eeg_module = EEGModule( 92 | n_layers_cnn=hps.model.eeg_module.n_layers_cnn, 93 | use_s4=hps.model.eeg_module.use_s4, 94 | n_layers_s4=hps.model.eeg_module.n_layers_s4, 95 | embedding_size=hps.model.inter_channels, 96 | is_mask=False, 97 | in_channels=hps.model.eeg_module.in_channels 98 | ).cuda(rank) 99 | 100 | net_g = SpeechDecoder( 101 | hps.data.filter_length // 2 + 1, 102 | hps.train.segment_size // hps.data.hop_length, 103 | **hps.model).cuda(rank) 104 | net_d = MultiPeriodDiscriminator(hps.model.use_spectral_norm).cuda(rank) 105 | 106 | optim_g = torch.optim.AdamW( 107 | net_g.parameters(), 108 | hps.train.learning_rate, 109 | betas=hps.train.betas, 110 | eps=hps.train.eps) 111 | optim_eeg = torch.optim.AdamW( 112 | eeg_module.parameters(), 113 | hps.train.learning_rate, 114 | betas=hps.train.betas, 115 | eps=hps.train.eps) 116 | optim_d = torch.optim.AdamW( 117 | net_d.parameters(), 118 | hps.train.learning_rate, 119 | betas=hps.train.betas, 120 | eps=hps.train.eps) 121 | net_g = DDP(net_g, device_ids=[rank], find_unused_parameters=True) 122 | net_d = DDP(net_d, device_ids=[rank], find_unused_parameters=True) 123 | eeg_module = DDP(eeg_module, device_ids=[rank], find_unused_parameters=True) 124 | 125 | if hps.train.pretrained_audio: 126 | print('loading pretrained audio generator from ' + hps.train.pretrained_audio) 127 | pretrained_audio_ckpt = torch.load(hps.train.pretrained_audio, map_location='cpu') 128 | pretrained_stated_dict = pretrained_audio_ckpt['model'] 129 | new_state_dict = {} 130 | for k, v in net_g.module.state_dict().items(): 131 | if k.split('.')[0] != 'enc_proj': 132 | new_state_dict[k] = pretrained_stated_dict[k] 133 | else: 134 | new_state_dict[k] = v 135 | net_g.module.load_state_dict(new_state_dict) 136 | 137 | print('finished loading pretrained audio generator') 138 | 139 | 140 | 141 | if hps.train.freeze_modules: 142 | if "dec" in hps.train.freeze_modules: 143 | for param in net_g.module.dec.parameters(): 144 | param.requires_grad = False 145 | if "enc_q" in hps.train.freeze_modules: 146 | for param in net_g.module.enc_q.parameters(): 147 | param.requires_grad = False 148 | if "dp" in hps.train.freeze_modules: 149 | for param in net_g.module.dp.parameters(): 150 | param.requires_grad = False 151 | if "flow" in hps.train.freeze_modules: 152 | for param in net_g.module.flow.parameters(): 153 | param.requires_grad = False 154 | if "eeg" in hps.train.freeze_modules: 155 | for param in eeg_module.module.parameters(): 156 | param.requires_grad = False 157 | print('freezing eeg module') 158 | 159 | if hps.train.pretrained_eeg: 160 | print('loading pretrained eeg module', hps.train.pretrained_eeg) 161 | _ = utils.load_checkpoint(hps.train.pretrained_eeg, eeg_module, None) 162 | if "eeg" in hps.train.freeze_modules: 163 | for param in eeg_module.module.parameters(): 164 | param.requires_grad = False 165 | print('freezing eeg module') 166 | 167 | 168 | epoch_str = 1 169 | global_step = 0 170 | 171 | if True: # continue from past run 172 | try: 173 | _, _, _, epoch_str = utils.load_checkpoint(utils.latest_checkpoint_path(hps.model_dir, "G_*.pth"), net_g, optim_g) 174 | _, _, _, epoch_str = utils.load_checkpoint(utils.latest_checkpoint_path(hps.model_dir, "D_*.pth"), net_d, optim_d) 175 | _, _, _, epoch_str = utils.load_checkpoint(utils.latest_checkpoint_path(hps.model_dir, "E_*.pth"), eeg_module, optim_eeg) 176 | global_step = (epoch_str - 1) * len(train_loader) 177 | except: 178 | pass 179 | 180 | 181 | scheduler_g = torch.optim.lr_scheduler.ExponentialLR(optim_g, gamma=hps.train.lr_decay, last_epoch=epoch_str-2) 182 | scheduler_d = torch.optim.lr_scheduler.ExponentialLR(optim_d, gamma=hps.train.lr_decay, last_epoch=epoch_str-2) 183 | scheduler_eeg = torch.optim.lr_scheduler.ExponentialLR(optim_eeg, gamma=hps.train.lr_decay, last_epoch=epoch_str-2) 184 | 185 | scaler = GradScaler(enabled=False) 186 | 187 | for epoch in range(epoch_str, hps.train.epochs + 1): 188 | if rank==0: 189 | train_and_evaluate(rank, epoch, hps, [net_g, net_d, eeg_module], [optim_g, optim_d, optim_eeg], [scheduler_g, scheduler_d, scheduler_eeg], scaler, [train_loader, eval_loaders], logger, [writer, writer_eval]) 190 | else: 191 | train_and_evaluate(rank, epoch, hps, [net_g, net_d, eeg_module], [optim_g, optim_d, optim_eeg], [scheduler_g, scheduler_d, scheduler_eeg], scaler, [train_loader, None], None, None) 192 | scheduler_g.step() 193 | scheduler_d.step() 194 | scheduler_eeg.step() 195 | 196 | 197 | def train_and_evaluate(rank, epoch, hps, nets, optims, schedulers, scaler, loaders, logger, writers): 198 | net_g, net_d, eeg_module = nets 199 | optim_g, optim_d, optim_eeg = optims 200 | scheduler_g, scheduler_d, scheduler_eeg = schedulers 201 | train_loader = loaders[0] 202 | if writers is not None: 203 | writer, writer_eval = writers 204 | 205 | global global_step 206 | 207 | net_g.train() 208 | net_d.train() 209 | eeg_module.train() 210 | 211 | 212 | 213 | for batch_idx, (x, x_lengths, spec, spec_lengths, y, y_lengths) in enumerate(train_loader): 214 | x, x_lengths = x.cuda(rank, non_blocking=True), x_lengths.cuda(rank, non_blocking=True) 215 | spec, spec_lengths = spec.cuda(rank, non_blocking=True), spec_lengths.cuda(rank, non_blocking=True) 216 | y, y_lengths = y.cuda(rank, non_blocking=True), y_lengths.cuda(rank, non_blocking=True) 217 | 218 | with autocast(enabled=False): 219 | x, x_mask_output, mid_output, eeg_decoder_out = eeg_module(x) 220 | mid_output_lengths = x_lengths.clone() * mid_output.size(2) / x.size(2) 221 | mid_output_lengths = mid_output_lengths.long() 222 | 223 | y_hat, _, _, ids_slice, x_mask, z_mask,\ 224 | (z, z_p, m_p, logs_p, m_q, logs_q) = net_g(mid_output.detach(), mid_output_lengths, spec, spec_lengths) 225 | 226 | mel = spec_to_mel_torch( 227 | spec, 228 | hps.data.filter_length, 229 | hps.data.n_mel_channels, 230 | hps.data.sampling_rate, 231 | hps.data.mel_fmin, 232 | hps.data.mel_fmax) 233 | y_mel = commons.slice_segments(mel, ids_slice, hps.train.segment_size // hps.data.hop_length) 234 | y_hat_mel = mel_spectrogram_torch( 235 | y_hat.squeeze(1), 236 | hps.data.filter_length, 237 | hps.data.n_mel_channels, 238 | hps.data.sampling_rate, 239 | hps.data.hop_length, 240 | hps.data.win_length, 241 | hps.data.mel_fmin, 242 | hps.data.mel_fmax 243 | ) 244 | 245 | y = commons.slice_segments(y, ids_slice * hps.data.hop_length, hps.train.segment_size) # slice 246 | 247 | # Discriminator 248 | y_d_hat_r, y_d_hat_g, _, _ = net_d(y, y_hat.detach()) 249 | with autocast(enabled=False): 250 | loss_disc, losses_disc_r, losses_disc_g = discriminator_loss(y_d_hat_r, y_d_hat_g) 251 | loss_disc_all = loss_disc 252 | optim_d.zero_grad() 253 | scaler.scale(loss_disc_all).backward() 254 | scaler.unscale_(optim_d) 255 | grad_norm_d = commons.clip_grad_value_(net_d.parameters(), None) 256 | scaler.step(optim_d) 257 | 258 | with autocast(enabled=False): 259 | # Generator 260 | y_d_hat_r, y_d_hat_g, fmap_r, fmap_g = net_d(y, y_hat) 261 | with autocast(enabled=False): 262 | loss_mel = F.l1_loss(y_mel, y_hat_mel) * hps.train.c_mel 263 | loss_kl = kl_loss(z_p, logs_q, m_p, logs_p, z_mask) * hps.train.c_kl 264 | 265 | loss_fm = feature_loss(fmap_r, fmap_g) 266 | loss_gen, losses_gen = generator_loss(y_d_hat_g) 267 | loss_gen_all = loss_gen + loss_fm + loss_mel + loss_kl 268 | 269 | loss_eeg_module = eeg_loss(x, eeg_decoder_out, x_lengths) 270 | 271 | optim_g.zero_grad() 272 | scaler.scale(loss_gen_all).backward() 273 | scaler.unscale_(optim_g) 274 | grad_norm_g = commons.clip_grad_value_(net_g.parameters(), None) 275 | scaler.step(optim_g) 276 | 277 | 278 | optim_eeg.zero_grad() 279 | if "eeg" not in hps.train.freeze_modules: 280 | scaler.scale(loss_eeg_module).backward() 281 | scaler.unscale_(optim_eeg) 282 | grad_norm_eeg_enc = commons.clip_grad_value_(eeg_module.parameters(), None) 283 | scaler.step(optim_eeg) 284 | 285 | scaler.update() 286 | 287 | if rank==0: 288 | if global_step % hps.train.log_interval == 0: 289 | lr = optim_g.param_groups[0]['lr'] 290 | losses = [loss_disc, loss_gen, loss_fm, loss_mel, loss_kl] 291 | logger.info('Train Epoch: {} [{:.0f}%]'.format( 292 | epoch, 293 | 100. * batch_idx / len(train_loader))) 294 | logger.info([x.item() for x in losses] + [global_step, lr]) 295 | 296 | scalar_dict = {"loss/g/total": loss_gen_all, "loss/d/total": loss_disc_all, "learning_rate": lr, "grad_norm_d": grad_norm_d, "grad_norm_g": grad_norm_g, "loss/eeg/total":loss_eeg_module 297 | } 298 | scalar_dict.update({"loss/g/fm": loss_fm, "loss/g/mel": loss_mel, "loss/g/kl": loss_kl}) 299 | 300 | scalar_dict.update({"loss/g/{}".format(i): v for i, v in enumerate(losses_gen)}) 301 | scalar_dict.update({"loss/d_r/{}".format(i): v for i, v in enumerate(losses_disc_r)}) 302 | scalar_dict.update({"loss/d_g/{}".format(i): v for i, v in enumerate(losses_disc_g)}) 303 | image_dict = { 304 | "slice/mel_org": utils.plot_spectrogram_to_numpy(y_mel[0].data.cpu().numpy()), 305 | "slice/mel_gen": utils.plot_spectrogram_to_numpy(y_hat_mel[0].data.cpu().numpy()), 306 | "all/mel": utils.plot_spectrogram_to_numpy(mel[0].data.cpu().numpy()), 307 | } 308 | utils.summarize( 309 | writer=writer, 310 | global_step=global_step, 311 | images=image_dict, 312 | scalars=scalar_dict) 313 | 314 | if hps.wandb: 315 | wandb.log(scalar_dict) 316 | image_dict_wandb = {k:wandb.Image(v, caption=k) for k,v in image_dict.items()} 317 | wandb.log(image_dict_wandb) 318 | 319 | if global_step % hps.train.eval_interval == 0: 320 | for val_type, eval_loader in list(zip(['both', 'audio', 'subject'], loaders[1])): 321 | evaluate(hps, [net_g, eeg_module], eval_loader, writer_eval, val_type) 322 | utils.save_checkpoint(net_g, optim_g, hps.train.learning_rate, epoch, os.path.join(hps.model_dir, "G_{}.pth".format(global_step))) 323 | utils.save_checkpoint(net_d, optim_d, hps.train.learning_rate, epoch, os.path.join(hps.model_dir, "D_{}.pth".format(global_step))) 324 | utils.save_checkpoint(eeg_module, optim_eeg, hps.train.learning_rate, epoch, os.path.join(hps.model_dir, "E_{}.pth".format(global_step))) 325 | 326 | global_step += 1 327 | 328 | if rank == 0: 329 | logger.info('====> Epoch: {}'.format(epoch)) 330 | 331 | 332 | def evaluate(hps, net_g, eval_loader, writer_eval, val_type): 333 | generator, eeg_module = net_g 334 | generator.eval() 335 | eeg_module.eval() 336 | 337 | with torch.no_grad(): 338 | for batch_idx, (x, x_lengths, spec, spec_lengths, y, y_lengths) in enumerate(eval_loader): 339 | x, x_lengths = x.cuda(0), x_lengths.cuda(0) 340 | spec, spec_lengths = spec.cuda(0), spec_lengths.cuda(0) 341 | y, y_lengths = y.cuda(0), y_lengths.cuda(0) 342 | 343 | # remove else 344 | x = x[:1] 345 | x_lengths = x_lengths[:1] 346 | spec = spec[:1] 347 | spec_lengths = spec_lengths[:1] 348 | y = y[:1] 349 | y_lengths = y_lengths[:1] 350 | break 351 | 352 | x, x_mask_output, mid_output, eeg_decoder_out = eeg_module(x) 353 | mid_output_lengths = x_lengths.clone() * mid_output.size(2) / x.size(2) 354 | mid_output_lengths = mid_output_lengths.long() 355 | 356 | y_hat, _, mask, *_ = generator.module.infer(mid_output, mid_output_lengths, max_len=1000) 357 | 358 | y_hat_lengths = mask.sum([1,2]).long() * hps.data.hop_length 359 | 360 | mel = spec_to_mel_torch( 361 | spec, 362 | hps.data.filter_length, 363 | hps.data.n_mel_channels, 364 | hps.data.sampling_rate, 365 | hps.data.mel_fmin, 366 | hps.data.mel_fmax) 367 | y_hat_mel = mel_spectrogram_torch( 368 | y_hat.squeeze(1).float(), 369 | hps.data.filter_length, 370 | hps.data.n_mel_channels, 371 | hps.data.sampling_rate, 372 | hps.data.hop_length, 373 | hps.data.win_length, 374 | hps.data.mel_fmin, 375 | hps.data.mel_fmax 376 | ) 377 | image_dict = { 378 | f"{val_type}/gen/mel": utils.plot_spectrogram_to_numpy(y_hat_mel[0].cpu().numpy()) 379 | } 380 | audio_dict = { 381 | f"{val_type}/audio": y_hat[0,:,:y_hat_lengths[0]].cpu().float().numpy() 382 | } 383 | 384 | if global_step == 0: 385 | image_dict.update({f"{val_type}/gt/mel": utils.plot_spectrogram_to_numpy(mel[0].cpu().numpy())}) 386 | audio_dict.update({f"{val_type}/gt/audio": y[0,:,:y_lengths[0]].cpu().float().numpy()}) 387 | 388 | 389 | utils.summarize( 390 | writer=writer_eval, 391 | global_step=global_step, 392 | images=image_dict, 393 | audios=audio_dict, 394 | audio_sampling_rate=hps.data.sampling_rate 395 | ) 396 | 397 | if hps.wandb: 398 | image_dict_wandb = {k:wandb.Image(v, caption=k) for k,v in image_dict.items()} 399 | audio_dict_wandb = {k:wandb.Audio(v[0], caption=k, sample_rate=hps.data.sampling_rate) for k,v in audio_dict.items()} 400 | wandb.log(image_dict_wandb) 401 | wandb.log(audio_dict_wandb) 402 | 403 | 404 | generator.train() 405 | eeg_module.train() 406 | 407 | 408 | if __name__ == "__main__": 409 | main() 410 | -------------------------------------------------------------------------------- /filelists/n400/filelist_test_unseen_audio_text.txt: -------------------------------------------------------------------------------- 1 | sub-01/sub-01-_-(LD5_Eliminated)NPC_live||The woods are where bears live. 2 | sub-01/sub-01-_-NPC_aisle||The bride walked down the aisle. 3 | sub-01/sub-01-_-NPC_animals||The sign said do not feed the animals. 4 | sub-01/sub-01-_-NPC_apple||His favorite fruit was a red crisp apple. 5 | sub-01/sub-01-_-NPC_arm||I have two legs and two arms. 6 | sub-01/sub-01-_-NPC_bag||He puts his lunch in a paper bag. 7 | sub-01/sub-01-_-NPC_bank||My mom puts money in the bank. 8 | sub-01/sub-01-_-NPC_button||On the remote he clicked the button. 9 | sub-01/sub-01-_-NPC_cart||At the store I pushed the cart. 10 | sub-01/sub-01-_-NPC_chores||My mom makes me do household chores. 11 | sub-01/sub-01-_-NPC_clothes||My parents fold my clothes. 12 | sub-01/sub-01-_-NPC_cribs||Babies sleep in cribs. 13 | sub-01/sub-01-_-NPC_cup||I drink from a cup. 14 | sub-01/sub-01-_-NPC_dream||She woke up after a bad dream. 15 | sub-01/sub-01-_-NPC_frog||Ribbit goes the frog. 16 | sub-01/sub-01-_-NPC_fur||Cats have soft fur. 17 | sub-01/sub-01-_-NPC_mouse||The cat chased the mouse. 18 | sub-01/sub-01-_-NPC_one||First place is number one. 19 | sub-01/sub-01-_-NPC_pink||Pigs are colored pink. 20 | sub-01/sub-01-_-NPC_snake||Hiss goes the snake. 21 | sub-01/sub-01-_-NPI_bear(pie)||The woods had a big brown scary pie. 22 | sub-01/sub-01-_-NPI_bite(north)||He had an itchy bug north. 23 | sub-01/sub-01-_-NPI_cages(woman)||Animals are locked in woman. 24 | sub-01/sub-01-_-NPI_card(fly)||I signed a birthday fly. 25 | sub-01/sub-01-_-NPI_cliff(sheep)||He stood at the edge of a sheep. 26 | sub-01/sub-01-_-NPI_cry(fork)||When I am sad I fork. 27 | sub-01/sub-01-_-NPI_dare(brick)||Lets play truth or brick. 28 | sub-01/sub-01-_-NPI_dice(art)||In games I roll the art. 29 | sub-01/sub-01-_-NPI_down(tie)||She told me to calm tie. 30 | sub-01/sub-01-_-NPI_fall(duck)||After summer is duck. 31 | sub-01/sub-01-_-NPI_fork(cleats)||He eats steak with a knife and cleats. 32 | sub-01/sub-01-_-NPI_gel(sign)||He styled his hair with sign. 33 | sub-01/sub-01-_-NPI_night(inch)||It is dark at inch. 34 | sub-01/sub-01-_-NPI_nine(cat)||The number before ten is cat. 35 | sub-01/sub-01-_-NPI_one(cold)||First place is number cold. 36 | sub-01/sub-01-_-NPI_out(ant)||Three strikes and you are ant. 37 | sub-01/sub-01-_-NPI_pink(door)||Pigs are colored door. 38 | sub-01/sub-01-_-NPI_plank(shark)||The pirate said walk the shark. 39 | sub-01/sub-01-_-NPI_popped(square)||The water balloon was so full it square. 40 | sub-01/sub-01-_-NPI_round(kiss)||The ball is not square but kiss. 41 | sub-02/sub-02-_-(LD5_Eliminated)NPC_live||The woods are where bears live. 42 | sub-02/sub-02-_-NPC_aisle||The bride walked down the aisle. 43 | sub-02/sub-02-_-NPC_animals||The sign said do not feed the animals. 44 | sub-02/sub-02-_-NPC_apple||His favorite fruit was a red crisp apple. 45 | sub-02/sub-02-_-NPC_arm||I have two legs and two arms. 46 | sub-02/sub-02-_-NPC_bag||He puts his lunch in a paper bag. 47 | sub-02/sub-02-_-NPC_bank||My mom puts money in the bank. 48 | sub-02/sub-02-_-NPC_button||On the remote he clicked the button. 49 | sub-02/sub-02-_-NPC_cart||At the store I pushed the cart. 50 | sub-02/sub-02-_-NPC_chores||My mom makes me do household chores. 51 | sub-02/sub-02-_-NPC_clothes||My parents fold my clothes. 52 | sub-02/sub-02-_-NPC_cribs||Babies sleep in cribs. 53 | sub-02/sub-02-_-NPC_cup||I drink from a cup. 54 | sub-02/sub-02-_-NPC_dream||She woke up after a bad dream. 55 | sub-02/sub-02-_-NPC_frog||Ribbit goes the frog. 56 | sub-02/sub-02-_-NPC_fur||Cats have soft fur. 57 | sub-02/sub-02-_-NPC_mouse||The cat chased the mouse. 58 | sub-02/sub-02-_-NPC_one||First place is number one. 59 | sub-02/sub-02-_-NPC_pink||Pigs are colored pink. 60 | sub-02/sub-02-_-NPC_snake||Hiss goes the snake. 61 | sub-02/sub-02-_-NPI_bear(pie)||The woods had a big brown scary pie. 62 | sub-02/sub-02-_-NPI_bite(north)||He had an itchy bug north. 63 | sub-02/sub-02-_-NPI_cages(woman)||Animals are locked in woman. 64 | sub-02/sub-02-_-NPI_card(fly)||I signed a birthday fly. 65 | sub-02/sub-02-_-NPI_cliff(sheep)||He stood at the edge of a sheep. 66 | sub-02/sub-02-_-NPI_cry(fork)||When I am sad I fork. 67 | sub-02/sub-02-_-NPI_dare(brick)||Lets play truth or brick. 68 | sub-02/sub-02-_-NPI_dice(art)||In games I roll the art. 69 | sub-02/sub-02-_-NPI_down(tie)||She told me to calm tie. 70 | sub-02/sub-02-_-NPI_fall(duck)||After summer is duck. 71 | sub-02/sub-02-_-NPI_fork(cleats)||He eats steak with a knife and cleats. 72 | sub-02/sub-02-_-NPI_gel(sign)||He styled his hair with sign. 73 | sub-02/sub-02-_-NPI_night(inch)||It is dark at inch. 74 | sub-02/sub-02-_-NPI_nine(cat)||The number before ten is cat. 75 | sub-02/sub-02-_-NPI_one(cold)||First place is number cold. 76 | sub-02/sub-02-_-NPI_out(ant)||Three strikes and you are ant. 77 | sub-02/sub-02-_-NPI_pink(door)||Pigs are colored door. 78 | sub-02/sub-02-_-NPI_plank(shark)||The pirate said walk the shark. 79 | sub-02/sub-02-_-NPI_popped(square)||The water balloon was so full it square. 80 | sub-02/sub-02-_-NPI_round(kiss)||The ball is not square but kiss. 81 | sub-03/sub-03-_-(LD5_Eliminated)NPC_live||The woods are where bears live. 82 | sub-03/sub-03-_-NPC_aisle||The bride walked down the aisle. 83 | sub-03/sub-03-_-NPC_animals||The sign said do not feed the animals. 84 | sub-03/sub-03-_-NPC_apple||His favorite fruit was a red crisp apple. 85 | sub-03/sub-03-_-NPC_arm||I have two legs and two arms. 86 | sub-03/sub-03-_-NPC_bag||He puts his lunch in a paper bag. 87 | sub-03/sub-03-_-NPC_bank||My mom puts money in the bank. 88 | sub-03/sub-03-_-NPC_button||On the remote he clicked the button. 89 | sub-03/sub-03-_-NPC_cart||At the store I pushed the cart. 90 | sub-03/sub-03-_-NPC_chores||My mom makes me do household chores. 91 | sub-03/sub-03-_-NPC_clothes||My parents fold my clothes. 92 | sub-03/sub-03-_-NPC_cribs||Babies sleep in cribs. 93 | sub-03/sub-03-_-NPC_cup||I drink from a cup. 94 | sub-03/sub-03-_-NPC_dream||She woke up after a bad dream. 95 | sub-03/sub-03-_-NPC_frog||Ribbit goes the frog. 96 | sub-03/sub-03-_-NPC_fur||Cats have soft fur. 97 | sub-03/sub-03-_-NPC_mouse||The cat chased the mouse. 98 | sub-03/sub-03-_-NPC_one||First place is number one. 99 | sub-03/sub-03-_-NPC_pink||Pigs are colored pink. 100 | sub-03/sub-03-_-NPC_snake||Hiss goes the snake. 101 | sub-03/sub-03-_-NPI_bear(pie)||The woods had a big brown scary pie. 102 | sub-03/sub-03-_-NPI_bite(north)||He had an itchy bug north. 103 | sub-03/sub-03-_-NPI_cages(woman)||Animals are locked in woman. 104 | sub-03/sub-03-_-NPI_card(fly)||I signed a birthday fly. 105 | sub-03/sub-03-_-NPI_cliff(sheep)||He stood at the edge of a sheep. 106 | sub-03/sub-03-_-NPI_cry(fork)||When I am sad I fork. 107 | sub-03/sub-03-_-NPI_dare(brick)||Lets play truth or brick. 108 | sub-03/sub-03-_-NPI_dice(art)||In games I roll the art. 109 | sub-03/sub-03-_-NPI_down(tie)||She told me to calm tie. 110 | sub-03/sub-03-_-NPI_fall(duck)||After summer is duck. 111 | sub-03/sub-03-_-NPI_fork(cleats)||He eats steak with a knife and cleats. 112 | sub-03/sub-03-_-NPI_gel(sign)||He styled his hair with sign. 113 | sub-03/sub-03-_-NPI_night(inch)||It is dark at inch. 114 | sub-03/sub-03-_-NPI_nine(cat)||The number before ten is cat. 115 | sub-03/sub-03-_-NPI_one(cold)||First place is number cold. 116 | sub-03/sub-03-_-NPI_out(ant)||Three strikes and you are ant. 117 | sub-03/sub-03-_-NPI_pink(door)||Pigs are colored door. 118 | sub-03/sub-03-_-NPI_plank(shark)||The pirate said walk the shark. 119 | sub-03/sub-03-_-NPI_popped(square)||The water balloon was so full it square. 120 | sub-03/sub-03-_-NPI_round(kiss)||The ball is not square but kiss. 121 | sub-04/sub-04-_-(LD5_Eliminated)NPC_live||The woods are where bears live. 122 | sub-04/sub-04-_-NPC_aisle||The bride walked down the aisle. 123 | sub-04/sub-04-_-NPC_animals||The sign said do not feed the animals. 124 | sub-04/sub-04-_-NPC_apple||His favorite fruit was a red crisp apple. 125 | sub-04/sub-04-_-NPC_arm||I have two legs and two arms. 126 | sub-04/sub-04-_-NPC_bag||He puts his lunch in a paper bag. 127 | sub-04/sub-04-_-NPC_bank||My mom puts money in the bank. 128 | sub-04/sub-04-_-NPC_button||On the remote he clicked the button. 129 | sub-04/sub-04-_-NPC_cart||At the store I pushed the cart. 130 | sub-04/sub-04-_-NPC_chores||My mom makes me do household chores. 131 | sub-04/sub-04-_-NPC_clothes||My parents fold my clothes. 132 | sub-04/sub-04-_-NPC_cribs||Babies sleep in cribs. 133 | sub-04/sub-04-_-NPC_cup||I drink from a cup. 134 | sub-04/sub-04-_-NPC_dream||She woke up after a bad dream. 135 | sub-04/sub-04-_-NPC_frog||Ribbit goes the frog. 136 | sub-04/sub-04-_-NPC_fur||Cats have soft fur. 137 | sub-04/sub-04-_-NPC_mouse||The cat chased the mouse. 138 | sub-04/sub-04-_-NPC_one||First place is number one. 139 | sub-04/sub-04-_-NPC_pink||Pigs are colored pink. 140 | sub-04/sub-04-_-NPC_snake||Hiss goes the snake. 141 | sub-04/sub-04-_-NPI_bear(pie)||The woods had a big brown scary pie. 142 | sub-04/sub-04-_-NPI_bite(north)||He had an itchy bug north. 143 | sub-04/sub-04-_-NPI_cages(woman)||Animals are locked in woman. 144 | sub-04/sub-04-_-NPI_card(fly)||I signed a birthday fly. 145 | sub-04/sub-04-_-NPI_cliff(sheep)||He stood at the edge of a sheep. 146 | sub-04/sub-04-_-NPI_cry(fork)||When I am sad I fork. 147 | sub-04/sub-04-_-NPI_dare(brick)||Lets play truth or brick. 148 | sub-04/sub-04-_-NPI_dice(art)||In games I roll the art. 149 | sub-04/sub-04-_-NPI_down(tie)||She told me to calm tie. 150 | sub-04/sub-04-_-NPI_fall(duck)||After summer is duck. 151 | sub-04/sub-04-_-NPI_fork(cleats)||He eats steak with a knife and cleats. 152 | sub-04/sub-04-_-NPI_gel(sign)||He styled his hair with sign. 153 | sub-04/sub-04-_-NPI_night(inch)||It is dark at inch. 154 | sub-04/sub-04-_-NPI_nine(cat)||The number before ten is cat. 155 | sub-04/sub-04-_-NPI_one(cold)||First place is number cold. 156 | sub-04/sub-04-_-NPI_out(ant)||Three strikes and you are ant. 157 | sub-04/sub-04-_-NPI_pink(door)||Pigs are colored door. 158 | sub-04/sub-04-_-NPI_plank(shark)||The pirate said walk the shark. 159 | sub-04/sub-04-_-NPI_popped(square)||The water balloon was so full it square. 160 | sub-04/sub-04-_-NPI_round(kiss)||The ball is not square but kiss. 161 | sub-06/sub-06-_-(LD5_Eliminated)NPC_live||The woods are where bears live. 162 | sub-06/sub-06-_-NPC_aisle||The bride walked down the aisle. 163 | sub-06/sub-06-_-NPC_animals||The sign said do not feed the animals. 164 | sub-06/sub-06-_-NPC_apple||His favorite fruit was a red crisp apple. 165 | sub-06/sub-06-_-NPC_arm||I have two legs and two arms. 166 | sub-06/sub-06-_-NPC_bag||He puts his lunch in a paper bag. 167 | sub-06/sub-06-_-NPC_bank||My mom puts money in the bank. 168 | sub-06/sub-06-_-NPC_button||On the remote he clicked the button. 169 | sub-06/sub-06-_-NPC_cart||At the store I pushed the cart. 170 | sub-06/sub-06-_-NPC_chores||My mom makes me do household chores. 171 | sub-06/sub-06-_-NPC_clothes||My parents fold my clothes. 172 | sub-06/sub-06-_-NPC_cribs||Babies sleep in cribs. 173 | sub-06/sub-06-_-NPC_cup||I drink from a cup. 174 | sub-06/sub-06-_-NPC_dream||She woke up after a bad dream. 175 | sub-06/sub-06-_-NPC_frog||Ribbit goes the frog. 176 | sub-06/sub-06-_-NPC_fur||Cats have soft fur. 177 | sub-06/sub-06-_-NPC_mouse||The cat chased the mouse. 178 | sub-06/sub-06-_-NPC_one||First place is number one. 179 | sub-06/sub-06-_-NPC_pink||Pigs are colored pink. 180 | sub-06/sub-06-_-NPC_snake||Hiss goes the snake. 181 | sub-06/sub-06-_-NPI_bear(pie)||The woods had a big brown scary pie. 182 | sub-06/sub-06-_-NPI_bite(north)||He had an itchy bug north. 183 | sub-06/sub-06-_-NPI_cages(woman)||Animals are locked in woman. 184 | sub-06/sub-06-_-NPI_card(fly)||I signed a birthday fly. 185 | sub-06/sub-06-_-NPI_cliff(sheep)||He stood at the edge of a sheep. 186 | sub-06/sub-06-_-NPI_cry(fork)||When I am sad I fork. 187 | sub-06/sub-06-_-NPI_dare(brick)||Lets play truth or brick. 188 | sub-06/sub-06-_-NPI_dice(art)||In games I roll the art. 189 | sub-06/sub-06-_-NPI_down(tie)||She told me to calm tie. 190 | sub-06/sub-06-_-NPI_fall(duck)||After summer is duck. 191 | sub-06/sub-06-_-NPI_fork(cleats)||He eats steak with a knife and cleats. 192 | sub-06/sub-06-_-NPI_gel(sign)||He styled his hair with sign. 193 | sub-06/sub-06-_-NPI_night(inch)||It is dark at inch. 194 | sub-06/sub-06-_-NPI_nine(cat)||The number before ten is cat. 195 | sub-06/sub-06-_-NPI_one(cold)||First place is number cold. 196 | sub-06/sub-06-_-NPI_out(ant)||Three strikes and you are ant. 197 | sub-06/sub-06-_-NPI_pink(door)||Pigs are colored door. 198 | sub-06/sub-06-_-NPI_plank(shark)||The pirate said walk the shark. 199 | sub-06/sub-06-_-NPI_popped(square)||The water balloon was so full it square. 200 | sub-06/sub-06-_-NPI_round(kiss)||The ball is not square but kiss. 201 | sub-07/sub-07-_-(LD5_Eliminated)NPC_live||The woods are where bears live. 202 | sub-07/sub-07-_-NPC_aisle||The bride walked down the aisle. 203 | sub-07/sub-07-_-NPC_animals||The sign said do not feed the animals. 204 | sub-07/sub-07-_-NPC_apple||His favorite fruit was a red crisp apple. 205 | sub-07/sub-07-_-NPC_arm||I have two legs and two arms. 206 | sub-07/sub-07-_-NPC_bag||He puts his lunch in a paper bag. 207 | sub-07/sub-07-_-NPC_bank||My mom puts money in the bank. 208 | sub-07/sub-07-_-NPC_button||On the remote he clicked the button. 209 | sub-07/sub-07-_-NPC_cart||At the store I pushed the cart. 210 | sub-07/sub-07-_-NPC_chores||My mom makes me do household chores. 211 | sub-07/sub-07-_-NPC_clothes||My parents fold my clothes. 212 | sub-07/sub-07-_-NPC_cribs||Babies sleep in cribs. 213 | sub-07/sub-07-_-NPC_cup||I drink from a cup. 214 | sub-07/sub-07-_-NPC_dream||She woke up after a bad dream. 215 | sub-07/sub-07-_-NPC_frog||Ribbit goes the frog. 216 | sub-07/sub-07-_-NPC_fur||Cats have soft fur. 217 | sub-07/sub-07-_-NPC_mouse||The cat chased the mouse. 218 | sub-07/sub-07-_-NPC_one||First place is number one. 219 | sub-07/sub-07-_-NPC_pink||Pigs are colored pink. 220 | sub-07/sub-07-_-NPC_snake||Hiss goes the snake. 221 | sub-07/sub-07-_-NPI_bear(pie)||The woods had a big brown scary pie. 222 | sub-07/sub-07-_-NPI_bite(north)||He had an itchy bug north. 223 | sub-07/sub-07-_-NPI_cages(woman)||Animals are locked in woman. 224 | sub-07/sub-07-_-NPI_card(fly)||I signed a birthday fly. 225 | sub-07/sub-07-_-NPI_cliff(sheep)||He stood at the edge of a sheep. 226 | sub-07/sub-07-_-NPI_cry(fork)||When I am sad I fork. 227 | sub-07/sub-07-_-NPI_dare(brick)||Lets play truth or brick. 228 | sub-07/sub-07-_-NPI_dice(art)||In games I roll the art. 229 | sub-07/sub-07-_-NPI_down(tie)||She told me to calm tie. 230 | sub-07/sub-07-_-NPI_fall(duck)||After summer is duck. 231 | sub-07/sub-07-_-NPI_fork(cleats)||He eats steak with a knife and cleats. 232 | sub-07/sub-07-_-NPI_gel(sign)||He styled his hair with sign. 233 | sub-07/sub-07-_-NPI_night(inch)||It is dark at inch. 234 | sub-07/sub-07-_-NPI_nine(cat)||The number before ten is cat. 235 | sub-07/sub-07-_-NPI_one(cold)||First place is number cold. 236 | sub-07/sub-07-_-NPI_out(ant)||Three strikes and you are ant. 237 | sub-07/sub-07-_-NPI_pink(door)||Pigs are colored door. 238 | sub-07/sub-07-_-NPI_plank(shark)||The pirate said walk the shark. 239 | sub-07/sub-07-_-NPI_popped(square)||The water balloon was so full it square. 240 | sub-07/sub-07-_-NPI_round(kiss)||The ball is not square but kiss. 241 | sub-08/sub-08-_-(LD5_Eliminated)NPC_live||The woods are where bears live. 242 | sub-08/sub-08-_-NPC_aisle||The bride walked down the aisle. 243 | sub-08/sub-08-_-NPC_animals||The sign said do not feed the animals. 244 | sub-08/sub-08-_-NPC_apple||His favorite fruit was a red crisp apple. 245 | sub-08/sub-08-_-NPC_arm||I have two legs and two arms. 246 | sub-08/sub-08-_-NPC_bag||He puts his lunch in a paper bag. 247 | sub-08/sub-08-_-NPC_bank||My mom puts money in the bank. 248 | sub-08/sub-08-_-NPC_button||On the remote he clicked the button. 249 | sub-08/sub-08-_-NPC_cart||At the store I pushed the cart. 250 | sub-08/sub-08-_-NPC_chores||My mom makes me do household chores. 251 | sub-08/sub-08-_-NPC_clothes||My parents fold my clothes. 252 | sub-08/sub-08-_-NPC_cribs||Babies sleep in cribs. 253 | sub-08/sub-08-_-NPC_cup||I drink from a cup. 254 | sub-08/sub-08-_-NPC_dream||She woke up after a bad dream. 255 | sub-08/sub-08-_-NPC_frog||Ribbit goes the frog. 256 | sub-08/sub-08-_-NPC_fur||Cats have soft fur. 257 | sub-08/sub-08-_-NPC_mouse||The cat chased the mouse. 258 | sub-08/sub-08-_-NPC_one||First place is number one. 259 | sub-08/sub-08-_-NPC_pink||Pigs are colored pink. 260 | sub-08/sub-08-_-NPC_snake||Hiss goes the snake. 261 | sub-08/sub-08-_-NPI_bear(pie)||The woods had a big brown scary pie. 262 | sub-08/sub-08-_-NPI_bite(north)||He had an itchy bug north. 263 | sub-08/sub-08-_-NPI_cages(woman)||Animals are locked in woman. 264 | sub-08/sub-08-_-NPI_card(fly)||I signed a birthday fly. 265 | sub-08/sub-08-_-NPI_cliff(sheep)||He stood at the edge of a sheep. 266 | sub-08/sub-08-_-NPI_cry(fork)||When I am sad I fork. 267 | sub-08/sub-08-_-NPI_dare(brick)||Lets play truth or brick. 268 | sub-08/sub-08-_-NPI_dice(art)||In games I roll the art. 269 | sub-08/sub-08-_-NPI_down(tie)||She told me to calm tie. 270 | sub-08/sub-08-_-NPI_fall(duck)||After summer is duck. 271 | sub-08/sub-08-_-NPI_fork(cleats)||He eats steak with a knife and cleats. 272 | sub-08/sub-08-_-NPI_gel(sign)||He styled his hair with sign. 273 | sub-08/sub-08-_-NPI_night(inch)||It is dark at inch. 274 | sub-08/sub-08-_-NPI_nine(cat)||The number before ten is cat. 275 | sub-08/sub-08-_-NPI_one(cold)||First place is number cold. 276 | sub-08/sub-08-_-NPI_out(ant)||Three strikes and you are ant. 277 | sub-08/sub-08-_-NPI_pink(door)||Pigs are colored door. 278 | sub-08/sub-08-_-NPI_plank(shark)||The pirate said walk the shark. 279 | sub-08/sub-08-_-NPI_popped(square)||The water balloon was so full it square. 280 | sub-08/sub-08-_-NPI_round(kiss)||The ball is not square but kiss. 281 | sub-09/sub-09-_-(LD5_Eliminated)NPC_live||The woods are where bears live. 282 | sub-09/sub-09-_-NPC_aisle||The bride walked down the aisle. 283 | sub-09/sub-09-_-NPC_animals||The sign said do not feed the animals. 284 | sub-09/sub-09-_-NPC_apple||His favorite fruit was a red crisp apple. 285 | sub-09/sub-09-_-NPC_arm||I have two legs and two arms. 286 | sub-09/sub-09-_-NPC_bag||He puts his lunch in a paper bag. 287 | sub-09/sub-09-_-NPC_bank||My mom puts money in the bank. 288 | sub-09/sub-09-_-NPC_button||On the remote he clicked the button. 289 | sub-09/sub-09-_-NPC_cart||At the store I pushed the cart. 290 | sub-09/sub-09-_-NPC_chores||My mom makes me do household chores. 291 | sub-09/sub-09-_-NPC_clothes||My parents fold my clothes. 292 | sub-09/sub-09-_-NPC_cribs||Babies sleep in cribs. 293 | sub-09/sub-09-_-NPC_cup||I drink from a cup. 294 | sub-09/sub-09-_-NPC_dream||She woke up after a bad dream. 295 | sub-09/sub-09-_-NPC_frog||Ribbit goes the frog. 296 | sub-09/sub-09-_-NPC_fur||Cats have soft fur. 297 | sub-09/sub-09-_-NPC_mouse||The cat chased the mouse. 298 | sub-09/sub-09-_-NPC_one||First place is number one. 299 | sub-09/sub-09-_-NPC_pink||Pigs are colored pink. 300 | sub-09/sub-09-_-NPC_snake||Hiss goes the snake. 301 | sub-09/sub-09-_-NPI_bear(pie)||The woods had a big brown scary pie. 302 | sub-09/sub-09-_-NPI_bite(north)||He had an itchy bug north. 303 | sub-09/sub-09-_-NPI_cages(woman)||Animals are locked in woman. 304 | sub-09/sub-09-_-NPI_card(fly)||I signed a birthday fly. 305 | sub-09/sub-09-_-NPI_cliff(sheep)||He stood at the edge of a sheep. 306 | sub-09/sub-09-_-NPI_cry(fork)||When I am sad I fork. 307 | sub-09/sub-09-_-NPI_dare(brick)||Lets play truth or brick. 308 | sub-09/sub-09-_-NPI_dice(art)||In games I roll the art. 309 | sub-09/sub-09-_-NPI_down(tie)||She told me to calm tie. 310 | sub-09/sub-09-_-NPI_fall(duck)||After summer is duck. 311 | sub-09/sub-09-_-NPI_fork(cleats)||He eats steak with a knife and cleats. 312 | sub-09/sub-09-_-NPI_gel(sign)||He styled his hair with sign. 313 | sub-09/sub-09-_-NPI_night(inch)||It is dark at inch. 314 | sub-09/sub-09-_-NPI_nine(cat)||The number before ten is cat. 315 | sub-09/sub-09-_-NPI_one(cold)||First place is number cold. 316 | sub-09/sub-09-_-NPI_out(ant)||Three strikes and you are ant. 317 | sub-09/sub-09-_-NPI_pink(door)||Pigs are colored door. 318 | sub-09/sub-09-_-NPI_plank(shark)||The pirate said walk the shark. 319 | sub-09/sub-09-_-NPI_popped(square)||The water balloon was so full it square. 320 | sub-09/sub-09-_-NPI_round(kiss)||The ball is not square but kiss. 321 | sub-11/sub-11-_-(LD5_Eliminated)NPC_live||The woods are where bears live. 322 | sub-11/sub-11-_-NPC_aisle||The bride walked down the aisle. 323 | sub-11/sub-11-_-NPC_animals||The sign said do not feed the animals. 324 | sub-11/sub-11-_-NPC_apple||His favorite fruit was a red crisp apple. 325 | sub-11/sub-11-_-NPC_arm||I have two legs and two arms. 326 | sub-11/sub-11-_-NPC_bag||He puts his lunch in a paper bag. 327 | sub-11/sub-11-_-NPC_bank||My mom puts money in the bank. 328 | sub-11/sub-11-_-NPC_button||On the remote he clicked the button. 329 | sub-11/sub-11-_-NPC_cart||At the store I pushed the cart. 330 | sub-11/sub-11-_-NPC_chores||My mom makes me do household chores. 331 | sub-11/sub-11-_-NPC_clothes||My parents fold my clothes. 332 | sub-11/sub-11-_-NPC_cribs||Babies sleep in cribs. 333 | sub-11/sub-11-_-NPC_cup||I drink from a cup. 334 | sub-11/sub-11-_-NPC_dream||She woke up after a bad dream. 335 | sub-11/sub-11-_-NPC_frog||Ribbit goes the frog. 336 | sub-11/sub-11-_-NPC_fur||Cats have soft fur. 337 | sub-11/sub-11-_-NPC_mouse||The cat chased the mouse. 338 | sub-11/sub-11-_-NPC_one||First place is number one. 339 | sub-11/sub-11-_-NPC_pink||Pigs are colored pink. 340 | sub-11/sub-11-_-NPC_snake||Hiss goes the snake. 341 | sub-11/sub-11-_-NPI_bear(pie)||The woods had a big brown scary pie. 342 | sub-11/sub-11-_-NPI_bite(north)||He had an itchy bug north. 343 | sub-11/sub-11-_-NPI_cages(woman)||Animals are locked in woman. 344 | sub-11/sub-11-_-NPI_card(fly)||I signed a birthday fly. 345 | sub-11/sub-11-_-NPI_cliff(sheep)||He stood at the edge of a sheep. 346 | sub-11/sub-11-_-NPI_cry(fork)||When I am sad I fork. 347 | sub-11/sub-11-_-NPI_dare(brick)||Lets play truth or brick. 348 | sub-11/sub-11-_-NPI_dice(art)||In games I roll the art. 349 | sub-11/sub-11-_-NPI_down(tie)||She told me to calm tie. 350 | sub-11/sub-11-_-NPI_fall(duck)||After summer is duck. 351 | sub-11/sub-11-_-NPI_fork(cleats)||He eats steak with a knife and cleats. 352 | sub-11/sub-11-_-NPI_gel(sign)||He styled his hair with sign. 353 | sub-11/sub-11-_-NPI_night(inch)||It is dark at inch. 354 | sub-11/sub-11-_-NPI_nine(cat)||The number before ten is cat. 355 | sub-11/sub-11-_-NPI_one(cold)||First place is number cold. 356 | sub-11/sub-11-_-NPI_out(ant)||Three strikes and you are ant. 357 | sub-11/sub-11-_-NPI_pink(door)||Pigs are colored door. 358 | sub-11/sub-11-_-NPI_plank(shark)||The pirate said walk the shark. 359 | sub-11/sub-11-_-NPI_popped(square)||The water balloon was so full it square. 360 | sub-11/sub-11-_-NPI_round(kiss)||The ball is not square but kiss. 361 | sub-12/sub-12-_-(LD5_Eliminated)NPC_live||The woods are where bears live. 362 | sub-12/sub-12-_-NPC_aisle||The bride walked down the aisle. 363 | sub-12/sub-12-_-NPC_animals||The sign said do not feed the animals. 364 | sub-12/sub-12-_-NPC_apple||His favorite fruit was a red crisp apple. 365 | sub-12/sub-12-_-NPC_arm||I have two legs and two arms. 366 | sub-12/sub-12-_-NPC_bag||He puts his lunch in a paper bag. 367 | sub-12/sub-12-_-NPC_bank||My mom puts money in the bank. 368 | sub-12/sub-12-_-NPC_button||On the remote he clicked the button. 369 | sub-12/sub-12-_-NPC_cart||At the store I pushed the cart. 370 | sub-12/sub-12-_-NPC_chores||My mom makes me do household chores. 371 | sub-12/sub-12-_-NPC_clothes||My parents fold my clothes. 372 | sub-12/sub-12-_-NPC_cribs||Babies sleep in cribs. 373 | sub-12/sub-12-_-NPC_cup||I drink from a cup. 374 | sub-12/sub-12-_-NPC_dream||She woke up after a bad dream. 375 | sub-12/sub-12-_-NPC_frog||Ribbit goes the frog. 376 | sub-12/sub-12-_-NPC_fur||Cats have soft fur. 377 | sub-12/sub-12-_-NPC_mouse||The cat chased the mouse. 378 | sub-12/sub-12-_-NPC_one||First place is number one. 379 | sub-12/sub-12-_-NPC_pink||Pigs are colored pink. 380 | sub-12/sub-12-_-NPC_snake||Hiss goes the snake. 381 | sub-12/sub-12-_-NPI_bear(pie)||The woods had a big brown scary pie. 382 | sub-12/sub-12-_-NPI_bite(north)||He had an itchy bug north. 383 | sub-12/sub-12-_-NPI_cages(woman)||Animals are locked in woman. 384 | sub-12/sub-12-_-NPI_card(fly)||I signed a birthday fly. 385 | sub-12/sub-12-_-NPI_cliff(sheep)||He stood at the edge of a sheep. 386 | sub-12/sub-12-_-NPI_cry(fork)||When I am sad I fork. 387 | sub-12/sub-12-_-NPI_dare(brick)||Lets play truth or brick. 388 | sub-12/sub-12-_-NPI_dice(art)||In games I roll the art. 389 | sub-12/sub-12-_-NPI_down(tie)||She told me to calm tie. 390 | sub-12/sub-12-_-NPI_fall(duck)||After summer is duck. 391 | sub-12/sub-12-_-NPI_fork(cleats)||He eats steak with a knife and cleats. 392 | sub-12/sub-12-_-NPI_gel(sign)||He styled his hair with sign. 393 | sub-12/sub-12-_-NPI_night(inch)||It is dark at inch. 394 | sub-12/sub-12-_-NPI_nine(cat)||The number before ten is cat. 395 | sub-12/sub-12-_-NPI_one(cold)||First place is number cold. 396 | sub-12/sub-12-_-NPI_out(ant)||Three strikes and you are ant. 397 | sub-12/sub-12-_-NPI_pink(door)||Pigs are colored door. 398 | sub-12/sub-12-_-NPI_plank(shark)||The pirate said walk the shark. 399 | sub-12/sub-12-_-NPI_popped(square)||The water balloon was so full it square. 400 | sub-12/sub-12-_-NPI_round(kiss)||The ball is not square but kiss. 401 | sub-13/sub-13-_-(LD5_Eliminated)NPC_live||The woods are where bears live. 402 | sub-13/sub-13-_-NPC_aisle||The bride walked down the aisle. 403 | sub-13/sub-13-_-NPC_animals||The sign said do not feed the animals. 404 | sub-13/sub-13-_-NPC_apple||His favorite fruit was a red crisp apple. 405 | sub-13/sub-13-_-NPC_arm||I have two legs and two arms. 406 | sub-13/sub-13-_-NPC_bag||He puts his lunch in a paper bag. 407 | sub-13/sub-13-_-NPC_bank||My mom puts money in the bank. 408 | sub-13/sub-13-_-NPC_button||On the remote he clicked the button. 409 | sub-13/sub-13-_-NPC_cart||At the store I pushed the cart. 410 | sub-13/sub-13-_-NPC_chores||My mom makes me do household chores. 411 | sub-13/sub-13-_-NPC_clothes||My parents fold my clothes. 412 | sub-13/sub-13-_-NPC_cribs||Babies sleep in cribs. 413 | sub-13/sub-13-_-NPC_cup||I drink from a cup. 414 | sub-13/sub-13-_-NPC_dream||She woke up after a bad dream. 415 | sub-13/sub-13-_-NPC_frog||Ribbit goes the frog. 416 | sub-13/sub-13-_-NPC_fur||Cats have soft fur. 417 | sub-13/sub-13-_-NPC_mouse||The cat chased the mouse. 418 | sub-13/sub-13-_-NPC_one||First place is number one. 419 | sub-13/sub-13-_-NPC_pink||Pigs are colored pink. 420 | sub-13/sub-13-_-NPC_snake||Hiss goes the snake. 421 | sub-13/sub-13-_-NPI_bear(pie)||The woods had a big brown scary pie. 422 | sub-13/sub-13-_-NPI_bite(north)||He had an itchy bug north. 423 | sub-13/sub-13-_-NPI_cages(woman)||Animals are locked in woman. 424 | sub-13/sub-13-_-NPI_card(fly)||I signed a birthday fly. 425 | sub-13/sub-13-_-NPI_cliff(sheep)||He stood at the edge of a sheep. 426 | sub-13/sub-13-_-NPI_cry(fork)||When I am sad I fork. 427 | sub-13/sub-13-_-NPI_dare(brick)||Lets play truth or brick. 428 | sub-13/sub-13-_-NPI_dice(art)||In games I roll the art. 429 | sub-13/sub-13-_-NPI_down(tie)||She told me to calm tie. 430 | sub-13/sub-13-_-NPI_fall(duck)||After summer is duck. 431 | sub-13/sub-13-_-NPI_fork(cleats)||He eats steak with a knife and cleats. 432 | sub-13/sub-13-_-NPI_gel(sign)||He styled his hair with sign. 433 | sub-13/sub-13-_-NPI_night(inch)||It is dark at inch. 434 | sub-13/sub-13-_-NPI_nine(cat)||The number before ten is cat. 435 | sub-13/sub-13-_-NPI_one(cold)||First place is number cold. 436 | sub-13/sub-13-_-NPI_out(ant)||Three strikes and you are ant. 437 | sub-13/sub-13-_-NPI_pink(door)||Pigs are colored door. 438 | sub-13/sub-13-_-NPI_plank(shark)||The pirate said walk the shark. 439 | sub-13/sub-13-_-NPI_popped(square)||The water balloon was so full it square. 440 | sub-13/sub-13-_-NPI_round(kiss)||The ball is not square but kiss. 441 | sub-14/sub-14-_-(LD5_Eliminated)NPC_live||The woods are where bears live. 442 | sub-14/sub-14-_-NPC_aisle||The bride walked down the aisle. 443 | sub-14/sub-14-_-NPC_animals||The sign said do not feed the animals. 444 | sub-14/sub-14-_-NPC_apple||His favorite fruit was a red crisp apple. 445 | sub-14/sub-14-_-NPC_arm||I have two legs and two arms. 446 | sub-14/sub-14-_-NPC_bag||He puts his lunch in a paper bag. 447 | sub-14/sub-14-_-NPC_bank||My mom puts money in the bank. 448 | sub-14/sub-14-_-NPC_button||On the remote he clicked the button. 449 | sub-14/sub-14-_-NPC_cart||At the store I pushed the cart. 450 | sub-14/sub-14-_-NPC_chores||My mom makes me do household chores. 451 | sub-14/sub-14-_-NPC_clothes||My parents fold my clothes. 452 | sub-14/sub-14-_-NPC_cribs||Babies sleep in cribs. 453 | sub-14/sub-14-_-NPC_cup||I drink from a cup. 454 | sub-14/sub-14-_-NPC_dream||She woke up after a bad dream. 455 | sub-14/sub-14-_-NPC_frog||Ribbit goes the frog. 456 | sub-14/sub-14-_-NPC_fur||Cats have soft fur. 457 | sub-14/sub-14-_-NPC_mouse||The cat chased the mouse. 458 | sub-14/sub-14-_-NPC_one||First place is number one. 459 | sub-14/sub-14-_-NPC_pink||Pigs are colored pink. 460 | sub-14/sub-14-_-NPC_snake||Hiss goes the snake. 461 | sub-14/sub-14-_-NPI_bear(pie)||The woods had a big brown scary pie. 462 | sub-14/sub-14-_-NPI_bite(north)||He had an itchy bug north. 463 | sub-14/sub-14-_-NPI_cages(woman)||Animals are locked in woman. 464 | sub-14/sub-14-_-NPI_card(fly)||I signed a birthday fly. 465 | sub-14/sub-14-_-NPI_cliff(sheep)||He stood at the edge of a sheep. 466 | sub-14/sub-14-_-NPI_cry(fork)||When I am sad I fork. 467 | sub-14/sub-14-_-NPI_dare(brick)||Lets play truth or brick. 468 | sub-14/sub-14-_-NPI_dice(art)||In games I roll the art. 469 | sub-14/sub-14-_-NPI_down(tie)||She told me to calm tie. 470 | sub-14/sub-14-_-NPI_fall(duck)||After summer is duck. 471 | sub-14/sub-14-_-NPI_fork(cleats)||He eats steak with a knife and cleats. 472 | sub-14/sub-14-_-NPI_gel(sign)||He styled his hair with sign. 473 | sub-14/sub-14-_-NPI_night(inch)||It is dark at inch. 474 | sub-14/sub-14-_-NPI_nine(cat)||The number before ten is cat. 475 | sub-14/sub-14-_-NPI_one(cold)||First place is number cold. 476 | sub-14/sub-14-_-NPI_out(ant)||Three strikes and you are ant. 477 | sub-14/sub-14-_-NPI_pink(door)||Pigs are colored door. 478 | sub-14/sub-14-_-NPI_plank(shark)||The pirate said walk the shark. 479 | sub-14/sub-14-_-NPI_popped(square)||The water balloon was so full it square. 480 | sub-14/sub-14-_-NPI_round(kiss)||The ball is not square but kiss. 481 | sub-16/sub-16-_-(LD5_Eliminated)NPC_live||The woods are where bears live. 482 | sub-16/sub-16-_-NPC_aisle||The bride walked down the aisle. 483 | sub-16/sub-16-_-NPC_animals||The sign said do not feed the animals. 484 | sub-16/sub-16-_-NPC_apple||His favorite fruit was a red crisp apple. 485 | sub-16/sub-16-_-NPC_arm||I have two legs and two arms. 486 | sub-16/sub-16-_-NPC_bag||He puts his lunch in a paper bag. 487 | sub-16/sub-16-_-NPC_bank||My mom puts money in the bank. 488 | sub-16/sub-16-_-NPC_button||On the remote he clicked the button. 489 | sub-16/sub-16-_-NPC_cart||At the store I pushed the cart. 490 | sub-16/sub-16-_-NPC_chores||My mom makes me do household chores. 491 | sub-16/sub-16-_-NPC_clothes||My parents fold my clothes. 492 | sub-16/sub-16-_-NPC_cribs||Babies sleep in cribs. 493 | sub-16/sub-16-_-NPC_cup||I drink from a cup. 494 | sub-16/sub-16-_-NPC_dream||She woke up after a bad dream. 495 | sub-16/sub-16-_-NPC_frog||Ribbit goes the frog. 496 | sub-16/sub-16-_-NPC_fur||Cats have soft fur. 497 | sub-16/sub-16-_-NPC_mouse||The cat chased the mouse. 498 | sub-16/sub-16-_-NPC_one||First place is number one. 499 | sub-16/sub-16-_-NPC_pink||Pigs are colored pink. 500 | sub-16/sub-16-_-NPC_snake||Hiss goes the snake. 501 | sub-16/sub-16-_-NPI_bear(pie)||The woods had a big brown scary pie. 502 | sub-16/sub-16-_-NPI_bite(north)||He had an itchy bug north. 503 | sub-16/sub-16-_-NPI_cages(woman)||Animals are locked in woman. 504 | sub-16/sub-16-_-NPI_card(fly)||I signed a birthday fly. 505 | sub-16/sub-16-_-NPI_cliff(sheep)||He stood at the edge of a sheep. 506 | sub-16/sub-16-_-NPI_cry(fork)||When I am sad I fork. 507 | sub-16/sub-16-_-NPI_dare(brick)||Lets play truth or brick. 508 | sub-16/sub-16-_-NPI_dice(art)||In games I roll the art. 509 | sub-16/sub-16-_-NPI_down(tie)||She told me to calm tie. 510 | sub-16/sub-16-_-NPI_fall(duck)||After summer is duck. 511 | sub-16/sub-16-_-NPI_fork(cleats)||He eats steak with a knife and cleats. 512 | sub-16/sub-16-_-NPI_gel(sign)||He styled his hair with sign. 513 | sub-16/sub-16-_-NPI_night(inch)||It is dark at inch. 514 | sub-16/sub-16-_-NPI_nine(cat)||The number before ten is cat. 515 | sub-16/sub-16-_-NPI_one(cold)||First place is number cold. 516 | sub-16/sub-16-_-NPI_out(ant)||Three strikes and you are ant. 517 | sub-16/sub-16-_-NPI_pink(door)||Pigs are colored door. 518 | sub-16/sub-16-_-NPI_plank(shark)||The pirate said walk the shark. 519 | sub-16/sub-16-_-NPI_popped(square)||The water balloon was so full it square. 520 | sub-16/sub-16-_-NPI_round(kiss)||The ball is not square but kiss. 521 | sub-17/sub-17-_-(LD5_Eliminated)NPC_live||The woods are where bears live. 522 | sub-17/sub-17-_-NPC_aisle||The bride walked down the aisle. 523 | sub-17/sub-17-_-NPC_animals||The sign said do not feed the animals. 524 | sub-17/sub-17-_-NPC_apple||His favorite fruit was a red crisp apple. 525 | sub-17/sub-17-_-NPC_arm||I have two legs and two arms. 526 | sub-17/sub-17-_-NPC_bag||He puts his lunch in a paper bag. 527 | sub-17/sub-17-_-NPC_bank||My mom puts money in the bank. 528 | sub-17/sub-17-_-NPC_button||On the remote he clicked the button. 529 | sub-17/sub-17-_-NPC_cart||At the store I pushed the cart. 530 | sub-17/sub-17-_-NPC_chores||My mom makes me do household chores. 531 | sub-17/sub-17-_-NPC_clothes||My parents fold my clothes. 532 | sub-17/sub-17-_-NPC_cribs||Babies sleep in cribs. 533 | sub-17/sub-17-_-NPC_cup||I drink from a cup. 534 | sub-17/sub-17-_-NPC_dream||She woke up after a bad dream. 535 | sub-17/sub-17-_-NPC_frog||Ribbit goes the frog. 536 | sub-17/sub-17-_-NPC_fur||Cats have soft fur. 537 | sub-17/sub-17-_-NPC_mouse||The cat chased the mouse. 538 | sub-17/sub-17-_-NPC_one||First place is number one. 539 | sub-17/sub-17-_-NPC_pink||Pigs are colored pink. 540 | sub-17/sub-17-_-NPC_snake||Hiss goes the snake. 541 | sub-17/sub-17-_-NPI_bear(pie)||The woods had a big brown scary pie. 542 | sub-17/sub-17-_-NPI_bite(north)||He had an itchy bug north. 543 | sub-17/sub-17-_-NPI_cages(woman)||Animals are locked in woman. 544 | sub-17/sub-17-_-NPI_card(fly)||I signed a birthday fly. 545 | sub-17/sub-17-_-NPI_cliff(sheep)||He stood at the edge of a sheep. 546 | sub-17/sub-17-_-NPI_cry(fork)||When I am sad I fork. 547 | sub-17/sub-17-_-NPI_dare(brick)||Lets play truth or brick. 548 | sub-17/sub-17-_-NPI_dice(art)||In games I roll the art. 549 | sub-17/sub-17-_-NPI_down(tie)||She told me to calm tie. 550 | sub-17/sub-17-_-NPI_fall(duck)||After summer is duck. 551 | sub-17/sub-17-_-NPI_fork(cleats)||He eats steak with a knife and cleats. 552 | sub-17/sub-17-_-NPI_gel(sign)||He styled his hair with sign. 553 | sub-17/sub-17-_-NPI_night(inch)||It is dark at inch. 554 | sub-17/sub-17-_-NPI_nine(cat)||The number before ten is cat. 555 | sub-17/sub-17-_-NPI_one(cold)||First place is number cold. 556 | sub-17/sub-17-_-NPI_out(ant)||Three strikes and you are ant. 557 | sub-17/sub-17-_-NPI_pink(door)||Pigs are colored door. 558 | sub-17/sub-17-_-NPI_plank(shark)||The pirate said walk the shark. 559 | sub-17/sub-17-_-NPI_popped(square)||The water balloon was so full it square. 560 | sub-17/sub-17-_-NPI_round(kiss)||The ball is not square but kiss. 561 | sub-19/sub-19-_-(LD5_Eliminated)NPC_live||The woods are where bears live. 562 | sub-19/sub-19-_-NPC_aisle||The bride walked down the aisle. 563 | sub-19/sub-19-_-NPC_animals||The sign said do not feed the animals. 564 | sub-19/sub-19-_-NPC_apple||His favorite fruit was a red crisp apple. 565 | sub-19/sub-19-_-NPC_arm||I have two legs and two arms. 566 | sub-19/sub-19-_-NPC_bag||He puts his lunch in a paper bag. 567 | sub-19/sub-19-_-NPC_bank||My mom puts money in the bank. 568 | sub-19/sub-19-_-NPC_button||On the remote he clicked the button. 569 | sub-19/sub-19-_-NPC_cart||At the store I pushed the cart. 570 | sub-19/sub-19-_-NPC_chores||My mom makes me do household chores. 571 | sub-19/sub-19-_-NPC_clothes||My parents fold my clothes. 572 | sub-19/sub-19-_-NPC_cribs||Babies sleep in cribs. 573 | sub-19/sub-19-_-NPC_cup||I drink from a cup. 574 | sub-19/sub-19-_-NPC_dream||She woke up after a bad dream. 575 | sub-19/sub-19-_-NPC_frog||Ribbit goes the frog. 576 | sub-19/sub-19-_-NPC_fur||Cats have soft fur. 577 | sub-19/sub-19-_-NPC_mouse||The cat chased the mouse. 578 | sub-19/sub-19-_-NPC_one||First place is number one. 579 | sub-19/sub-19-_-NPC_pink||Pigs are colored pink. 580 | sub-19/sub-19-_-NPC_snake||Hiss goes the snake. 581 | sub-19/sub-19-_-NPI_bear(pie)||The woods had a big brown scary pie. 582 | sub-19/sub-19-_-NPI_bite(north)||He had an itchy bug north. 583 | sub-19/sub-19-_-NPI_cages(woman)||Animals are locked in woman. 584 | sub-19/sub-19-_-NPI_card(fly)||I signed a birthday fly. 585 | sub-19/sub-19-_-NPI_cliff(sheep)||He stood at the edge of a sheep. 586 | sub-19/sub-19-_-NPI_cry(fork)||When I am sad I fork. 587 | sub-19/sub-19-_-NPI_dare(brick)||Lets play truth or brick. 588 | sub-19/sub-19-_-NPI_dice(art)||In games I roll the art. 589 | sub-19/sub-19-_-NPI_down(tie)||She told me to calm tie. 590 | sub-19/sub-19-_-NPI_fall(duck)||After summer is duck. 591 | sub-19/sub-19-_-NPI_fork(cleats)||He eats steak with a knife and cleats. 592 | sub-19/sub-19-_-NPI_gel(sign)||He styled his hair with sign. 593 | sub-19/sub-19-_-NPI_night(inch)||It is dark at inch. 594 | sub-19/sub-19-_-NPI_nine(cat)||The number before ten is cat. 595 | sub-19/sub-19-_-NPI_one(cold)||First place is number cold. 596 | sub-19/sub-19-_-NPI_out(ant)||Three strikes and you are ant. 597 | sub-19/sub-19-_-NPI_pink(door)||Pigs are colored door. 598 | sub-19/sub-19-_-NPI_plank(shark)||The pirate said walk the shark. 599 | sub-19/sub-19-_-NPI_popped(square)||The water balloon was so full it square. 600 | sub-19/sub-19-_-NPI_round(kiss)||The ball is not square but kiss. 601 | sub-20/sub-20-_-(LD5_Eliminated)NPC_live||The woods are where bears live. 602 | sub-20/sub-20-_-NPC_aisle||The bride walked down the aisle. 603 | sub-20/sub-20-_-NPC_animals||The sign said do not feed the animals. 604 | sub-20/sub-20-_-NPC_apple||His favorite fruit was a red crisp apple. 605 | sub-20/sub-20-_-NPC_arm||I have two legs and two arms. 606 | sub-20/sub-20-_-NPC_bag||He puts his lunch in a paper bag. 607 | sub-20/sub-20-_-NPC_bank||My mom puts money in the bank. 608 | sub-20/sub-20-_-NPC_button||On the remote he clicked the button. 609 | sub-20/sub-20-_-NPC_cart||At the store I pushed the cart. 610 | sub-20/sub-20-_-NPC_chores||My mom makes me do household chores. 611 | sub-20/sub-20-_-NPC_clothes||My parents fold my clothes. 612 | sub-20/sub-20-_-NPC_cribs||Babies sleep in cribs. 613 | sub-20/sub-20-_-NPC_cup||I drink from a cup. 614 | sub-20/sub-20-_-NPC_dream||She woke up after a bad dream. 615 | sub-20/sub-20-_-NPC_frog||Ribbit goes the frog. 616 | sub-20/sub-20-_-NPC_fur||Cats have soft fur. 617 | sub-20/sub-20-_-NPC_mouse||The cat chased the mouse. 618 | sub-20/sub-20-_-NPC_one||First place is number one. 619 | sub-20/sub-20-_-NPC_pink||Pigs are colored pink. 620 | sub-20/sub-20-_-NPC_snake||Hiss goes the snake. 621 | sub-20/sub-20-_-NPI_bear(pie)||The woods had a big brown scary pie. 622 | sub-20/sub-20-_-NPI_bite(north)||He had an itchy bug north. 623 | sub-20/sub-20-_-NPI_cages(woman)||Animals are locked in woman. 624 | sub-20/sub-20-_-NPI_card(fly)||I signed a birthday fly. 625 | sub-20/sub-20-_-NPI_cliff(sheep)||He stood at the edge of a sheep. 626 | sub-20/sub-20-_-NPI_cry(fork)||When I am sad I fork. 627 | sub-20/sub-20-_-NPI_dare(brick)||Lets play truth or brick. 628 | sub-20/sub-20-_-NPI_dice(art)||In games I roll the art. 629 | sub-20/sub-20-_-NPI_down(tie)||She told me to calm tie. 630 | sub-20/sub-20-_-NPI_fall(duck)||After summer is duck. 631 | sub-20/sub-20-_-NPI_fork(cleats)||He eats steak with a knife and cleats. 632 | sub-20/sub-20-_-NPI_gel(sign)||He styled his hair with sign. 633 | sub-20/sub-20-_-NPI_night(inch)||It is dark at inch. 634 | sub-20/sub-20-_-NPI_nine(cat)||The number before ten is cat. 635 | sub-20/sub-20-_-NPI_one(cold)||First place is number cold. 636 | sub-20/sub-20-_-NPI_out(ant)||Three strikes and you are ant. 637 | sub-20/sub-20-_-NPI_pink(door)||Pigs are colored door. 638 | sub-20/sub-20-_-NPI_plank(shark)||The pirate said walk the shark. 639 | sub-20/sub-20-_-NPI_popped(square)||The water balloon was so full it square. 640 | sub-20/sub-20-_-NPI_round(kiss)||The ball is not square but kiss. 641 | sub-21/sub-21-_-(LD5_Eliminated)NPC_live||The woods are where bears live. 642 | sub-21/sub-21-_-NPC_aisle||The bride walked down the aisle. 643 | sub-21/sub-21-_-NPC_animals||The sign said do not feed the animals. 644 | sub-21/sub-21-_-NPC_apple||His favorite fruit was a red crisp apple. 645 | sub-21/sub-21-_-NPC_arm||I have two legs and two arms. 646 | sub-21/sub-21-_-NPC_bag||He puts his lunch in a paper bag. 647 | sub-21/sub-21-_-NPC_bank||My mom puts money in the bank. 648 | sub-21/sub-21-_-NPC_button||On the remote he clicked the button. 649 | sub-21/sub-21-_-NPC_cart||At the store I pushed the cart. 650 | sub-21/sub-21-_-NPC_chores||My mom makes me do household chores. 651 | sub-21/sub-21-_-NPC_clothes||My parents fold my clothes. 652 | sub-21/sub-21-_-NPC_cribs||Babies sleep in cribs. 653 | sub-21/sub-21-_-NPC_cup||I drink from a cup. 654 | sub-21/sub-21-_-NPC_dream||She woke up after a bad dream. 655 | sub-21/sub-21-_-NPC_frog||Ribbit goes the frog. 656 | sub-21/sub-21-_-NPC_fur||Cats have soft fur. 657 | sub-21/sub-21-_-NPC_mouse||The cat chased the mouse. 658 | sub-21/sub-21-_-NPC_one||First place is number one. 659 | sub-21/sub-21-_-NPC_pink||Pigs are colored pink. 660 | sub-21/sub-21-_-NPC_snake||Hiss goes the snake. 661 | sub-21/sub-21-_-NPI_bear(pie)||The woods had a big brown scary pie. 662 | sub-21/sub-21-_-NPI_bite(north)||He had an itchy bug north. 663 | sub-21/sub-21-_-NPI_cages(woman)||Animals are locked in woman. 664 | sub-21/sub-21-_-NPI_card(fly)||I signed a birthday fly. 665 | sub-21/sub-21-_-NPI_cliff(sheep)||He stood at the edge of a sheep. 666 | sub-21/sub-21-_-NPI_cry(fork)||When I am sad I fork. 667 | sub-21/sub-21-_-NPI_dare(brick)||Lets play truth or brick. 668 | sub-21/sub-21-_-NPI_dice(art)||In games I roll the art. 669 | sub-21/sub-21-_-NPI_down(tie)||She told me to calm tie. 670 | sub-21/sub-21-_-NPI_fall(duck)||After summer is duck. 671 | sub-21/sub-21-_-NPI_fork(cleats)||He eats steak with a knife and cleats. 672 | sub-21/sub-21-_-NPI_gel(sign)||He styled his hair with sign. 673 | sub-21/sub-21-_-NPI_night(inch)||It is dark at inch. 674 | sub-21/sub-21-_-NPI_nine(cat)||The number before ten is cat. 675 | sub-21/sub-21-_-NPI_one(cold)||First place is number cold. 676 | sub-21/sub-21-_-NPI_out(ant)||Three strikes and you are ant. 677 | sub-21/sub-21-_-NPI_pink(door)||Pigs are colored door. 678 | sub-21/sub-21-_-NPI_plank(shark)||The pirate said walk the shark. 679 | sub-21/sub-21-_-NPI_popped(square)||The water balloon was so full it square. 680 | sub-21/sub-21-_-NPI_round(kiss)||The ball is not square but kiss. 681 | sub-22/sub-22-_-(LD5_Eliminated)NPC_live||The woods are where bears live. 682 | sub-22/sub-22-_-NPC_aisle||The bride walked down the aisle. 683 | sub-22/sub-22-_-NPC_animals||The sign said do not feed the animals. 684 | sub-22/sub-22-_-NPC_apple||His favorite fruit was a red crisp apple. 685 | sub-22/sub-22-_-NPC_arm||I have two legs and two arms. 686 | sub-22/sub-22-_-NPC_bag||He puts his lunch in a paper bag. 687 | sub-22/sub-22-_-NPC_bank||My mom puts money in the bank. 688 | sub-22/sub-22-_-NPC_button||On the remote he clicked the button. 689 | sub-22/sub-22-_-NPC_cart||At the store I pushed the cart. 690 | sub-22/sub-22-_-NPC_chores||My mom makes me do household chores. 691 | sub-22/sub-22-_-NPC_clothes||My parents fold my clothes. 692 | sub-22/sub-22-_-NPC_cribs||Babies sleep in cribs. 693 | sub-22/sub-22-_-NPC_cup||I drink from a cup. 694 | sub-22/sub-22-_-NPC_dream||She woke up after a bad dream. 695 | sub-22/sub-22-_-NPC_frog||Ribbit goes the frog. 696 | sub-22/sub-22-_-NPC_fur||Cats have soft fur. 697 | sub-22/sub-22-_-NPC_mouse||The cat chased the mouse. 698 | sub-22/sub-22-_-NPC_one||First place is number one. 699 | sub-22/sub-22-_-NPC_pink||Pigs are colored pink. 700 | sub-22/sub-22-_-NPC_snake||Hiss goes the snake. 701 | sub-22/sub-22-_-NPI_bear(pie)||The woods had a big brown scary pie. 702 | sub-22/sub-22-_-NPI_bite(north)||He had an itchy bug north. 703 | sub-22/sub-22-_-NPI_cages(woman)||Animals are locked in woman. 704 | sub-22/sub-22-_-NPI_card(fly)||I signed a birthday fly. 705 | sub-22/sub-22-_-NPI_cliff(sheep)||He stood at the edge of a sheep. 706 | sub-22/sub-22-_-NPI_cry(fork)||When I am sad I fork. 707 | sub-22/sub-22-_-NPI_dare(brick)||Lets play truth or brick. 708 | sub-22/sub-22-_-NPI_dice(art)||In games I roll the art. 709 | sub-22/sub-22-_-NPI_down(tie)||She told me to calm tie. 710 | sub-22/sub-22-_-NPI_fall(duck)||After summer is duck. 711 | sub-22/sub-22-_-NPI_fork(cleats)||He eats steak with a knife and cleats. 712 | sub-22/sub-22-_-NPI_gel(sign)||He styled his hair with sign. 713 | sub-22/sub-22-_-NPI_night(inch)||It is dark at inch. 714 | sub-22/sub-22-_-NPI_nine(cat)||The number before ten is cat. 715 | sub-22/sub-22-_-NPI_one(cold)||First place is number cold. 716 | sub-22/sub-22-_-NPI_out(ant)||Three strikes and you are ant. 717 | sub-22/sub-22-_-NPI_pink(door)||Pigs are colored door. 718 | sub-22/sub-22-_-NPI_plank(shark)||The pirate said walk the shark. 719 | sub-22/sub-22-_-NPI_popped(square)||The water balloon was so full it square. 720 | sub-22/sub-22-_-NPI_round(kiss)||The ball is not square but kiss. -------------------------------------------------------------------------------- /filelists/n400/filelist_test_unseen_subject_text.txt: -------------------------------------------------------------------------------- 1 | sub-23/sub-23-_-(LD5_Eliminated)NPC_beard||Santa has a big white beard. 2 | sub-23/sub-23-_-(LD5_Eliminated)NPC_bee||I was stung by a buzzing bee. 3 | sub-23/sub-23-_-(LD5_Eliminated)NPC_bench||He liked to sit on the park bench. 4 | sub-23/sub-23-_-(LD5_Eliminated)NPC_chain||Prisoners walk with a ball and chain. 5 | sub-23/sub-23-_-(LD5_Eliminated)NPC_cleats||For soccer you wear cleats. 6 | sub-23/sub-23-_-(LD5_Eliminated)NPC_coal||Santa gives bad kids coal. 7 | sub-23/sub-23-_-(LD5_Eliminated)NPC_cut||I got a hair cut. 8 | sub-23/sub-23-_-(LD5_Eliminated)NPC_cute||She thinks all baby animals are cute. 9 | sub-23/sub-23-_-(LD5_Eliminated)NPC_eggs||For breakfast she has bacon and eggs. 10 | sub-23/sub-23-_-(LD5_Eliminated)NPC_found||My hat was in the lost and found. 11 | sub-23/sub-23-_-(LD5_Eliminated)NPC_fried||He likes his chicken deep fried. 12 | sub-23/sub-23-_-(LD5_Eliminated)NPC_holes||My dog digs holes. 13 | sub-23/sub-23-_-(LD5_Eliminated)NPC_pole||Santa lives at the north pole. 14 | sub-23/sub-23-_-(LD5_Eliminated)NPC_rings||Before school starts the bell rings. 15 | sub-23/sub-23-_-(LD5_Eliminated)NPC_roar||The lion says roar. 16 | sub-23/sub-23-_-(LD5_Eliminated)NPC_school||Jeff is too cool for school. 17 | sub-23/sub-23-_-(LD5_Eliminated)NPC_warm||The water was luke warm. 18 | sub-23/sub-23-_-(LD5_Eliminated)NPC_water||The boat floats on the water. 19 | sub-23/sub-23-_-(LD5_Eliminated)NPC_weight||The girl gained weight. 20 | sub-23/sub-23-_-(LD5_Eliminated)NPI_beard(catch)||Santa has a big white catch. 21 | sub-23/sub-23-_-(LD5_Eliminated)NPI_bee(burn)||I was stung by a buzzing burn. 22 | sub-23/sub-23-_-(LD5_Eliminated)NPI_bench(grade)||He liked to sit on the park grade. 23 | sub-23/sub-23-_-(LD5_Eliminated)NPI_chain(flood)||Prisoners walk with a ball and flood. 24 | sub-23/sub-23-_-(LD5_Eliminated)NPI_cleats(wood)||For soccer you wear wood. 25 | sub-23/sub-23-_-(LD5_Eliminated)NPI_coal(chain)||Santa gives bad kids chain. 26 | sub-23/sub-23-_-(LD5_Eliminated)NPI_cut(boat)||I got a hair boat. 27 | sub-23/sub-23-_-(LD5_Eliminated)NPI_cute(fence)||She thinks all baby animals are fence. 28 | sub-23/sub-23-_-(LD5_Eliminated)NPI_eggs(brake)||For breakfast she has bacon and brake. 29 | sub-23/sub-23-_-(LD5_Eliminated)NPI_found(goat)||My hat was in the lost and goat. 30 | sub-23/sub-23-_-(LD5_Eliminated)NPI_fried(mom)||He likes his chicken deep mom. 31 | sub-23/sub-23-_-(LD5_Eliminated)NPI_holes(spring)||My dog digs spring. 32 | sub-23/sub-23-_-(LD5_Eliminated)NPI_live(truck)||The woods are where bears truck. 33 | sub-23/sub-23-_-(LD5_Eliminated)NPI_pole(ring)||Santa lives at the north ring. 34 | sub-23/sub-23-_-(LD5_Eliminated)NPI_rings(style)||Before school starts the bell style. 35 | sub-23/sub-23-_-(LD5_Eliminated)NPI_roar(mold)||The lion says mold. 36 | sub-23/sub-23-_-(LD5_Eliminated)NPI_school(cut)||Jeff is too cool for cut. 37 | sub-23/sub-23-_-(LD5_Eliminated)NPI_warm(skunk)||The water was luke skunk. 38 | sub-23/sub-23-_-(LD5_Eliminated)NPI_water(fur)||The boat floats on the fur. 39 | sub-23/sub-23-_-(LD5_Eliminated)NPI_weight(sky)||The girl gained sky. 40 | sub-23/sub-23-_-NPC_ants||The picnic was ruined by crawling ants. 41 | sub-23/sub-23-_-NPC_axe||My dad chops wood with an axe. 42 | sub-23/sub-23-_-NPC_bake||The cake is in the oven to bake. 43 | sub-23/sub-23-_-NPC_barn||The farmer's animals live in the barn. 44 | sub-23/sub-23-_-NPC_bat||The boy hit the ball with a bat. 45 | sub-23/sub-23-_-NPC_bath||To get clean I take a bath. 46 | sub-23/sub-23-_-NPC_bear||The woods had a big brown scary bear. 47 | sub-23/sub-23-_-NPC_beat||The drummer keeps the beat. 48 | sub-23/sub-23-_-NPC_bed||At night I sleep in my bed. 49 | sub-23/sub-23-_-NPC_belt||Around my pants I wear a belt. 50 | sub-23/sub-23-_-NPC_bird||Tweet goes the bird. 51 | sub-23/sub-23-_-NPC_bite||He had an itchy bug bite. 52 | sub-23/sub-23-_-NPC_blinking||She lost the staring contest by blinking. 53 | sub-23/sub-23-_-NPC_blocks||The baby liked to stack blocks. 54 | sub-23/sub-23-_-NPC_bloom||In spring the flowers bloom. 55 | sub-23/sub-23-_-NPC_blue||The color of the sky is blue. 56 | sub-23/sub-23-_-NPC_boat||We went fishing on a boat. 57 | sub-23/sub-23-_-NPC_bones||The Museum showed dinosaur bones. 58 | sub-23/sub-23-_-NPC_books||He likes to read books. 59 | sub-23/sub-23-_-NPC_bottle||The baby drinks from his bottle. 60 | sub-23/sub-23-_-NPC_bow||On the present I tied a bow. 61 | sub-23/sub-23-_-NPC_bowl||He pours cereal into a bowl. 62 | sub-23/sub-23-_-NPC_brakes||The car stopped with its brakes. 63 | sub-23/sub-23-_-NPC_bread||She baked a loaf of bread. 64 | sub-23/sub-23-_-NPC_bricks||My fireplace is made of red bricks. 65 | sub-23/sub-23-_-NPC_bridge||Boats sail under the bridge. 66 | sub-23/sub-23-_-NPC_bright||I wear sunglasses because the sun is bright. 67 | sub-23/sub-23-_-NPC_broke||He wore a cast because his arm broke. 68 | sub-23/sub-23-_-NPC_broom||I sweep the floor with a broom. 69 | sub-23/sub-23-_-NPC_brown||A tree trunk is colored brown. 70 | sub-23/sub-23-_-NPC_bruise||I hit my arm and got a bruise. 71 | sub-23/sub-23-_-NPC_bulb||Tom replaced the light bulb. 72 | sub-23/sub-23-_-NPC_bus||To school I ride the yellow bus. 73 | sub-23/sub-23-_-NPC_bushes||Outside a man trimmed the bushes. 74 | sub-23/sub-23-_-NPC_cages||Animals are locked in cages. 75 | sub-23/sub-23-_-NPC_camping||We bring a tent to go camping. 76 | sub-23/sub-23-_-NPC_candles||She blew out the birthday candles. 77 | sub-23/sub-23-_-NPC_cane||The old man walked with a cane. 78 | sub-23/sub-23-_-NPC_cap||He screwed off the bottle cap. 79 | sub-23/sub-23-_-NPC_cape||Superman wears a red cape. 80 | sub-23/sub-23-_-NPC_car||He puts gas in his car. 81 | sub-23/sub-23-_-NPC_card||I signed a birthday card. 82 | sub-23/sub-23-_-NPC_castle||A princess lives in a castle. 83 | sub-23/sub-23-_-NPC_cat||Meow goes the cat. 84 | sub-23/sub-23-_-NPC_catch||I throw and you catch. 85 | sub-23/sub-23-_-NPC_caves||Bats live in caves. 86 | sub-23/sub-23-_-NPC_chalk||On the sidewalk I draw with chalk. 87 | sub-23/sub-23-_-NPC_cheese||Her favorite food is mac and cheese. 88 | sub-23/sub-23-_-NPC_chest||My heart was pounding in my chest. 89 | sub-23/sub-23-_-NPC_chick||A baby chicken is a chick. 90 | sub-23/sub-23-_-NPC_claws||The cat has sharp claws. 91 | sub-23/sub-23-_-NPC_clay||In art class we mold clay. 92 | sub-23/sub-23-_-NPC_clean||I keep my room clean. 93 | sub-23/sub-23-_-NPC_cliff||He stood at the edge of a cliff. 94 | sub-23/sub-23-_-NPC_clock||Tick tock goes the clock. 95 | sub-23/sub-23-_-NPC_clouds||In the sky are white fluffy clouds. 96 | sub-23/sub-23-_-NPC_clown||I laughed at the circus clown. 97 | sub-23/sub-23-_-NPC_clues||The detectives found many clues. 98 | sub-23/sub-23-_-NPC_coach||I was taught by my soccer coach. 99 | sub-23/sub-23-_-NPC_coat||I put on my rain coat. 100 | sub-23/sub-23-_-NPC_cob||She eats corn on the cob. 101 | sub-23/sub-23-_-NPC_cold||Winter is very cold. 102 | sub-23/sub-23-_-NPC_cone||He eats his ice cream on a cone. 103 | sub-23/sub-23-_-NPC_cop||He was pulled over by a cop. 104 | sub-23/sub-23-_-NPC_couch||I watch tv and sit on the couch. 105 | sub-23/sub-23-_-NPC_cough||My cold made me sneeze and cough. 106 | sub-23/sub-23-_-NPC_cow||Farmers milk the cows. 107 | sub-23/sub-23-_-NPC_crafts||I like arts and crafts. 108 | sub-23/sub-23-_-NPC_cream||I spray whipped cream. 109 | sub-23/sub-23-_-NPC_crown||A king wears a crown. 110 | sub-23/sub-23-_-NPC_crust||His mom makes her own pie crust. 111 | sub-23/sub-23-_-NPC_cry||When I am sad I cry. 112 | sub-23/sub-23-_-NPC_dancers||Ballerinas are great dancers. 113 | sub-23/sub-23-_-NPC_dare||Lets play truth or dare. 114 | sub-23/sub-23-_-NPC_deep||The pool was six feet deep. 115 | sub-23/sub-23-_-NPC_desk||In school I sit at a desk. 116 | sub-23/sub-23-_-NPC_dice||In games I roll the dice. 117 | sub-23/sub-23-_-NPC_dip||I love chips and dip. 118 | sub-23/sub-23-_-NPC_dirt||Worms crawl in the dirt. 119 | sub-23/sub-23-_-NPC_dog||Bark says the dog. 120 | sub-23/sub-23-_-NPC_dolls||My sister plays with Barbie dolls. 121 | sub-23/sub-23-_-NPC_door||He shut the door. 122 | sub-23/sub-23-_-NPC_down||She told me to calm down. 123 | sub-23/sub-23-_-NPC_dress||The bride wears a white dress. 124 | sub-23/sub-23-_-NPC_drill||At school there was a fire drill. 125 | sub-23/sub-23-_-NPC_duck||Quack goes the duck. 126 | sub-23/sub-23-_-NPC_dust||The old table was covered in dust. 127 | sub-23/sub-23-_-NPC_ears||I hear with my ears. 128 | sub-23/sub-23-_-NPC_earth||We live on planet earth. 129 | sub-23/sub-23-_-NPC_eats||The baby wears a bib when he eats. 130 | sub-23/sub-23-_-NPC_eyes||I see with my eyes. 131 | sub-23/sub-23-_-NPC_fake||The diamond was not real it was fake. 132 | sub-23/sub-23-_-NPC_fall||After summer is fall. 133 | sub-23/sub-23-_-NPC_farm||Cows live on a farm. 134 | sub-23/sub-23-_-NPC_fast||Cheetahs are really fast. 135 | sub-23/sub-23-_-NPC_fat||Goats are skinny and pigs are fat. 136 | sub-23/sub-23-_-NPC_fear||The brave man had no fear. 137 | sub-23/sub-23-_-NPC_fight||The angry boys got in a fight. 138 | sub-23/sub-23-_-NPC_fin||Sharks have a tail fin. 139 | sub-23/sub-23-_-NPC_fingers||On one hand I have five fingers. 140 | sub-23/sub-23-_-NPC_fish||On the boat my dad caught a fish. 141 | sub-23/sub-23-_-NPC_fist||I punched the bag with my fist. 142 | sub-23/sub-23-_-NPC_fixed||The toy broke and needed to be fixed. 143 | sub-23/sub-23-_-NPC_flag||He waves an American flag. 144 | sub-23/sub-23-_-NPC_flat||The earth is round not flat. 145 | sub-23/sub-23-_-NPC_flip||On the trampoline he did a flip. 146 | sub-23/sub-23-_-NPC_flop||At the pool he did a belly flop. 147 | sub-23/sub-23-_-NPC_fly||She swatted the fly. 148 | sub-23/sub-23-_-NPC_foot||I put the sock on my foot. 149 | sub-23/sub-23-_-NPC_fork||He eats steak with a knife and fork. 150 | sub-23/sub-23-_-NPC_freeze||In the winter the pond will freeze. 151 | sub-23/sub-23-_-NPC_fridge||The Jell-O had to chill in the fridge. 152 | sub-23/sub-23-_-NPC_friend||I play with my best friend. 153 | sub-23/sub-23-_-NPC_fruit||He likes to eat fresh fruit. 154 | sub-23/sub-23-_-NPC_game||His dad watches the football game. 155 | sub-23/sub-23-_-NPC_gel||He styled his hair with gel. 156 | sub-23/sub-23-_-NPC_ghosts||The house was haunted by ghosts. 157 | sub-23/sub-23-_-NPC_gills||Fish breathe through gills. 158 | sub-23/sub-23-_-NPC_glass||Windows are made of glass. 159 | sub-23/sub-23-_-NPC_gloves||On his hands are a pair of gloves. 160 | sub-23/sub-23-_-NPC_glue||The paper sticks together like glue. 161 | sub-23/sub-23-_-NPC_goal||The soccer team scored a goal. 162 | sub-23/sub-23-_-NPC_goat||Bah says the sheep and goat. 163 | sub-23/sub-23-_-NPC_gold||Wedding rings are made of gold. 164 | sub-23/sub-23-_-NPC_gone||The cookies were eaten until they were gone. 165 | sub-23/sub-23-_-NPC_goose||We played duck duck goose. 166 | sub-23/sub-23-_-NPC_grade||In school he is in first grade. 167 | sub-23/sub-23-_-NPC_grapes||Raisins are dried grapes. 168 | sub-23/sub-23-_-NPC_gray||The rainy sky was colored gray. 169 | sub-23/sub-23-_-NPC_green||The grass is colored green. 170 | sub-23/sub-23-_-NPC_grill||He cooked hotdogs outside on the grill. 171 | sub-23/sub-23-_-NPC_guess||On an answer he made a lucky guess. 172 | sub-23/sub-23-_-NPC_gum||He is chewing bubble gum. 173 | sub-23/sub-23-_-NPC_hair||Bald people do not have hair. 174 | sub-23/sub-23-_-NPC_hands||I clap my hands. 175 | sub-23/sub-23-_-NPC_hug||Charlie wanted a kiss and a hug. 176 | sub-23/sub-23-_-NPC_inches||This year he has grown two inches. 177 | sub-23/sub-23-_-NPC_job||His mom quit her job. 178 | sub-23/sub-23-_-NPC_kitchen||She cooks in the kitchen. 179 | sub-23/sub-23-_-NPC_lawn||My dad mows the lawn. 180 | sub-23/sub-23-_-NPC_legs||Dogs have four legs. 181 | sub-23/sub-23-_-NPC_lie||Not telling the truth is telling a lie. 182 | sub-23/sub-23-_-NPC_nests||Birds like to build nests. 183 | sub-23/sub-23-_-NPC_nice||The opposite of mean is nice. 184 | sub-23/sub-23-_-NPC_night||It is dark at night. 185 | sub-23/sub-23-_-NPC_nine||The number before ten is nine. 186 | sub-23/sub-23-_-NPC_out||Three strikes and you are out! 187 | sub-23/sub-23-_-NPC_outside||He put on his shoes to go outside. 188 | sub-23/sub-23-_-NPC_pants||He pulled up his pants. 189 | sub-23/sub-23-_-NPC_party||He had fun at the birthday party. 190 | sub-23/sub-23-_-NPC_phone||He called Helen on the phone. 191 | sub-23/sub-23-_-NPC_pictures||I like to draw pictures. 192 | sub-23/sub-23-_-NPC_pipe||His grandpa smoked from a pipe. 193 | sub-23/sub-23-_-NPC_plank||The pirate said walk the plank! 194 | sub-23/sub-23-_-NPC_popped||The water balloon was so full it popped. 195 | sub-23/sub-23-_-NPC_present||For his birthday I wrapped a present. 196 | sub-23/sub-23-_-NPC_round||The ball is not square but round. 197 | sub-23/sub-23-_-NPC_shot||The doctor gave him a flu shot. 198 | sub-23/sub-23-_-NPC_skunk||His dog was sprayed by a skunk. 199 | sub-23/sub-23-_-NPC_solve||He had a mystery to solve. 200 | sub-23/sub-23-_-NPC_spider||There are eight legs on a spider. 201 | sub-23/sub-23-_-NPC_steal||A thief likes to steal. 202 | sub-23/sub-23-_-NPC_store||We buy toys at the toy store. 203 | sub-23/sub-23-_-NPC_stove||He burned his hand on the stove. 204 | sub-23/sub-23-_-NPC_string||His guitar had a broken string. 205 | sub-23/sub-23-_-NPC_study||He failed the test because he didn't study. 206 | sub-23/sub-23-_-NPC_sun||The clouds hid the sun. 207 | sub-23/sub-23-_-NPC_tank||Mom filled up her gas tank. 208 | sub-23/sub-23-_-NPC_tear||Down the sad woman's cheek came a tear. 209 | sub-23/sub-23-_-NPC_test||I cheated on the test. 210 | sub-23/sub-23-_-NPC_thorns||The rose had sharp thorns. 211 | sub-23/sub-23-_-NPC_three||I count one two three. 212 | sub-23/sub-23-_-NPC_toilet||I flush the toilet. 213 | sub-23/sub-23-_-NPC_trees||A monkey climbs trees. 214 | sub-23/sub-23-_-NPC_truck||His dad drove a delivery truck. 215 | sub-23/sub-23-_-NPC_walk||Babies crawl before they walk. 216 | sub-23/sub-23-_-NPC_wear||She had no clothes to wear. 217 | sub-23/sub-23-_-NPC_wet||The water got her hair wet. 218 | sub-23/sub-23-_-NPC_whales||The biggest animals in the sea are whales. 219 | sub-23/sub-23-_-NPC_wind||My hair was blown by the wind. 220 | sub-23/sub-23-_-NPC_wood||He chopped the wood. 221 | sub-23/sub-23-_-NPI_aisle(chicken)||The bride walked down the chicken. 222 | sub-23/sub-23-_-NPI_animals(trampoline)||The sign said do not feed the trampoline. 223 | sub-23/sub-23-_-NPI_ants(fish)||The picnic was ruined by crawling fish. 224 | sub-23/sub-23-_-NPI_apple(balloon)||His favorite fruit was a red crisp balloon. 225 | sub-23/sub-23-_-NPI_arm(stack)||I have two legs and two stack. 226 | sub-23/sub-23-_-NPI_axe(grass)||My dad chops wood with an grass. 227 | sub-23/sub-23-_-NPI_bag(bell)||He puts his lunch in a paper bell. 228 | sub-23/sub-23-_-NPI_bake(milk)||The cake is in the oven to milk. 229 | sub-23/sub-23-_-NPI_bank(hike)||My mom puts money in the hike. 230 | sub-23/sub-23-_-NPI_barn(claw)||The farmer's animals live in the claw. 231 | sub-23/sub-23-_-NPI_bat(gel)||The boy hit the ball with a gel. 232 | sub-23/sub-23-_-NPI_bath(roar)||To get clean I take a roar. 233 | sub-23/sub-23-_-NPI_beat(sneeze)||The drummer keeps the sneeze. 234 | sub-23/sub-23-_-NPI_bed(frog)||At night I sleep in my frog. 235 | sub-23/sub-23-_-NPI_belt(bed)||Around my pants I wear a bed. 236 | sub-23/sub-23-_-NPI_bird(thorn)||Tweet goes the thorn. 237 | sub-23/sub-23-_-NPI_blinking(belly)||She lost the staring contest by belly. 238 | sub-23/sub-23-_-NPI_blocks(night)||The baby liked to stack night. 239 | sub-23/sub-23-_-NPI_bloom(school)||In spring the flowers school. 240 | sub-23/sub-23-_-NPI_blue(arm)||The color of the sky is arm. 241 | sub-23/sub-23-_-NPI_boat(cop)||We went fishing on a cop. 242 | sub-23/sub-23-_-NPI_bones(wind)||The Museum showed dinosaur wind. 243 | sub-23/sub-23-_-NPI_books(tear)||He likes to read tear. 244 | sub-23/sub-23-_-NPI_bottle(money)||The baby drinks from his money. 245 | sub-23/sub-23-_-NPI_bow(eye)||On the present I tied a eye. 246 | sub-23/sub-23-_-NPI_bowl(test)||He pours cereal into a test. 247 | sub-23/sub-23-_-NPI_brakes(cob)||The car stopped with its cob. 248 | sub-23/sub-23-_-NPI_bread(sea)||She baked a loaf of sea. 249 | sub-23/sub-23-_-NPI_bricks(leg)||My fireplace is made of red leg. 250 | sub-23/sub-23-_-NPI_bridge(steak)||Boats sail under the steak. 251 | sub-23/sub-23-_-NPI_bright(gills)||I wear sunglasses because the sun is gills. 252 | sub-23/sub-23-_-NPI_broke(pond)||He wore a cast because his arm pond. 253 | sub-23/sub-23-_-NPI_broom(bus)||I sweep the floor with a bus. 254 | sub-23/sub-23-_-NPI_brown(job)||A tree trunk is colored job. 255 | sub-23/sub-23-_-NPI_bruise(rain)||I hit my arm and got a rain. 256 | sub-23/sub-23-_-NPI_bulb(farm)||Tom replaced the light farm. 257 | sub-23/sub-23-_-NPI_bus(egg)||To school I ride the yellow egg. 258 | sub-23/sub-23-_-NPI_bushes(oven)||Outside a man trimmed the oven. 259 | sub-23/sub-23-_-NPI_button(hotdog)||On the remote he clicked the hotdog. 260 | sub-23/sub-23-_-NPI_cake(clue)||He baked a birthday clue. 261 | sub-23/sub-23-_-NPI_camping(bacon)||We bring a tent to go bacon. 262 | sub-23/sub-23-_-NPI_candles(planet)||She blew out the birthday planet. 263 | sub-23/sub-23-_-NPI_cane(fin)||The old man walked with a fin. 264 | sub-23/sub-23-_-NPI_cap(chore)||He screwed off the bottle chore. 265 | sub-23/sub-23-_-NPI_cape(pole)||Superman wears a red pole. 266 | sub-23/sub-23-_-NPI_car(chest)||He puts gas in his chest. 267 | sub-23/sub-23-_-NPI_cart(score)||At the store I pushed the score. 268 | sub-23/sub-23-_-NPI_castle(doctor)||A princess lives in a doctor. 269 | sub-23/sub-23-_-NPI_cat(spray)||Meow goes the spray. 270 | sub-23/sub-23-_-NPI_catch(road)||I throw and you road. 271 | sub-23/sub-23-_-NPI_caves(string)||Bats live in string. 272 | sub-23/sub-23-_-NPI_chalk(fat)||On the sidewalk I draw with fat. 273 | sub-23/sub-23-_-NPI_cheese(clock)||Her favorite food is mac and clock. 274 | sub-23/sub-23-_-NPI_chest(sock)||My heart was pounding in my sock. 275 | sub-23/sub-23-_-NPI_chick(bruise)||A baby chicken is a bruise. 276 | sub-23/sub-23-_-NPI_chores(cave)||My mom makes me do household cave. 277 | sub-23/sub-23-_-NPI_claws(floor)||The cat has sharp floor. 278 | sub-23/sub-23-_-NPI_clay(gas)||In art class we mold gas. 279 | sub-23/sub-23-_-NPI_clean(cheek)||I keep my room cheek. 280 | sub-23/sub-23-_-NPI_clock(tree)||Tick tock goes the tree. 281 | sub-23/sub-23-_-NPI_clothes(bite)||My parents fold my bite. 282 | sub-23/sub-23-_-NPI_clouds(dirt)||In the sky are white fluffy dirt. 283 | sub-23/sub-23-_-NPI_clown(dice)||I laughed at the circus dice. 284 | sub-23/sub-23-_-NPI_clues(year)||The detectives found many year. 285 | sub-23/sub-23-_-NPI_coach(goose)||I was taught by my soccer goose. 286 | sub-23/sub-23-_-NPI_coat(loaf)||I put on my rain loaf. 287 | sub-23/sub-23-_-NPI_cob(lie)||She eats corn on the lie. 288 | sub-23/sub-23-_-NPI_cold(bib)||Winter is very bib. 289 | sub-23/sub-23-_-NPI_cone(hug)||He eats his ice cream on a hug. 290 | sub-23/sub-23-_-NPI_cop(cake)||He was pulled over by a cake. 291 | sub-23/sub-23-_-NPI_couch(bulb)||I watch tv and sit on the bulb. 292 | sub-23/sub-23-_-NPI_cough(friend)||My cold made me sneeze and friend. 293 | sub-23/sub-23-_-NPI_cow(axe)||Farmers milk the axe. 294 | sub-23/sub-23-_-NPI_crafts(tank)||I like arts and tank. 295 | sub-23/sub-23-_-NPI_cream(couch)||I spray whipped couch. 296 | sub-23/sub-23-_-NPI_cribs(shot)||Babies sleep in shot. 297 | sub-23/sub-23-_-NPI_crown(bench)||A king wears a bench. 298 | sub-23/sub-23-_-NPI_crust(foot)||His mom makes her own pie foot. 299 | sub-23/sub-23-_-NPI_dancers(pirate)||Ballerinas are great pirate. 300 | sub-23/sub-23-_-NPI_deep(bone)||The pool was six feet bone. 301 | sub-23/sub-23-_-NPI_desk(ice)||In school I sit at a ice. 302 | sub-23/sub-23-_-NPI_dip(cage)||I love chips and cage. 303 | sub-23/sub-23-_-NPI_dirt(screw)||Worms crawl in the screw. 304 | sub-23/sub-23-_-NPI_dog(chalk)||Bark says the chalk. 305 | sub-23/sub-23-_-NPI_dolls(worm)||My sister plays with Barbie worm. 306 | sub-23/sub-23-_-NPI_door(smoke)||He shut the smoke. 307 | sub-23/sub-23-_-NPI_dream(lawn)||She woke up after a bad lawn. 308 | sub-23/sub-23-_-NPI_dress(fun)||The bride wears a white fun. 309 | sub-23/sub-23-_-NPI_drill(bowl)||At school there was a fire bowl. 310 | sub-23/sub-23-_-NPI_duck(bank)||Quack goes the bank. 311 | sub-23/sub-23-_-NPI_dust(cook)||The old table was covered in cook. 312 | sub-23/sub-23-_-NPI_ears(bee)||I hear with my bee. 313 | sub-23/sub-23-_-NPI_earth(rose)||We live on planet rose. 314 | sub-23/sub-23-_-NPI_eats(ball)||The baby wears a bib when he ball. 315 | sub-23/sub-23-_-NPI_eyes(hip)||I see with my hip. 316 | sub-23/sub-23-_-NPI_fake(crust)||The diamond was not real it was crust. 317 | sub-23/sub-23-_-NPI_farm(bird)||Cows live on a bird. 318 | sub-23/sub-23-_-NPI_fast(thief)||Cheetahs are really thief. 319 | sub-23/sub-23-_-NPI_fat(glass)||Goats are skinny and pigs are glass. 320 | sub-23/sub-23-_-NPI_fear(walk)||The brave man had no walk. 321 | sub-23/sub-23-_-NPI_fight(mouse)||The angry boys got in a mouse. 322 | sub-23/sub-23-_-NPI_fin(desk)||Sharks have a tail desk. 323 | sub-23/sub-23-_-NPI_fingers(kitchen)||On one hand I have five kitchen. 324 | sub-23/sub-23-_-NPI_fish(cry)||On the boat my dad caught a cry. 325 | sub-23/sub-23-_-NPI_fist(bath)||I punched the bag with my bath. 326 | sub-23/sub-23-_-NPI_fixed(dog)||The toy broke and needed to be dog. 327 | sub-23/sub-23-_-NPI_flag(touch)||He waves an American touch. 328 | sub-23/sub-23-_-NPI_flat(cow)||The earth is round not cow. 329 | sub-23/sub-23-_-NPI_flip(dark)||On the trampoline he did a dark. 330 | sub-23/sub-23-_-NPI_flop(toy)||At the pool he did a belly toy. 331 | sub-23/sub-23-_-NPI_fly(edge)||She swatted the edge. 332 | sub-23/sub-23-_-NPI_foot(shout)||I put the sock on my shout. 333 | sub-23/sub-23-_-NPI_freeze(sun)||In the winter the pond will sun. 334 | sub-23/sub-23-_-NPI_fridge(pig)||The Jell-O had to chill in the pig. 335 | sub-23/sub-23-_-NPI_friend(pound)||I play with my best pound. 336 | sub-23/sub-23-_-NPI_frog(glove)||Ribbit goes the glove. 337 | sub-23/sub-23-_-NPI_fruit(tent)||He likes to eat fresh tent. 338 | sub-23/sub-23-_-NPI_fur(dad)||Cats have soft dad. 339 | sub-23/sub-23-_-NPI_game(coal)||His dad watches the football coal. 340 | sub-23/sub-23-_-NPI_ghosts(strike)||The house was haunted by strike. 341 | sub-23/sub-23-_-NPI_gills(doll)||Fish breathe through doll. 342 | sub-23/sub-23-_-NPI_glass(shoe)||Windows are made of shoe. 343 | sub-23/sub-23-_-NPI_gloves(car)||On his hands are a pair of car. 344 | sub-23/sub-23-_-NPI_glue(cliff)||The paper sticks together like cliff. 345 | sub-23/sub-23-_-NPI_goal(hair)||The soccer team scored a hair. 346 | sub-23/sub-23-_-NPI_goat(bread)||Bah says the sheep and bread. 347 | sub-23/sub-23-_-NPI_gold(fridge)||Wedding rings are made of fridge. 348 | sub-23/sub-23-_-NPI_gone(feet)||The cookies were eaten until they were feet. 349 | sub-23/sub-23-_-NPI_goose(pipe)||We played duck duck pipe. 350 | sub-23/sub-23-_-NPI_grade(grill)||In school he is in first grill. 351 | sub-23/sub-23-_-NPI_grapes(man)||Raisins are dried man. 352 | sub-23/sub-23-_-NPI_gray(bald)||The rainy sky was colored bald. 353 | sub-23/sub-23-_-NPI_green(lock)||The grass is colored lock. 354 | sub-23/sub-23-_-NPI_grill(heart)||He cooked hotdogs outside on the heart. 355 | sub-23/sub-23-_-NPI_guess(grape)||On an answer he made a lucky grape. 356 | sub-23/sub-23-_-NPI_gum(coat)||He is chewing bubble coat. 357 | sub-23/sub-23-_-NPI_hair(nest)||Bald people do not have nest. 358 | sub-23/sub-23-_-NPI_hands(king)||I clap my king. 359 | sub-23/sub-23-_-NPI_hug(fear)||Charlie wanted a kiss and a fear. 360 | sub-23/sub-23-_-NPI_inches(cookie)||This year he has grown two cookie. 361 | sub-23/sub-23-_-NPI_job(pants)||His mom quit her pants. 362 | sub-23/sub-23-_-NPI_kitchen(baby)||She cooks in the baby. 363 | sub-23/sub-23-_-NPI_lawn(weight)||My dad mows the weight. 364 | sub-23/sub-23-_-NPI_legs(barn)||Dogs have four barn. 365 | sub-23/sub-23-_-NPI_lie(cup)||Not telling the truth is telling a cup. 366 | sub-23/sub-23-_-NPI_mouse(store)||The cat chased the store. 367 | sub-23/sub-23-_-NPI_nests(drill)||Birds like to build drill. 368 | sub-23/sub-23-_-NPI_nice(gum)||The opposite of mean is gum. 369 | sub-23/sub-23-_-NPI_outside(apple)||He put on his shoes to go apple. 370 | sub-23/sub-23-_-NPI_pants(woods)||He pulled up his woods. 371 | sub-23/sub-23-_-NPI_party(sidewalk)||He had fun at the birthday sidewalk. 372 | sub-23/sub-23-_-NPI_phone(snake)||He called Helen on the snake. 373 | sub-23/sub-23-_-NPI_pictures(contest)||I like to draw contest. 374 | sub-23/sub-23-_-NPI_pipe(phone)||His grandpa smoked from a phone. 375 | sub-23/sub-23-_-NPI_present(sister)||For his birthday I wrapped a sister. 376 | sub-23/sub-23-_-NPI_shot(crown)||The doctor gave him a flu crown. 377 | sub-23/sub-23-_-NPI_skunk(broom)||His dog was sprayed by a broom. 378 | sub-23/sub-23-_-NPI_snake(play)||Hiss goes the play. 379 | sub-23/sub-23-_-NPI_solve(cloud)||He had a mystery to cloud. 380 | sub-23/sub-23-_-NPI_spider(summer)||There are eight legs on a summer. 381 | sub-23/sub-23-_-NPI_steal(earth)||A thief likes to earth. 382 | sub-23/sub-23-_-NPI_store(stick)||We buy toys at the toy stick. 383 | sub-23/sub-23-_-NPI_stove(kid)||He burned his hand on the kid. 384 | sub-23/sub-23-_-NPI_string(chick)||His guitar had a broken chick. 385 | sub-23/sub-23-_-NPI_study(princess)||He failed the test because he didn't princess. 386 | sub-23/sub-23-_-NPI_sun(lunch)||The clouds hid the lunch. 387 | sub-23/sub-23-_-NPI_tank(truth)||Mom filled up her gas truth. 388 | sub-23/sub-23-_-NPI_tear(corn)||Down the sad woman's cheek came a corn. 389 | sub-23/sub-23-_-NPI_test(dust)||I cheated on the dust. 390 | sub-23/sub-23-_-NPI_thorns(house)||The rose had sharp house. 391 | sub-23/sub-23-_-NPI_three(boy)||I count one two boy. 392 | sub-23/sub-23-_-NPI_toilet(window)||I flush the window. 393 | sub-23/sub-23-_-NPI_trees(book)||A monkey climbs book. 394 | sub-23/sub-23-_-NPI_truck(cap)||His dad drove a delivery break. 395 | sub-23/sub-23-_-NPI_walk(class)||Babies crawl before they class. 396 | sub-23/sub-23-_-NPI_wear(chief)||She had no clothes to chief. 397 | sub-23/sub-23-_-NPI_wet(whale)||The water got her hair whale. 398 | sub-23/sub-23-_-NPI_whales(dress)||The biggest animals in the sea are dress. 399 | sub-23/sub-23-_-NPI_wind(cheese)||My hair was blown by the cheese. 400 | sub-23/sub-23-_-NPI_wood(cough)||He chopped the cough. 401 | sub-24/sub-24-_-(LD5_Eliminated)NPC_beard||Santa has a big white beard. 402 | sub-24/sub-24-_-(LD5_Eliminated)NPC_bee||I was stung by a buzzing bee. 403 | sub-24/sub-24-_-(LD5_Eliminated)NPC_bench||He liked to sit on the park bench. 404 | sub-24/sub-24-_-(LD5_Eliminated)NPC_chain||Prisoners walk with a ball and chain. 405 | sub-24/sub-24-_-(LD5_Eliminated)NPC_cleats||For soccer you wear cleats. 406 | sub-24/sub-24-_-(LD5_Eliminated)NPC_coal||Santa gives bad kids coal. 407 | sub-24/sub-24-_-(LD5_Eliminated)NPC_cut||I got a hair cut. 408 | sub-24/sub-24-_-(LD5_Eliminated)NPC_cute||She thinks all baby animals are cute. 409 | sub-24/sub-24-_-(LD5_Eliminated)NPC_eggs||For breakfast she has bacon and eggs. 410 | sub-24/sub-24-_-(LD5_Eliminated)NPC_found||My hat was in the lost and found. 411 | sub-24/sub-24-_-(LD5_Eliminated)NPC_fried||He likes his chicken deep fried. 412 | sub-24/sub-24-_-(LD5_Eliminated)NPC_holes||My dog digs holes. 413 | sub-24/sub-24-_-(LD5_Eliminated)NPC_pole||Santa lives at the north pole. 414 | sub-24/sub-24-_-(LD5_Eliminated)NPC_rings||Before school starts the bell rings. 415 | sub-24/sub-24-_-(LD5_Eliminated)NPC_roar||The lion says roar. 416 | sub-24/sub-24-_-(LD5_Eliminated)NPC_school||Jeff is too cool for school. 417 | sub-24/sub-24-_-(LD5_Eliminated)NPC_warm||The water was luke warm. 418 | sub-24/sub-24-_-(LD5_Eliminated)NPC_water||The boat floats on the water. 419 | sub-24/sub-24-_-(LD5_Eliminated)NPC_weight||The girl gained weight. 420 | sub-24/sub-24-_-(LD5_Eliminated)NPI_beard(catch)||Santa has a big white catch. 421 | sub-24/sub-24-_-(LD5_Eliminated)NPI_bee(burn)||I was stung by a buzzing burn. 422 | sub-24/sub-24-_-(LD5_Eliminated)NPI_bench(grade)||He liked to sit on the park grade. 423 | sub-24/sub-24-_-(LD5_Eliminated)NPI_chain(flood)||Prisoners walk with a ball and flood. 424 | sub-24/sub-24-_-(LD5_Eliminated)NPI_cleats(wood)||For soccer you wear wood. 425 | sub-24/sub-24-_-(LD5_Eliminated)NPI_coal(chain)||Santa gives bad kids chain. 426 | sub-24/sub-24-_-(LD5_Eliminated)NPI_cut(boat)||I got a hair boat. 427 | sub-24/sub-24-_-(LD5_Eliminated)NPI_cute(fence)||She thinks all baby animals are fence. 428 | sub-24/sub-24-_-(LD5_Eliminated)NPI_eggs(brake)||For breakfast she has bacon and brake. 429 | sub-24/sub-24-_-(LD5_Eliminated)NPI_found(goat)||My hat was in the lost and goat. 430 | sub-24/sub-24-_-(LD5_Eliminated)NPI_fried(mom)||He likes his chicken deep mom. 431 | sub-24/sub-24-_-(LD5_Eliminated)NPI_holes(spring)||My dog digs spring. 432 | sub-24/sub-24-_-(LD5_Eliminated)NPI_live(truck)||The woods are where bears truck. 433 | sub-24/sub-24-_-(LD5_Eliminated)NPI_pole(ring)||Santa lives at the north ring. 434 | sub-24/sub-24-_-(LD5_Eliminated)NPI_rings(style)||Before school starts the bell style. 435 | sub-24/sub-24-_-(LD5_Eliminated)NPI_roar(mold)||The lion says mold. 436 | sub-24/sub-24-_-(LD5_Eliminated)NPI_school(cut)||Jeff is too cool for cut. 437 | sub-24/sub-24-_-(LD5_Eliminated)NPI_warm(skunk)||The water was luke skunk. 438 | sub-24/sub-24-_-(LD5_Eliminated)NPI_water(fur)||The boat floats on the fur. 439 | sub-24/sub-24-_-(LD5_Eliminated)NPI_weight(sky)||The girl gained sky. 440 | sub-24/sub-24-_-NPC_ants||The picnic was ruined by crawling ants. 441 | sub-24/sub-24-_-NPC_axe||My dad chops wood with an axe. 442 | sub-24/sub-24-_-NPC_bake||The cake is in the oven to bake. 443 | sub-24/sub-24-_-NPC_barn||The farmer's animals live in the barn. 444 | sub-24/sub-24-_-NPC_bat||The boy hit the ball with a bat. 445 | sub-24/sub-24-_-NPC_bath||To get clean I take a bath. 446 | sub-24/sub-24-_-NPC_bear||The woods had a big brown scary bear. 447 | sub-24/sub-24-_-NPC_beat||The drummer keeps the beat. 448 | sub-24/sub-24-_-NPC_bed||At night I sleep in my bed. 449 | sub-24/sub-24-_-NPC_belt||Around my pants I wear a belt. 450 | sub-24/sub-24-_-NPC_bird||Tweet goes the bird. 451 | sub-24/sub-24-_-NPC_bite||He had an itchy bug bite. 452 | sub-24/sub-24-_-NPC_blinking||She lost the staring contest by blinking. 453 | sub-24/sub-24-_-NPC_blocks||The baby liked to stack blocks. 454 | sub-24/sub-24-_-NPC_bloom||In spring the flowers bloom. 455 | sub-24/sub-24-_-NPC_blue||The color of the sky is blue. 456 | sub-24/sub-24-_-NPC_boat||We went fishing on a boat. 457 | sub-24/sub-24-_-NPC_bones||The Museum showed dinosaur bones. 458 | sub-24/sub-24-_-NPC_books||He likes to read books. 459 | sub-24/sub-24-_-NPC_bottle||The baby drinks from his bottle. 460 | sub-24/sub-24-_-NPC_bow||On the present I tied a bow. 461 | sub-24/sub-24-_-NPC_bowl||He pours cereal into a bowl. 462 | sub-24/sub-24-_-NPC_brakes||The car stopped with its brakes. 463 | sub-24/sub-24-_-NPC_bread||She baked a loaf of bread. 464 | sub-24/sub-24-_-NPC_bricks||My fireplace is made of red bricks. 465 | sub-24/sub-24-_-NPC_bridge||Boats sail under the bridge. 466 | sub-24/sub-24-_-NPC_bright||I wear sunglasses because the sun is bright. 467 | sub-24/sub-24-_-NPC_broke||He wore a cast because his arm broke. 468 | sub-24/sub-24-_-NPC_broom||I sweep the floor with a broom. 469 | sub-24/sub-24-_-NPC_brown||A tree trunk is colored brown. 470 | sub-24/sub-24-_-NPC_bruise||I hit my arm and got a bruise. 471 | sub-24/sub-24-_-NPC_bulb||Tom replaced the light bulb. 472 | sub-24/sub-24-_-NPC_bus||To school I ride the yellow bus. 473 | sub-24/sub-24-_-NPC_bushes||Outside a man trimmed the bushes. 474 | sub-24/sub-24-_-NPC_cages||Animals are locked in cages. 475 | sub-24/sub-24-_-NPC_camping||We bring a tent to go camping. 476 | sub-24/sub-24-_-NPC_candles||She blew out the birthday candles. 477 | sub-24/sub-24-_-NPC_cane||The old man walked with a cane. 478 | sub-24/sub-24-_-NPC_cap||He screwed off the bottle cap. 479 | sub-24/sub-24-_-NPC_cape||Superman wears a red cape. 480 | sub-24/sub-24-_-NPC_car||He puts gas in his car. 481 | sub-24/sub-24-_-NPC_card||I signed a birthday card. 482 | sub-24/sub-24-_-NPC_castle||A princess lives in a castle. 483 | sub-24/sub-24-_-NPC_cat||Meow goes the cat. 484 | sub-24/sub-24-_-NPC_catch||I throw and you catch. 485 | sub-24/sub-24-_-NPC_caves||Bats live in caves. 486 | sub-24/sub-24-_-NPC_chalk||On the sidewalk I draw with chalk. 487 | sub-24/sub-24-_-NPC_cheese||Her favorite food is mac and cheese. 488 | sub-24/sub-24-_-NPC_chest||My heart was pounding in my chest. 489 | sub-24/sub-24-_-NPC_chick||A baby chicken is a chick. 490 | sub-24/sub-24-_-NPC_claws||The cat has sharp claws. 491 | sub-24/sub-24-_-NPC_clay||In art class we mold clay. 492 | sub-24/sub-24-_-NPC_clean||I keep my room clean. 493 | sub-24/sub-24-_-NPC_cliff||He stood at the edge of a cliff. 494 | sub-24/sub-24-_-NPC_clock||Tick tock goes the clock. 495 | sub-24/sub-24-_-NPC_clouds||In the sky are white fluffy clouds. 496 | sub-24/sub-24-_-NPC_clown||I laughed at the circus clown. 497 | sub-24/sub-24-_-NPC_clues||The detectives found many clues. 498 | sub-24/sub-24-_-NPC_coach||I was taught by my soccer coach. 499 | sub-24/sub-24-_-NPC_coat||I put on my rain coat. 500 | sub-24/sub-24-_-NPC_cob||She eats corn on the cob. 501 | sub-24/sub-24-_-NPC_cold||Winter is very cold. 502 | sub-24/sub-24-_-NPC_cone||He eats his ice cream on a cone. 503 | sub-24/sub-24-_-NPC_cop||He was pulled over by a cop. 504 | sub-24/sub-24-_-NPC_couch||I watch tv and sit on the couch. 505 | sub-24/sub-24-_-NPC_cough||My cold made me sneeze and cough. 506 | sub-24/sub-24-_-NPC_cow||Farmers milk the cows. 507 | sub-24/sub-24-_-NPC_crafts||I like arts and crafts. 508 | sub-24/sub-24-_-NPC_cream||I spray whipped cream. 509 | sub-24/sub-24-_-NPC_crown||A king wears a crown. 510 | sub-24/sub-24-_-NPC_crust||His mom makes her own pie crust. 511 | sub-24/sub-24-_-NPC_cry||When I am sad I cry. 512 | sub-24/sub-24-_-NPC_dancers||Ballerinas are great dancers. 513 | sub-24/sub-24-_-NPC_dare||Lets play truth or dare. 514 | sub-24/sub-24-_-NPC_deep||The pool was six feet deep. 515 | sub-24/sub-24-_-NPC_desk||In school I sit at a desk. 516 | sub-24/sub-24-_-NPC_dice||In games I roll the dice. 517 | sub-24/sub-24-_-NPC_dip||I love chips and dip. 518 | sub-24/sub-24-_-NPC_dirt||Worms crawl in the dirt. 519 | sub-24/sub-24-_-NPC_dog||Bark says the dog. 520 | sub-24/sub-24-_-NPC_dolls||My sister plays with Barbie dolls. 521 | sub-24/sub-24-_-NPC_door||He shut the door. 522 | sub-24/sub-24-_-NPC_down||She told me to calm down. 523 | sub-24/sub-24-_-NPC_dress||The bride wears a white dress. 524 | sub-24/sub-24-_-NPC_drill||At school there was a fire drill. 525 | sub-24/sub-24-_-NPC_duck||Quack goes the duck. 526 | sub-24/sub-24-_-NPC_dust||The old table was covered in dust. 527 | sub-24/sub-24-_-NPC_ears||I hear with my ears. 528 | sub-24/sub-24-_-NPC_earth||We live on planet earth. 529 | sub-24/sub-24-_-NPC_eats||The baby wears a bib when he eats. 530 | sub-24/sub-24-_-NPC_eyes||I see with my eyes. 531 | sub-24/sub-24-_-NPC_fake||The diamond was not real it was fake. 532 | sub-24/sub-24-_-NPC_fall||After summer is fall. 533 | sub-24/sub-24-_-NPC_farm||Cows live on a farm. 534 | sub-24/sub-24-_-NPC_fast||Cheetahs are really fast. 535 | sub-24/sub-24-_-NPC_fat||Goats are skinny and pigs are fat. 536 | sub-24/sub-24-_-NPC_fear||The brave man had no fear. 537 | sub-24/sub-24-_-NPC_fight||The angry boys got in a fight. 538 | sub-24/sub-24-_-NPC_fin||Sharks have a tail fin. 539 | sub-24/sub-24-_-NPC_fingers||On one hand I have five fingers. 540 | sub-24/sub-24-_-NPC_fish||On the boat my dad caught a fish. 541 | sub-24/sub-24-_-NPC_fist||I punched the bag with my fist. 542 | sub-24/sub-24-_-NPC_fixed||The toy broke and needed to be fixed. 543 | sub-24/sub-24-_-NPC_flag||He waves an American flag. 544 | sub-24/sub-24-_-NPC_flat||The earth is round not flat. 545 | sub-24/sub-24-_-NPC_flip||On the trampoline he did a flip. 546 | sub-24/sub-24-_-NPC_flop||At the pool he did a belly flop. 547 | sub-24/sub-24-_-NPC_fly||She swatted the fly. 548 | sub-24/sub-24-_-NPC_foot||I put the sock on my foot. 549 | sub-24/sub-24-_-NPC_fork||He eats steak with a knife and fork. 550 | sub-24/sub-24-_-NPC_freeze||In the winter the pond will freeze. 551 | sub-24/sub-24-_-NPC_fridge||The Jell-O had to chill in the fridge. 552 | sub-24/sub-24-_-NPC_friend||I play with my best friend. 553 | sub-24/sub-24-_-NPC_fruit||He likes to eat fresh fruit. 554 | sub-24/sub-24-_-NPC_game||His dad watches the football game. 555 | sub-24/sub-24-_-NPC_gel||He styled his hair with gel. 556 | sub-24/sub-24-_-NPC_ghosts||The house was haunted by ghosts. 557 | sub-24/sub-24-_-NPC_gills||Fish breathe through gills. 558 | sub-24/sub-24-_-NPC_glass||Windows are made of glass. 559 | sub-24/sub-24-_-NPC_gloves||On his hands are a pair of gloves. 560 | sub-24/sub-24-_-NPC_glue||The paper sticks together like glue. 561 | sub-24/sub-24-_-NPC_goal||The soccer team scored a goal. 562 | sub-24/sub-24-_-NPC_goat||Bah says the sheep and goat. 563 | sub-24/sub-24-_-NPC_gold||Wedding rings are made of gold. 564 | sub-24/sub-24-_-NPC_gone||The cookies were eaten until they were gone. 565 | sub-24/sub-24-_-NPC_goose||We played duck duck goose. 566 | sub-24/sub-24-_-NPC_grade||In school he is in first grade. 567 | sub-24/sub-24-_-NPC_grapes||Raisins are dried grapes. 568 | sub-24/sub-24-_-NPC_gray||The rainy sky was colored gray. 569 | sub-24/sub-24-_-NPC_green||The grass is colored green. 570 | sub-24/sub-24-_-NPC_grill||He cooked hotdogs outside on the grill. 571 | sub-24/sub-24-_-NPC_guess||On an answer he made a lucky guess. 572 | sub-24/sub-24-_-NPC_gum||He is chewing bubble gum. 573 | sub-24/sub-24-_-NPC_hair||Bald people do not have hair. 574 | sub-24/sub-24-_-NPC_hands||I clap my hands. 575 | sub-24/sub-24-_-NPC_hug||Charlie wanted a kiss and a hug. 576 | sub-24/sub-24-_-NPC_inches||This year he has grown two inches. 577 | sub-24/sub-24-_-NPC_job||His mom quit her job. 578 | sub-24/sub-24-_-NPC_kitchen||She cooks in the kitchen. 579 | sub-24/sub-24-_-NPC_lawn||My dad mows the lawn. 580 | sub-24/sub-24-_-NPC_legs||Dogs have four legs. 581 | sub-24/sub-24-_-NPC_lie||Not telling the truth is telling a lie. 582 | sub-24/sub-24-_-NPC_nests||Birds like to build nests. 583 | sub-24/sub-24-_-NPC_nice||The opposite of mean is nice. 584 | sub-24/sub-24-_-NPC_night||It is dark at night. 585 | sub-24/sub-24-_-NPC_nine||The number before ten is nine. 586 | sub-24/sub-24-_-NPC_out||Three strikes and you are out! 587 | sub-24/sub-24-_-NPC_outside||He put on his shoes to go outside. 588 | sub-24/sub-24-_-NPC_pants||He pulled up his pants. 589 | sub-24/sub-24-_-NPC_party||He had fun at the birthday party. 590 | sub-24/sub-24-_-NPC_phone||He called Helen on the phone. 591 | sub-24/sub-24-_-NPC_pictures||I like to draw pictures. 592 | sub-24/sub-24-_-NPC_pipe||His grandpa smoked from a pipe. 593 | sub-24/sub-24-_-NPC_plank||The pirate said walk the plank! 594 | sub-24/sub-24-_-NPC_popped||The water balloon was so full it popped. 595 | sub-24/sub-24-_-NPC_present||For his birthday I wrapped a present. 596 | sub-24/sub-24-_-NPC_round||The ball is not square but round. 597 | sub-24/sub-24-_-NPC_shot||The doctor gave him a flu shot. 598 | sub-24/sub-24-_-NPC_skunk||His dog was sprayed by a skunk. 599 | sub-24/sub-24-_-NPC_solve||He had a mystery to solve. 600 | sub-24/sub-24-_-NPC_spider||There are eight legs on a spider. 601 | sub-24/sub-24-_-NPC_steal||A thief likes to steal. 602 | sub-24/sub-24-_-NPC_store||We buy toys at the toy store. 603 | sub-24/sub-24-_-NPC_stove||He burned his hand on the stove. 604 | sub-24/sub-24-_-NPC_string||His guitar had a broken string. 605 | sub-24/sub-24-_-NPC_study||He failed the test because he didn't study. 606 | sub-24/sub-24-_-NPC_sun||The clouds hid the sun. 607 | sub-24/sub-24-_-NPC_tank||Mom filled up her gas tank. 608 | sub-24/sub-24-_-NPC_tear||Down the sad woman's cheek came a tear. 609 | sub-24/sub-24-_-NPC_test||I cheated on the test. 610 | sub-24/sub-24-_-NPC_thorns||The rose had sharp thorns. 611 | sub-24/sub-24-_-NPC_three||I count one two three. 612 | sub-24/sub-24-_-NPC_toilet||I flush the toilet. 613 | sub-24/sub-24-_-NPC_trees||A monkey climbs trees. 614 | sub-24/sub-24-_-NPC_truck||His dad drove a delivery truck. 615 | sub-24/sub-24-_-NPC_walk||Babies crawl before they walk. 616 | sub-24/sub-24-_-NPC_wear||She had no clothes to wear. 617 | sub-24/sub-24-_-NPC_wet||The water got her hair wet. 618 | sub-24/sub-24-_-NPC_whales||The biggest animals in the sea are whales. 619 | sub-24/sub-24-_-NPC_wind||My hair was blown by the wind. 620 | sub-24/sub-24-_-NPC_wood||He chopped the wood. 621 | sub-24/sub-24-_-NPI_aisle(chicken)||The bride walked down the chicken. 622 | sub-24/sub-24-_-NPI_animals(trampoline)||The sign said do not feed the trampoline. 623 | sub-24/sub-24-_-NPI_ants(fish)||The picnic was ruined by crawling fish. 624 | sub-24/sub-24-_-NPI_apple(balloon)||His favorite fruit was a red crisp balloon. 625 | sub-24/sub-24-_-NPI_arm(stack)||I have two legs and two stack. 626 | sub-24/sub-24-_-NPI_axe(grass)||My dad chops wood with an grass. 627 | sub-24/sub-24-_-NPI_bag(bell)||He puts his lunch in a paper bell. 628 | sub-24/sub-24-_-NPI_bake(milk)||The cake is in the oven to milk. 629 | sub-24/sub-24-_-NPI_bank(hike)||My mom puts money in the hike. 630 | sub-24/sub-24-_-NPI_barn(claw)||The farmer's animals live in the claw. 631 | sub-24/sub-24-_-NPI_bat(gel)||The boy hit the ball with a gel. 632 | sub-24/sub-24-_-NPI_bath(roar)||To get clean I take a roar. 633 | sub-24/sub-24-_-NPI_beat(sneeze)||The drummer keeps the sneeze. 634 | sub-24/sub-24-_-NPI_bed(frog)||At night I sleep in my frog. 635 | sub-24/sub-24-_-NPI_belt(bed)||Around my pants I wear a bed. 636 | sub-24/sub-24-_-NPI_bird(thorn)||Tweet goes the thorn. 637 | sub-24/sub-24-_-NPI_blinking(belly)||She lost the staring contest by belly. 638 | sub-24/sub-24-_-NPI_blocks(night)||The baby liked to stack night. 639 | sub-24/sub-24-_-NPI_bloom(school)||In spring the flowers school. 640 | sub-24/sub-24-_-NPI_blue(arm)||The color of the sky is arm. 641 | sub-24/sub-24-_-NPI_boat(cop)||We went fishing on a cop. 642 | sub-24/sub-24-_-NPI_bones(wind)||The Museum showed dinosaur wind. 643 | sub-24/sub-24-_-NPI_books(tear)||He likes to read tear. 644 | sub-24/sub-24-_-NPI_bottle(money)||The baby drinks from his money. 645 | sub-24/sub-24-_-NPI_bow(eye)||On the present I tied a eye. 646 | sub-24/sub-24-_-NPI_bowl(test)||He pours cereal into a test. 647 | sub-24/sub-24-_-NPI_brakes(cob)||The car stopped with its cob. 648 | sub-24/sub-24-_-NPI_bread(sea)||She baked a loaf of sea. 649 | sub-24/sub-24-_-NPI_bricks(leg)||My fireplace is made of red leg. 650 | sub-24/sub-24-_-NPI_bridge(steak)||Boats sail under the steak. 651 | sub-24/sub-24-_-NPI_bright(gills)||I wear sunglasses because the sun is gills. 652 | sub-24/sub-24-_-NPI_broke(pond)||He wore a cast because his arm pond. 653 | sub-24/sub-24-_-NPI_broom(bus)||I sweep the floor with a bus. 654 | sub-24/sub-24-_-NPI_brown(job)||A tree trunk is colored job. 655 | sub-24/sub-24-_-NPI_bruise(rain)||I hit my arm and got a rain. 656 | sub-24/sub-24-_-NPI_bulb(farm)||Tom replaced the light farm. 657 | sub-24/sub-24-_-NPI_bus(egg)||To school I ride the yellow egg. 658 | sub-24/sub-24-_-NPI_bushes(oven)||Outside a man trimmed the oven. 659 | sub-24/sub-24-_-NPI_button(hotdog)||On the remote he clicked the hotdog. 660 | sub-24/sub-24-_-NPI_cake(clue)||He baked a birthday clue. 661 | sub-24/sub-24-_-NPI_camping(bacon)||We bring a tent to go bacon. 662 | sub-24/sub-24-_-NPI_candles(planet)||She blew out the birthday planet. 663 | sub-24/sub-24-_-NPI_cane(fin)||The old man walked with a fin. 664 | sub-24/sub-24-_-NPI_cap(chore)||He screwed off the bottle chore. 665 | sub-24/sub-24-_-NPI_cape(pole)||Superman wears a red pole. 666 | sub-24/sub-24-_-NPI_car(chest)||He puts gas in his chest. 667 | sub-24/sub-24-_-NPI_cart(score)||At the store I pushed the score. 668 | sub-24/sub-24-_-NPI_castle(doctor)||A princess lives in a doctor. 669 | sub-24/sub-24-_-NPI_cat(spray)||Meow goes the spray. 670 | sub-24/sub-24-_-NPI_catch(road)||I throw and you road. 671 | sub-24/sub-24-_-NPI_caves(string)||Bats live in string. 672 | sub-24/sub-24-_-NPI_chalk(fat)||On the sidewalk I draw with fat. 673 | sub-24/sub-24-_-NPI_cheese(clock)||Her favorite food is mac and clock. 674 | sub-24/sub-24-_-NPI_chest(sock)||My heart was pounding in my sock. 675 | sub-24/sub-24-_-NPI_chick(bruise)||A baby chicken is a bruise. 676 | sub-24/sub-24-_-NPI_chores(cave)||My mom makes me do household cave. 677 | sub-24/sub-24-_-NPI_claws(floor)||The cat has sharp floor. 678 | sub-24/sub-24-_-NPI_clay(gas)||In art class we mold gas. 679 | sub-24/sub-24-_-NPI_clean(cheek)||I keep my room cheek. 680 | sub-24/sub-24-_-NPI_clock(tree)||Tick tock goes the tree. 681 | sub-24/sub-24-_-NPI_clothes(bite)||My parents fold my bite. 682 | sub-24/sub-24-_-NPI_clouds(dirt)||In the sky are white fluffy dirt. 683 | sub-24/sub-24-_-NPI_clown(dice)||I laughed at the circus dice. 684 | sub-24/sub-24-_-NPI_clues(year)||The detectives found many year. 685 | sub-24/sub-24-_-NPI_coach(goose)||I was taught by my soccer goose. 686 | sub-24/sub-24-_-NPI_coat(loaf)||I put on my rain loaf. 687 | sub-24/sub-24-_-NPI_cob(lie)||She eats corn on the lie. 688 | sub-24/sub-24-_-NPI_cold(bib)||Winter is very bib. 689 | sub-24/sub-24-_-NPI_cone(hug)||He eats his ice cream on a hug. 690 | sub-24/sub-24-_-NPI_cop(cake)||He was pulled over by a cake. 691 | sub-24/sub-24-_-NPI_couch(bulb)||I watch tv and sit on the bulb. 692 | sub-24/sub-24-_-NPI_cough(friend)||My cold made me sneeze and friend. 693 | sub-24/sub-24-_-NPI_cow(axe)||Farmers milk the axe. 694 | sub-24/sub-24-_-NPI_crafts(tank)||I like arts and tank. 695 | sub-24/sub-24-_-NPI_cream(couch)||I spray whipped couch. 696 | sub-24/sub-24-_-NPI_cribs(shot)||Babies sleep in shot. 697 | sub-24/sub-24-_-NPI_crown(bench)||A king wears a bench. 698 | sub-24/sub-24-_-NPI_crust(foot)||His mom makes her own pie foot. 699 | sub-24/sub-24-_-NPI_dancers(pirate)||Ballerinas are great pirate. 700 | sub-24/sub-24-_-NPI_deep(bone)||The pool was six feet bone. 701 | sub-24/sub-24-_-NPI_desk(ice)||In school I sit at a ice. 702 | sub-24/sub-24-_-NPI_dip(cage)||I love chips and cage. 703 | sub-24/sub-24-_-NPI_dirt(screw)||Worms crawl in the screw. 704 | sub-24/sub-24-_-NPI_dog(chalk)||Bark says the chalk. 705 | sub-24/sub-24-_-NPI_dolls(worm)||My sister plays with Barbie worm. 706 | sub-24/sub-24-_-NPI_door(smoke)||He shut the smoke. 707 | sub-24/sub-24-_-NPI_dream(lawn)||She woke up after a bad lawn. 708 | sub-24/sub-24-_-NPI_dress(fun)||The bride wears a white fun. 709 | sub-24/sub-24-_-NPI_drill(bowl)||At school there was a fire bowl. 710 | sub-24/sub-24-_-NPI_duck(bank)||Quack goes the bank. 711 | sub-24/sub-24-_-NPI_dust(cook)||The old table was covered in cook. 712 | sub-24/sub-24-_-NPI_ears(bee)||I hear with my bee. 713 | sub-24/sub-24-_-NPI_earth(rose)||We live on planet rose. 714 | sub-24/sub-24-_-NPI_eats(ball)||The baby wears a bib when he ball. 715 | sub-24/sub-24-_-NPI_eyes(hip)||I see with my hip. 716 | sub-24/sub-24-_-NPI_fake(crust)||The diamond was not real it was crust. 717 | sub-24/sub-24-_-NPI_farm(bird)||Cows live on a bird. 718 | sub-24/sub-24-_-NPI_fast(thief)||Cheetahs are really thief. 719 | sub-24/sub-24-_-NPI_fat(glass)||Goats are skinny and pigs are glass. 720 | sub-24/sub-24-_-NPI_fear(walk)||The brave man had no walk. 721 | sub-24/sub-24-_-NPI_fight(mouse)||The angry boys got in a mouse. 722 | sub-24/sub-24-_-NPI_fin(desk)||Sharks have a tail desk. 723 | sub-24/sub-24-_-NPI_fingers(kitchen)||On one hand I have five kitchen. 724 | sub-24/sub-24-_-NPI_fish(cry)||On the boat my dad caught a cry. 725 | sub-24/sub-24-_-NPI_fist(bath)||I punched the bag with my bath. 726 | sub-24/sub-24-_-NPI_fixed(dog)||The toy broke and needed to be dog. 727 | sub-24/sub-24-_-NPI_flag(touch)||He waves an American touch. 728 | sub-24/sub-24-_-NPI_flat(cow)||The earth is round not cow. 729 | sub-24/sub-24-_-NPI_flip(dark)||On the trampoline he did a dark. 730 | sub-24/sub-24-_-NPI_flop(toy)||At the pool he did a belly toy. 731 | sub-24/sub-24-_-NPI_fly(edge)||She swatted the edge. 732 | sub-24/sub-24-_-NPI_foot(shout)||I put the sock on my shout. 733 | sub-24/sub-24-_-NPI_freeze(sun)||In the winter the pond will sun. 734 | sub-24/sub-24-_-NPI_fridge(pig)||The Jell-O had to chill in the pig. 735 | sub-24/sub-24-_-NPI_friend(pound)||I play with my best pound. 736 | sub-24/sub-24-_-NPI_frog(glove)||Ribbit goes the glove. 737 | sub-24/sub-24-_-NPI_fruit(tent)||He likes to eat fresh tent. 738 | sub-24/sub-24-_-NPI_fur(dad)||Cats have soft dad. 739 | sub-24/sub-24-_-NPI_game(coal)||His dad watches the football coal. 740 | sub-24/sub-24-_-NPI_ghosts(strike)||The house was haunted by strike. 741 | sub-24/sub-24-_-NPI_gills(doll)||Fish breathe through doll. 742 | sub-24/sub-24-_-NPI_glass(shoe)||Windows are made of shoe. 743 | sub-24/sub-24-_-NPI_gloves(car)||On his hands are a pair of car. 744 | sub-24/sub-24-_-NPI_glue(cliff)||The paper sticks together like cliff. 745 | sub-24/sub-24-_-NPI_goal(hair)||The soccer team scored a hair. 746 | sub-24/sub-24-_-NPI_goat(bread)||Bah says the sheep and bread. 747 | sub-24/sub-24-_-NPI_gold(fridge)||Wedding rings are made of fridge. 748 | sub-24/sub-24-_-NPI_gone(feet)||The cookies were eaten until they were feet. 749 | sub-24/sub-24-_-NPI_goose(pipe)||We played duck duck pipe. 750 | sub-24/sub-24-_-NPI_grade(grill)||In school he is in first grill. 751 | sub-24/sub-24-_-NPI_grapes(man)||Raisins are dried man. 752 | sub-24/sub-24-_-NPI_gray(bald)||The rainy sky was colored bald. 753 | sub-24/sub-24-_-NPI_green(lock)||The grass is colored lock. 754 | sub-24/sub-24-_-NPI_grill(heart)||He cooked hotdogs outside on the heart. 755 | sub-24/sub-24-_-NPI_guess(grape)||On an answer he made a lucky grape. 756 | sub-24/sub-24-_-NPI_gum(coat)||He is chewing bubble coat. 757 | sub-24/sub-24-_-NPI_hair(nest)||Bald people do not have nest. 758 | sub-24/sub-24-_-NPI_hands(king)||I clap my king. 759 | sub-24/sub-24-_-NPI_hug(fear)||Charlie wanted a kiss and a fear. 760 | sub-24/sub-24-_-NPI_inches(cookie)||This year he has grown two cookie. 761 | sub-24/sub-24-_-NPI_job(pants)||His mom quit her pants. 762 | sub-24/sub-24-_-NPI_kitchen(baby)||She cooks in the baby. 763 | sub-24/sub-24-_-NPI_lawn(weight)||My dad mows the weight. 764 | sub-24/sub-24-_-NPI_legs(barn)||Dogs have four barn. 765 | sub-24/sub-24-_-NPI_lie(cup)||Not telling the truth is telling a cup. 766 | sub-24/sub-24-_-NPI_mouse(store)||The cat chased the store. 767 | sub-24/sub-24-_-NPI_nests(drill)||Birds like to build drill. 768 | sub-24/sub-24-_-NPI_nice(gum)||The opposite of mean is gum. 769 | sub-24/sub-24-_-NPI_outside(apple)||He put on his shoes to go apple. 770 | sub-24/sub-24-_-NPI_pants(woods)||He pulled up his woods. 771 | sub-24/sub-24-_-NPI_party(sidewalk)||He had fun at the birthday sidewalk. 772 | sub-24/sub-24-_-NPI_phone(snake)||He called Helen on the snake. 773 | sub-24/sub-24-_-NPI_pictures(contest)||I like to draw contest. 774 | sub-24/sub-24-_-NPI_pipe(phone)||His grandpa smoked from a phone. 775 | sub-24/sub-24-_-NPI_present(sister)||For his birthday I wrapped a sister. 776 | sub-24/sub-24-_-NPI_shot(crown)||The doctor gave him a flu crown. 777 | sub-24/sub-24-_-NPI_skunk(broom)||His dog was sprayed by a broom. 778 | sub-24/sub-24-_-NPI_snake(play)||Hiss goes the play. 779 | sub-24/sub-24-_-NPI_solve(cloud)||He had a mystery to cloud. 780 | sub-24/sub-24-_-NPI_spider(summer)||There are eight legs on a summer. 781 | sub-24/sub-24-_-NPI_steal(earth)||A thief likes to earth. 782 | sub-24/sub-24-_-NPI_store(stick)||We buy toys at the toy stick. 783 | sub-24/sub-24-_-NPI_stove(kid)||He burned his hand on the kid. 784 | sub-24/sub-24-_-NPI_string(chick)||His guitar had a broken chick. 785 | sub-24/sub-24-_-NPI_study(princess)||He failed the test because he didn't princess. 786 | sub-24/sub-24-_-NPI_sun(lunch)||The clouds hid the lunch. 787 | sub-24/sub-24-_-NPI_tank(truth)||Mom filled up her gas truth. 788 | sub-24/sub-24-_-NPI_tear(corn)||Down the sad woman's cheek came a corn. 789 | sub-24/sub-24-_-NPI_test(dust)||I cheated on the dust. 790 | sub-24/sub-24-_-NPI_thorns(house)||The rose had sharp house. 791 | sub-24/sub-24-_-NPI_three(boy)||I count one two boy. 792 | sub-24/sub-24-_-NPI_toilet(window)||I flush the window. 793 | sub-24/sub-24-_-NPI_trees(book)||A monkey climbs book. 794 | sub-24/sub-24-_-NPI_truck(cap)||His dad drove a delivery break. 795 | sub-24/sub-24-_-NPI_walk(class)||Babies crawl before they class. 796 | sub-24/sub-24-_-NPI_wear(chief)||She had no clothes to chief. 797 | sub-24/sub-24-_-NPI_wet(whale)||The water got her hair whale. 798 | sub-24/sub-24-_-NPI_whales(dress)||The biggest animals in the sea are dress. 799 | sub-24/sub-24-_-NPI_wind(cheese)||My hair was blown by the cheese. 800 | sub-24/sub-24-_-NPI_wood(cough)||He chopped the cough. --------------------------------------------------------------------------------