├── LICENSE ├── README.md ├── commons.py ├── configs ├── cvc-eng-ppgs-three-emo-cycleloss.json ├── cvc-eng-ppgs-three-emo.json ├── cvc-whispers-multi.json └── cvc-whispers-three-emo.json ├── cvc627.png ├── data_utils_engppg.py ├── data_utils_whisper.py ├── losses.py ├── mel_processing.py ├── models.py ├── modules.py ├── ppg.py ├── ppgemoconvert_exp.py ├── preprocess_ppg.py ├── train_eng_ppg_emo_loss.py ├── train_whisper_emo.py ├── utils.py ├── whisper ├── LICENSE ├── README.md ├── __init__.py ├── __pycache__ │ ├── __init__.cpython-37.pyc │ ├── audio.cpython-37.pyc │ ├── decoding.cpython-37.pyc │ ├── model.cpython-37.pyc │ ├── tokenizer.cpython-37.pyc │ └── utils.cpython-37.pyc ├── assets │ ├── gpt2 │ │ ├── merges.txt │ │ ├── special_tokens_map.json │ │ ├── tokenizer_config.json │ │ └── vocab.json │ ├── mel_filters.npz │ └── multilingual │ │ ├── added_tokens.json │ │ ├── merges.txt │ │ ├── special_tokens_map.json │ │ ├── tokenizer_config.json │ │ └── vocab.json ├── audio.py ├── decoding.py ├── model.py ├── normalizers │ ├── __init__.py │ ├── basic.py │ ├── english.json │ └── english.py ├── tokenizer.py └── utils.py ├── whisperconvert_exp.py └── whisperconvert_longaudio.py /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 ConsistencyVC 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ConsistencyVC-voive-conversion 2 | 3 | ## Using joint training speaker encoder with consistency loss to achieve cross-lingual voice conversion and expressive voice conversion 4 | 5 | Demo page: https://consistencyvc.github.io/ConsistencyVC-demo-page 6 | 7 | The whisper medium model can be downloaded here: https://drive.google.com/file/d/1PZsfQg3PUZuu1k6nHvavd6OcOB_8m1Aa/view?usp=drive_link 8 | 9 | The pre-trained models are available here:https://drive.google.com/drive/folders/1KvMN1V8BWCzJd-N8hfyP283rLQBKIbig?usp=sharing 10 | 11 | Note: The audio needs to be 16KHz for train and inference. 12 | 13 | cvc 14 | 15 | 16 | 17 | 18 | # Inference with the pre-trained models (use WEO as example) 19 | 20 | Generate the WEO of the source speech in [src](https://github.com/ConsistencyVC/ConsistencyVC-voive-conversion/blob/467ed5e632b2b328d01c87cb73e92b26b36deb05/whisperconvert_exp.py#L39C1-L39C1) by preprocess_ppg.py. 21 | 22 | Copy the root of the reference speech to [tgt](https://github.com/ConsistencyVC/ConsistencyVC-voive-conversion/blob/467ed5e632b2b328d01c87cb73e92b26b36deb05/whisperconvert_exp.py#L47) 23 | 24 | Use whisperconvert_exp.py to achieve voice conversion using WEO as content information. 25 | 26 | For ConsistencyEVC, use ppgemoconvert_exp.py to achieve voice conversion using ppg as content information. 27 | 28 | # Inference for the long audio 29 | I uploaded a new py file for the inference of long audio. 30 | You don't need to run the whisper by another file, just change [this part](https://github.com/ConsistencyVC/ConsistencyVC-voive-conversion/blob/83f72b0801240e7d932c9314431df6e75f2d1c22/whisperconvert_longaudio.py#L41) and run this py file. 31 | 32 | # Train models by your dataset 33 | 34 | Use ppg.py to generate the PPG. 35 | 36 | Use preprocess_ppg.py to generate the WEO. 37 | 38 | ## If you want to use WEO to train a cross-lingual voice conversion model: 39 | 40 | First you need to train the model without speaker consistency loss for 100k steps: 41 | 42 | change [this line](https://github.com/ConsistencyVC/ConsistencyVC-voive-conversion/blob/b5e8e984dffd5a12910d1846e25b128298933e40/train_whisper_emo.py#L214C11-L214C11) to 43 | 44 | ```python 45 | loss_gen_all = loss_gen + loss_fm + loss_mel + loss_kl# + loss_emo 46 | ``` 47 | 48 | run the py file: 49 | 50 | ```python 51 | python train_whisper_emo.py -c configs/cvc-whispers-multi.json -m cvc-whispers-three 52 | ``` 53 | 54 | Then change [this line](https://github.com/ConsistencyVC/ConsistencyVC-voive-conversion/blob/71cf17a5b65c12987ea7fba74d1d173ea1aae5cb/train_whisper_emo.py#L214) back to finetune this model with speaker consistency loss 55 | 56 | ```python 57 | python train_whisper_emo.py -c configs/cvc-whispers-three-emo.json -m cvc-whispers-three 58 | ``` 59 | 60 | ## If you want to use PPG to train an expressive voice conversion model: 61 | 62 | First you need to train the model without speaker consistency loss for 100k steps: 63 | 64 | change [this line](https://github.com/ConsistencyVC/ConsistencyVC-voive-conversion/blob/71cf17a5b65c12987ea7fba74d1d173ea1aae5cb/train_eng_ppg_emo_loss.py#L311) to 65 | 66 | ```python 67 | loss_gen_all = loss_gen + loss_fm + loss_mel + loss_kl# + loss_emo 68 | ``` 69 | 70 | run the py file: 71 | 72 | ```python 73 | python train_eng_ppg_emo_loss.py -c configs/cvc-eng-ppgs-three-emo.json -m cvc-eng-ppgs-three-emo 74 | ``` 75 | 76 | Then change [this line](https://github.com/ConsistencyVC/ConsistencyVC-voive-conversion/blob/71cf17a5b65c12987ea7fba74d1d173ea1aae5cb/train_eng_ppg_emo_loss.py#L311) back to finetune this model with speaker consistency loss 77 | 78 | ```python 79 | python train_eng_ppg_emo_loss.py -c configs/cvc-eng-ppgs-three-emo-cycleloss.json -m cvc-eng-ppgs-three-emo 80 | ``` 81 | 82 | 83 | # Reference 84 | 85 | The code structure is based on [FreeVC-s](https://github.com/OlaWod/FreeVC). Suggestion: please follow the instruction of FreeVC to install python requirements. 86 | 87 | The WEO content feature is based on [LoraSVC](https://github.com/PlayVoice/lora-svc). 88 | 89 | The PPG is from the [phoneme recognition model](https://huggingface.co/speech31/wav2vec2-large-english-TIMIT-phoneme_v3). 90 | 91 | 92 | -------------------------------------------------------------------------------- /commons.py: -------------------------------------------------------------------------------- 1 | import math 2 | import numpy as np 3 | import torch 4 | from torch import nn 5 | from torch.nn import functional as F 6 | 7 | 8 | def init_weights(m, mean=0.0, std=0.01): 9 | classname = m.__class__.__name__ 10 | if classname.find("Conv") != -1: 11 | m.weight.data.normal_(mean, std) 12 | 13 | 14 | def get_padding(kernel_size, dilation=1): 15 | return int((kernel_size*dilation - dilation)/2) 16 | 17 | 18 | def convert_pad_shape(pad_shape): 19 | l = pad_shape[::-1] 20 | pad_shape = [item for sublist in l for item in sublist] 21 | return pad_shape 22 | 23 | 24 | def intersperse(lst, item): 25 | result = [item] * (len(lst) * 2 + 1) 26 | result[1::2] = lst 27 | return result 28 | 29 | 30 | def kl_divergence(m_p, logs_p, m_q, logs_q): 31 | """KL(P||Q)""" 32 | kl = (logs_q - logs_p) - 0.5 33 | kl += 0.5 * (torch.exp(2. * logs_p) + ((m_p - m_q)**2)) * torch.exp(-2. * logs_q) 34 | return kl 35 | 36 | 37 | def rand_gumbel(shape): 38 | """Sample from the Gumbel distribution, protect from overflows.""" 39 | uniform_samples = torch.rand(shape) * 0.99998 + 0.00001 40 | return -torch.log(-torch.log(uniform_samples)) 41 | 42 | 43 | def rand_gumbel_like(x): 44 | g = rand_gumbel(x.size()).to(dtype=x.dtype, device=x.device) 45 | return g 46 | 47 | 48 | def slice_segments(x, ids_str, segment_size=4): 49 | ret = torch.zeros_like(x[:, :, :segment_size]) 50 | for i in range(x.size(0)): 51 | idx_str = ids_str[i] 52 | idx_end = idx_str + segment_size 53 | ret[i] = x[i, :, idx_str:idx_end] 54 | return ret 55 | 56 | 57 | def rand_slice_segments(x, x_lengths=None, segment_size=4): 58 | b, d, t = x.size() 59 | if x_lengths is None: 60 | x_lengths = t 61 | ids_str_max = x_lengths - segment_size + 1 62 | ids_str = (torch.rand([b]).to(device=x.device) * ids_str_max).to(dtype=torch.long) 63 | ret = slice_segments(x, ids_str, segment_size) 64 | return ret, ids_str 65 | 66 | 67 | def rand_spec_segments(x, x_lengths=None, segment_size=4): 68 | b, d, t = x.size() 69 | if x_lengths is None: 70 | x_lengths = t 71 | ids_str_max = x_lengths - segment_size 72 | ids_str = (torch.rand([b]).to(device=x.device) * ids_str_max).to(dtype=torch.long) 73 | ret = slice_segments(x, ids_str, segment_size) 74 | return ret, ids_str 75 | 76 | 77 | def get_timing_signal_1d( 78 | length, channels, min_timescale=1.0, max_timescale=1.0e4): 79 | position = torch.arange(length, dtype=torch.float) 80 | num_timescales = channels // 2 81 | log_timescale_increment = ( 82 | math.log(float(max_timescale) / float(min_timescale)) / 83 | (num_timescales - 1)) 84 | inv_timescales = min_timescale * torch.exp( 85 | torch.arange(num_timescales, dtype=torch.float) * -log_timescale_increment) 86 | scaled_time = position.unsqueeze(0) * inv_timescales.unsqueeze(1) 87 | signal = torch.cat([torch.sin(scaled_time), torch.cos(scaled_time)], 0) 88 | signal = F.pad(signal, [0, 0, 0, channels % 2]) 89 | signal = signal.view(1, channels, length) 90 | return signal 91 | 92 | 93 | def add_timing_signal_1d(x, min_timescale=1.0, max_timescale=1.0e4): 94 | b, channels, length = x.size() 95 | signal = get_timing_signal_1d(length, channels, min_timescale, max_timescale) 96 | return x + signal.to(dtype=x.dtype, device=x.device) 97 | 98 | 99 | def cat_timing_signal_1d(x, min_timescale=1.0, max_timescale=1.0e4, axis=1): 100 | b, channels, length = x.size() 101 | signal = get_timing_signal_1d(length, channels, min_timescale, max_timescale) 102 | return torch.cat([x, signal.to(dtype=x.dtype, device=x.device)], axis) 103 | 104 | 105 | def subsequent_mask(length): 106 | mask = torch.tril(torch.ones(length, length)).unsqueeze(0).unsqueeze(0) 107 | return mask 108 | 109 | 110 | @torch.jit.script 111 | def fused_add_tanh_sigmoid_multiply(input_a, input_b, n_channels): 112 | n_channels_int = n_channels[0] 113 | in_act = input_a + input_b 114 | t_act = torch.tanh(in_act[:, :n_channels_int, :]) 115 | s_act = torch.sigmoid(in_act[:, n_channels_int:, :]) 116 | acts = t_act * s_act 117 | return acts 118 | 119 | 120 | def convert_pad_shape(pad_shape): 121 | l = pad_shape[::-1] 122 | pad_shape = [item for sublist in l for item in sublist] 123 | return pad_shape 124 | 125 | 126 | def shift_1d(x): 127 | x = F.pad(x, convert_pad_shape([[0, 0], [0, 0], [1, 0]]))[:, :, :-1] 128 | return x 129 | 130 | 131 | def sequence_mask(length, max_length=None): 132 | if max_length is None: 133 | max_length = length.max() 134 | x = torch.arange(max_length, dtype=length.dtype, device=length.device) 135 | return x.unsqueeze(0) < length.unsqueeze(1) 136 | 137 | 138 | def generate_path(duration, mask): 139 | """ 140 | duration: [b, 1, t_x] 141 | mask: [b, 1, t_y, t_x] 142 | """ 143 | device = duration.device 144 | 145 | b, _, t_y, t_x = mask.shape 146 | cum_duration = torch.cumsum(duration, -1) 147 | 148 | cum_duration_flat = cum_duration.view(b * t_x) 149 | path = sequence_mask(cum_duration_flat, t_y).to(mask.dtype) 150 | path = path.view(b, t_x, t_y) 151 | path = path - F.pad(path, convert_pad_shape([[0, 0], [1, 0], [0, 0]]))[:, :-1] 152 | path = path.unsqueeze(1).transpose(2,3) * mask 153 | return path 154 | 155 | 156 | def clip_grad_value_(parameters, clip_value, norm_type=2): 157 | if isinstance(parameters, torch.Tensor): 158 | parameters = [parameters] 159 | parameters = list(filter(lambda p: p.grad is not None, parameters)) 160 | norm_type = float(norm_type) 161 | if clip_value is not None: 162 | clip_value = float(clip_value) 163 | 164 | total_norm = 0 165 | for p in parameters: 166 | param_norm = p.grad.data.norm(norm_type) 167 | total_norm += param_norm.item() ** norm_type 168 | if clip_value is not None: 169 | p.grad.data.clamp_(min=-clip_value, max=clip_value) 170 | total_norm = total_norm ** (1. / norm_type) 171 | return total_norm 172 | -------------------------------------------------------------------------------- /configs/cvc-eng-ppgs-three-emo-cycleloss.json: -------------------------------------------------------------------------------- 1 | { 2 | "train": { 3 | "log_interval": 200, 4 | "eval_interval": 6000, 5 | "seed": 1235, 6 | "epochs": 10000, 7 | "learning_rate": 2e-4, 8 | "betas": [0.8, 0.99], 9 | "eps": 1e-9, 10 | "batch_size": 42, 11 | "fp16_run": true, 12 | "lr_decay": 0.999875, 13 | "segment_size": 24000, 14 | "init_lr_ratio": 1, 15 | "warmup_epochs": 0, 16 | "c_mel": 45, 17 | "c_kl": 1.0, 18 | "use_sr": false, 19 | "max_speclen": 300, 20 | "port": "8006" 21 | }, 22 | "data": { 23 | "training_files":"train.txt", 24 | "validation_files":"test.txt", 25 | "max_wav_value": 32768.0, 26 | "sampling_rate": 16000, 27 | "filter_length": 1024, 28 | "hop_length": 320, 29 | "win_length": 1024, 30 | "n_mel_channels": 80, 31 | "mel_fmin": 0.0, 32 | "mel_fmax": null 33 | }, 34 | "model": { 35 | "inter_channels": 192, 36 | "hidden_channels": 192, 37 | "filter_channels": 768, 38 | "n_heads": 2, 39 | "n_layers": 6, 40 | "kernel_size": 3, 41 | "p_dropout": 0.1, 42 | "resblock": "1", 43 | "resblock_kernel_sizes": [3,7,11], 44 | "resblock_dilation_sizes": [[1,3,5], [1,3,5], [1,3,5]], 45 | "upsample_rates": [10,8,2,2], 46 | "upsample_initial_channel": 512, 47 | "upsample_kernel_sizes": [20,16,4,4], 48 | "n_layers_q": 3, 49 | "use_spectral_norm": false, 50 | "gin_channels": 256, 51 | "ssl_dim": 44, 52 | "use_spk": false 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /configs/cvc-eng-ppgs-three-emo.json: -------------------------------------------------------------------------------- 1 | { 2 | "train": { 3 | "log_interval": 200, 4 | "eval_interval": 6000, 5 | "seed": 1235, 6 | "epochs": 10000, 7 | "learning_rate": 2e-4, 8 | "betas": [0.8, 0.99], 9 | "eps": 1e-9, 10 | "batch_size": 108, 11 | "fp16_run": true, 12 | "lr_decay": 0.999875, 13 | "segment_size": 8960, 14 | "init_lr_ratio": 1, 15 | "warmup_epochs": 0, 16 | "c_mel": 45, 17 | "c_kl": 1.0, 18 | "use_sr": false, 19 | "max_speclen": 300, 20 | "port": "8006" 21 | }, 22 | "data": { 23 | "training_files":"train.txt", 24 | "validation_files":"test.txt", 25 | "max_wav_value": 32768.0, 26 | "sampling_rate": 16000, 27 | "filter_length": 1024, 28 | "hop_length": 320, 29 | "win_length": 1024, 30 | "n_mel_channels": 80, 31 | "mel_fmin": 0.0, 32 | "mel_fmax": null 33 | }, 34 | "model": { 35 | "inter_channels": 192, 36 | "hidden_channels": 192, 37 | "filter_channels": 768, 38 | "n_heads": 2, 39 | "n_layers": 6, 40 | "kernel_size": 3, 41 | "p_dropout": 0.1, 42 | "resblock": "1", 43 | "resblock_kernel_sizes": [3,7,11], 44 | "resblock_dilation_sizes": [[1,3,5], [1,3,5], [1,3,5]], 45 | "upsample_rates": [10,8,2,2], 46 | "upsample_initial_channel": 512, 47 | "upsample_kernel_sizes": [20,16,4,4], 48 | "n_layers_q": 3, 49 | "use_spectral_norm": false, 50 | "gin_channels": 256, 51 | "ssl_dim": 44, 52 | "use_spk": false 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /configs/cvc-whispers-multi.json: -------------------------------------------------------------------------------- 1 | { 2 | "train": { 3 | "log_interval": 200, 4 | "eval_interval": 6000, 5 | "seed": 1235, 6 | "epochs": 10000, 7 | "learning_rate": 2e-4, 8 | "betas": [0.8, 0.99], 9 | "eps": 1e-9, 10 | "batch_size": 108, 11 | "fp16_run": true, 12 | "lr_decay": 0.999875, 13 | "segment_size": 8960, 14 | "init_lr_ratio": 1, 15 | "warmup_epochs": 0, 16 | "c_mel": 45, 17 | "c_kl": 1.0, 18 | "use_sr": false, 19 | "max_speclen": 300, 20 | "port": "8001" 21 | }, 22 | "data": { 23 | "training_files":"train.txt", 24 | "validation_files":"test.txt", 25 | "max_wav_value": 32768.0, 26 | "sampling_rate": 16000, 27 | "filter_length": 1024, 28 | "hop_length": 320, 29 | "win_length": 1024, 30 | "n_mel_channels": 80, 31 | "mel_fmin": 0.0, 32 | "mel_fmax": null 33 | }, 34 | "model": { 35 | "inter_channels": 192, 36 | "hidden_channels": 192, 37 | "filter_channels": 768, 38 | "n_heads": 2, 39 | "n_layers": 6, 40 | "kernel_size": 3, 41 | "p_dropout": 0.1, 42 | "resblock": "1", 43 | "resblock_kernel_sizes": [3,7,11], 44 | "resblock_dilation_sizes": [[1,3,5], [1,3,5], [1,3,5]], 45 | "upsample_rates": [10,8,2,2], 46 | "upsample_initial_channel": 512, 47 | "upsample_kernel_sizes": [20,16,4,4], 48 | "n_layers_q": 3, 49 | "use_spectral_norm": false, 50 | "gin_channels": 256, 51 | "ssl_dim": 1024, 52 | "use_spk": false 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /configs/cvc-whispers-three-emo.json: -------------------------------------------------------------------------------- 1 | { 2 | "train": { 3 | "log_interval": 200, 4 | "eval_interval": 2500, 5 | "seed": 1235, 6 | "epochs": 10000, 7 | "learning_rate": 2e-4, 8 | "betas": [0.8, 0.99], 9 | "eps": 1e-9, 10 | "batch_size": 42, 11 | "fp16_run": true, 12 | "lr_decay": 0.999875, 13 | "segment_size": 24000, 14 | "init_lr_ratio": 1, 15 | "warmup_epochs": 0, 16 | "c_mel": 45, 17 | "c_kl": 1.0, 18 | "use_sr": false, 19 | "max_speclen": 300, 20 | "port": "8006" 21 | }, 22 | "data": { 23 | "training_files":"train.txt", 24 | "validation_files":"test.txt", 25 | "max_wav_value": 32768.0, 26 | "sampling_rate": 16000, 27 | "filter_length": 1024, 28 | "hop_length": 320, 29 | "win_length": 1024, 30 | "n_mel_channels": 80, 31 | "mel_fmin": 0.0, 32 | "mel_fmax": null 33 | }, 34 | "model": { 35 | "inter_channels": 192, 36 | "hidden_channels": 192, 37 | "filter_channels": 768, 38 | "n_heads": 2, 39 | "n_layers": 6, 40 | "kernel_size": 3, 41 | "p_dropout": 0.1, 42 | "resblock": "1", 43 | "resblock_kernel_sizes": [3,7,11], 44 | "resblock_dilation_sizes": [[1,3,5], [1,3,5], [1,3,5]], 45 | "upsample_rates": [10,8,2,2], 46 | "upsample_initial_channel": 512, 47 | "upsample_kernel_sizes": [20,16,4,4], 48 | "n_layers_q": 3, 49 | "use_spectral_norm": false, 50 | "gin_channels": 256, 51 | "ssl_dim": 1024, 52 | "use_spk": false 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /cvc627.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConsistencyVC/ConsistencyVC-voive-conversion/b1506a2c4c68337de922b624c0df1a1dd034419d/cvc627.png -------------------------------------------------------------------------------- /data_utils_engppg.py: -------------------------------------------------------------------------------- 1 | import time 2 | import os 3 | import random 4 | import numpy as np 5 | import torch 6 | import torch.utils.data 7 | 8 | import commons 9 | from mel_processing import spectrogram_torch, spec_to_mel_torch 10 | from utils import load_wav_to_torch, load_filepaths_and_text, transform 11 | #import h5py 12 | 13 | 14 | """Multi speaker version""" 15 | class TextAudioSpeakerLoader(torch.utils.data.Dataset): 16 | """ 17 | 1) loads audio, speaker_id, text pairs 18 | 2) normalizes text and converts them to sequences of integers 19 | 3) computes spectrograms from audio files. 20 | """ 21 | def __init__(self, audiopaths, hparams): 22 | self.audiopaths = load_filepaths_and_text(audiopaths) 23 | self.max_wav_value = hparams.data.max_wav_value 24 | self.sampling_rate = hparams.data.sampling_rate 25 | self.filter_length = hparams.data.filter_length 26 | self.hop_length = hparams.data.hop_length 27 | self.win_length = hparams.data.win_length 28 | self.sampling_rate = hparams.data.sampling_rate 29 | self.use_sr = hparams.train.use_sr 30 | self.use_spk = hparams.model.use_spk 31 | self.spec_len = hparams.train.max_speclen 32 | 33 | random.seed(1235) 34 | random.shuffle(self.audiopaths) 35 | self._filter() 36 | 37 | def _filter(self): 38 | """ 39 | Filter text & store spec lengths 40 | """ 41 | # Store spectrogram lengths for Bucketing 42 | # wav_length ~= file_size / (wav_channels * Bytes per dim) = file_size / (1 * 2) 43 | # spec_length = wav_length // hop_length 44 | 45 | lengths = [] 46 | for audiopath in self.audiopaths: 47 | lengths.append(os.path.getsize(audiopath[0]) // (2 * self.hop_length)) 48 | self.lengths = lengths 49 | 50 | def get_audio(self, filename): 51 | audio, sampling_rate = load_wav_to_torch(filename) 52 | if sampling_rate != self.sampling_rate: 53 | raise ValueError("{} SR doesn't match target {} SR,the audio is{}".format( 54 | sampling_rate, self.sampling_rate,filename)) 55 | audio_norm = audio / self.max_wav_value 56 | audio_norm = audio_norm.unsqueeze(0) 57 | spec_filename = filename.replace(".wav", ".f{}h{}w{}spec.pt".format(self.filter_length, self.hop_length, self.win_length)) 58 | if os.path.exists(spec_filename): 59 | try: 60 | spec = torch.load(spec_filename) 61 | except: 62 | print(spec_filename,"不存在") 63 | spec = spectrogram_torch(audio_norm, self.filter_length, 64 | self.sampling_rate, self.hop_length, self.win_length, 65 | center=False) 66 | spec = torch.squeeze(spec, 0) 67 | torch.save(spec, spec_filename) 68 | else: 69 | #import sys 70 | #print(spec_filename,"不存在") 71 | #sys.exit() 72 | spec = spectrogram_torch(audio_norm, self.filter_length, 73 | self.sampling_rate, self.hop_length, self.win_length, 74 | center=False) 75 | spec = torch.squeeze(spec, 0) 76 | torch.save(spec, spec_filename) 77 | 78 | if self.use_spk: 79 | spk_filename = filename.replace(".wav", ".npy") 80 | spk_filename = spk_filename.replace("DUMMY", "dataset/spk") 81 | spk = torch.from_numpy(np.load(spk_filename)) 82 | 83 | if not self.use_sr: 84 | c_filename = filename.replace(".wav", "_eng_ppg.pt") 85 | #c_filename = c_filename.replace("DUMMY", "dataset/wavlm") 86 | c = torch.load(c_filename).squeeze(0) 87 | 88 | else: 89 | i = random.randint(68,92) 90 | ''' 91 | basename = os.path.basename(filename)[:-4] 92 | spkname = basename[:4] 93 | #print(basename, spkname) 94 | with h5py.File(f"dataset/rs/wavlm/{spkname}/{i}.hdf5","r") as f: 95 | c = torch.from_numpy(f[basename][()]).squeeze(0) 96 | #print(c) 97 | ''' 98 | import sys 99 | sys.exit() 100 | c_filename = filename.replace(".wav", f"_{i}.pt") 101 | c_filename = c_filename.replace("DUMMY", "dataset/sr/wavlm") 102 | c = torch.load(c_filename).squeeze(0) 103 | 104 | # 2023.01.10 update: code below can deteriorate model performance 105 | # I added these code during cleaning up, thinking that it can offer better performance than my provided checkpoints, but actually it does the opposite. 106 | # What an act of 'adding legs to a snake'! 107 | ''' 108 | lmin = min(c.size(-1), spec.size(-1)) 109 | spec, c = spec[:, :lmin], c[:, :lmin] 110 | audio_norm = audio_norm[:, :lmin*self.hop_length] 111 | _spec, _c, _audio_norm = spec, c, audio_norm 112 | while spec.size(-1) < self.spec_len: 113 | spec = torch.cat((spec, _spec), -1) 114 | c = torch.cat((c, _c), -1) 115 | audio_norm = torch.cat((audio_norm, _audio_norm), -1) 116 | start = random.randint(0, spec.size(-1) - self.spec_len) 117 | end = start + self.spec_len 118 | spec = spec[:, start:end] 119 | c = c[:, start:end] 120 | audio_norm = audio_norm[:, start*self.hop_length:end*self.hop_length] 121 | ''' 122 | 123 | #if self.use_spk: 124 | # return c, spec, audio_norm, spk 125 | #else: 126 | return c, spec, audio_norm 127 | 128 | def __getitem__(self, index): 129 | return self.get_audio(self.audiopaths[index][0]) 130 | 131 | def __len__(self): 132 | return len(self.audiopaths) 133 | 134 | 135 | class TextAudioSpeakerCollate(): 136 | """ Zero-pads model inputs and targets 137 | """ 138 | def __init__(self, hps): 139 | self.hps = hps 140 | self.use_sr = hps.train.use_sr 141 | self.use_spk = hps.model.use_spk 142 | 143 | def __call__(self, batch): 144 | """Collate's training batch from normalized text, audio and speaker identities 145 | PARAMS 146 | ------ 147 | batch: [text_normalized, spec_normalized, wav_normalized, sid] 148 | """ 149 | # Right zero-pad all one-hot text sequences to max input length 150 | _, ids_sorted_decreasing = torch.sort( 151 | torch.LongTensor([x[0].size(1) for x in batch]), 152 | dim=0, descending=True) 153 | 154 | max_spec_len = max([x[1].size(1) for x in batch]) 155 | max_wav_len = max([x[2].size(1) for x in batch]) 156 | 157 | spec_lengths = torch.LongTensor(len(batch)) 158 | wav_lengths = torch.LongTensor(len(batch)) 159 | if self.use_spk: 160 | spks = torch.FloatTensor(len(batch), batch[0][3].size(0)) 161 | else: 162 | spks = None 163 | 164 | c_padded = torch.FloatTensor(len(batch), batch[0][0].size(0), max_spec_len) 165 | spec_padded = torch.FloatTensor(len(batch), batch[0][1].size(0), max_spec_len) 166 | wav_padded = torch.FloatTensor(len(batch), 1, max_wav_len) 167 | c_padded.zero_() 168 | spec_padded.zero_() 169 | wav_padded.zero_() 170 | 171 | for i in range(len(ids_sorted_decreasing)): 172 | row = batch[ids_sorted_decreasing[i]] 173 | 174 | c = row[0] 175 | c_padded[i, :, :c.size(1)] = c 176 | 177 | spec = row[1] 178 | spec_padded[i, :, :spec.size(1)] = spec 179 | spec_lengths[i] = spec.size(1) 180 | 181 | wav = row[2] 182 | wav_padded[i, :, :wav.size(1)] = wav 183 | wav_lengths[i] = wav.size(1) 184 | 185 | if self.use_spk: 186 | spks[i] = row[3] 187 | 188 | spec_seglen = spec_lengths[-1] if spec_lengths[-1] < self.hps.train.max_speclen + 1 else self.hps.train.max_speclen + 1 189 | wav_seglen = spec_seglen * self.hps.data.hop_length 190 | 191 | spec_padded, ids_slice = commons.rand_spec_segments(spec_padded, spec_lengths, spec_seglen) 192 | wav_padded = commons.slice_segments(wav_padded, ids_slice * self.hps.data.hop_length, wav_seglen) 193 | 194 | c_padded = commons.slice_segments(c_padded, ids_slice, spec_seglen)[:,:,:-1] 195 | 196 | spec_padded = spec_padded[:,:,:-1] 197 | wav_padded = wav_padded[:,:,:-self.hps.data.hop_length] 198 | 199 | if self.use_spk: 200 | return c_padded, spec_padded, wav_padded, spks 201 | else: 202 | return c_padded, spec_padded, wav_padded 203 | 204 | 205 | class DistributedBucketSampler(torch.utils.data.distributed.DistributedSampler): 206 | """ 207 | Maintain similar input lengths in a batch. 208 | Length groups are specified by boundaries. 209 | Ex) boundaries = [b1, b2, b3] -> any batch is included either {x | b1 < length(x) <=b2} or {x | b2 < length(x) <= b3}. 210 | 211 | It removes samples which are not included in the boundaries. 212 | Ex) boundaries = [b1, b2, b3] -> any x s.t. length(x) <= b1 or length(x) > b3 are discarded. 213 | """ 214 | def __init__(self, dataset, batch_size, boundaries, num_replicas=None, rank=None, shuffle=True): 215 | super().__init__(dataset, num_replicas=num_replicas, rank=rank, shuffle=shuffle) 216 | self.lengths = dataset.lengths 217 | self.batch_size = batch_size 218 | self.boundaries = boundaries 219 | 220 | self.buckets, self.num_samples_per_bucket = self._create_buckets() 221 | print(self.num_samples_per_bucket) 222 | self.total_size = sum(self.num_samples_per_bucket) 223 | self.num_samples = self.total_size // self.num_replicas 224 | 225 | def _create_buckets(self): 226 | buckets = [[] for _ in range(len(self.boundaries) - 1)] 227 | for i in range(len(self.lengths)): 228 | length = self.lengths[i] 229 | idx_bucket = self._bisect(length) 230 | if idx_bucket != -1: 231 | buckets[idx_bucket].append(i) 232 | 233 | for i in range(len(buckets) - 1, 0, -1): 234 | if len(buckets[i]) == 0: 235 | buckets.pop(i) 236 | self.boundaries.pop(i+1) 237 | 238 | num_samples_per_bucket = [] 239 | for i in range(len(buckets)): 240 | len_bucket = len(buckets[i]) 241 | total_batch_size = self.num_replicas * self.batch_size 242 | rem = (total_batch_size - (len_bucket % total_batch_size)) % total_batch_size 243 | num_samples_per_bucket.append(len_bucket + rem) 244 | return buckets, num_samples_per_bucket 245 | 246 | def __iter__(self): 247 | # deterministically shuffle based on epoch 248 | g = torch.Generator() 249 | g.manual_seed(self.epoch) 250 | 251 | indices = [] 252 | if self.shuffle: 253 | for bucket in self.buckets: 254 | indices.append(torch.randperm(len(bucket), generator=g).tolist()) 255 | else: 256 | for bucket in self.buckets: 257 | indices.append(list(range(len(bucket)))) 258 | 259 | batches = [] 260 | for i in range(len(self.buckets)): 261 | bucket = self.buckets[i] 262 | len_bucket = len(bucket) 263 | ids_bucket = indices[i] 264 | num_samples_bucket = self.num_samples_per_bucket[i] 265 | 266 | # add extra samples to make it evenly divisible 267 | rem = num_samples_bucket - len_bucket 268 | ids_bucket = ids_bucket + ids_bucket * (rem // len_bucket) + ids_bucket[:(rem % len_bucket)] 269 | 270 | # subsample 271 | ids_bucket = ids_bucket[self.rank::self.num_replicas] 272 | 273 | # batching 274 | for j in range(len(ids_bucket) // self.batch_size): 275 | batch = [bucket[idx] for idx in ids_bucket[j*self.batch_size:(j+1)*self.batch_size]] 276 | batches.append(batch) 277 | 278 | if self.shuffle: 279 | batch_ids = torch.randperm(len(batches), generator=g).tolist() 280 | batches = [batches[i] for i in batch_ids] 281 | self.batches = batches 282 | 283 | assert len(self.batches) * self.batch_size == self.num_samples 284 | return iter(self.batches) 285 | 286 | def _bisect(self, x, lo=0, hi=None): 287 | if hi is None: 288 | hi = len(self.boundaries) - 1 289 | 290 | if hi > lo: 291 | mid = (hi + lo) // 2 292 | if self.boundaries[mid] < x and x <= self.boundaries[mid+1]: 293 | return mid 294 | elif x <= self.boundaries[mid]: 295 | return self._bisect(x, lo, mid) 296 | else: 297 | return self._bisect(x, mid + 1, hi) 298 | else: 299 | return -1 300 | 301 | def __len__(self): 302 | return self.num_samples // self.batch_size 303 | -------------------------------------------------------------------------------- /data_utils_whisper.py: -------------------------------------------------------------------------------- 1 | import time 2 | import os 3 | import random 4 | import numpy as np 5 | import torch 6 | import torch.utils.data 7 | 8 | import commons 9 | from mel_processing import spectrogram_torch, spec_to_mel_torch 10 | from utils import load_wav_to_torch, load_filepaths_and_text, transform 11 | #import h5py 12 | 13 | 14 | """Multi speaker version""" 15 | class TextAudioSpeakerLoader(torch.utils.data.Dataset): 16 | """ 17 | 1) loads audio, speaker_id, text pairs 18 | 2) normalizes text and converts them to sequences of integers 19 | 3) computes spectrograms from audio files. 20 | """ 21 | def __init__(self, audiopaths, hparams): 22 | self.audiopaths = load_filepaths_and_text(audiopaths) 23 | self.max_wav_value = hparams.data.max_wav_value 24 | self.sampling_rate = hparams.data.sampling_rate 25 | self.filter_length = hparams.data.filter_length 26 | self.hop_length = hparams.data.hop_length 27 | self.win_length = hparams.data.win_length 28 | self.sampling_rate = hparams.data.sampling_rate 29 | self.use_sr = hparams.train.use_sr 30 | self.use_spk = hparams.model.use_spk 31 | self.spec_len = hparams.train.max_speclen 32 | 33 | random.seed(1235) 34 | random.shuffle(self.audiopaths) 35 | self._filter() 36 | 37 | def _filter(self): 38 | """ 39 | Filter text & store spec lengths 40 | """ 41 | # Store spectrogram lengths for Bucketing 42 | # wav_length ~= file_size / (wav_channels * Bytes per dim) = file_size / (1 * 2) 43 | # spec_length = wav_length // hop_length 44 | 45 | lengths = [] 46 | for audiopath in self.audiopaths: 47 | lengths.append(os.path.getsize(audiopath[0]) // (2 * self.hop_length)) 48 | self.lengths = lengths 49 | 50 | def get_audio(self, filename): 51 | audio, sampling_rate = load_wav_to_torch(filename) 52 | if sampling_rate != self.sampling_rate: 53 | raise ValueError("{} SR doesn't match target {} SR,the audio is{}".format( 54 | sampling_rate, self.sampling_rate,filename)) 55 | audio_norm = audio / self.max_wav_value 56 | audio_norm = audio_norm.unsqueeze(0) 57 | #spec_filename = filename.replace(".wav", ".spec.pt") 58 | spec_filename = filename.replace(".wav", ".f{}h{}w{}spec.pt".format(self.filter_length, self.hop_length, self.win_length)) 59 | if os.path.exists(spec_filename): 60 | try: 61 | spec = torch.load(spec_filename) 62 | except: 63 | print(spec_filename) 64 | spec = spectrogram_torch(audio_norm, self.filter_length, 65 | self.sampling_rate, self.hop_length, self.win_length, 66 | center=False) 67 | spec = torch.squeeze(spec, 0) 68 | torch.save(spec, spec_filename) 69 | else: 70 | print(spec_filename) 71 | spec = spectrogram_torch(audio_norm, self.filter_length, 72 | self.sampling_rate, self.hop_length, self.win_length, 73 | center=False) 74 | spec = torch.squeeze(spec, 0) 75 | torch.save(spec, spec_filename) 76 | 77 | if self.use_spk: 78 | spk_filename = filename.replace(".wav", ".npy") 79 | spk_filename = spk_filename.replace("DUMMY", "dataset/spk") 80 | spk = torch.from_numpy(np.load(spk_filename)) 81 | 82 | if not self.use_sr: 83 | c_filename = filename.replace(".wav", "whisper.pt.npy") 84 | #c_filename = c_filename.replace("DUMMY", "dataset/wavlm") 85 | #c = torch.load(c_filename).squeeze(0) 86 | c=torch.from_numpy(np.load(c_filename)) 87 | c=c.transpose(1,0) 88 | else: 89 | i = random.randint(68,92) 90 | ''' 91 | basename = os.path.basename(filename)[:-4] 92 | spkname = basename[:4] 93 | #print(basename, spkname) 94 | with h5py.File(f"dataset/rs/wavlm/{spkname}/{i}.hdf5","r") as f: 95 | c = torch.from_numpy(f[basename][()]).squeeze(0) 96 | #print(c) 97 | ''' 98 | c_filename = filename.replace(".wav", f"_{i}.pt") 99 | c_filename = c_filename.replace("DUMMY", "dataset/sr/wavlm") 100 | c = torch.load(c_filename).squeeze(0) 101 | 102 | # 2023.01.10 update: code below can deteriorate model performance 103 | # I added these code during cleaning up, thinking that it can offer better performance than my provided checkpoints, but actually it does the opposite. 104 | # What an act of 'adding legs to a snake'! 105 | ''' 106 | lmin = min(c.size(-1), spec.size(-1)) 107 | spec, c = spec[:, :lmin], c[:, :lmin] 108 | audio_norm = audio_norm[:, :lmin*self.hop_length] 109 | _spec, _c, _audio_norm = spec, c, audio_norm 110 | while spec.size(-1) < self.spec_len: 111 | spec = torch.cat((spec, _spec), -1) 112 | c = torch.cat((c, _c), -1) 113 | audio_norm = torch.cat((audio_norm, _audio_norm), -1) 114 | start = random.randint(0, spec.size(-1) - self.spec_len) 115 | end = start + self.spec_len 116 | spec = spec[:, start:end] 117 | c = c[:, start:end] 118 | audio_norm = audio_norm[:, start*self.hop_length:end*self.hop_length] 119 | ''' 120 | 121 | #if self.use_spk: 122 | # return c, spec, audio_norm, spk 123 | #else: 124 | return c, spec, audio_norm 125 | 126 | def __getitem__(self, index): 127 | return self.get_audio(self.audiopaths[index][0]) 128 | 129 | def __len__(self): 130 | return len(self.audiopaths) 131 | 132 | 133 | class TextAudioSpeakerCollate(): 134 | """ Zero-pads model inputs and targets 135 | """ 136 | def __init__(self, hps): 137 | self.hps = hps 138 | self.use_sr = hps.train.use_sr 139 | self.use_spk = hps.model.use_spk 140 | 141 | def __call__(self, batch): 142 | """Collate's training batch from normalized text, audio and speaker identities 143 | PARAMS 144 | ------ 145 | batch: [text_normalized, spec_normalized, wav_normalized, sid] 146 | """ 147 | # Right zero-pad all one-hot text sequences to max input length 148 | _, ids_sorted_decreasing = torch.sort( 149 | torch.LongTensor([x[0].size(1) for x in batch]), 150 | dim=0, descending=True) 151 | 152 | max_spec_len = max([x[1].size(1) for x in batch]) 153 | max_wav_len = max([x[2].size(1) for x in batch]) 154 | 155 | spec_lengths = torch.LongTensor(len(batch)) 156 | wav_lengths = torch.LongTensor(len(batch)) 157 | if self.use_spk: 158 | spks = torch.FloatTensor(len(batch), batch[0][3].size(0)) 159 | else: 160 | spks = None 161 | 162 | c_padded = torch.FloatTensor(len(batch), batch[0][0].size(0), max_spec_len) 163 | spec_padded = torch.FloatTensor(len(batch), batch[0][1].size(0), max_spec_len) 164 | wav_padded = torch.FloatTensor(len(batch), 1, max_wav_len) 165 | c_padded.zero_() 166 | spec_padded.zero_() 167 | wav_padded.zero_() 168 | 169 | for i in range(len(ids_sorted_decreasing)): 170 | row = batch[ids_sorted_decreasing[i]] 171 | 172 | c = row[0] 173 | c_padded[i, :, :c.size(1)] = c 174 | 175 | spec = row[1] 176 | spec_padded[i, :, :spec.size(1)] = spec 177 | spec_lengths[i] = spec.size(1) 178 | 179 | wav = row[2] 180 | wav_padded[i, :, :wav.size(1)] = wav 181 | wav_lengths[i] = wav.size(1) 182 | 183 | if self.use_spk: 184 | spks[i] = row[3] 185 | 186 | spec_seglen = spec_lengths[-1] if spec_lengths[-1] < self.hps.train.max_speclen + 1 else self.hps.train.max_speclen + 1 187 | wav_seglen = spec_seglen * self.hps.data.hop_length 188 | 189 | spec_padded, ids_slice = commons.rand_spec_segments(spec_padded, spec_lengths, spec_seglen) 190 | wav_padded = commons.slice_segments(wav_padded, ids_slice * self.hps.data.hop_length, wav_seglen) 191 | 192 | c_padded = commons.slice_segments(c_padded, ids_slice, spec_seglen)[:,:,:-1] 193 | 194 | spec_padded = spec_padded[:,:,:-1] 195 | wav_padded = wav_padded[:,:,:-self.hps.data.hop_length] 196 | 197 | if self.use_spk: 198 | return c_padded, spec_padded, wav_padded, spks 199 | else: 200 | return c_padded, spec_padded, wav_padded 201 | 202 | 203 | class DistributedBucketSampler(torch.utils.data.distributed.DistributedSampler): 204 | """ 205 | Maintain similar input lengths in a batch. 206 | Length groups are specified by boundaries. 207 | Ex) boundaries = [b1, b2, b3] -> any batch is included either {x | b1 < length(x) <=b2} or {x | b2 < length(x) <= b3}. 208 | 209 | It removes samples which are not included in the boundaries. 210 | Ex) boundaries = [b1, b2, b3] -> any x s.t. length(x) <= b1 or length(x) > b3 are discarded. 211 | """ 212 | def __init__(self, dataset, batch_size, boundaries, num_replicas=None, rank=None, shuffle=True): 213 | super().__init__(dataset, num_replicas=num_replicas, rank=rank, shuffle=shuffle) 214 | self.lengths = dataset.lengths 215 | self.batch_size = batch_size 216 | self.boundaries = boundaries 217 | 218 | self.buckets, self.num_samples_per_bucket = self._create_buckets() 219 | print(self.num_samples_per_bucket) 220 | self.total_size = sum(self.num_samples_per_bucket) 221 | self.num_samples = self.total_size // self.num_replicas 222 | 223 | def _create_buckets(self): 224 | buckets = [[] for _ in range(len(self.boundaries) - 1)] 225 | for i in range(len(self.lengths)): 226 | length = self.lengths[i] 227 | idx_bucket = self._bisect(length) 228 | if idx_bucket != -1: 229 | buckets[idx_bucket].append(i) 230 | 231 | for i in range(len(buckets) - 1, 0, -1): 232 | if len(buckets[i]) == 0: 233 | buckets.pop(i) 234 | self.boundaries.pop(i+1) 235 | 236 | num_samples_per_bucket = [] 237 | for i in range(len(buckets)): 238 | len_bucket = len(buckets[i]) 239 | total_batch_size = self.num_replicas * self.batch_size 240 | rem = (total_batch_size - (len_bucket % total_batch_size)) % total_batch_size 241 | num_samples_per_bucket.append(len_bucket + rem) 242 | return buckets, num_samples_per_bucket 243 | 244 | def __iter__(self): 245 | # deterministically shuffle based on epoch 246 | g = torch.Generator() 247 | g.manual_seed(self.epoch) 248 | 249 | indices = [] 250 | if self.shuffle: 251 | for bucket in self.buckets: 252 | indices.append(torch.randperm(len(bucket), generator=g).tolist()) 253 | else: 254 | for bucket in self.buckets: 255 | indices.append(list(range(len(bucket)))) 256 | 257 | batches = [] 258 | for i in range(len(self.buckets)): 259 | bucket = self.buckets[i] 260 | len_bucket = len(bucket) 261 | ids_bucket = indices[i] 262 | num_samples_bucket = self.num_samples_per_bucket[i] 263 | 264 | # add extra samples to make it evenly divisible 265 | rem = num_samples_bucket - len_bucket 266 | ids_bucket = ids_bucket + ids_bucket * (rem // len_bucket) + ids_bucket[:(rem % len_bucket)] 267 | 268 | # subsample 269 | ids_bucket = ids_bucket[self.rank::self.num_replicas] 270 | 271 | # batching 272 | for j in range(len(ids_bucket) // self.batch_size): 273 | batch = [bucket[idx] for idx in ids_bucket[j*self.batch_size:(j+1)*self.batch_size]] 274 | batches.append(batch) 275 | 276 | if self.shuffle: 277 | batch_ids = torch.randperm(len(batches), generator=g).tolist() 278 | batches = [batches[i] for i in batch_ids] 279 | self.batches = batches 280 | 281 | assert len(self.batches) * self.batch_size == self.num_samples 282 | return iter(self.batches) 283 | 284 | def _bisect(self, x, lo=0, hi=None): 285 | if hi is None: 286 | hi = len(self.boundaries) - 1 287 | 288 | if hi > lo: 289 | mid = (hi + lo) // 2 290 | if self.boundaries[mid] < x and x <= self.boundaries[mid+1]: 291 | return mid 292 | elif x <= self.boundaries[mid]: 293 | return self._bisect(x, lo, mid) 294 | else: 295 | return self._bisect(x, mid + 1, hi) 296 | else: 297 | return -1 298 | 299 | def __len__(self): 300 | return self.num_samples // self.batch_size 301 | -------------------------------------------------------------------------------- /losses.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch.nn import functional as F 3 | 4 | import commons 5 | 6 | 7 | def feature_loss(fmap_r, fmap_g): 8 | loss = 0 9 | for dr, dg in zip(fmap_r, fmap_g): 10 | for rl, gl in zip(dr, dg): 11 | rl = rl.float().detach() 12 | gl = gl.float() 13 | loss += torch.mean(torch.abs(rl - gl)) 14 | 15 | return loss * 2 16 | 17 | 18 | def discriminator_loss(disc_real_outputs, disc_generated_outputs): 19 | loss = 0 20 | r_losses = [] 21 | g_losses = [] 22 | for dr, dg in zip(disc_real_outputs, disc_generated_outputs): 23 | dr = dr.float() 24 | dg = dg.float() 25 | r_loss = torch.mean((1-dr)**2) 26 | g_loss = torch.mean(dg**2) 27 | loss += (r_loss + g_loss) 28 | r_losses.append(r_loss.item()) 29 | g_losses.append(g_loss.item()) 30 | 31 | return loss, r_losses, g_losses 32 | 33 | 34 | def generator_loss(disc_outputs): 35 | loss = 0 36 | gen_losses = [] 37 | for dg in disc_outputs: 38 | dg = dg.float() 39 | l = torch.mean((1-dg)**2) 40 | gen_losses.append(l) 41 | loss += l 42 | 43 | return loss, gen_losses 44 | 45 | 46 | def kl_loss(z_p, logs_q, m_p, logs_p, z_mask): 47 | """ 48 | z_p, logs_q: [b, h, t_t] 49 | m_p, logs_p: [b, h, t_t] 50 | """ 51 | z_p = z_p.float() 52 | logs_q = logs_q.float() 53 | m_p = m_p.float() 54 | logs_p = logs_p.float() 55 | z_mask = z_mask.float() 56 | #print(logs_p) 57 | kl = logs_p - logs_q - 0.5 58 | kl += 0.5 * ((z_p - m_p)**2) * torch.exp(-2. * logs_p) 59 | kl = torch.sum(kl * z_mask) 60 | l = kl / torch.sum(z_mask) 61 | return l 62 | -------------------------------------------------------------------------------- /mel_processing.py: -------------------------------------------------------------------------------- 1 | import math 2 | import os 3 | import random 4 | import torch 5 | from torch import nn 6 | import torch.nn.functional as F 7 | import torch.utils.data 8 | import numpy as np 9 | import librosa 10 | import librosa.util as librosa_util 11 | from librosa.util import normalize, pad_center, tiny 12 | from scipy.signal import get_window 13 | from scipy.io.wavfile import read 14 | from librosa.filters import mel as librosa_mel_fn 15 | 16 | MAX_WAV_VALUE = 32768.0 17 | 18 | 19 | def dynamic_range_compression_torch(x, C=1, clip_val=1e-5): 20 | """ 21 | PARAMS 22 | ------ 23 | C: compression factor 24 | """ 25 | return torch.log(torch.clamp(x, min=clip_val) * C) 26 | 27 | 28 | def dynamic_range_decompression_torch(x, C=1): 29 | """ 30 | PARAMS 31 | ------ 32 | C: compression factor used to compress 33 | """ 34 | return torch.exp(x) / C 35 | 36 | 37 | def spectral_normalize_torch(magnitudes): 38 | output = dynamic_range_compression_torch(magnitudes) 39 | return output 40 | 41 | 42 | def spectral_de_normalize_torch(magnitudes): 43 | output = dynamic_range_decompression_torch(magnitudes) 44 | return output 45 | 46 | 47 | mel_basis = {} 48 | hann_window = {} 49 | 50 | 51 | def spectrogram_torch(y, n_fft, sampling_rate, hop_size, win_size, center=False): 52 | if torch.min(y) < -1.: 53 | print('min value is ', torch.min(y)) 54 | if torch.max(y) > 1.: 55 | print('max value is ', torch.max(y)) 56 | 57 | global hann_window 58 | dtype_device = str(y.dtype) + '_' + str(y.device) 59 | wnsize_dtype_device = str(win_size) + '_' + dtype_device 60 | if wnsize_dtype_device not in hann_window: 61 | hann_window[wnsize_dtype_device] = torch.hann_window(win_size).to(dtype=y.dtype, device=y.device) 62 | 63 | y = torch.nn.functional.pad(y.unsqueeze(1), (int((n_fft-hop_size)/2), int((n_fft-hop_size)/2)), mode='reflect') 64 | y = y.squeeze(1) 65 | 66 | spec = torch.stft(y, n_fft, hop_length=hop_size, win_length=win_size, window=hann_window[wnsize_dtype_device], 67 | center=center, pad_mode='reflect', normalized=False, onesided=True, return_complex=False) 68 | 69 | spec = torch.sqrt(spec.pow(2).sum(-1) + 1e-6) 70 | return spec 71 | 72 | 73 | def spec_to_mel_torch(spec, n_fft, num_mels, sampling_rate, fmin, fmax): 74 | global mel_basis 75 | dtype_device = str(spec.dtype) + '_' + str(spec.device) 76 | fmax_dtype_device = str(fmax) + '_' + dtype_device 77 | if fmax_dtype_device not in mel_basis: 78 | mel = librosa_mel_fn(sr=sampling_rate, n_fft=n_fft, n_mels=num_mels, fmin=fmin, fmax=fmax) 79 | mel_basis[fmax_dtype_device] = torch.from_numpy(mel).to(dtype=spec.dtype, device=spec.device) 80 | spec = torch.matmul(mel_basis[fmax_dtype_device], spec) 81 | spec = spectral_normalize_torch(spec) 82 | return spec 83 | 84 | 85 | def mel_spectrogram_torch(y, n_fft, num_mels, sampling_rate, hop_size, win_size, fmin, fmax, center=False): 86 | if torch.min(y) < -1.: 87 | print('min value is ', torch.min(y)) 88 | if torch.max(y) > 1.: 89 | print('max value is ', torch.max(y)) 90 | 91 | global mel_basis, hann_window 92 | dtype_device = str(y.dtype) + '_' + str(y.device) 93 | fmax_dtype_device = str(fmax) + '_' + dtype_device 94 | wnsize_dtype_device = str(win_size) + '_' + dtype_device 95 | if fmax_dtype_device not in mel_basis: 96 | mel = librosa_mel_fn(sr=sampling_rate, n_fft=n_fft, n_mels=num_mels, fmin=fmin, fmax=fmax) 97 | mel_basis[fmax_dtype_device] = torch.from_numpy(mel).to(dtype=y.dtype, device=y.device) 98 | if wnsize_dtype_device not in hann_window: 99 | hann_window[wnsize_dtype_device] = torch.hann_window(win_size).to(dtype=y.dtype, device=y.device) 100 | 101 | y = torch.nn.functional.pad(y.unsqueeze(1), (int((n_fft-hop_size)/2), int((n_fft-hop_size)/2)), mode='reflect') 102 | y = y.squeeze(1) 103 | 104 | spec = torch.stft(y, n_fft, hop_length=hop_size, win_length=win_size, window=hann_window[wnsize_dtype_device], 105 | center=center, pad_mode='reflect', normalized=False, onesided=True, return_complex=False) 106 | 107 | spec = torch.sqrt(spec.pow(2).sum(-1) + 1e-6) 108 | 109 | spec = torch.matmul(mel_basis[fmax_dtype_device], spec) 110 | spec = spectral_normalize_torch(spec) 111 | 112 | return spec 113 | -------------------------------------------------------------------------------- /models.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import math 3 | import torch 4 | from torch import nn 5 | from torch.nn import functional as F 6 | 7 | import commons 8 | import modules 9 | 10 | from torch.nn import Conv1d, ConvTranspose1d, AvgPool1d, Conv2d 11 | from torch.nn.utils import weight_norm, remove_weight_norm, spectral_norm 12 | from commons import init_weights, get_padding 13 | 14 | 15 | class ResidualCouplingBlock(nn.Module): 16 | def __init__(self, 17 | channels, 18 | hidden_channels, 19 | kernel_size, 20 | dilation_rate, 21 | n_layers, 22 | n_flows=4, 23 | gin_channels=0): 24 | super().__init__() 25 | self.channels = channels 26 | self.hidden_channels = hidden_channels 27 | self.kernel_size = kernel_size 28 | self.dilation_rate = dilation_rate 29 | self.n_layers = n_layers 30 | self.n_flows = n_flows 31 | self.gin_channels = gin_channels 32 | 33 | self.flows = nn.ModuleList() 34 | for i in range(n_flows): 35 | self.flows.append(modules.ResidualCouplingLayer(channels, hidden_channels, kernel_size, dilation_rate, n_layers, gin_channels=gin_channels, mean_only=True)) 36 | self.flows.append(modules.Flip()) 37 | 38 | def forward(self, x, x_mask, g=None, reverse=False): 39 | if not reverse: 40 | for flow in self.flows: 41 | x, _ = flow(x, x_mask, g=g, reverse=reverse) 42 | else: 43 | for flow in reversed(self.flows): 44 | x = flow(x, x_mask, g=g, reverse=reverse) 45 | return x 46 | 47 | 48 | class Encoder(nn.Module): 49 | def __init__(self, 50 | in_channels, 51 | out_channels, 52 | hidden_channels, 53 | kernel_size, 54 | dilation_rate, 55 | n_layers, 56 | gin_channels=0): 57 | super().__init__() 58 | self.in_channels = in_channels 59 | self.out_channels = out_channels 60 | self.hidden_channels = hidden_channels 61 | self.kernel_size = kernel_size 62 | self.dilation_rate = dilation_rate 63 | self.n_layers = n_layers 64 | self.gin_channels = gin_channels 65 | 66 | self.pre = nn.Conv1d(in_channels, hidden_channels, 1) 67 | self.enc = modules.WN(hidden_channels, kernel_size, dilation_rate, n_layers, gin_channels=gin_channels) 68 | self.proj = nn.Conv1d(hidden_channels, out_channels * 2, 1) 69 | 70 | def forward(self, x, x_lengths, g=None): 71 | x_mask = torch.unsqueeze(commons.sequence_mask(x_lengths, x.size(2)), 1).to(x.dtype) 72 | x = self.pre(x) * x_mask 73 | x = self.enc(x, x_mask, g=g) 74 | stats = self.proj(x) * x_mask 75 | m, logs = torch.split(stats, self.out_channels, dim=1) 76 | z = (m + torch.randn_like(m) * torch.exp(logs)) * x_mask 77 | return z, m, logs, x_mask 78 | 79 | 80 | class Generator(torch.nn.Module): 81 | def __init__(self, initial_channel, resblock, resblock_kernel_sizes, resblock_dilation_sizes, upsample_rates, upsample_initial_channel, upsample_kernel_sizes, gin_channels=0): 82 | super(Generator, self).__init__() 83 | self.num_kernels = len(resblock_kernel_sizes) 84 | self.num_upsamples = len(upsample_rates) 85 | self.conv_pre = Conv1d(initial_channel, upsample_initial_channel, 7, 1, padding=3) 86 | resblock = modules.ResBlock1 if resblock == '1' else modules.ResBlock2 87 | 88 | self.ups = nn.ModuleList() 89 | for i, (u, k) in enumerate(zip(upsample_rates, upsample_kernel_sizes)): 90 | self.ups.append(weight_norm( 91 | ConvTranspose1d(upsample_initial_channel//(2**i), upsample_initial_channel//(2**(i+1)), 92 | k, u, padding=(k-u)//2))) 93 | 94 | self.resblocks = nn.ModuleList() 95 | for i in range(len(self.ups)): 96 | ch = upsample_initial_channel//(2**(i+1)) 97 | for j, (k, d) in enumerate(zip(resblock_kernel_sizes, resblock_dilation_sizes)): 98 | self.resblocks.append(resblock(ch, k, d)) 99 | 100 | self.conv_post = Conv1d(ch, 1, 7, 1, padding=3, bias=False) 101 | self.ups.apply(init_weights) 102 | 103 | if gin_channels != 0: 104 | self.cond = nn.Conv1d(gin_channels, upsample_initial_channel, 1) 105 | 106 | def forward(self, x, g=None): 107 | x = self.conv_pre(x) 108 | if g is not None: 109 | x = x + self.cond(g) 110 | 111 | for i in range(self.num_upsamples): 112 | x = F.leaky_relu(x, modules.LRELU_SLOPE) 113 | x = self.ups[i](x) 114 | xs = None 115 | for j in range(self.num_kernels): 116 | if xs is None: 117 | xs = self.resblocks[i*self.num_kernels+j](x) 118 | else: 119 | xs += self.resblocks[i*self.num_kernels+j](x) 120 | x = xs / self.num_kernels 121 | x = F.leaky_relu(x) 122 | x = self.conv_post(x) 123 | x = torch.tanh(x) 124 | 125 | return x 126 | 127 | def remove_weight_norm(self): 128 | print('Removing weight norm...') 129 | for l in self.ups: 130 | remove_weight_norm(l) 131 | for l in self.resblocks: 132 | l.remove_weight_norm() 133 | 134 | 135 | class DiscriminatorP(torch.nn.Module): 136 | def __init__(self, period, kernel_size=5, stride=3, use_spectral_norm=False): 137 | super(DiscriminatorP, self).__init__() 138 | self.period = period 139 | self.use_spectral_norm = use_spectral_norm 140 | norm_f = weight_norm if use_spectral_norm == False else spectral_norm 141 | self.convs = nn.ModuleList([ 142 | norm_f(Conv2d(1, 32, (kernel_size, 1), (stride, 1), padding=(get_padding(kernel_size, 1), 0))), 143 | norm_f(Conv2d(32, 128, (kernel_size, 1), (stride, 1), padding=(get_padding(kernel_size, 1), 0))), 144 | norm_f(Conv2d(128, 512, (kernel_size, 1), (stride, 1), padding=(get_padding(kernel_size, 1), 0))), 145 | norm_f(Conv2d(512, 1024, (kernel_size, 1), (stride, 1), padding=(get_padding(kernel_size, 1), 0))), 146 | norm_f(Conv2d(1024, 1024, (kernel_size, 1), 1, padding=(get_padding(kernel_size, 1), 0))), 147 | ]) 148 | self.conv_post = norm_f(Conv2d(1024, 1, (3, 1), 1, padding=(1, 0))) 149 | 150 | def forward(self, x): 151 | fmap = [] 152 | 153 | # 1d to 2d 154 | b, c, t = x.shape 155 | if t % self.period != 0: # pad first 156 | n_pad = self.period - (t % self.period) 157 | x = F.pad(x, (0, n_pad), "reflect") 158 | t = t + n_pad 159 | x = x.view(b, c, t // self.period, self.period) 160 | 161 | for l in self.convs: 162 | x = l(x) 163 | x = F.leaky_relu(x, modules.LRELU_SLOPE) 164 | fmap.append(x) 165 | x = self.conv_post(x) 166 | fmap.append(x) 167 | x = torch.flatten(x, 1, -1) 168 | 169 | return x, fmap 170 | 171 | 172 | class DiscriminatorS(torch.nn.Module): 173 | def __init__(self, use_spectral_norm=False): 174 | super(DiscriminatorS, self).__init__() 175 | norm_f = weight_norm if use_spectral_norm == False else spectral_norm 176 | self.convs = nn.ModuleList([ 177 | norm_f(Conv1d(1, 16, 15, 1, padding=7)), 178 | norm_f(Conv1d(16, 64, 41, 4, groups=4, padding=20)), 179 | norm_f(Conv1d(64, 256, 41, 4, groups=16, padding=20)), 180 | norm_f(Conv1d(256, 1024, 41, 4, groups=64, padding=20)), 181 | norm_f(Conv1d(1024, 1024, 41, 4, groups=256, padding=20)), 182 | norm_f(Conv1d(1024, 1024, 5, 1, padding=2)), 183 | ]) 184 | self.conv_post = norm_f(Conv1d(1024, 1, 3, 1, padding=1)) 185 | 186 | def forward(self, x): 187 | fmap = [] 188 | 189 | for l in self.convs: 190 | x = l(x) 191 | x = F.leaky_relu(x, modules.LRELU_SLOPE) 192 | fmap.append(x) 193 | x = self.conv_post(x) 194 | fmap.append(x) 195 | x = torch.flatten(x, 1, -1) 196 | 197 | return x, fmap 198 | 199 | 200 | class MultiPeriodDiscriminator(torch.nn.Module): 201 | def __init__(self, use_spectral_norm=False): 202 | super(MultiPeriodDiscriminator, self).__init__() 203 | periods = [2,3,5,7,11] 204 | 205 | discs = [DiscriminatorS(use_spectral_norm=use_spectral_norm)] 206 | discs = discs + [DiscriminatorP(i, use_spectral_norm=use_spectral_norm) for i in periods] 207 | self.discriminators = nn.ModuleList(discs) 208 | 209 | def forward(self, y, y_hat): 210 | y_d_rs = [] 211 | y_d_gs = [] 212 | fmap_rs = [] 213 | fmap_gs = [] 214 | for i, d in enumerate(self.discriminators): 215 | y_d_r, fmap_r = d(y) 216 | y_d_g, fmap_g = d(y_hat) 217 | y_d_rs.append(y_d_r) 218 | y_d_gs.append(y_d_g) 219 | fmap_rs.append(fmap_r) 220 | fmap_gs.append(fmap_g) 221 | 222 | return y_d_rs, y_d_gs, fmap_rs, fmap_gs 223 | 224 | 225 | class SpeakerEncoder(torch.nn.Module): 226 | def __init__(self, mel_n_channels=80, model_num_layers=3, model_hidden_size=256, model_embedding_size=256): 227 | super(SpeakerEncoder, self).__init__() 228 | self.lstm = nn.LSTM(mel_n_channels, model_hidden_size, model_num_layers, batch_first=True) 229 | self.linear = nn.Linear(model_hidden_size, model_embedding_size) 230 | self.relu = nn.ReLU() 231 | 232 | def forward(self, mels): 233 | self.lstm.flatten_parameters() 234 | _, (hidden, _) = self.lstm(mels) 235 | embeds_raw = self.relu(self.linear(hidden[-1])) 236 | return embeds_raw / torch.norm(embeds_raw, dim=1, keepdim=True) 237 | 238 | def compute_partial_slices(self, total_frames, partial_frames, partial_hop): 239 | mel_slices = [] 240 | for i in range(0, total_frames-partial_frames, partial_hop): 241 | mel_range = torch.arange(i, i+partial_frames) 242 | mel_slices.append(mel_range) 243 | 244 | return mel_slices 245 | 246 | def embed_utterance(self, mel, partial_frames=128, partial_hop=64): 247 | mel_len = mel.size(1) 248 | last_mel = mel[:,-partial_frames:] 249 | 250 | if mel_len > partial_frames: 251 | mel_slices = self.compute_partial_slices(mel_len, partial_frames, partial_hop) 252 | mels = list(mel[:,s] for s in mel_slices) 253 | mels.append(last_mel) 254 | mels = torch.stack(tuple(mels), 0).squeeze(1) 255 | 256 | with torch.no_grad(): 257 | partial_embeds = self(mels) 258 | embed = torch.mean(partial_embeds, axis=0).unsqueeze(0) 259 | #embed = embed / torch.linalg.norm(embed, 2) 260 | else: 261 | with torch.no_grad(): 262 | embed = self(last_mel) 263 | 264 | return embed 265 | 266 | 267 | class SynthesizerTrn(nn.Module): 268 | """ 269 | Synthesizer for Training 270 | """ 271 | 272 | def __init__(self, 273 | spec_channels, 274 | segment_size, 275 | inter_channels, 276 | hidden_channels, 277 | filter_channels, 278 | n_heads, 279 | n_layers, 280 | kernel_size, 281 | p_dropout, 282 | resblock, 283 | resblock_kernel_sizes, 284 | resblock_dilation_sizes, 285 | upsample_rates, 286 | upsample_initial_channel, 287 | upsample_kernel_sizes, 288 | gin_channels, 289 | ssl_dim, 290 | use_spk, 291 | **kwargs): 292 | 293 | super().__init__() 294 | self.spec_channels = spec_channels 295 | self.inter_channels = inter_channels 296 | self.hidden_channels = hidden_channels 297 | self.filter_channels = filter_channels 298 | self.n_heads = n_heads 299 | self.n_layers = n_layers 300 | self.kernel_size = kernel_size 301 | self.p_dropout = p_dropout 302 | self.resblock = resblock 303 | self.resblock_kernel_sizes = resblock_kernel_sizes 304 | self.resblock_dilation_sizes = resblock_dilation_sizes 305 | self.upsample_rates = upsample_rates 306 | self.upsample_initial_channel = upsample_initial_channel 307 | self.upsample_kernel_sizes = upsample_kernel_sizes 308 | self.segment_size = segment_size 309 | self.gin_channels = gin_channels 310 | self.ssl_dim = ssl_dim 311 | self.use_spk = use_spk 312 | 313 | self.enc_p = Encoder(ssl_dim, inter_channels, hidden_channels, 5, 1, 16) 314 | self.dec = Generator(inter_channels, resblock, resblock_kernel_sizes, resblock_dilation_sizes, upsample_rates, upsample_initial_channel, upsample_kernel_sizes, gin_channels=gin_channels) 315 | self.enc_q = Encoder(spec_channels, inter_channels, hidden_channels, 5, 1, 16, gin_channels=gin_channels) 316 | self.flow = ResidualCouplingBlock(inter_channels, hidden_channels, 5, 1, 4, gin_channels=gin_channels) 317 | 318 | if not self.use_spk: 319 | self.enc_spk = SpeakerEncoder(model_hidden_size=gin_channels, model_embedding_size=gin_channels) 320 | 321 | def forward(self, c, spec, g=None, mel=None, c_lengths=None, spec_lengths=None): 322 | if c_lengths == None: 323 | c_lengths = (torch.ones(c.size(0)) * c.size(-1)).to(c.device) 324 | if spec_lengths == None: 325 | spec_lengths = (torch.ones(spec.size(0)) * spec.size(-1)).to(spec.device) 326 | 327 | if not self.use_spk: 328 | #print(torch.max(mel),torch.min(mel),mel.size()) 329 | g_raw = self.enc_spk(mel.transpose(1,2)) 330 | g = g_raw.unsqueeze(-1) 331 | 332 | _, m_p, logs_p, _ = self.enc_p(c, c_lengths)#这里也输入一下g会不会更好?models_g_content.py里面有模型代码 333 | z, m_q, logs_q, spec_mask = self.enc_q(spec, spec_lengths, g=g) 334 | z_p = self.flow(z, spec_mask, g=g) 335 | #print(z.size(),spec_lengths, self.segment_size) 336 | z_slice, ids_slice = commons.rand_slice_segments(z, spec_lengths, self.segment_size) 337 | #print(z_slice.size()) 338 | o = self.dec(z_slice, g=g) 339 | #with torch.no_grad(): 340 | # g_hat = self.enc_spk(mel.transpose(1,2)) 341 | 342 | return o, ids_slice, spec_mask, (z, z_p, m_p, logs_p, m_q, logs_q),g_raw#,g_hat 343 | 344 | def infer(self, c, g=None, mel=None, c_lengths=None): 345 | if c_lengths == None: 346 | c_lengths = (torch.ones(c.size(0)) * c.size(-1)).to(c.device) 347 | if not self.use_spk: 348 | #g = self.enc_spk.embed_utterance(mel.transpose(1,2)) 349 | g = self.enc_spk(mel.transpose(1,2)) 350 | g = g.unsqueeze(-1) 351 | 352 | z_p, m_p, logs_p, c_mask = self.enc_p(c, c_lengths) 353 | z = self.flow(z_p, c_mask, g=g, reverse=True) 354 | o = self.dec(z * c_mask, g=g) 355 | 356 | return o 357 | -------------------------------------------------------------------------------- /modules.py: -------------------------------------------------------------------------------- 1 | import copy 2 | import math 3 | import numpy as np 4 | import scipy 5 | import torch 6 | from torch import nn 7 | from torch.nn import functional as F 8 | 9 | from torch.nn import Conv1d, ConvTranspose1d, AvgPool1d, Conv2d 10 | from torch.nn.utils import weight_norm, remove_weight_norm 11 | 12 | import commons 13 | from commons import init_weights, get_padding 14 | 15 | 16 | LRELU_SLOPE = 0.1 17 | 18 | 19 | class LayerNorm(nn.Module): 20 | def __init__(self, channels, eps=1e-5): 21 | super().__init__() 22 | self.channels = channels 23 | self.eps = eps 24 | 25 | self.gamma = nn.Parameter(torch.ones(channels)) 26 | self.beta = nn.Parameter(torch.zeros(channels)) 27 | 28 | def forward(self, x): 29 | x = x.transpose(1, -1) 30 | x = F.layer_norm(x, (self.channels,), self.gamma, self.beta, self.eps) 31 | return x.transpose(1, -1) 32 | 33 | 34 | class ConvReluNorm(nn.Module): 35 | def __init__(self, in_channels, hidden_channels, out_channels, kernel_size, n_layers, p_dropout): 36 | super().__init__() 37 | self.in_channels = in_channels 38 | self.hidden_channels = hidden_channels 39 | self.out_channels = out_channels 40 | self.kernel_size = kernel_size 41 | self.n_layers = n_layers 42 | self.p_dropout = p_dropout 43 | assert n_layers > 1, "Number of layers should be larger than 0." 44 | 45 | self.conv_layers = nn.ModuleList() 46 | self.norm_layers = nn.ModuleList() 47 | self.conv_layers.append(nn.Conv1d(in_channels, hidden_channels, kernel_size, padding=kernel_size//2)) 48 | self.norm_layers.append(LayerNorm(hidden_channels)) 49 | self.relu_drop = nn.Sequential( 50 | nn.ReLU(), 51 | nn.Dropout(p_dropout)) 52 | for _ in range(n_layers-1): 53 | self.conv_layers.append(nn.Conv1d(hidden_channels, hidden_channels, kernel_size, padding=kernel_size//2)) 54 | self.norm_layers.append(LayerNorm(hidden_channels)) 55 | self.proj = nn.Conv1d(hidden_channels, out_channels, 1) 56 | self.proj.weight.data.zero_() 57 | self.proj.bias.data.zero_() 58 | 59 | def forward(self, x, x_mask): 60 | x_org = x 61 | for i in range(self.n_layers): 62 | x = self.conv_layers[i](x * x_mask) 63 | x = self.norm_layers[i](x) 64 | x = self.relu_drop(x) 65 | x = x_org + self.proj(x) 66 | return x * x_mask 67 | 68 | 69 | class DDSConv(nn.Module): 70 | """ 71 | Dialted and Depth-Separable Convolution 72 | """ 73 | def __init__(self, channels, kernel_size, n_layers, p_dropout=0.): 74 | super().__init__() 75 | self.channels = channels 76 | self.kernel_size = kernel_size 77 | self.n_layers = n_layers 78 | self.p_dropout = p_dropout 79 | 80 | self.drop = nn.Dropout(p_dropout) 81 | self.convs_sep = nn.ModuleList() 82 | self.convs_1x1 = nn.ModuleList() 83 | self.norms_1 = nn.ModuleList() 84 | self.norms_2 = nn.ModuleList() 85 | for i in range(n_layers): 86 | dilation = kernel_size ** i 87 | padding = (kernel_size * dilation - dilation) // 2 88 | self.convs_sep.append(nn.Conv1d(channels, channels, kernel_size, 89 | groups=channels, dilation=dilation, padding=padding 90 | )) 91 | self.convs_1x1.append(nn.Conv1d(channels, channels, 1)) 92 | self.norms_1.append(LayerNorm(channels)) 93 | self.norms_2.append(LayerNorm(channels)) 94 | 95 | def forward(self, x, x_mask, g=None): 96 | if g is not None: 97 | x = x + g 98 | for i in range(self.n_layers): 99 | y = self.convs_sep[i](x * x_mask) 100 | y = self.norms_1[i](y) 101 | y = F.gelu(y) 102 | y = self.convs_1x1[i](y) 103 | y = self.norms_2[i](y) 104 | y = F.gelu(y) 105 | y = self.drop(y) 106 | x = x + y 107 | return x * x_mask 108 | 109 | 110 | class WN(torch.nn.Module): 111 | def __init__(self, hidden_channels, kernel_size, dilation_rate, n_layers, gin_channels=0, p_dropout=0): 112 | super(WN, self).__init__() 113 | assert(kernel_size % 2 == 1) 114 | self.hidden_channels =hidden_channels 115 | self.kernel_size = kernel_size, 116 | self.dilation_rate = dilation_rate 117 | self.n_layers = n_layers 118 | self.gin_channels = gin_channels 119 | self.p_dropout = p_dropout 120 | 121 | self.in_layers = torch.nn.ModuleList() 122 | self.res_skip_layers = torch.nn.ModuleList() 123 | self.drop = nn.Dropout(p_dropout) 124 | 125 | if gin_channels != 0: 126 | cond_layer = torch.nn.Conv1d(gin_channels, 2*hidden_channels*n_layers, 1) 127 | self.cond_layer = torch.nn.utils.weight_norm(cond_layer, name='weight') 128 | 129 | for i in range(n_layers): 130 | dilation = dilation_rate ** i 131 | padding = int((kernel_size * dilation - dilation) / 2) 132 | in_layer = torch.nn.Conv1d(hidden_channels, 2*hidden_channels, kernel_size, 133 | dilation=dilation, padding=padding) 134 | in_layer = torch.nn.utils.weight_norm(in_layer, name='weight') 135 | self.in_layers.append(in_layer) 136 | 137 | # last one is not necessary 138 | if i < n_layers - 1: 139 | res_skip_channels = 2 * hidden_channels 140 | else: 141 | res_skip_channels = hidden_channels 142 | 143 | res_skip_layer = torch.nn.Conv1d(hidden_channels, res_skip_channels, 1) 144 | res_skip_layer = torch.nn.utils.weight_norm(res_skip_layer, name='weight') 145 | self.res_skip_layers.append(res_skip_layer) 146 | 147 | def forward(self, x, x_mask, g=None, **kwargs): 148 | output = torch.zeros_like(x) 149 | n_channels_tensor = torch.IntTensor([self.hidden_channels]) 150 | 151 | if g is not None: 152 | g = self.cond_layer(g) 153 | 154 | for i in range(self.n_layers): 155 | x_in = self.in_layers[i](x) 156 | if g is not None: 157 | cond_offset = i * 2 * self.hidden_channels 158 | g_l = g[:,cond_offset:cond_offset+2*self.hidden_channels,:] 159 | else: 160 | g_l = torch.zeros_like(x_in) 161 | 162 | acts = commons.fused_add_tanh_sigmoid_multiply( 163 | x_in, 164 | g_l, 165 | n_channels_tensor) 166 | acts = self.drop(acts) 167 | 168 | res_skip_acts = self.res_skip_layers[i](acts) 169 | if i < self.n_layers - 1: 170 | res_acts = res_skip_acts[:,:self.hidden_channels,:] 171 | x = (x + res_acts) * x_mask 172 | output = output + res_skip_acts[:,self.hidden_channels:,:] 173 | else: 174 | output = output + res_skip_acts 175 | return output * x_mask 176 | 177 | def remove_weight_norm(self): 178 | if self.gin_channels != 0: 179 | torch.nn.utils.remove_weight_norm(self.cond_layer) 180 | for l in self.in_layers: 181 | torch.nn.utils.remove_weight_norm(l) 182 | for l in self.res_skip_layers: 183 | torch.nn.utils.remove_weight_norm(l) 184 | 185 | 186 | class ResBlock1(torch.nn.Module): 187 | def __init__(self, channels, kernel_size=3, dilation=(1, 3, 5)): 188 | super(ResBlock1, self).__init__() 189 | self.convs1 = nn.ModuleList([ 190 | weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[0], 191 | padding=get_padding(kernel_size, dilation[0]))), 192 | weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[1], 193 | padding=get_padding(kernel_size, dilation[1]))), 194 | weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[2], 195 | padding=get_padding(kernel_size, dilation[2]))) 196 | ]) 197 | self.convs1.apply(init_weights) 198 | 199 | self.convs2 = nn.ModuleList([ 200 | weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=1, 201 | padding=get_padding(kernel_size, 1))), 202 | weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=1, 203 | padding=get_padding(kernel_size, 1))), 204 | weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=1, 205 | padding=get_padding(kernel_size, 1))) 206 | ]) 207 | self.convs2.apply(init_weights) 208 | 209 | def forward(self, x, x_mask=None): 210 | for c1, c2 in zip(self.convs1, self.convs2): 211 | xt = F.leaky_relu(x, LRELU_SLOPE) 212 | if x_mask is not None: 213 | xt = xt * x_mask 214 | xt = c1(xt) 215 | xt = F.leaky_relu(xt, LRELU_SLOPE) 216 | if x_mask is not None: 217 | xt = xt * x_mask 218 | xt = c2(xt) 219 | x = xt + x 220 | if x_mask is not None: 221 | x = x * x_mask 222 | return x 223 | 224 | def remove_weight_norm(self): 225 | for l in self.convs1: 226 | remove_weight_norm(l) 227 | for l in self.convs2: 228 | remove_weight_norm(l) 229 | 230 | 231 | class ResBlock2(torch.nn.Module): 232 | def __init__(self, channels, kernel_size=3, dilation=(1, 3)): 233 | super(ResBlock2, self).__init__() 234 | self.convs = nn.ModuleList([ 235 | weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[0], 236 | padding=get_padding(kernel_size, dilation[0]))), 237 | weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[1], 238 | padding=get_padding(kernel_size, dilation[1]))) 239 | ]) 240 | self.convs.apply(init_weights) 241 | 242 | def forward(self, x, x_mask=None): 243 | for c in self.convs: 244 | xt = F.leaky_relu(x, LRELU_SLOPE) 245 | if x_mask is not None: 246 | xt = xt * x_mask 247 | xt = c(xt) 248 | x = xt + x 249 | if x_mask is not None: 250 | x = x * x_mask 251 | return x 252 | 253 | def remove_weight_norm(self): 254 | for l in self.convs: 255 | remove_weight_norm(l) 256 | 257 | 258 | class Log(nn.Module): 259 | def forward(self, x, x_mask, reverse=False, **kwargs): 260 | if not reverse: 261 | y = torch.log(torch.clamp_min(x, 1e-5)) * x_mask 262 | logdet = torch.sum(-y, [1, 2]) 263 | return y, logdet 264 | else: 265 | x = torch.exp(x) * x_mask 266 | return x 267 | 268 | 269 | class Flip(nn.Module): 270 | def forward(self, x, *args, reverse=False, **kwargs): 271 | x = torch.flip(x, [1]) 272 | if not reverse: 273 | logdet = torch.zeros(x.size(0)).to(dtype=x.dtype, device=x.device) 274 | return x, logdet 275 | else: 276 | return x 277 | 278 | 279 | class ElementwiseAffine(nn.Module): 280 | def __init__(self, channels): 281 | super().__init__() 282 | self.channels = channels 283 | self.m = nn.Parameter(torch.zeros(channels,1)) 284 | self.logs = nn.Parameter(torch.zeros(channels,1)) 285 | 286 | def forward(self, x, x_mask, reverse=False, **kwargs): 287 | if not reverse: 288 | y = self.m + torch.exp(self.logs) * x 289 | y = y * x_mask 290 | logdet = torch.sum(self.logs * x_mask, [1,2]) 291 | return y, logdet 292 | else: 293 | x = (x - self.m) * torch.exp(-self.logs) * x_mask 294 | return x 295 | 296 | 297 | class ResidualCouplingLayer(nn.Module): 298 | def __init__(self, 299 | channels, 300 | hidden_channels, 301 | kernel_size, 302 | dilation_rate, 303 | n_layers, 304 | p_dropout=0, 305 | gin_channels=0, 306 | mean_only=False): 307 | assert channels % 2 == 0, "channels should be divisible by 2" 308 | super().__init__() 309 | self.channels = channels 310 | self.hidden_channels = hidden_channels 311 | self.kernel_size = kernel_size 312 | self.dilation_rate = dilation_rate 313 | self.n_layers = n_layers 314 | self.half_channels = channels // 2 315 | self.mean_only = mean_only 316 | 317 | self.pre = nn.Conv1d(self.half_channels, hidden_channels, 1) 318 | self.enc = WN(hidden_channels, kernel_size, dilation_rate, n_layers, p_dropout=p_dropout, gin_channels=gin_channels) 319 | self.post = nn.Conv1d(hidden_channels, self.half_channels * (2 - mean_only), 1) 320 | self.post.weight.data.zero_() 321 | self.post.bias.data.zero_() 322 | 323 | def forward(self, x, x_mask, g=None, reverse=False): 324 | x0, x1 = torch.split(x, [self.half_channels]*2, 1) 325 | h = self.pre(x0) * x_mask 326 | h = self.enc(h, x_mask, g=g) 327 | stats = self.post(h) * x_mask 328 | if not self.mean_only: 329 | m, logs = torch.split(stats, [self.half_channels]*2, 1) 330 | else: 331 | m = stats 332 | logs = torch.zeros_like(m) 333 | 334 | if not reverse: 335 | x1 = m + x1 * torch.exp(logs) * x_mask 336 | x = torch.cat([x0, x1], 1) 337 | logdet = torch.sum(logs, [1,2]) 338 | return x, logdet 339 | else: 340 | x1 = (x1 - m) * torch.exp(-logs) * x_mask 341 | x = torch.cat([x0, x1], 1) 342 | return x 343 | -------------------------------------------------------------------------------- /ppg.py: -------------------------------------------------------------------------------- 1 | from transformers import Wav2Vec2Processor, Wav2Vec2ForCTC 2 | #from datasets import load_dataset 3 | import torch 4 | import soundfile as sf 5 | from glob import glob 6 | import glob2 7 | from tqdm import tqdm 8 | import matplotlib.pyplot as plt 9 | # load model and processor 10 | processor = Wav2Vec2Processor.from_pretrained("speech31/wav2vec2-large-english-TIMIT-phoneme_v3") 11 | model = Wav2Vec2ForCTC.from_pretrained("speech31/wav2vec2-large-english-TIMIT-phoneme_v3").cuda().eval() 12 | files=glob2.glob(r".\dataset\crosslingual_emo_dataset\**\*.wav") 13 | files=sorted(files) 14 | print(len(files)) 15 | for file in tqdm(files): 16 | ppg_file=file.replace(r".wav",r"_eng_ppg.pt") 17 | # Read and process the input 18 | audio_input, sample_rate = sf.read(file) 19 | #audio_input=audio_input[:25000] 20 | #sf.write('3752_slice.wav',audio_input,sample_rate) 21 | inputs = processor(audio_input, sampling_rate=16_000, return_tensors="pt", padding=True) 22 | #print(inputs) 23 | #import matplotlib.pyplot as plt 24 | with torch.no_grad(): 25 | logits = model(inputs.input_values.cuda()).logits 26 | logits_trans=logits.transpose(2,1) 27 | #print(logits_trans.size()) 28 | #print(file,ppg_file) 29 | 30 | torch.save(logits_trans.cpu(),ppg_file) 31 | 32 | #lo=logits.squeeze(0).numpy() 33 | #print(lo) 34 | #lo=lo.transpose(1,0) 35 | #plt.imshow(lo) 36 | #plt.show() 37 | #hidden_states=model(inputs.input_values).hidden_states 38 | #print(hidden_states) 39 | #hidden=hidden_states.squeeze(0).numpy() 40 | #print(hidden) 41 | #plt.imshow(hidden) 42 | #plt.show() 43 | # Decode id into string 44 | #print(logits) 45 | ##predicted_ids = torch.argmax(logits, axis=-1) 46 | ##print(predicted_ids) 47 | ##predicted_sentences = processor.batch_decode(predicted_ids) 48 | ##print(predicted_sentences) 49 | -------------------------------------------------------------------------------- /ppgemoconvert_exp.py: -------------------------------------------------------------------------------- 1 | import os 2 | import argparse 3 | import torch 4 | import librosa 5 | import time 6 | from scipy.io.wavfile import write 7 | from tqdm import tqdm 8 | import soundfile as sf 9 | import utils 10 | from models import SynthesizerTrn 11 | from mel_processing import mel_spectrogram_torch 12 | import logging 13 | logging.getLogger('numba').setLevel(logging.WARNING) 14 | 15 | 16 | if __name__ == "__main__": 17 | parser = argparse.ArgumentParser() 18 | parser.add_argument("--hpfile", type=str, default="./logs/cvc-44ppg-emoloss/config.json", help="path to json config file") 19 | parser.add_argument("--ptfile", type=str, default="./logs/cvc-44ppg-emoloss/G_cvc-44ppg-emoloss.pth", help="path to pth file") 20 | parser.add_argument("--outdir", type=str, default="output_exp/20_exp_cvc-44ppg-emoloss", help="path to output dir") 21 | parser.add_argument("--use_timestamp", default=False, action="store_true") 22 | args = parser.parse_args() 23 | 24 | os.makedirs(args.outdir, exist_ok=True) 25 | hps = utils.get_hparams_from_file(args.hpfile) 26 | 27 | print("Loading model...") 28 | net_g = SynthesizerTrn( 29 | hps.data.filter_length // 2 + 1, 30 | hps.train.segment_size // hps.data.hop_length, 31 | **hps.model).cuda() 32 | _ = net_g.eval() 33 | print("Loading checkpoint...") 34 | _ = utils.load_checkpoint(args.ptfile, net_g, None, True) 35 | 36 | src_wavs=[r".\dataset\ESD16k\0011\Neutral\evaluation\0011_000004.wav", 37 | r".\dataset\ESD16k\0016\Neutral\test\0016_000031.wav"] 38 | 39 | 40 | tgt_wavs=[r".\dataset\ESD16k\0012\Happy\train\0012_000897.wav", 41 | r".\dataset\ESD16k\0012\Angry\test\0012_000374.wav", 42 | r".\dataset\ESD16k\0012\Sad\train\0012_001188.wav", 43 | r".\dataset\ESD16k\0012\Surprise\train\0012_001504.wav", 44 | r".\dataset\ESD16k\0015\Happy\train\0015_000875.wav", 45 | r".\dataset\ESD16k\0015\Angry\train\0015_000619.wav", 46 | r".\dataset\ESD16k\0015\Sad\train\0015_001233.wav", 47 | r".\dataset\ESD16k\0015\Surprise\train\0015_001656.wav", 48 | r".\ravdess_ref\act_11_03-01-05-02-01-01-11_man_angry.wav", 49 | r".\ravdess_ref\act2-03-01-05-01-02-02-02-womanangry.wav"] 50 | print("Processing text...") 51 | titles, srcs, tgts = [], [], [] 52 | for src_wav in src_wavs: 53 | for tgt_wav in tgt_wavs: 54 | src_wav_name=os.path.basename(src_wav)[:-4] 55 | tgt_wav_name=os.path.basename(tgt_wav)[:-4] 56 | title="{}_to_{}".format(src_wav_name,tgt_wav_name) 57 | titles.append(title) 58 | srcs.append(src_wav) 59 | tgts.append(tgt_wav) 60 | print(srcs) 61 | print(tgts) 62 | print(titles) 63 | #import sys 64 | #sys.exit() 65 | """ 66 | with open(args.txtpath, "r") as f: 67 | for rawline in f.readlines(): 68 | print(rawline) 69 | title, src, tgt = rawline.strip().split("|") 70 | titles.append(title) 71 | srcs.append(src) 72 | tgts.append(tgt) 73 | """ 74 | 75 | print("Synthesizing...") 76 | with torch.no_grad(): 77 | for line in tqdm(zip(titles, srcs, tgts)): 78 | title, src, tgt = line 79 | srcname,tgtname=title.split("to") 80 | # tgt 81 | wav_tgt, _ = librosa.load(tgt, sr=hps.data.sampling_rate) 82 | sf.write(os.path.join(args.outdir, f"{tgtname}.wav"), wav_tgt, hps.data.sampling_rate) 83 | #wav_tgt, _ = librosa.effects.trim(wav_tgt, top_db=20) 84 | 85 | wav_tgt = torch.from_numpy(wav_tgt).unsqueeze(0).cuda() 86 | mel_tgt = mel_spectrogram_torch( 87 | wav_tgt, 88 | hps.data.filter_length, 89 | hps.data.n_mel_channels, 90 | hps.data.sampling_rate, 91 | hps.data.hop_length, 92 | hps.data.win_length, 93 | hps.data.mel_fmin, 94 | hps.data.mel_fmax 95 | ) 96 | # src 97 | wav_src, _ = librosa.load(src, sr=hps.data.sampling_rate) 98 | sf.write(os.path.join(args.outdir, f"{srcname}.wav"), wav_src, hps.data.sampling_rate) 99 | wav_src = torch.from_numpy(wav_src).unsqueeze(0).cuda() 100 | #c = utils.get_content(cmodel, wav_src) 101 | c_filename = src.replace(".wav", "ppg.pt") 102 | print(src, tgt,c_filename) 103 | c = torch.load(c_filename)#.squeeze(0) 104 | 105 | print(c.size(),mel_tgt.size()) 106 | audio = net_g.infer(c.cuda(), mel=mel_tgt) 107 | audio = audio[0][0].data.cpu().float().numpy() 108 | if args.use_timestamp: 109 | timestamp = time.strftime("%m-%d_%H-%M", time.localtime()) 110 | write(os.path.join(args.outdir, "{}.wav".format(timestamp+"_"+title)), hps.data.sampling_rate, audio) 111 | else: 112 | write(os.path.join(args.outdir, f"{title}.wav"), hps.data.sampling_rate, audio) 113 | 114 | -------------------------------------------------------------------------------- /preprocess_ppg.py: -------------------------------------------------------------------------------- 1 | import os 2 | import numpy as np 3 | import argparse 4 | import torch 5 | from glob import glob 6 | from tqdm import tqdm 7 | from whisper.model import Whisper, ModelDimensions 8 | from whisper.audio import load_audio, pad_or_trim, log_mel_spectrogram 9 | import librosa 10 | import soundfile as sf 11 | def load_model(path) -> Whisper: 12 | device = "cuda" if torch.cuda.is_available() else "cpu" 13 | checkpoint = torch.load(path, map_location=device) 14 | dims = ModelDimensions(**checkpoint["dims"]) 15 | model = Whisper(dims) 16 | model.load_state_dict(checkpoint["model_state_dict"]) 17 | return model.to(device) 18 | 19 | 20 | def pred_ppg(whisper: Whisper, wavPath, ppgPath): 21 | audio, sr = librosa.load(wavPath,sr=None) 22 | if len(audio) >= sr * 29: 23 | print(wavPath,"cut to 29s") 24 | audio = audio[:sr * 29] 25 | #librosa.output.write_wav("your_audio_file.wav", audio, sr) 26 | sf.write(wavPath, audio, sr) 27 | audio = load_audio(wavPath) 28 | audln = audio.shape[0] 29 | ppgln = audln // 320 30 | # audio = pad_or_trim(audio) 31 | mel = log_mel_spectrogram(audio).to(whisper.device) 32 | with torch.no_grad(): 33 | ppg = whisper.encoder(mel.unsqueeze(0)).squeeze().data.cpu().float().numpy() 34 | 35 | if ppgln>ppg.shape[0]: 36 | print("ppgln>ppg.shape[0]") 37 | ppg = ppg[:ppgln,] # [length, dim=1024] 38 | #if audln // 320 Epoch: {}'.format(epoch)) 353 | 354 | 355 | def evaluate(hps, generator, eval_loader, writer_eval): 356 | generator.eval() 357 | with torch.no_grad(): 358 | for batch_idx, items in enumerate(eval_loader): 359 | if hps.model.use_spk: 360 | c, spec, y, spk = items 361 | g = spk[:1].cuda(0) 362 | else: 363 | c, spec, y = items 364 | g = None 365 | spec, y = spec[:1].cuda(0), y[:1].cuda(0) 366 | c = c[:1].cuda(0) 367 | break 368 | mel = spec_to_mel_torch( 369 | spec, 370 | hps.data.filter_length, 371 | hps.data.n_mel_channels, 372 | hps.data.sampling_rate, 373 | hps.data.mel_fmin, 374 | hps.data.mel_fmax) 375 | y_hat = generator.infer(c, g=g, mel=mel)#generator.module.infer(c, g=g, mel=mel) 376 | print(torch.max(y_hat),torch.min(y_hat)) 377 | y_hat_mel = mel_spectrogram_torch( 378 | y_hat.squeeze(1).float(), 379 | hps.data.filter_length, 380 | hps.data.n_mel_channels, 381 | hps.data.sampling_rate, 382 | hps.data.hop_length, 383 | hps.data.win_length, 384 | hps.data.mel_fmin, 385 | hps.data.mel_fmax 386 | ) 387 | image_dict = { 388 | "gen/mel": utils.plot_spectrogram_to_numpy(y_hat_mel[0].cpu().numpy()), 389 | "gt/mel": utils.plot_spectrogram_to_numpy(mel[0].cpu().numpy()) 390 | } 391 | audio_dict = { 392 | "gen/audio": y_hat[0], 393 | "gt/audio": y[0] 394 | } 395 | utils.summarize( 396 | writer=writer_eval, 397 | global_step=global_step, 398 | images=image_dict, 399 | audios=audio_dict, 400 | audio_sampling_rate=hps.data.sampling_rate 401 | ) 402 | generator.train() 403 | 404 | 405 | if __name__ == "__main__": 406 | main() 407 | -------------------------------------------------------------------------------- /train_whisper_emo.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | import argparse 4 | import itertools 5 | import math 6 | import torch 7 | from torch import nn, optim 8 | from torch.nn import functional as F 9 | from torch.utils.data import DataLoader 10 | print("import tensorboard") 11 | from torch.utils.tensorboard import SummaryWriter 12 | #import torch.multiprocessing as mp 13 | #import torch.distributed as dist 14 | #from torch.nn.parallel import DistributedDataParallel as DDP 15 | from torch.cuda.amp import autocast, GradScaler 16 | #带情感loss，用whisper作为内容特征#python train_whisper_emo.py -c configs/freevc-whispers-three-emo.json -m freevc-whispers-three-emo 17 | import commons 18 | import utils 19 | from data_utils_whisper import ( 20 | TextAudioSpeakerLoader, 21 | TextAudioSpeakerCollate, 22 | DistributedBucketSampler 23 | ) 24 | from models import ( 25 | SynthesizerTrn, 26 | MultiPeriodDiscriminator, 27 | ) 28 | from losses import ( 29 | generator_loss, 30 | discriminator_loss, 31 | feature_loss, 32 | kl_loss 33 | ) 34 | from mel_processing import mel_spectrogram_torch, spec_to_mel_torch 35 | 36 | torch.backends.cudnn.benchmark = True 37 | global_step = 0 38 | #os.environ['TORCH_DISTRIBUTED_DEBUG'] = 'INFO' 39 | 40 | 41 | def main(): 42 | """Assume Single Node Multi GPUs Training Only""" 43 | assert torch.cuda.is_available(), "CPU training is not allowed." 44 | hps = utils.get_hparams() 45 | 46 | n_gpus = torch.cuda.device_count() 47 | #os.environ['MASTER_ADDR'] = 'localhost' 48 | #os.environ['MASTER_PORT'] = hps.train.port 49 | print("start run") 50 | run(0,n_gpus, hps) 51 | #mp.spawn(run, nprocs=n_gpus, args=(n_gpus, hps,)) 52 | 53 | 54 | def run(rank, n_gpus, hps): 55 | global global_step 56 | if rank == 0: 57 | logger = utils.get_logger(hps.model_dir) 58 | logger.info(hps) 59 | utils.check_git_hash(hps.model_dir) 60 | writer = SummaryWriter(log_dir=hps.model_dir) 61 | writer_eval = SummaryWriter(log_dir=os.path.join(hps.model_dir, "eval")) 62 | 63 | #dist.init_process_group(backend='nccl', init_method='env://', world_size=n_gpus, rank=rank) 64 | torch.manual_seed(hps.train.seed) 65 | torch.cuda.set_device(rank) 66 | 67 | train_dataset = TextAudioSpeakerLoader(hps.data.training_files, hps) 68 | train_sampler = DistributedBucketSampler( 69 | train_dataset, 70 | hps.train.batch_size, 71 | [75,100,125,150,175,200,225,250,300,350,400,450,500,550,600,650,700,750,800,850,900,950,1000,1100,1200,1300,1400,1500,2000,3000,4000,5000], 72 | num_replicas=n_gpus, 73 | rank=rank, 74 | shuffle=True) 75 | collate_fn = TextAudioSpeakerCollate(hps) 76 | train_loader = DataLoader(train_dataset, num_workers=12, shuffle=False, pin_memory=True, 77 | collate_fn=collate_fn, batch_sampler=train_sampler) 78 | if rank == 0: 79 | eval_dataset = TextAudioSpeakerLoader(hps.data.validation_files, hps) 80 | eval_loader = DataLoader(eval_dataset, num_workers=2, shuffle=True, 81 | batch_size=hps.train.batch_size, pin_memory=False, 82 | drop_last=False, collate_fn=collate_fn) 83 | 84 | net_g = SynthesizerTrn( 85 | hps.data.filter_length // 2 + 1, 86 | hps.train.segment_size // hps.data.hop_length, 87 | **hps.model).cuda(rank) 88 | net_d = MultiPeriodDiscriminator(hps.model.use_spectral_norm).cuda(rank) 89 | optim_g = torch.optim.AdamW( 90 | net_g.parameters(), 91 | hps.train.learning_rate, 92 | betas=hps.train.betas, 93 | eps=hps.train.eps) 94 | optim_d = torch.optim.AdamW( 95 | net_d.parameters(), 96 | hps.train.learning_rate, 97 | betas=hps.train.betas, 98 | eps=hps.train.eps) 99 | #net_g = DDP(net_g, device_ids=[rank])#, find_unused_parameters=True) 100 | #net_d = DDP(net_d, device_ids=[rank]) 101 | 102 | try: 103 | _, _, _, epoch_str = utils.load_checkpoint(utils.latest_checkpoint_path(hps.model_dir, "G_*.pth"), net_g, optim_g) 104 | _, _, _, epoch_str = utils.load_checkpoint(utils.latest_checkpoint_path(hps.model_dir, "D_*.pth"), net_d, optim_d) 105 | global_step = (epoch_str - 1) * len(train_loader) 106 | except: 107 | epoch_str = 1 108 | global_step = 0 109 | 110 | scheduler_g = torch.optim.lr_scheduler.ExponentialLR(optim_g, gamma=hps.train.lr_decay, last_epoch=epoch_str-2) 111 | scheduler_d = torch.optim.lr_scheduler.ExponentialLR(optim_d, gamma=hps.train.lr_decay, last_epoch=epoch_str-2) 112 | 113 | scaler = GradScaler(enabled=hps.train.fp16_run) 114 | 115 | for epoch in range(epoch_str, hps.train.epochs + 1): 116 | if rank==0: 117 | train_and_evaluate(rank, epoch, hps, [net_g, net_d], [optim_g, optim_d], [scheduler_g, scheduler_d], scaler, [train_loader, eval_loader], logger, [writer, writer_eval]) 118 | else: 119 | train_and_evaluate(rank, epoch, hps, [net_g, net_d], [optim_g, optim_d], [scheduler_g, scheduler_d], scaler, [train_loader, None], None, None) 120 | scheduler_g.step() 121 | scheduler_d.step() 122 | 123 | 124 | def train_and_evaluate(rank, epoch, hps, nets, optims, schedulers, scaler, loaders, logger, writers): 125 | 126 | net_g, net_d = nets 127 | optim_g, optim_d = optims 128 | scheduler_g, scheduler_d = schedulers 129 | train_loader, eval_loader = loaders 130 | if writers is not None: 131 | writer, writer_eval = writers 132 | 133 | train_loader.batch_sampler.set_epoch(epoch) 134 | global global_step 135 | 136 | net_g.train() 137 | net_d.train() 138 | for batch_idx, items in enumerate(train_loader): 139 | if hps.model.use_spk: 140 | c, spec, y, spk = items 141 | g = spk.cuda(rank, non_blocking=True) 142 | else: 143 | c, spec, y = items 144 | g = None 145 | spec, y = spec.cuda(rank, non_blocking=True), y.cuda(rank, non_blocking=True) 146 | c = c.cuda(rank, non_blocking=True) 147 | mel = spec_to_mel_torch( 148 | spec, 149 | hps.data.filter_length, 150 | hps.data.n_mel_channels, 151 | hps.data.sampling_rate, 152 | hps.data.mel_fmin, 153 | hps.data.mel_fmax) 154 | real_mel = mel_spectrogram_torch( 155 | y.squeeze(1), 156 | hps.data.filter_length, 157 | hps.data.n_mel_channels, 158 | hps.data.sampling_rate, 159 | hps.data.hop_length, 160 | hps.data.win_length, 161 | hps.data.mel_fmin, 162 | hps.data.mel_fmax 163 | ) 164 | #print(torch.max(mel),torch.min(mel),torch.max(real_mel),torch.min(real_mel)) 165 | with autocast(enabled=hps.train.fp16_run): 166 | y_hat, ids_slice, z_mask,\ 167 | (z, z_p, m_p, logs_p, m_q, logs_q),emo_y = net_g(c, spec, g=g, mel=mel) 168 | #print(torch.max(y),torch.min(y),torch.max(y_hat),torch.min(y_hat)) 169 | y_mel = commons.slice_segments(mel, ids_slice, hps.train.segment_size // hps.data.hop_length) 170 | y_hat_mel = mel_spectrogram_torch( 171 | y_hat.squeeze(1), 172 | hps.data.filter_length, 173 | hps.data.n_mel_channels, 174 | hps.data.sampling_rate, 175 | hps.data.hop_length, 176 | hps.data.win_length, 177 | hps.data.mel_fmin, 178 | hps.data.mel_fmax 179 | ) 180 | y = commons.slice_segments(y, ids_slice * hps.data.hop_length, hps.train.segment_size) # slice 181 | 182 | # Discriminator 183 | y_d_hat_r, y_d_hat_g, _, _ = net_d(y, y_hat.detach()) 184 | with autocast(enabled=False): 185 | loss_disc, losses_disc_r, losses_disc_g = discriminator_loss(y_d_hat_r, y_d_hat_g) 186 | loss_disc_all = loss_disc 187 | optim_d.zero_grad() 188 | scaler.scale(loss_disc_all).backward() 189 | scaler.unscale_(optim_d) 190 | grad_norm_d = commons.clip_grad_value_(net_d.parameters(), None) 191 | scaler.step(optim_d) 192 | 193 | with autocast(enabled=hps.train.fp16_run): 194 | # Generator 195 | y_d_hat_r, y_d_hat_g, fmap_r, fmap_g = net_d(y, y_hat) 196 | #print(torch.max(mel),torch.min(mel),mel.size(),torch.max(y_hat_mel),torch.min(y_hat_mel),y_hat_mel.size()) 197 | #y_hat_mel=torch.rand_like(y_hat_mel) 198 | #emo_y_hat=net_g.enc_spk(y_hat_mel.transpose(1,2))#process_func(y_input=y_hat,embeddings=True) 199 | #y=torch.rand_like(y) 200 | emo_y_hat=net_g.enc_spk(y_hat_mel.transpose(1,2))#process_func(y_input=y,embeddings=True) 201 | #print(torch.max(emo_y),torch.min(emo_y),emo_y.size(),torch.max(emo_y_hat),torch.min(emo_y_hat),emo_y_hat.size()) 202 | #print(emo_y_hat.size(),emo_y.size()) 203 | with autocast(enabled=False): 204 | loss_mel = F.l1_loss(y_hat_mel, y_mel) * hps.train.c_mel 205 | #print("loss_mel",loss_mel) 206 | loss_kl = kl_loss(z_p, logs_q, m_p, logs_p, z_mask) * hps.train.c_kl 207 | #print("loss_kl",loss_kl) 208 | loss_fm = feature_loss(fmap_r, fmap_g) * 0.5 209 | #print("loss_fm",loss_fm) 210 | loss_gen, losses_gen = generator_loss(y_d_hat_g) 211 | #print("loss_gen, losses_gen",loss_gen, losses_gen) 212 | loss_emo=F.l1_loss(emo_y_hat,emo_y) * hps.train.c_mel * 0.5 213 | #print("loss_emo",loss_emo) 214 | loss_gen_all = loss_gen + loss_fm + loss_mel + loss_kl + loss_emo 215 | optim_g.zero_grad() 216 | scaler.scale(loss_gen_all).backward() 217 | scaler.unscale_(optim_g) 218 | grad_norm_g = commons.clip_grad_value_(net_g.parameters(), None) 219 | scaler.step(optim_g) 220 | scaler.update() 221 | 222 | if rank==0: 223 | if global_step % hps.train.log_interval == 0: 224 | lr = optim_g.param_groups[0]['lr'] 225 | losses = [loss_disc, loss_gen, loss_fm, loss_mel, loss_kl,loss_emo] 226 | logger.info('Train Epoch: {} [{:.0f}%]'.format( 227 | epoch, 228 | 100. * batch_idx / len(train_loader))) 229 | logger.info([x.item() for x in losses] + [global_step, lr]) 230 | 231 | scalar_dict = {"loss/g/total": loss_gen_all, "loss/d/total": loss_disc_all, "learning_rate": lr, "grad_norm_d": grad_norm_d, "grad_norm_g": grad_norm_g} 232 | scalar_dict.update({"loss/g/fm": loss_fm, "loss/g/mel": loss_mel, "loss/g/kl": loss_kl, "loss/g/emo": loss_emo}) 233 | 234 | scalar_dict.update({"loss/g/{}".format(i): v for i, v in enumerate(losses_gen)}) 235 | scalar_dict.update({"loss/d_r/{}".format(i): v for i, v in enumerate(losses_disc_r)}) 236 | scalar_dict.update({"loss/d_g/{}".format(i): v for i, v in enumerate(losses_disc_g)}) 237 | image_dict = { 238 | "slice/mel_org": utils.plot_spectrogram_to_numpy(y_mel[0].data.cpu().numpy()), 239 | "slice/mel_gen": utils.plot_spectrogram_to_numpy(y_hat_mel[0].data.cpu().numpy()), 240 | "all/mel": utils.plot_spectrogram_to_numpy(mel[0].data.cpu().numpy()), 241 | } 242 | utils.summarize( 243 | writer=writer, 244 | global_step=global_step, 245 | images=image_dict, 246 | scalars=scalar_dict) 247 | 248 | if global_step % hps.train.eval_interval == 0: 249 | evaluate(hps, net_g, eval_loader, writer_eval) 250 | utils.save_checkpoint(net_g, optim_g, hps.train.learning_rate, epoch, os.path.join(hps.model_dir, "G_{}.pth".format(global_step))) 251 | utils.save_checkpoint(net_d, optim_d, hps.train.learning_rate, epoch, os.path.join(hps.model_dir, "D_{}.pth".format(global_step))) 252 | global_step += 1 253 | 254 | if rank == 0: 255 | logger.info('====> Epoch: {}'.format(epoch)) 256 | 257 | 258 | def evaluate(hps, generator, eval_loader, writer_eval): 259 | generator.eval() 260 | with torch.no_grad(): 261 | for batch_idx, items in enumerate(eval_loader): 262 | if hps.model.use_spk: 263 | c, spec, y, spk = items 264 | g = spk[:1].cuda(0) 265 | else: 266 | c, spec, y = items 267 | g = None 268 | spec, y = spec[:1].cuda(0), y[:1].cuda(0) 269 | c = c[:1].cuda(0) 270 | break 271 | mel = spec_to_mel_torch( 272 | spec, 273 | hps.data.filter_length, 274 | hps.data.n_mel_channels, 275 | hps.data.sampling_rate, 276 | hps.data.mel_fmin, 277 | hps.data.mel_fmax) 278 | y_hat = generator.infer(c, g=g, mel=mel)#generator.module.infer(c, g=g, mel=mel) 279 | print(torch.max(y_hat),torch.min(y_hat)) 280 | y_hat_mel = mel_spectrogram_torch( 281 | y_hat.squeeze(1).float(), 282 | hps.data.filter_length, 283 | hps.data.n_mel_channels, 284 | hps.data.sampling_rate, 285 | hps.data.hop_length, 286 | hps.data.win_length, 287 | hps.data.mel_fmin, 288 | hps.data.mel_fmax 289 | ) 290 | image_dict = { 291 | "gen/mel": utils.plot_spectrogram_to_numpy(y_hat_mel[0].cpu().numpy()), 292 | "gt/mel": utils.plot_spectrogram_to_numpy(mel[0].cpu().numpy()) 293 | } 294 | audio_dict = { 295 | "gen/audio": y_hat[0], 296 | "gt/audio": y[0] 297 | } 298 | utils.summarize( 299 | writer=writer_eval, 300 | global_step=global_step, 301 | images=image_dict, 302 | audios=audio_dict, 303 | audio_sampling_rate=hps.data.sampling_rate 304 | ) 305 | generator.train() 306 | 307 | 308 | if __name__ == "__main__": 309 | main() 310 | -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import glob 3 | import sys 4 | import argparse 5 | import logging 6 | import json 7 | import subprocess 8 | import numpy as np 9 | from scipy.io.wavfile import read 10 | import torch 11 | import torchvision 12 | from torch.nn import functional as F 13 | from commons import sequence_mask 14 | 15 | 16 | MATPLOTLIB_FLAG = False 17 | 18 | logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) 19 | logger = logging 20 | 21 | 22 | 23 | 24 | 25 | def get_content(cmodel, y): 26 | with torch.no_grad(): 27 | c = cmodel.extract_features(y.squeeze(1))[0] 28 | c = c.transpose(1, 2) 29 | return c 30 | 31 | 32 | 33 | 34 | 35 | def transform(mel, height): # 68-92 36 | #r = np.random.random() 37 | #rate = r * 0.3 + 0.85 # 0.85-1.15 38 | #height = int(mel.size(-2) * rate) 39 | tgt = torchvision.transforms.functional.resize(mel, (height, mel.size(-1))) 40 | if height >= mel.size(-2): 41 | return tgt[:, :mel.size(-2), :] 42 | else: 43 | silence = tgt[:,-1:,:].repeat(1,mel.size(-2)-height,1) 44 | silence += torch.randn_like(silence) / 10 45 | return torch.cat((tgt, silence), 1) 46 | 47 | 48 | def stretch(mel, width): # 0.5-2 49 | return torchvision.transforms.functional.resize(mel, (mel.size(-2), width)) 50 | 51 | 52 | def load_checkpoint(checkpoint_path, model, optimizer=None, strict=False): 53 | assert os.path.isfile(checkpoint_path) 54 | checkpoint_dict = torch.load(checkpoint_path, map_location='cpu') 55 | iteration = checkpoint_dict['iteration'] 56 | learning_rate = checkpoint_dict['learning_rate'] 57 | if optimizer is not None: 58 | optimizer.load_state_dict(checkpoint_dict['optimizer']) 59 | saved_state_dict = checkpoint_dict['model'] 60 | if hasattr(model, 'module'): 61 | state_dict = model.module.state_dict() 62 | else: 63 | state_dict = model.state_dict() 64 | if strict: 65 | assert state_dict.keys() == saved_state_dict.keys(), "Mismatched model config and checkpoint." 66 | new_state_dict= {} 67 | for k, v in state_dict.items(): 68 | try: 69 | new_state_dict[k] = saved_state_dict[k] 70 | except: 71 | logger.info("%s is not in the checkpoint" % k) 72 | new_state_dict[k] = v 73 | if hasattr(model, 'module'): 74 | model.module.load_state_dict(new_state_dict) 75 | else: 76 | model.load_state_dict(new_state_dict) 77 | logger.info("Loaded checkpoint '{}' (iteration {})" .format( 78 | checkpoint_path, iteration)) 79 | return model, optimizer, learning_rate, iteration 80 | 81 | 82 | def save_checkpoint(model, optimizer, learning_rate, iteration, checkpoint_path): 83 | logger.info("Saving model and optimizer state at iteration {} to {}".format( 84 | iteration, checkpoint_path)) 85 | if hasattr(model, 'module'): 86 | state_dict = model.module.state_dict() 87 | else: 88 | state_dict = model.state_dict() 89 | torch.save({'model': state_dict, 90 | 'iteration': iteration, 91 | 'optimizer': optimizer.state_dict(), 92 | 'learning_rate': learning_rate}, checkpoint_path) 93 | 94 | 95 | def summarize(writer, global_step, scalars={}, histograms={}, images={}, audios={}, audio_sampling_rate=22050): 96 | for k, v in scalars.items(): 97 | writer.add_scalar(k, v, global_step) 98 | for k, v in histograms.items(): 99 | writer.add_histogram(k, v, global_step) 100 | for k, v in images.items(): 101 | writer.add_image(k, v, global_step, dataformats='HWC') 102 | for k, v in audios.items(): 103 | writer.add_audio(k, v, global_step, audio_sampling_rate) 104 | 105 | 106 | def latest_checkpoint_path(dir_path, regex="G_*.pth"): 107 | f_list = glob.glob(os.path.join(dir_path, regex)) 108 | f_list.sort(key=lambda f: int("".join(filter(str.isdigit, f)))) 109 | x = f_list[-1] 110 | print(x) 111 | return x 112 | 113 | 114 | def plot_spectrogram_to_numpy(spectrogram): 115 | global MATPLOTLIB_FLAG 116 | if not MATPLOTLIB_FLAG: 117 | import matplotlib 118 | matplotlib.use("Agg") 119 | MATPLOTLIB_FLAG = True 120 | mpl_logger = logging.getLogger('matplotlib') 121 | mpl_logger.setLevel(logging.WARNING) 122 | import matplotlib.pylab as plt 123 | import numpy as np 124 | 125 | fig, ax = plt.subplots(figsize=(10,2)) 126 | im = ax.imshow(spectrogram, aspect="auto", origin="lower", 127 | interpolation='none') 128 | plt.colorbar(im, ax=ax) 129 | plt.xlabel("Frames") 130 | plt.ylabel("Channels") 131 | plt.tight_layout() 132 | 133 | fig.canvas.draw() 134 | data = np.fromstring(fig.canvas.tostring_rgb(), dtype=np.uint8, sep='') 135 | data = data.reshape(fig.canvas.get_width_height()[::-1] + (3,)) 136 | plt.close() 137 | return data 138 | 139 | 140 | def plot_alignment_to_numpy(alignment, info=None): 141 | global MATPLOTLIB_FLAG 142 | if not MATPLOTLIB_FLAG: 143 | import matplotlib 144 | matplotlib.use("Agg") 145 | MATPLOTLIB_FLAG = True 146 | mpl_logger = logging.getLogger('matplotlib') 147 | mpl_logger.setLevel(logging.WARNING) 148 | import matplotlib.pylab as plt 149 | import numpy as np 150 | 151 | fig, ax = plt.subplots(figsize=(6, 4)) 152 | im = ax.imshow(alignment.transpose(), aspect='auto', origin='lower', 153 | interpolation='none') 154 | fig.colorbar(im, ax=ax) 155 | xlabel = 'Decoder timestep' 156 | if info is not None: 157 | xlabel += '\n\n' + info 158 | plt.xlabel(xlabel) 159 | plt.ylabel('Encoder timestep') 160 | plt.tight_layout() 161 | 162 | fig.canvas.draw() 163 | data = np.fromstring(fig.canvas.tostring_rgb(), dtype=np.uint8, sep='') 164 | data = data.reshape(fig.canvas.get_width_height()[::-1] + (3,)) 165 | plt.close() 166 | return data 167 | 168 | 169 | def load_wav_to_torch(full_path): 170 | sampling_rate, data = read(full_path) 171 | return torch.FloatTensor(data.astype(np.float32)), sampling_rate 172 | 173 | 174 | def load_filepaths_and_text(filename, split="|"): 175 | with open(filename, encoding='utf-8') as f: 176 | filepaths_and_text = [line.strip().split(split) for line in f] 177 | return filepaths_and_text 178 | 179 | 180 | def get_hparams(init=True): 181 | parser = argparse.ArgumentParser() 182 | parser.add_argument('-c', '--config', type=str, default="./configs/base.json", 183 | help='JSON file for configuration') 184 | parser.add_argument('-m', '--model', type=str, required=True, 185 | help='Model name') 186 | 187 | args = parser.parse_args() 188 | model_dir = os.path.join("./logs", args.model) 189 | 190 | if not os.path.exists(model_dir): 191 | os.makedirs(model_dir) 192 | 193 | config_path = args.config 194 | config_save_path = os.path.join(model_dir, "config.json") 195 | if init: 196 | with open(config_path, "r") as f: 197 | data = f.read() 198 | with open(config_save_path, "w") as f: 199 | f.write(data) 200 | else: 201 | with open(config_save_path, "r") as f: 202 | data = f.read() 203 | config = json.loads(data) 204 | 205 | hparams = HParams(**config) 206 | hparams.model_dir = model_dir 207 | return hparams 208 | 209 | 210 | def get_hparams_from_dir(model_dir): 211 | config_save_path = os.path.join(model_dir, "config.json") 212 | with open(config_save_path, "r") as f: 213 | data = f.read() 214 | config = json.loads(data) 215 | 216 | hparams =HParams(**config) 217 | hparams.model_dir = model_dir 218 | return hparams 219 | 220 | 221 | def get_hparams_from_file(config_path): 222 | with open(config_path, "r") as f: 223 | data = f.read() 224 | config = json.loads(data) 225 | 226 | hparams =HParams(**config) 227 | return hparams 228 | 229 | 230 | def check_git_hash(model_dir): 231 | source_dir = os.path.dirname(os.path.realpath(__file__)) 232 | if not os.path.exists(os.path.join(source_dir, ".git")): 233 | logger.warn("{} is not a git repository, therefore hash value comparison will be ignored.".format( 234 | source_dir 235 | )) 236 | return 237 | 238 | cur_hash = subprocess.getoutput("git rev-parse HEAD") 239 | 240 | path = os.path.join(model_dir, "githash") 241 | if os.path.exists(path): 242 | saved_hash = open(path).read() 243 | if saved_hash != cur_hash: 244 | logger.warn("git hash values are different. {}(saved) != {}(current)".format( 245 | saved_hash[:8], cur_hash[:8])) 246 | else: 247 | open(path, "w").write(cur_hash) 248 | 249 | 250 | def get_logger(model_dir, filename="train.log"): 251 | global logger 252 | logger = logging.getLogger(os.path.basename(model_dir)) 253 | logger.setLevel(logging.DEBUG) 254 | 255 | formatter = logging.Formatter("%(asctime)s\t%(name)s\t%(levelname)s\t%(message)s") 256 | if not os.path.exists(model_dir): 257 | os.makedirs(model_dir) 258 | h = logging.FileHandler(os.path.join(model_dir, filename)) 259 | h.setLevel(logging.DEBUG) 260 | h.setFormatter(formatter) 261 | logger.addHandler(h) 262 | return logger 263 | 264 | 265 | class HParams(): 266 | def __init__(self, **kwargs): 267 | for k, v in kwargs.items(): 268 | if type(v) == dict: 269 | v = HParams(**v) 270 | self[k] = v 271 | 272 | def keys(self): 273 | return self.__dict__.keys() 274 | 275 | def items(self): 276 | return self.__dict__.items() 277 | 278 | def values(self): 279 | return self.__dict__.values() 280 | 281 | def __len__(self): 282 | return len(self.__dict__) 283 | 284 | def __getitem__(self, key): 285 | return getattr(self, key) 286 | 287 | def __setitem__(self, key, value): 288 | return setattr(self, key, value) 289 | 290 | def __contains__(self, key): 291 | return key in self.__dict__ 292 | 293 | def __repr__(self): 294 | return self.__dict__.__repr__() 295 | -------------------------------------------------------------------------------- /whisper/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 OpenAI 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /whisper/README.md: -------------------------------------------------------------------------------- 1 | # Whisper 2 | 3 | [[Blog]](https://openai.com/blog/whisper) 4 | [[Paper]](https://arxiv.org/abs/2212.04356) 5 | [[Model card]](https://github.com/openai/whisper/blob/main/model-card.md) 6 | [[Colab example]](https://colab.research.google.com/github/openai/whisper/blob/master/notebooks/LibriSpeech.ipynb) 7 | 8 | Whisper is a general-purpose speech recognition model. It is trained on a large dataset of diverse audio and is also a multitasking model that can perform multilingual speech recognition, speech translation, and language identification. 9 | 10 | 11 | ## Approach 12 | 13 | ![Approach](https://raw.githubusercontent.com/openai/whisper/main/approach.png) 14 | 15 | A Transformer sequence-to-sequence model is trained on various speech processing tasks, including multilingual speech recognition, speech translation, spoken language identification, and voice activity detection. These tasks are jointly represented as a sequence of tokens to be predicted by the decoder, allowing a single model to replace many stages of a traditional speech-processing pipeline. The multitask training format uses a set of special tokens that serve as task specifiers or classification targets. 16 | 17 | 18 | ## Setup 19 | 20 | We used Python 3.9.9 and [PyTorch](https://pytorch.org/) 1.10.1 to train and test our models, but the codebase is expected to be compatible with Python 3.8-3.10 and recent PyTorch versions. The codebase also depends on a few Python packages, most notably [HuggingFace Transformers](https://huggingface.co/docs/transformers/index) for their fast tokenizer implementation and [ffmpeg-python](https://github.com/kkroening/ffmpeg-python) for reading audio files. You can download and install (or update to) the latest release of Whisper with the following command: 21 | 22 | pip install -U openai-whisper 23 | 24 | Alternatively, the following command will pull and install the latest commit from this repository, along with its Python dependencies: 25 | 26 | pip install git+https://github.com/openai/whisper.git 27 | 28 | To update the package to the latest version of this repository, please run: 29 | 30 | pip install --upgrade --no-deps --force-reinstall git+https://github.com/openai/whisper.git 31 | 32 | It also requires the command-line tool [`ffmpeg`](https://ffmpeg.org/) to be installed on your system, which is available from most package managers: 33 | 34 | ```bash 35 | # on Ubuntu or Debian 36 | sudo apt update && sudo apt install ffmpeg 37 | 38 | # on Arch Linux 39 | sudo pacman -S ffmpeg 40 | 41 | # on MacOS using Homebrew (https://brew.sh/) 42 | brew install ffmpeg 43 | 44 | # on Windows using Chocolatey (https://chocolatey.org/) 45 | choco install ffmpeg 46 | 47 | # on Windows using Scoop (https://scoop.sh/) 48 | scoop install ffmpeg 49 | ``` 50 | 51 | You may need [`rust`](http://rust-lang.org) installed as well, in case [tokenizers](https://pypi.org/project/tokenizers/) does not provide a pre-built wheel for your platform. If you see installation errors during the `pip install` command above, please follow the [Getting started page](https://www.rust-lang.org/learn/get-started) to install Rust development environment. Additionally, you may need to configure the `PATH` environment variable, e.g. `export PATH="$HOME/.cargo/bin:$PATH"`. If the installation fails with `No module named 'setuptools_rust'`, you need to install `setuptools_rust`, e.g. by running: 52 | 53 | ```bash 54 | pip install setuptools-rust 55 | ``` 56 | 57 | 58 | ## Available models and languages 59 | 60 | There are five model sizes, four with English-only versions, offering speed and accuracy tradeoffs. Below are the names of the available models and their approximate memory requirements and relative speed. 61 | 62 | 63 | | Size | Parameters | English-only model | Multilingual model | Required VRAM | Relative speed | 64 | |:------:|:----------:|:------------------:|:------------------:|:-------------:|:--------------:| 65 | | tiny | 39 M | `tiny.en` | `tiny` | ~1 GB | ~32x | 66 | | base | 74 M | `base.en` | `base` | ~1 GB | ~16x | 67 | | small | 244 M | `small.en` | `small` | ~2 GB | ~6x | 68 | | medium | 769 M | `medium.en` | `medium` | ~5 GB | ~2x | 69 | | large | 1550 M | N/A | `large` | ~10 GB | 1x | 70 | 71 | The `.en` models for English-only applications tend to perform better, especially for the `tiny.en` and `base.en` models. We observed that the difference becomes less significant for the `small.en` and `medium.en` models. 72 | 73 | Whisper's performance varies widely depending on the language. The figure below shows a WER (Word Error Rate) breakdown by languages of the Fleurs dataset using the `large-v2` model. More WER and BLEU scores corresponding to the other models and datasets can be found in Appendix D in [the paper](https://arxiv.org/abs/2212.04356). The smaller, the better. 74 | 75 | ![WER breakdown by language](https://raw.githubusercontent.com/openai/whisper/main/language-breakdown.svg) 76 | 77 | 78 | 79 | ## Command-line usage 80 | 81 | The following command will transcribe speech in audio files, using the `medium` model: 82 | 83 | whisper audio.flac audio.mp3 audio.wav --model medium 84 | 85 | The default setting (which selects the `small` model) works well for transcribing English. To transcribe an audio file containing non-English speech, you can specify the language using the `--language` option: 86 | 87 | whisper japanese.wav --language Japanese 88 | 89 | Adding `--task translate` will translate the speech into English: 90 | 91 | whisper japanese.wav --language Japanese --task translate 92 | 93 | Run the following to view all available options: 94 | 95 | whisper --help 96 | 97 | See [tokenizer.py](https://github.com/openai/whisper/blob/main/whisper/tokenizer.py) for the list of all available languages. 98 | 99 | 100 | ## Python usage 101 | 102 | Transcription can also be performed within Python: 103 | 104 | ```python 105 | import whisper 106 | 107 | model = whisper.load_model("base") 108 | result = model.transcribe("audio.mp3") 109 | print(result["text"]) 110 | ``` 111 | 112 | Internally, the `transcribe()` method reads the entire file and processes the audio with a sliding 30-second window, performing autoregressive sequence-to-sequence predictions on each window. 113 | 114 | Below is an example usage of `whisper.detect_language()` and `whisper.decode()` which provide lower-level access to the model. 115 | 116 | ```python 117 | import whisper 118 | 119 | model = whisper.load_model("base") 120 | 121 | # load audio and pad/trim it to fit 30 seconds 122 | audio = whisper.load_audio("audio.mp3") 123 | audio = whisper.pad_or_trim(audio) 124 | 125 | # make log-Mel spectrogram and move to the same device as the model 126 | mel = whisper.log_mel_spectrogram(audio).to(model.device) 127 | 128 | # detect the spoken language 129 | _, probs = model.detect_language(mel) 130 | print(f"Detected language: {max(probs, key=probs.get)}") 131 | 132 | # decode the audio 133 | options = whisper.DecodingOptions() 134 | result = whisper.decode(model, mel, options) 135 | 136 | # print the recognized text 137 | print(result.text) 138 | ``` 139 | 140 | ## More examples 141 | 142 | Please use the [🙌 Show and tell](https://github.com/openai/whisper/discussions/categories/show-and-tell) category in Discussions for sharing more example usages of Whisper and third-party extensions such as web demos, integrations with other tools, ports for different platforms, etc. 143 | 144 | 145 | ## License 146 | 147 | Whisper's code and model weights are released under the MIT License. See [LICENSE](https://github.com/openai/whisper/blob/main/LICENSE) for further details. -------------------------------------------------------------------------------- /whisper/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConsistencyVC/ConsistencyVC-voive-conversion/b1506a2c4c68337de922b624c0df1a1dd034419d/whisper/__init__.py -------------------------------------------------------------------------------- /whisper/__pycache__/__init__.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConsistencyVC/ConsistencyVC-voive-conversion/b1506a2c4c68337de922b624c0df1a1dd034419d/whisper/__pycache__/__init__.cpython-37.pyc -------------------------------------------------------------------------------- /whisper/__pycache__/audio.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConsistencyVC/ConsistencyVC-voive-conversion/b1506a2c4c68337de922b624c0df1a1dd034419d/whisper/__pycache__/audio.cpython-37.pyc -------------------------------------------------------------------------------- /whisper/__pycache__/decoding.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConsistencyVC/ConsistencyVC-voive-conversion/b1506a2c4c68337de922b624c0df1a1dd034419d/whisper/__pycache__/decoding.cpython-37.pyc -------------------------------------------------------------------------------- /whisper/__pycache__/model.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConsistencyVC/ConsistencyVC-voive-conversion/b1506a2c4c68337de922b624c0df1a1dd034419d/whisper/__pycache__/model.cpython-37.pyc -------------------------------------------------------------------------------- /whisper/__pycache__/tokenizer.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConsistencyVC/ConsistencyVC-voive-conversion/b1506a2c4c68337de922b624c0df1a1dd034419d/whisper/__pycache__/tokenizer.cpython-37.pyc -------------------------------------------------------------------------------- /whisper/__pycache__/utils.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConsistencyVC/ConsistencyVC-voive-conversion/b1506a2c4c68337de922b624c0df1a1dd034419d/whisper/__pycache__/utils.cpython-37.pyc -------------------------------------------------------------------------------- /whisper/assets/gpt2/special_tokens_map.json: -------------------------------------------------------------------------------- 1 | {"bos_token": "<|endoftext|>", "eos_token": "<|endoftext|>", "unk_token": "<|endoftext|>"} -------------------------------------------------------------------------------- /whisper/assets/gpt2/tokenizer_config.json: -------------------------------------------------------------------------------- 1 | {"unk_token": "<|endoftext|>", "bos_token": "<|endoftext|>", "eos_token": "<|endoftext|>", "add_prefix_space": false, "model_max_length": 1024, "special_tokens_map_file": null, "name_or_path": "gpt2", "tokenizer_class": "GPT2Tokenizer"} -------------------------------------------------------------------------------- /whisper/assets/mel_filters.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConsistencyVC/ConsistencyVC-voive-conversion/b1506a2c4c68337de922b624c0df1a1dd034419d/whisper/assets/mel_filters.npz -------------------------------------------------------------------------------- /whisper/assets/multilingual/added_tokens.json: -------------------------------------------------------------------------------- 1 | {"<|endoftext|>": 50257} 2 | -------------------------------------------------------------------------------- /whisper/assets/multilingual/special_tokens_map.json: -------------------------------------------------------------------------------- 1 | {"bos_token": "<|endoftext|>", "eos_token": "<|endoftext|>", "unk_token": "<|endoftext|>"} -------------------------------------------------------------------------------- /whisper/assets/multilingual/tokenizer_config.json: -------------------------------------------------------------------------------- 1 | {"unk_token": {"content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "bos_token": {"content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "eos_token": {"content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "add_prefix_space": false, "model_max_length": 1024, "special_tokens_map_file": null, "name_or_path": "multilingual", "errors": "replace", "tokenizer_class": "GPT2Tokenizer"} -------------------------------------------------------------------------------- /whisper/audio.py: -------------------------------------------------------------------------------- 1 | import os 2 | from functools import lru_cache 3 | from typing import Union 4 | 5 | import ffmpeg 6 | import numpy as np 7 | import torch 8 | import torch.nn.functional as F 9 | 10 | from .utils import exact_div 11 | 12 | # hard-coded audio hyperparameters 13 | SAMPLE_RATE = 16000 14 | N_FFT = 400 15 | N_MELS = 80 16 | HOP_LENGTH = 160 17 | CHUNK_LENGTH = 30 18 | N_SAMPLES = CHUNK_LENGTH * SAMPLE_RATE # 480000: number of samples in a chunk 19 | N_FRAMES = exact_div(N_SAMPLES, HOP_LENGTH) # 3000: number of frames in a mel spectrogram input 20 | 21 | 22 | def load_audio(file: str, sr: int = SAMPLE_RATE): 23 | """ 24 | Open an audio file and read as mono waveform, resampling as necessary 25 | 26 | Parameters 27 | ---------- 28 | file: str 29 | The audio file to open 30 | 31 | sr: int 32 | The sample rate to resample the audio if necessary 33 | 34 | Returns 35 | ------- 36 | A NumPy array containing the audio waveform, in float32 dtype. 37 | """ 38 | try: 39 | # This launches a subprocess to decode audio while down-mixing and resampling as necessary. 40 | # Requires the ffmpeg CLI and `ffmpeg-python` package to be installed. 41 | out, _ = ( 42 | ffmpeg.input(file, threads=0) 43 | .output("-", format="s16le", acodec="pcm_s16le", ac=1, ar=sr) 44 | .run(cmd=["ffmpeg", "-nostdin"], capture_stdout=True, capture_stderr=True) 45 | ) 46 | except ffmpeg.Error as e: 47 | raise RuntimeError(f"Failed to load audio: {e.stderr.decode()}") from e 48 | 49 | return np.frombuffer(out, np.int16).flatten().astype(np.float32) / 32768.0 50 | 51 | 52 | def pad_or_trim(array, length: int = N_SAMPLES, *, axis: int = -1): 53 | """ 54 | Pad or trim the audio array to N_SAMPLES, as expected by the encoder. 55 | """ 56 | if torch.is_tensor(array): 57 | if array.shape[axis] > length: 58 | array = array.index_select(dim=axis, index=torch.arange(length, device=array.device)) 59 | 60 | if array.shape[axis] < length: 61 | pad_widths = [(0, 0)] * array.ndim 62 | pad_widths[axis] = (0, length - array.shape[axis]) 63 | array = F.pad(array, [pad for sizes in pad_widths[::-1] for pad in sizes]) 64 | else: 65 | if array.shape[axis] > length: 66 | array = array.take(indices=range(length), axis=axis) 67 | 68 | if array.shape[axis] < length: 69 | pad_widths = [(0, 0)] * array.ndim 70 | pad_widths[axis] = (0, length - array.shape[axis]) 71 | array = np.pad(array, pad_widths) 72 | 73 | return array 74 | 75 | 76 | @lru_cache(maxsize=None) 77 | def mel_filters(device, n_mels: int = N_MELS) -> torch.Tensor: 78 | """ 79 | load the mel filterbank matrix for projecting STFT into a Mel spectrogram. 80 | Allows decoupling librosa dependency; saved using: 81 | 82 | np.savez_compressed( 83 | "mel_filters.npz", 84 | mel_80=librosa.filters.mel(sr=16000, n_fft=400, n_mels=80), 85 | ) 86 | """ 87 | assert n_mels == 80, f"Unsupported n_mels: {n_mels}" 88 | with np.load(os.path.join(os.path.dirname(__file__), "assets", "mel_filters.npz")) as f: 89 | return torch.from_numpy(f[f"mel_{n_mels}"]).to(device) 90 | 91 | 92 | def log_mel_spectrogram(audio: Union[str, np.ndarray, torch.Tensor], n_mels: int = N_MELS): 93 | """ 94 | Compute the log-Mel spectrogram of 95 | 96 | Parameters 97 | ---------- 98 | audio: Union[str, np.ndarray, torch.Tensor], shape = (*) 99 | The path to audio or either a NumPy array or Tensor containing the audio waveform in 16 kHz 100 | 101 | n_mels: int 102 | The number of Mel-frequency filters, only 80 is supported 103 | 104 | Returns 105 | ------- 106 | torch.Tensor, shape = (80, n_frames) 107 | A Tensor that contains the Mel spectrogram 108 | """ 109 | if not torch.is_tensor(audio): 110 | if isinstance(audio, str): 111 | audio = load_audio(audio) 112 | audio = torch.from_numpy(audio) 113 | 114 | window = torch.hann_window(N_FFT).to(audio.device) 115 | stft = torch.stft(audio, N_FFT, HOP_LENGTH, window=window, return_complex=True) 116 | magnitudes = stft[..., :-1].abs() ** 2 117 | 118 | filters = mel_filters(audio.device, n_mels) 119 | mel_spec = filters @ magnitudes 120 | 121 | log_spec = torch.clamp(mel_spec, min=1e-10).log10() 122 | log_spec = torch.maximum(log_spec, log_spec.max() - 8.0) 123 | log_spec = (log_spec + 4.0) / 4.0 124 | return log_spec 125 | -------------------------------------------------------------------------------- /whisper/model.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import Dict 3 | from typing import Iterable, Optional 4 | 5 | import numpy as np 6 | import torch 7 | import torch.nn.functional as F 8 | from torch import Tensor 9 | from torch import nn 10 | 11 | from .decoding import detect_language as detect_language_function, decode as decode_function 12 | 13 | 14 | @dataclass 15 | class ModelDimensions: 16 | n_mels: int 17 | n_audio_ctx: int 18 | n_audio_state: int 19 | n_audio_head: int 20 | n_audio_layer: int 21 | n_vocab: int 22 | n_text_ctx: int 23 | n_text_state: int 24 | n_text_head: int 25 | n_text_layer: int 26 | 27 | 28 | class LayerNorm(nn.LayerNorm): 29 | def forward(self, x: Tensor) -> Tensor: 30 | return super().forward(x.float()).type(x.dtype) 31 | 32 | 33 | class Linear(nn.Linear): 34 | def forward(self, x: Tensor) -> Tensor: 35 | return F.linear( 36 | x, self.weight.to(x.dtype), None if self.bias is None else self.bias.to(x.dtype) 37 | ) 38 | 39 | 40 | class Conv1d(nn.Conv1d): 41 | def _conv_forward(self, x: Tensor, weight: Tensor, bias: Optional[Tensor]) -> Tensor: 42 | return super()._conv_forward( 43 | x, weight.to(x.dtype), None if bias is None else bias.to(x.dtype) 44 | ) 45 | 46 | 47 | def sinusoids(length, channels, max_timescale=10000): 48 | """Returns sinusoids for positional embedding""" 49 | assert channels % 2 == 0 50 | log_timescale_increment = np.log(max_timescale) / (channels // 2 - 1) 51 | inv_timescales = torch.exp(-log_timescale_increment * torch.arange(channels // 2)) 52 | scaled_time = torch.arange(length)[:, np.newaxis] * inv_timescales[np.newaxis, :] 53 | return torch.cat([torch.sin(scaled_time), torch.cos(scaled_time)], dim=1) 54 | 55 | 56 | class MultiHeadAttention(nn.Module): 57 | def __init__(self, n_state: int, n_head: int): 58 | super().__init__() 59 | self.n_head = n_head 60 | self.query = Linear(n_state, n_state) 61 | self.key = Linear(n_state, n_state, bias=False) 62 | self.value = Linear(n_state, n_state) 63 | self.out = Linear(n_state, n_state) 64 | 65 | def forward( 66 | self, 67 | x: Tensor, 68 | xa: Optional[Tensor] = None, 69 | mask: Optional[Tensor] = None, 70 | kv_cache: Optional[dict] = None, 71 | ): 72 | q = self.query(x) 73 | 74 | if kv_cache is None or xa is None or self.key not in kv_cache: 75 | # hooks, if installed (i.e. kv_cache is not None), will prepend the cached kv tensors; 76 | # otherwise, perform key/value projections for self- or cross-attention as usual. 77 | k = self.key(x if xa is None else xa) 78 | v = self.value(x if xa is None else xa) 79 | else: 80 | # for cross-attention, calculate keys and values once and reuse in subsequent calls. 81 | k = kv_cache[self.key] 82 | v = kv_cache[self.value] 83 | 84 | wv, qk = self.qkv_attention(q, k, v, mask) 85 | return self.out(wv), qk 86 | 87 | def qkv_attention(self, q: Tensor, k: Tensor, v: Tensor, mask: Optional[Tensor] = None): 88 | n_batch, n_ctx, n_state = q.shape 89 | scale = (n_state // self.n_head) ** -0.25 90 | q = q.view(*q.shape[:2], self.n_head, -1).permute(0, 2, 1, 3) * scale 91 | k = k.view(*k.shape[:2], self.n_head, -1).permute(0, 2, 3, 1) * scale 92 | v = v.view(*v.shape[:2], self.n_head, -1).permute(0, 2, 1, 3) 93 | 94 | qk = q @ k 95 | if mask is not None: 96 | qk = qk + mask[:n_ctx, :n_ctx] 97 | qk = qk.float() 98 | 99 | w = F.softmax(qk, dim=-1).to(q.dtype) 100 | return (w @ v).permute(0, 2, 1, 3).flatten(start_dim=2), qk.detach() 101 | 102 | 103 | class ResidualAttentionBlock(nn.Module): 104 | def __init__(self, n_state: int, n_head: int, cross_attention: bool = False): 105 | super().__init__() 106 | 107 | self.attn = MultiHeadAttention(n_state, n_head) 108 | self.attn_ln = LayerNorm(n_state) 109 | 110 | self.cross_attn = MultiHeadAttention(n_state, n_head) if cross_attention else None 111 | self.cross_attn_ln = LayerNorm(n_state) if cross_attention else None 112 | 113 | n_mlp = n_state * 4 114 | self.mlp = nn.Sequential(Linear(n_state, n_mlp), nn.GELU(), Linear(n_mlp, n_state)) 115 | self.mlp_ln = LayerNorm(n_state) 116 | 117 | def forward( 118 | self, 119 | x: Tensor, 120 | xa: Optional[Tensor] = None, 121 | mask: Optional[Tensor] = None, 122 | kv_cache: Optional[dict] = None, 123 | ): 124 | x = x + self.attn(self.attn_ln(x), mask=mask, kv_cache=kv_cache)[0] 125 | if self.cross_attn: 126 | x = x + self.cross_attn(self.cross_attn_ln(x), xa, kv_cache=kv_cache)[0] 127 | x = x + self.mlp(self.mlp_ln(x)) 128 | return x 129 | 130 | 131 | class AudioEncoder(nn.Module): 132 | def __init__(self, n_mels: int, n_ctx: int, n_state: int, n_head: int, n_layer: int): 133 | super().__init__() 134 | self.conv1 = Conv1d(n_mels, n_state, kernel_size=3, padding=1) 135 | self.conv2 = Conv1d(n_state, n_state, kernel_size=3, stride=2, padding=1) 136 | self.register_buffer("positional_embedding", sinusoids(n_ctx, n_state)) 137 | 138 | self.blocks: Iterable[ResidualAttentionBlock] = nn.ModuleList( 139 | [ResidualAttentionBlock(n_state, n_head) for _ in range(n_layer)] 140 | ) 141 | self.ln_post = LayerNorm(n_state) 142 | 143 | def forward(self, x: Tensor): 144 | """ 145 | x : torch.Tensor, shape = (batch_size, n_mels, n_ctx) 146 | the mel spectrogram of the audio 147 | """ 148 | x = F.gelu(self.conv1(x)) 149 | x = F.gelu(self.conv2(x)) 150 | x = x.permute(0, 2, 1) 151 | 152 | len_x = x.shape[1] 153 | len_e = self.positional_embedding.shape[0] 154 | assert len_x <= len_e, "incorrect audio shape" 155 | pos_e = self.positional_embedding[:len_x, :] 156 | x = (x + pos_e).to(x.dtype) 157 | 158 | for block in self.blocks: 159 | x = block(x) 160 | 161 | x = self.ln_post(x) 162 | return x 163 | 164 | 165 | class TextDecoder(nn.Module): 166 | def __init__(self, n_vocab: int, n_ctx: int, n_state: int, n_head: int, n_layer: int): 167 | super().__init__() 168 | 169 | self.token_embedding = nn.Embedding(n_vocab, n_state) 170 | self.positional_embedding = nn.Parameter(torch.empty(n_ctx, n_state)) 171 | 172 | self.blocks: Iterable[ResidualAttentionBlock] = nn.ModuleList( 173 | [ResidualAttentionBlock(n_state, n_head, cross_attention=True) for _ in range(n_layer)] 174 | ) 175 | self.ln = LayerNorm(n_state) 176 | 177 | mask = torch.empty(n_ctx, n_ctx).fill_(-np.inf).triu_(1) 178 | self.register_buffer("mask", mask, persistent=False) 179 | 180 | def forward(self, x: Tensor, xa: Tensor, kv_cache: Optional[dict] = None): 181 | """ 182 | x : torch.LongTensor, shape = (batch_size, <= n_ctx) 183 | the text tokens 184 | xa : torch.Tensor, shape = (batch_size, n_mels, n_audio_ctx) 185 | the encoded audio features to be attended on 186 | """ 187 | offset = next(iter(kv_cache.values())).shape[1] if kv_cache else 0 188 | x = self.token_embedding(x) + self.positional_embedding[offset : offset + x.shape[-1]] 189 | x = x.to(xa.dtype) 190 | 191 | for block in self.blocks: 192 | x = block(x, xa, mask=self.mask, kv_cache=kv_cache) 193 | 194 | x = self.ln(x) 195 | logits = (x @ torch.transpose(self.token_embedding.weight.to(x.dtype), 0, 1)).float() 196 | 197 | return logits 198 | 199 | 200 | class Whisper(nn.Module): 201 | def __init__(self, dims: ModelDimensions): 202 | super().__init__() 203 | self.dims = dims 204 | self.encoder = AudioEncoder( 205 | self.dims.n_mels, 206 | self.dims.n_audio_ctx, 207 | self.dims.n_audio_state, 208 | self.dims.n_audio_head, 209 | self.dims.n_audio_layer, 210 | ) 211 | self.decoder = TextDecoder( 212 | self.dims.n_vocab, 213 | self.dims.n_text_ctx, 214 | self.dims.n_text_state, 215 | self.dims.n_text_head, 216 | self.dims.n_text_layer, 217 | ) 218 | 219 | def embed_audio(self, mel: torch.Tensor): 220 | return self.encoder(mel) 221 | 222 | def logits(self, tokens: torch.Tensor, audio_features: torch.Tensor): 223 | return self.decoder(tokens, audio_features) 224 | 225 | def forward(self, mel: torch.Tensor, tokens: torch.Tensor) -> Dict[str, torch.Tensor]: 226 | return self.decoder(tokens, self.encoder(mel)) 227 | 228 | @property 229 | def device(self): 230 | return next(self.parameters()).device 231 | 232 | @property 233 | def is_multilingual(self): 234 | return self.dims.n_vocab == 51865 235 | 236 | def install_kv_cache_hooks(self, cache: Optional[dict] = None): 237 | """ 238 | The `MultiHeadAttention` module optionally accepts `kv_cache` which stores the key and value 239 | tensors calculated for the previous positions. This method returns a dictionary that stores 240 | all caches, and the necessary hooks for the key and value projection modules that save the 241 | intermediate tensors to be reused during later calculations. 242 | 243 | Returns 244 | ------- 245 | cache : Dict[nn.Module, torch.Tensor] 246 | A dictionary object mapping the key/value projection modules to its cache 247 | hooks : List[RemovableHandle] 248 | List of PyTorch RemovableHandle objects to stop the hooks to be called 249 | """ 250 | cache = {**cache} if cache is not None else {} 251 | hooks = [] 252 | 253 | def save_to_cache(module, _, output): 254 | if module not in cache or output.shape[1] > self.decoder.positional_embedding.shape[0]: 255 | cache[module] = output # save as-is, for the first token or cross attention 256 | else: 257 | cache[module] = torch.cat([cache[module], output], dim=1).detach() 258 | return cache[module] 259 | 260 | def install_hooks(layer: nn.Module): 261 | if isinstance(layer, MultiHeadAttention): 262 | hooks.append(layer.key.register_forward_hook(save_to_cache)) 263 | hooks.append(layer.value.register_forward_hook(save_to_cache)) 264 | 265 | self.decoder.apply(install_hooks) 266 | return cache, hooks 267 | 268 | detect_language = detect_language_function 269 | decode = decode_function 270 | -------------------------------------------------------------------------------- /whisper/normalizers/__init__.py: -------------------------------------------------------------------------------- 1 | from .basic import BasicTextNormalizer 2 | from .english import EnglishTextNormalizer 3 | -------------------------------------------------------------------------------- /whisper/normalizers/basic.py: -------------------------------------------------------------------------------- 1 | import re 2 | import unicodedata 3 | 4 | import regex 5 | 6 | # non-ASCII letters that are not separated by "NFKD" normalization 7 | ADDITIONAL_DIACRITICS = { 8 | "œ": "oe", 9 | "Œ": "OE", 10 | "ø": "o", 11 | "Ø": "O", 12 | "æ": "ae", 13 | "Æ": "AE", 14 | "ß": "ss", 15 | "ẞ": "SS", 16 | "đ": "d", 17 | "Đ": "D", 18 | "ð": "d", 19 | "Ð": "D", 20 | "þ": "th", 21 | "Þ": "th", 22 | "ł": "l", 23 | "Ł": "L", 24 | } 25 | 26 | 27 | def remove_symbols_and_diacritics(s: str, keep=""): 28 | """ 29 | Replace any other markers, symbols, and punctuations with a space, 30 | and drop any diacritics (category 'Mn' and some manual mappings) 31 | """ 32 | return "".join( 33 | c 34 | if c in keep 35 | else ADDITIONAL_DIACRITICS[c] 36 | if c in ADDITIONAL_DIACRITICS 37 | else "" 38 | if unicodedata.category(c) == "Mn" 39 | else " " 40 | if unicodedata.category(c)[0] in "MSP" 41 | else c 42 | for c in unicodedata.normalize("NFKD", s) 43 | ) 44 | 45 | 46 | def remove_symbols(s: str): 47 | """ 48 | Replace any other markers, symbols, punctuations with a space, keeping diacritics 49 | """ 50 | return "".join( 51 | " " if unicodedata.category(c)[0] in "MSP" else c for c in unicodedata.normalize("NFKC", s) 52 | ) 53 | 54 | 55 | class BasicTextNormalizer: 56 | def __init__(self, remove_diacritics: bool = False, split_letters: bool = False): 57 | self.clean = remove_symbols_and_diacritics if remove_diacritics else remove_symbols 58 | self.split_letters = split_letters 59 | 60 | def __call__(self, s: str): 61 | s = s.lower() 62 | s = re.sub(r"[<\[][^>\]]*[>\]]", "", s) # remove words between brackets 63 | s = re.sub(r"\(([^)]+?)\)", "", s) # remove words between parenthesis 64 | s = self.clean(s).lower() 65 | 66 | if self.split_letters: 67 | s = " ".join(regex.findall(r"\X", s, regex.U)) 68 | 69 | s = re.sub(r"\s+", " ", s) # replace any successive whitespace characters with a space 70 | 71 | return s 72 | -------------------------------------------------------------------------------- /whisper/normalizers/english.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import re 4 | from fractions import Fraction 5 | from typing import Iterator, List, Match, Optional, Union 6 | 7 | from more_itertools import windowed 8 | 9 | from .basic import remove_symbols_and_diacritics 10 | 11 | 12 | class EnglishNumberNormalizer: 13 | """ 14 | Convert any spelled-out numbers into arabic numbers, while handling: 15 | 16 | - remove any commas 17 | - keep the suffixes such as: `1960s`, `274th`, `32nd`, etc. 18 | - spell out currency symbols after the number. e.g. `$20 million` -> `20000000 dollars` 19 | - spell out `one` and `ones` 20 | - interpret successive single-digit numbers as nominal: `one oh one` -> `101` 21 | """ 22 | 23 | def __init__(self): 24 | super().__init__() 25 | 26 | self.zeros = {"o", "oh", "zero"} 27 | self.ones = { 28 | name: i 29 | for i, name in enumerate( 30 | [ 31 | "one", 32 | "two", 33 | "three", 34 | "four", 35 | "five", 36 | "six", 37 | "seven", 38 | "eight", 39 | "nine", 40 | "ten", 41 | "eleven", 42 | "twelve", 43 | "thirteen", 44 | "fourteen", 45 | "fifteen", 46 | "sixteen", 47 | "seventeen", 48 | "eighteen", 49 | "nineteen", 50 | ], 51 | start=1, 52 | ) 53 | } 54 | self.ones_plural = { 55 | "sixes" if name == "six" else name + "s": (value, "s") 56 | for name, value in self.ones.items() 57 | } 58 | self.ones_ordinal = { 59 | "zeroth": (0, "th"), 60 | "first": (1, "st"), 61 | "second": (2, "nd"), 62 | "third": (3, "rd"), 63 | "fifth": (5, "th"), 64 | "twelfth": (12, "th"), 65 | **{ 66 | name + ("h" if name.endswith("t") else "th"): (value, "th") 67 | for name, value in self.ones.items() 68 | if value > 3 and value != 5 and value != 12 69 | }, 70 | } 71 | self.ones_suffixed = {**self.ones_plural, **self.ones_ordinal} 72 | 73 | self.tens = { 74 | "twenty": 20, 75 | "thirty": 30, 76 | "forty": 40, 77 | "fifty": 50, 78 | "sixty": 60, 79 | "seventy": 70, 80 | "eighty": 80, 81 | "ninety": 90, 82 | } 83 | self.tens_plural = { 84 | name.replace("y", "ies"): (value, "s") for name, value in self.tens.items() 85 | } 86 | self.tens_ordinal = { 87 | name.replace("y", "ieth"): (value, "th") for name, value in self.tens.items() 88 | } 89 | self.tens_suffixed = {**self.tens_plural, **self.tens_ordinal} 90 | 91 | self.multipliers = { 92 | "hundred": 100, 93 | "thousand": 1_000, 94 | "million": 1_000_000, 95 | "billion": 1_000_000_000, 96 | "trillion": 1_000_000_000_000, 97 | "quadrillion": 1_000_000_000_000_000, 98 | "quintillion": 1_000_000_000_000_000_000, 99 | "sextillion": 1_000_000_000_000_000_000_000, 100 | "septillion": 1_000_000_000_000_000_000_000_000, 101 | "octillion": 1_000_000_000_000_000_000_000_000_000, 102 | "nonillion": 1_000_000_000_000_000_000_000_000_000_000, 103 | "decillion": 1_000_000_000_000_000_000_000_000_000_000_000, 104 | } 105 | self.multipliers_plural = { 106 | name + "s": (value, "s") for name, value in self.multipliers.items() 107 | } 108 | self.multipliers_ordinal = { 109 | name + "th": (value, "th") for name, value in self.multipliers.items() 110 | } 111 | self.multipliers_suffixed = {**self.multipliers_plural, **self.multipliers_ordinal} 112 | self.decimals = {*self.ones, *self.tens, *self.zeros} 113 | 114 | self.preceding_prefixers = { 115 | "minus": "-", 116 | "negative": "-", 117 | "plus": "+", 118 | "positive": "+", 119 | } 120 | self.following_prefixers = { 121 | "pound": "£", 122 | "pounds": "£", 123 | "euro": "€", 124 | "euros": "€", 125 | "dollar": "$", 126 | "dollars": "$", 127 | "cent": "¢", 128 | "cents": "¢", 129 | } 130 | self.prefixes = set( 131 | list(self.preceding_prefixers.values()) + list(self.following_prefixers.values()) 132 | ) 133 | self.suffixers = { 134 | "per": {"cent": "%"}, 135 | "percent": "%", 136 | } 137 | self.specials = {"and", "double", "triple", "point"} 138 | 139 | self.words = set( 140 | [ 141 | key 142 | for mapping in [ 143 | self.zeros, 144 | self.ones, 145 | self.ones_suffixed, 146 | self.tens, 147 | self.tens_suffixed, 148 | self.multipliers, 149 | self.multipliers_suffixed, 150 | self.preceding_prefixers, 151 | self.following_prefixers, 152 | self.suffixers, 153 | self.specials, 154 | ] 155 | for key in mapping 156 | ] 157 | ) 158 | self.literal_words = {"one", "ones"} 159 | 160 | def process_words(self, words: List[str]) -> Iterator[str]: 161 | prefix: Optional[str] = None 162 | value: Optional[Union[str, int]] = None 163 | skip = False 164 | 165 | def to_fraction(s: str): 166 | try: 167 | return Fraction(s) 168 | except ValueError: 169 | return None 170 | 171 | def output(result: Union[str, int]): 172 | nonlocal prefix, value 173 | result = str(result) 174 | if prefix is not None: 175 | result = prefix + result 176 | value = None 177 | prefix = None 178 | return result 179 | 180 | if len(words) == 0: 181 | return 182 | 183 | for prev, current, next in windowed([None] + words + [None], 3): 184 | if skip: 185 | skip = False 186 | continue 187 | 188 | next_is_numeric = next is not None and re.match(r"^\d+(\.\d+)?$", next) 189 | has_prefix = current[0] in self.prefixes 190 | current_without_prefix = current[1:] if has_prefix else current 191 | if re.match(r"^\d+(\.\d+)?$", current_without_prefix): 192 | # arabic numbers (potentially with signs and fractions) 193 | f = to_fraction(current_without_prefix) 194 | assert f is not None 195 | if value is not None: 196 | if isinstance(value, str) and value.endswith("."): 197 | # concatenate decimals / ip address components 198 | value = str(value) + str(current) 199 | continue 200 | else: 201 | yield output(value) 202 | 203 | prefix = current[0] if has_prefix else prefix 204 | if f.denominator == 1: 205 | value = f.numerator # store integers as int 206 | else: 207 | value = current_without_prefix 208 | elif current not in self.words: 209 | # non-numeric words 210 | if value is not None: 211 | yield output(value) 212 | yield output(current) 213 | elif current in self.zeros: 214 | value = str(value or "") + "0" 215 | elif current in self.ones: 216 | ones = self.ones[current] 217 | 218 | if value is None: 219 | value = ones 220 | elif isinstance(value, str) or prev in self.ones: 221 | if prev in self.tens and ones < 10: # replace the last zero with the digit 222 | assert value[-1] == "0" 223 | value = value[:-1] + str(ones) 224 | else: 225 | value = str(value) + str(ones) 226 | elif ones < 10: 227 | if value % 10 == 0: 228 | value += ones 229 | else: 230 | value = str(value) + str(ones) 231 | else: # eleven to nineteen 232 | if value % 100 == 0: 233 | value += ones 234 | else: 235 | value = str(value) + str(ones) 236 | elif current in self.ones_suffixed: 237 | # ordinal or cardinal; yield the number right away 238 | ones, suffix = self.ones_suffixed[current] 239 | if value is None: 240 | yield output(str(ones) + suffix) 241 | elif isinstance(value, str) or prev in self.ones: 242 | if prev in self.tens and ones < 10: 243 | assert value[-1] == "0" 244 | yield output(value[:-1] + str(ones) + suffix) 245 | else: 246 | yield output(str(value) + str(ones) + suffix) 247 | elif ones < 10: 248 | if value % 10 == 0: 249 | yield output(str(value + ones) + suffix) 250 | else: 251 | yield output(str(value) + str(ones) + suffix) 252 | else: # eleven to nineteen 253 | if value % 100 == 0: 254 | yield output(str(value + ones) + suffix) 255 | else: 256 | yield output(str(value) + str(ones) + suffix) 257 | value = None 258 | elif current in self.tens: 259 | tens = self.tens[current] 260 | if value is None: 261 | value = tens 262 | elif isinstance(value, str): 263 | value = str(value) + str(tens) 264 | else: 265 | if value % 100 == 0: 266 | value += tens 267 | else: 268 | value = str(value) + str(tens) 269 | elif current in self.tens_suffixed: 270 | # ordinal or cardinal; yield the number right away 271 | tens, suffix = self.tens_suffixed[current] 272 | if value is None: 273 | yield output(str(tens) + suffix) 274 | elif isinstance(value, str): 275 | yield output(str(value) + str(tens) + suffix) 276 | else: 277 | if value % 100 == 0: 278 | yield output(str(value + tens) + suffix) 279 | else: 280 | yield output(str(value) + str(tens) + suffix) 281 | elif current in self.multipliers: 282 | multiplier = self.multipliers[current] 283 | if value is None: 284 | value = multiplier 285 | elif isinstance(value, str) or value == 0: 286 | f = to_fraction(value) 287 | p = f * multiplier if f is not None else None 288 | if f is not None and p.denominator == 1: 289 | value = p.numerator 290 | else: 291 | yield output(value) 292 | value = multiplier 293 | else: 294 | before = value // 1000 * 1000 295 | residual = value % 1000 296 | value = before + residual * multiplier 297 | elif current in self.multipliers_suffixed: 298 | multiplier, suffix = self.multipliers_suffixed[current] 299 | if value is None: 300 | yield output(str(multiplier) + suffix) 301 | elif isinstance(value, str): 302 | f = to_fraction(value) 303 | p = f * multiplier if f is not None else None 304 | if f is not None and p.denominator == 1: 305 | yield output(str(p.numerator) + suffix) 306 | else: 307 | yield output(value) 308 | yield output(str(multiplier) + suffix) 309 | else: # int 310 | before = value // 1000 * 1000 311 | residual = value % 1000 312 | value = before + residual * multiplier 313 | yield output(str(value) + suffix) 314 | value = None 315 | elif current in self.preceding_prefixers: 316 | # apply prefix (positive, minus, etc.) if it precedes a number 317 | if value is not None: 318 | yield output(value) 319 | 320 | if next in self.words or next_is_numeric: 321 | prefix = self.preceding_prefixers[current] 322 | else: 323 | yield output(current) 324 | elif current in self.following_prefixers: 325 | # apply prefix (dollars, cents, etc.) only after a number 326 | if value is not None: 327 | prefix = self.following_prefixers[current] 328 | yield output(value) 329 | else: 330 | yield output(current) 331 | elif current in self.suffixers: 332 | # apply suffix symbols (percent -> '%') 333 | if value is not None: 334 | suffix = self.suffixers[current] 335 | if isinstance(suffix, dict): 336 | if next in suffix: 337 | yield output(str(value) + suffix[next]) 338 | skip = True 339 | else: 340 | yield output(value) 341 | yield output(current) 342 | else: 343 | yield output(str(value) + suffix) 344 | else: 345 | yield output(current) 346 | elif current in self.specials: 347 | if next not in self.words and not next_is_numeric: 348 | # apply special handling only if the next word can be numeric 349 | if value is not None: 350 | yield output(value) 351 | yield output(current) 352 | elif current == "and": 353 | # ignore "and" after hundreds, thousands, etc. 354 | if prev not in self.multipliers: 355 | if value is not None: 356 | yield output(value) 357 | yield output(current) 358 | elif current == "double" or current == "triple": 359 | if next in self.ones or next in self.zeros: 360 | repeats = 2 if current == "double" else 3 361 | ones = self.ones.get(next, 0) 362 | value = str(value or "") + str(ones) * repeats 363 | skip = True 364 | else: 365 | if value is not None: 366 | yield output(value) 367 | yield output(current) 368 | elif current == "point": 369 | if next in self.decimals or next_is_numeric: 370 | value = str(value or "") + "." 371 | else: 372 | # should all have been covered at this point 373 | raise ValueError(f"Unexpected token: {current}") 374 | else: 375 | # all should have been covered at this point 376 | raise ValueError(f"Unexpected token: {current}") 377 | 378 | if value is not None: 379 | yield output(value) 380 | 381 | def preprocess(self, s: str): 382 | # replace " and a half" with " point five" 383 | results = [] 384 | 385 | segments = re.split(r"\band\s+a\s+half\b", s) 386 | for i, segment in enumerate(segments): 387 | if len(segment.strip()) == 0: 388 | continue 389 | if i == len(segments) - 1: 390 | results.append(segment) 391 | else: 392 | results.append(segment) 393 | last_word = segment.rsplit(maxsplit=2)[-1] 394 | if last_word in self.decimals or last_word in self.multipliers: 395 | results.append("point five") 396 | else: 397 | results.append("and a half") 398 | 399 | s = " ".join(results) 400 | 401 | # put a space at number/letter boundary 402 | s = re.sub(r"([a-z])([0-9])", r"\1 \2", s) 403 | s = re.sub(r"([0-9])([a-z])", r"\1 \2", s) 404 | 405 | # but remove spaces which could be a suffix 406 | s = re.sub(r"([0-9])\s+(st|nd|rd|th|s)\b", r"\1\2", s) 407 | 408 | return s 409 | 410 | def postprocess(self, s: str): 411 | def combine_cents(m: Match): 412 | try: 413 | currency = m.group(1) 414 | integer = m.group(2) 415 | cents = int(m.group(3)) 416 | return f"{currency}{integer}.{cents:02d}" 417 | except ValueError: 418 | return m.string 419 | 420 | def extract_cents(m: Match): 421 | try: 422 | return f"¢{int(m.group(1))}" 423 | except ValueError: 424 | return m.string 425 | 426 | # apply currency postprocessing; "$2 and ¢7" -> "$2.07" 427 | s = re.sub(r"([€£$])([0-9]+) (?:and )?¢([0-9]{1,2})\b", combine_cents, s) 428 | s = re.sub(r"[€£$]0.([0-9]{1,2})\b", extract_cents, s) 429 | 430 | # write "one(s)" instead of "1(s)", just for the readability 431 | s = re.sub(r"\b1(s?)\b", r"one\1", s) 432 | 433 | return s 434 | 435 | def __call__(self, s: str): 436 | s = self.preprocess(s) 437 | s = " ".join(word for word in self.process_words(s.split()) if word is not None) 438 | s = self.postprocess(s) 439 | 440 | return s 441 | 442 | 443 | class EnglishSpellingNormalizer: 444 | """ 445 | Applies British-American spelling mappings as listed in [1]. 446 | 447 | [1] https://www.tysto.com/uk-us-spelling-list.html 448 | """ 449 | 450 | def __init__(self): 451 | mapping_path = os.path.join(os.path.dirname(__file__), "english.json") 452 | self.mapping = json.load(open(mapping_path)) 453 | 454 | def __call__(self, s: str): 455 | return " ".join(self.mapping.get(word, word) for word in s.split()) 456 | 457 | 458 | class EnglishTextNormalizer: 459 | def __init__(self): 460 | self.ignore_patterns = r"\b(hmm|mm|mhm|mmm|uh|um)\b" 461 | self.replacers = { 462 | # common contractions 463 | r"\bwon't\b": "will not", 464 | r"\bcan't\b": "can not", 465 | r"\blet's\b": "let us", 466 | r"\bain't\b": "aint", 467 | r"\by'all\b": "you all", 468 | r"\bwanna\b": "want to", 469 | r"\bgotta\b": "got to", 470 | r"\bgonna\b": "going to", 471 | r"\bi'ma\b": "i am going to", 472 | r"\bimma\b": "i am going to", 473 | r"\bwoulda\b": "would have", 474 | r"\bcoulda\b": "could have", 475 | r"\bshoulda\b": "should have", 476 | r"\bma'am\b": "madam", 477 | # contractions in titles/prefixes 478 | r"\bmr\b": "mister ", 479 | r"\bmrs\b": "missus ", 480 | r"\bst\b": "saint ", 481 | r"\bdr\b": "doctor ", 482 | r"\bprof\b": "professor ", 483 | r"\bcapt\b": "captain ", 484 | r"\bgov\b": "governor ", 485 | r"\bald\b": "alderman ", 486 | r"\bgen\b": "general ", 487 | r"\bsen\b": "senator ", 488 | r"\brep\b": "representative ", 489 | r"\bpres\b": "president ", 490 | r"\brev\b": "reverend ", 491 | r"\bhon\b": "honorable ", 492 | r"\basst\b": "assistant ", 493 | r"\bassoc\b": "associate ", 494 | r"\blt\b": "lieutenant ", 495 | r"\bcol\b": "colonel ", 496 | r"\bjr\b": "junior ", 497 | r"\bsr\b": "senior ", 498 | r"\besq\b": "esquire ", 499 | # prefect tenses, ideally it should be any past participles, but it's harder.. 500 | r"'d been\b": " had been", 501 | r"'s been\b": " has been", 502 | r"'d gone\b": " had gone", 503 | r"'s gone\b": " has gone", 504 | r"'d done\b": " had done", # "'s done" is ambiguous 505 | r"'s got\b": " has got", 506 | # general contractions 507 | r"n't\b": " not", 508 | r"'re\b": " are", 509 | r"'s\b": " is", 510 | r"'d\b": " would", 511 | r"'ll\b": " will", 512 | r"'t\b": " not", 513 | r"'ve\b": " have", 514 | r"'m\b": " am", 515 | } 516 | self.standardize_numbers = EnglishNumberNormalizer() 517 | self.standardize_spellings = EnglishSpellingNormalizer() 518 | 519 | def __call__(self, s: str): 520 | s = s.lower() 521 | 522 | s = re.sub(r"[<\[][^>\]]*[>\]]", "", s) # remove words between brackets 523 | s = re.sub(r"\(([^)]+?)\)", "", s) # remove words between parenthesis 524 | s = re.sub(self.ignore_patterns, "", s) 525 | s = re.sub(r"\s+'", "'", s) # standardize when there's a space before an apostrophe 526 | 527 | for pattern, replacement in self.replacers.items(): 528 | s = re.sub(pattern, replacement, s) 529 | 530 | s = re.sub(r"(\d),(\d)", r"\1\2", s) # remove commas between digits 531 | s = re.sub(r"\.([^0-9]|$)", r" \1", s) # remove periods not followed by numbers 532 | s = remove_symbols_and_diacritics(s, keep=".%$¢€£") # keep some symbols for numerics 533 | 534 | s = self.standardize_numbers(s) 535 | s = self.standardize_spellings(s) 536 | 537 | # now remove prefix/suffix symbols that are not preceded/followed by numbers 538 | s = re.sub(r"[.$¢€£]([^0-9])", r" \1", s) 539 | s = re.sub(r"([^0-9])%", r"\1 ", s) 540 | 541 | s = re.sub(r"\s+", " ", s) # replace any successive whitespace characters with a space 542 | 543 | return s 544 | -------------------------------------------------------------------------------- /whisper/tokenizer.py: -------------------------------------------------------------------------------- 1 | import os 2 | from dataclasses import dataclass 3 | from functools import lru_cache 4 | from typing import List, Optional, Tuple, Union 5 | 6 | import numpy as np 7 | import torch 8 | from transformers import GPT2TokenizerFast 9 | 10 | LANGUAGES = { 11 | "en": "english", 12 | "zh": "chinese", 13 | "de": "german", 14 | "es": "spanish", 15 | "ru": "russian", 16 | "ko": "korean", 17 | "fr": "french", 18 | "ja": "japanese", 19 | "pt": "portuguese", 20 | "tr": "turkish", 21 | "pl": "polish", 22 | "ca": "catalan", 23 | "nl": "dutch", 24 | "ar": "arabic", 25 | "sv": "swedish", 26 | "it": "italian", 27 | "id": "indonesian", 28 | "hi": "hindi", 29 | "fi": "finnish", 30 | "vi": "vietnamese", 31 | "he": "hebrew", 32 | "uk": "ukrainian", 33 | "el": "greek", 34 | "ms": "malay", 35 | "cs": "czech", 36 | "ro": "romanian", 37 | "da": "danish", 38 | "hu": "hungarian", 39 | "ta": "tamil", 40 | "no": "norwegian", 41 | "th": "thai", 42 | "ur": "urdu", 43 | "hr": "croatian", 44 | "bg": "bulgarian", 45 | "lt": "lithuanian", 46 | "la": "latin", 47 | "mi": "maori", 48 | "ml": "malayalam", 49 | "cy": "welsh", 50 | "sk": "slovak", 51 | "te": "telugu", 52 | "fa": "persian", 53 | "lv": "latvian", 54 | "bn": "bengali", 55 | "sr": "serbian", 56 | "az": "azerbaijani", 57 | "sl": "slovenian", 58 | "kn": "kannada", 59 | "et": "estonian", 60 | "mk": "macedonian", 61 | "br": "breton", 62 | "eu": "basque", 63 | "is": "icelandic", 64 | "hy": "armenian", 65 | "ne": "nepali", 66 | "mn": "mongolian", 67 | "bs": "bosnian", 68 | "kk": "kazakh", 69 | "sq": "albanian", 70 | "sw": "swahili", 71 | "gl": "galician", 72 | "mr": "marathi", 73 | "pa": "punjabi", 74 | "si": "sinhala", 75 | "km": "khmer", 76 | "sn": "shona", 77 | "yo": "yoruba", 78 | "so": "somali", 79 | "af": "afrikaans", 80 | "oc": "occitan", 81 | "ka": "georgian", 82 | "be": "belarusian", 83 | "tg": "tajik", 84 | "sd": "sindhi", 85 | "gu": "gujarati", 86 | "am": "amharic", 87 | "yi": "yiddish", 88 | "lo": "lao", 89 | "uz": "uzbek", 90 | "fo": "faroese", 91 | "ht": "haitian creole", 92 | "ps": "pashto", 93 | "tk": "turkmen", 94 | "nn": "nynorsk", 95 | "mt": "maltese", 96 | "sa": "sanskrit", 97 | "lb": "luxembourgish", 98 | "my": "myanmar", 99 | "bo": "tibetan", 100 | "tl": "tagalog", 101 | "mg": "malagasy", 102 | "as": "assamese", 103 | "tt": "tatar", 104 | "haw": "hawaiian", 105 | "ln": "lingala", 106 | "ha": "hausa", 107 | "ba": "bashkir", 108 | "jw": "javanese", 109 | "su": "sundanese", 110 | } 111 | 112 | # language code lookup by name, with a few language aliases 113 | TO_LANGUAGE_CODE = { 114 | **{language: code for code, language in LANGUAGES.items()}, 115 | "burmese": "my", 116 | "valencian": "ca", 117 | "flemish": "nl", 118 | "haitian": "ht", 119 | "letzeburgesch": "lb", 120 | "pushto": "ps", 121 | "panjabi": "pa", 122 | "moldavian": "ro", 123 | "moldovan": "ro", 124 | "sinhalese": "si", 125 | "castilian": "es", 126 | } 127 | 128 | 129 | @dataclass(frozen=True) 130 | class Tokenizer: 131 | """A thin wrapper around `GPT2TokenizerFast` providing quick access to special tokens""" 132 | 133 | tokenizer: "GPT2TokenizerFast" 134 | language: Optional[str] 135 | sot_sequence: Tuple[int] 136 | 137 | def encode(self, text, **kwargs): 138 | return self.tokenizer.encode(text, **kwargs) 139 | 140 | def decode(self, token_ids: Union[int, List[int], np.ndarray, torch.Tensor], **kwargs): 141 | return self.tokenizer.decode(token_ids, **kwargs) 142 | 143 | def decode_with_timestamps(self, tokens) -> str: 144 | """ 145 | Timestamp tokens are above the special tokens' id range and are ignored by `decode()`. 146 | This method decodes given tokens with timestamps tokens annotated, e.g. "<|1.08|>". 147 | """ 148 | outputs = [[]] 149 | for token in tokens: 150 | if token >= self.timestamp_begin: 151 | timestamp = f"<|{(token - self.timestamp_begin) * 0.02:.2f}|>" 152 | outputs.append(timestamp) 153 | outputs.append([]) 154 | else: 155 | outputs[-1].append(token) 156 | outputs = [s if isinstance(s, str) else self.tokenizer.decode(s) for s in outputs] 157 | return "".join(outputs) 158 | 159 | @property 160 | @lru_cache() 161 | def eot(self) -> int: 162 | return self.tokenizer.eos_token_id 163 | 164 | @property 165 | @lru_cache() 166 | def sot(self) -> int: 167 | return self._get_single_token_id("<|startoftranscript|>") 168 | 169 | @property 170 | @lru_cache() 171 | def sot_lm(self) -> int: 172 | return self._get_single_token_id("<|startoflm|>") 173 | 174 | @property 175 | @lru_cache() 176 | def sot_prev(self) -> int: 177 | return self._get_single_token_id("<|startofprev|>") 178 | 179 | @property 180 | @lru_cache() 181 | def no_speech(self) -> int: 182 | return self._get_single_token_id("<|nospeech|>") 183 | 184 | @property 185 | @lru_cache() 186 | def no_timestamps(self) -> int: 187 | return self._get_single_token_id("<|notimestamps|>") 188 | 189 | @property 190 | @lru_cache() 191 | def timestamp_begin(self) -> int: 192 | return self.tokenizer.all_special_ids[-1] + 1 193 | 194 | @property 195 | @lru_cache() 196 | def language_token(self) -> int: 197 | """Returns the token id corresponding to the value of the `language` field""" 198 | if self.language is None: 199 | raise ValueError(f"This tokenizer does not have language token configured") 200 | 201 | additional_tokens = dict( 202 | zip( 203 | self.tokenizer.additional_special_tokens, 204 | self.tokenizer.additional_special_tokens_ids, 205 | ) 206 | ) 207 | candidate = f"<|{self.language}|>" 208 | if candidate in additional_tokens: 209 | return additional_tokens[candidate] 210 | 211 | raise KeyError(f"Language {self.language} not found in tokenizer.") 212 | 213 | @property 214 | @lru_cache() 215 | def all_language_tokens(self) -> Tuple[int]: 216 | result = [] 217 | for token, token_id in zip( 218 | self.tokenizer.additional_special_tokens, 219 | self.tokenizer.additional_special_tokens_ids, 220 | ): 221 | if token.strip("<|>") in LANGUAGES: 222 | result.append(token_id) 223 | return tuple(result) 224 | 225 | @property 226 | @lru_cache() 227 | def all_language_codes(self) -> Tuple[str]: 228 | return tuple(self.decode([l]).strip("<|>") for l in self.all_language_tokens) 229 | 230 | @property 231 | @lru_cache() 232 | def sot_sequence_including_notimestamps(self) -> Tuple[int]: 233 | return tuple(list(self.sot_sequence) + [self.no_timestamps]) 234 | 235 | @property 236 | @lru_cache() 237 | def non_speech_tokens(self) -> Tuple[int]: 238 | """ 239 | Returns the list of tokens to suppress in order to avoid any speaker tags or non-speech 240 | annotations, to prevent sampling texts that are not actually spoken in the audio, e.g. 241 | 242 | - ♪♪♪ 243 | - ( SPEAKING FOREIGN LANGUAGE ) 244 | - [DAVID] Hey there, 245 | 246 | keeping basic punctuations like commas, periods, question marks, exclamation points, etc. 247 | """ 248 | symbols = list("\"#()*+/:;<=>@[\\]^_`{|}~「」『』") 249 | symbols += "<< >> <<< >>> -- --- -( -[ (' (\" (( )) ((( ))) [[ ]] {{ }} ♪♪ ♪♪♪".split() 250 | 251 | # symbols that may be a single token or multiple tokens depending on the tokenizer. 252 | # In case they're multiple tokens, suppress the first token, which is safe because: 253 | # These are between U+2640 and U+267F miscellaneous symbols that are okay to suppress 254 | # in generations, and in the 3-byte UTF-8 representation they share the first two bytes. 255 | miscellaneous = set("♩♪♫♬♭♮♯") 256 | assert all(0x2640 <= ord(c) <= 0x267F for c in miscellaneous) 257 | 258 | # allow hyphens "-" and single quotes "'" between words, but not at the beginning of a word 259 | result = {self.tokenizer.encode(" -")[0], self.tokenizer.encode(" '")[0]} 260 | for symbol in symbols + list(miscellaneous): 261 | for tokens in [self.tokenizer.encode(symbol), self.tokenizer.encode(" " + symbol)]: 262 | if len(tokens) == 1 or symbol in miscellaneous: 263 | result.add(tokens[0]) 264 | 265 | return tuple(sorted(result)) 266 | 267 | def _get_single_token_id(self, text) -> int: 268 | tokens = self.tokenizer.encode(text) 269 | assert len(tokens) == 1, f"{text} is not encoded as a single token" 270 | return tokens[0] 271 | 272 | 273 | @lru_cache(maxsize=None) 274 | def build_tokenizer(name: str = "gpt2"): 275 | os.environ["TOKENIZERS_PARALLELISM"] = "false" 276 | path = os.path.join(os.path.dirname(__file__), "assets", name) 277 | tokenizer = GPT2TokenizerFast.from_pretrained(path) 278 | 279 | specials = [ 280 | "<|startoftranscript|>", 281 | *[f"<|{lang}|>" for lang in LANGUAGES.keys()], 282 | "<|translate|>", 283 | "<|transcribe|>", 284 | "<|startoflm|>", 285 | "<|startofprev|>", 286 | "<|nospeech|>", 287 | "<|notimestamps|>", 288 | ] 289 | 290 | tokenizer.add_special_tokens(dict(additional_special_tokens=specials)) 291 | return tokenizer 292 | 293 | 294 | @lru_cache(maxsize=None) 295 | def get_tokenizer( 296 | multilingual: bool, 297 | *, 298 | task: Optional[str] = None, # Literal["transcribe", "translate", None] 299 | language: Optional[str] = None, 300 | ) -> Tokenizer: 301 | if language is not None: 302 | language = language.lower() 303 | if language not in LANGUAGES: 304 | if language in TO_LANGUAGE_CODE: 305 | language = TO_LANGUAGE_CODE[language] 306 | else: 307 | raise ValueError(f"Unsupported language: {language}") 308 | 309 | if multilingual: 310 | tokenizer_name = "multilingual" 311 | task = task or "transcribe" 312 | language = language or "en" 313 | else: 314 | tokenizer_name = "gpt2" 315 | task = None 316 | language = None 317 | 318 | tokenizer = build_tokenizer(name=tokenizer_name) 319 | all_special_ids: List[int] = tokenizer.all_special_ids 320 | sot: int = all_special_ids[1] 321 | translate: int = all_special_ids[-6] 322 | transcribe: int = all_special_ids[-5] 323 | 324 | langs = tuple(LANGUAGES.keys()) 325 | sot_sequence = [sot] 326 | if language is not None: 327 | sot_sequence.append(sot + 1 + langs.index(language)) 328 | if task is not None: 329 | sot_sequence.append(transcribe if task == "transcribe" else translate) 330 | 331 | return Tokenizer(tokenizer=tokenizer, language=language, sot_sequence=tuple(sot_sequence)) 332 | -------------------------------------------------------------------------------- /whisper/utils.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import sys 4 | import zlib 5 | from typing import Callable, TextIO 6 | 7 | system_encoding = sys.getdefaultencoding() 8 | 9 | if system_encoding != "utf-8": 10 | def make_safe(string): 11 | # replaces any character not representable using the system default encoding with an '?', 12 | # avoiding UnicodeEncodeError (https://github.com/openai/whisper/discussions/729). 13 | return string.encode(system_encoding, errors="replace").decode(system_encoding) 14 | else: 15 | def make_safe(string): 16 | # utf-8 can encode any Unicode code point, so no need to do the round-trip encoding 17 | return string 18 | 19 | 20 | def exact_div(x, y): 21 | assert x % y == 0 22 | return x // y 23 | 24 | 25 | def str2bool(string): 26 | str2val = {"True": True, "False": False} 27 | if string in str2val: 28 | return str2val[string] 29 | else: 30 | raise ValueError(f"Expected one of {set(str2val.keys())}, got {string}") 31 | 32 | 33 | def optional_int(string): 34 | return None if string == "None" else int(string) 35 | 36 | 37 | def optional_float(string): 38 | return None if string == "None" else float(string) 39 | 40 | 41 | def compression_ratio(text) -> float: 42 | text_bytes = text.encode("utf-8") 43 | return len(text_bytes) / len(zlib.compress(text_bytes)) 44 | 45 | 46 | def format_timestamp(seconds: float, always_include_hours: bool = False, decimal_marker: str = '.'): 47 | assert seconds >= 0, "non-negative timestamp expected" 48 | milliseconds = round(seconds * 1000.0) 49 | 50 | hours = milliseconds // 3_600_000 51 | milliseconds -= hours * 3_600_000 52 | 53 | minutes = milliseconds // 60_000 54 | milliseconds -= minutes * 60_000 55 | 56 | seconds = milliseconds // 1_000 57 | milliseconds -= seconds * 1_000 58 | 59 | hours_marker = f"{hours:02d}:" if always_include_hours or hours > 0 else "" 60 | return f"{hours_marker}{minutes:02d}:{seconds:02d}{decimal_marker}{milliseconds:03d}" 61 | 62 | 63 | class ResultWriter: 64 | extension: str 65 | 66 | def __init__(self, output_dir: str): 67 | self.output_dir = output_dir 68 | 69 | def __call__(self, result: dict, audio_path: str): 70 | audio_basename = os.path.basename(audio_path) 71 | output_path = os.path.join(self.output_dir, audio_basename + "." + self.extension) 72 | 73 | with open(output_path, "w", encoding="utf-8") as f: 74 | self.write_result(result, file=f) 75 | 76 | def write_result(self, result: dict, file: TextIO): 77 | raise NotImplementedError 78 | 79 | 80 | class WriteTXT(ResultWriter): 81 | extension: str = "txt" 82 | 83 | def write_result(self, result: dict, file: TextIO): 84 | for segment in result["segments"]: 85 | print(segment['text'].strip(), file=file, flush=True) 86 | 87 | 88 | class WriteVTT(ResultWriter): 89 | extension: str = "vtt" 90 | 91 | def write_result(self, result: dict, file: TextIO): 92 | print("WEBVTT\n", file=file) 93 | for segment in result["segments"]: 94 | print( 95 | f"{format_timestamp(segment['start'])} --> {format_timestamp(segment['end'])}\n" 96 | f"{segment['text'].strip().replace('-->', '->')}\n", 97 | file=file, 98 | flush=True, 99 | ) 100 | 101 | 102 | class WriteSRT(ResultWriter): 103 | extension: str = "srt" 104 | 105 | def write_result(self, result: dict, file: TextIO): 106 | for i, segment in enumerate(result["segments"], start=1): 107 | # write srt lines 108 | print( 109 | f"{i}\n" 110 | f"{format_timestamp(segment['start'], always_include_hours=True, decimal_marker=',')} --> " 111 | f"{format_timestamp(segment['end'], always_include_hours=True, decimal_marker=',')}\n" 112 | f"{segment['text'].strip().replace('-->', '->')}\n", 113 | file=file, 114 | flush=True, 115 | ) 116 | 117 | 118 | class WriteTSV(ResultWriter): 119 | """ 120 | Write a transcript to a file in TSV (tab-separated values) format containing lines like: 121 | \t\t 122 | 123 | Using integer milliseconds as start and end times means there's no chance of interference from 124 | an environment setting a language encoding that causes the decimal in a floating point number 125 | to appear as a comma; also is faster and more efficient to parse & store, e.g., in C++. 126 | """ 127 | extension: str = "tsv" 128 | 129 | def write_result(self, result: dict, file: TextIO): 130 | print("start", "end", "text", sep="\t", file=file) 131 | for segment in result["segments"]: 132 | print(round(1000 * segment['start']), file=file, end="\t") 133 | print(round(1000 * segment['end']), file=file, end="\t") 134 | print(segment['text'].strip().replace("\t", " "), file=file, flush=True) 135 | 136 | 137 | class WriteJSON(ResultWriter): 138 | extension: str = "json" 139 | 140 | def write_result(self, result: dict, file: TextIO): 141 | json.dump(result, file) 142 | 143 | 144 | def get_writer(output_format: str, output_dir: str) -> Callable[[dict, TextIO], None]: 145 | writers = { 146 | "txt": WriteTXT, 147 | "vtt": WriteVTT, 148 | "srt": WriteSRT, 149 | "tsv": WriteTSV, 150 | "json": WriteJSON, 151 | } 152 | 153 | if output_format == "all": 154 | all_writers = [writer(output_dir) for writer in writers.values()] 155 | 156 | def write_all(result: dict, file: TextIO): 157 | for writer in all_writers: 158 | writer(result, file) 159 | 160 | return write_all 161 | 162 | return writers[output_format](output_dir) 163 | 164 | -------------------------------------------------------------------------------- /whisperconvert_exp.py: -------------------------------------------------------------------------------- 1 | import os 2 | import argparse 3 | import torch 4 | import librosa 5 | import time 6 | from scipy.io.wavfile import write 7 | from tqdm import tqdm 8 | import soundfile as sf 9 | import utils 10 | from models import SynthesizerTrn 11 | from mel_processing import mel_spectrogram_torch 12 | import logging 13 | logging.getLogger('numba').setLevel(logging.WARNING) 14 | 15 | 16 | if __name__ == "__main__": 17 | 18 | parser = argparse.ArgumentParser() 19 | parser.add_argument("--hpfile", type=str, default="./logs/cvc-whispers-three-emo-loss/config.json", help="path to json config file") 20 | parser.add_argument("--ptfile", type=str, default="./logs/cvc-whispers-three-emo-loss/G_cvc-whispers-three-emo-loss.pth", help="path to pth file") 21 | parser.add_argument("--outdir", type=str, default="output/60_exp_crosslingual_whispers-three-emo-loss", help="path to output dir") 22 | parser.add_argument("--use_timestamp", default=False, action="store_true") 23 | args = parser.parse_args() 24 | 25 | os.makedirs(args.outdir, exist_ok=True) 26 | hps = utils.get_hparams_from_file(args.hpfile) 27 | 28 | print("Loading model...") 29 | net_g = SynthesizerTrn( 30 | hps.data.filter_length // 2 + 1, 31 | hps.train.segment_size // hps.data.hop_length, 32 | **hps.model).cuda() 33 | _ = net_g.eval() 34 | print("Loading checkpoint...") 35 | _ = utils.load_checkpoint(args.ptfile, net_g, None, True) 36 | 37 | 38 | 39 | src_wavs=[r".\dataset\crosslingual_emo_dataset\LibriTTS100\911\128684\911_128684_000004_000001.wav", 40 | r".\dataset\crosslingual_emo_dataset\LibriTTS100\730\359\730_359_000004_000001.wav", 41 | r".\dataset\crosslingual_emo_dataset\aishell3\wav\SSB0246\SSB02460001.wav", 42 | r".\dataset\crosslingual_emo_dataset\aishell3\wav\SSB1863\SSB18630001.wav", 43 | r".\dataset\crosslingual_emo_dataset\jvs\jvs003\nonpara30\wav24kHz16bit\BASIC5000_0440.wav", 44 | r".\dataset\crosslingual_emo_dataset\jvs\jvs014\nonpara30\wav24kHz16bit\BASIC5000_0318.wav"] 45 | 46 | 47 | tgt_wavs=[r".\dataset\crosslingual_emo_dataset\LibriTTS100\27\123349\27_123349_000003_000002.wav", 48 | r".\dataset\crosslingual_emo_dataset\LibriTTS100\87\121553\87_121553_000254_000000.wav", 49 | r".\dataset\crosslingual_emo_dataset\aishell3\wav\SSB1935\SSB19350001.wav", 50 | r".\dataset\crosslingual_emo_dataset\aishell3\wav\SSB1759\SSB17590008.wav", 51 | r".\dataset\crosslingual_emo_dataset\jvs\jvs009\nonpara30\wav24kHz16bit\BASIC5000_0155.wav", 52 | r".\dataset\crosslingual_emo_dataset\jvs\jvs010\nonpara30\wav24kHz16bit\BASIC5000_0113.wav", 53 | r".\dataset\vctk-16k\p304\p304_007.wav", 54 | r".\jecs_ref\JECS0000_JA.wav", 55 | r".\aishell1_ref\BAC009S0655W0493.wav"] 56 | print("Processing text...") 57 | titles, srcs, tgts = [], [], [] 58 | for src_wav in src_wavs: 59 | for tgt_wav in tgt_wavs: 60 | src_wav_name=os.path.basename(src_wav)[:-4] 61 | tgt_wav_name=os.path.basename(tgt_wav)[:-4] 62 | title="{}_to_{}".format(src_wav_name,tgt_wav_name) 63 | titles.append(title) 64 | srcs.append(src_wav) 65 | tgts.append(tgt_wav) 66 | print(srcs) 67 | print(tgts) 68 | print(titles) 69 | #import sys 70 | #sys.exit() 71 | """ 72 | with open(args.txtpath, "r") as f: 73 | for rawline in f.readlines(): 74 | print(rawline) 75 | title, src, tgt = rawline.strip().split("|") 76 | titles.append(title) 77 | srcs.append(src) 78 | tgts.append(tgt) 79 | """ 80 | 81 | print("Synthesizing...") 82 | with torch.no_grad(): 83 | for line in tqdm(zip(titles, srcs, tgts)): 84 | title, src, tgt = line 85 | srcname,tgtname=title.split("to") 86 | # tgt 87 | wav_tgt, _ = librosa.load(tgt, sr=hps.data.sampling_rate) 88 | sf.write(os.path.join(args.outdir, f"{tgtname}.wav"), wav_tgt, hps.data.sampling_rate) 89 | #wav_tgt, _ = librosa.effects.trim(wav_tgt, top_db=20) 90 | 91 | wav_tgt = torch.from_numpy(wav_tgt).unsqueeze(0).cuda() 92 | mel_tgt = mel_spectrogram_torch( 93 | wav_tgt, 94 | hps.data.filter_length, 95 | hps.data.n_mel_channels, 96 | hps.data.sampling_rate, 97 | hps.data.hop_length, 98 | hps.data.win_length, 99 | hps.data.mel_fmin, 100 | hps.data.mel_fmax 101 | ) 102 | # src 103 | wav_src, _ = librosa.load(src, sr=hps.data.sampling_rate) 104 | sf.write(os.path.join(args.outdir, f"{srcname}.wav"), wav_src, hps.data.sampling_rate) 105 | wav_src = torch.from_numpy(wav_src).unsqueeze(0).cuda() 106 | #c = utils.get_content(cmodel, wav_src) 107 | c_filename = src.replace(".wav", "whisper.pt.npy") 108 | #print(src, tgt,c_filename) 109 | #c = torch.load(c_filename)#.squeeze(0) 110 | import numpy as np 111 | c=torch.from_numpy(np.load(c_filename)) 112 | c=c.transpose(1,0) 113 | c=c.unsqueeze(0) 114 | 115 | print(c.size(),mel_tgt.size()) 116 | audio = net_g.infer(c.cuda(), mel=mel_tgt) 117 | audio = audio[0][0].data.cpu().float().numpy() 118 | if args.use_timestamp: 119 | timestamp = time.strftime("%m-%d_%H-%M", time.localtime()) 120 | write(os.path.join(args.outdir, "{}.wav".format(timestamp+"_"+title)), hps.data.sampling_rate, audio) 121 | else: 122 | write(os.path.join(args.outdir, f"{title}.wav"), hps.data.sampling_rate, audio) 123 | 124 | -------------------------------------------------------------------------------- /whisperconvert_longaudio.py: -------------------------------------------------------------------------------- 1 | import os 2 | import argparse 3 | import torch 4 | import librosa 5 | import time 6 | from scipy.io.wavfile import write 7 | from tqdm import tqdm 8 | import soundfile as sf 9 | import utils 10 | from models import SynthesizerTrn 11 | from mel_processing import mel_spectrogram_torch 12 | import logging 13 | logging.getLogger('numba').setLevel(logging.WARNING) 14 | import librosa # Optional. Use any library you like to read audio files. 15 | import soundfile # Optional. Use any library you like to write audio files. 16 | from preprocess_ppg import pred_ppg_c,load_model 17 | 18 | if __name__ == "__main__": 19 | 20 | parser = argparse.ArgumentParser() 21 | parser.add_argument("--hpfile", type=str, default="ConsistencyVC-voive-conversion/logs/config.json", help="path to json config file") 22 | parser.add_argument("--ptfile", type=str, default="ConsistencyVC-voive-conversion/logs/G_cvc-whispers-three-emo-loss.pth", help="path to pth file") 23 | parser.add_argument("--outdir", type=str, default="output/long", help="path to output dir") 24 | parser.add_argument("--use_timestamp", default=False, action="store_true") 25 | args = parser.parse_args() 26 | 27 | os.makedirs(args.outdir, exist_ok=True) 28 | hps = utils.get_hparams_from_file(args.hpfile) 29 | 30 | print("Loading model...") 31 | net_g = SynthesizerTrn( 32 | hps.data.filter_length // 2 + 1, 33 | hps.train.segment_size // hps.data.hop_length, 34 | **hps.model).cuda() 35 | _ = net_g.eval() 36 | print("Loading checkpoint...") 37 | _ = utils.load_checkpoint(args.ptfile, net_g, None, True) 38 | 39 | 40 | 41 | src_wavs=[r"longaudio1.wav", 42 | r"longaudio2.wav"] 43 | 44 | 45 | tgt_wavs=["tgt_slice/20230712-092103-296_1.wav"] 46 | print("Processing text...") 47 | titles, srcs, tgts = [], [], [] 48 | for src_wav in src_wavs: 49 | 50 | for tgt_wav in tgt_wavs: 51 | src_wav_name=os.path.basename(src_wav)[:-4] 52 | tgt_wav_name=os.path.basename(tgt_wav)[:-4] 53 | title="{}_to_{}".format(src_wav_name,tgt_wav_name) 54 | titles.append(title) 55 | srcs.append(src_wav) 56 | tgts.append(tgt_wav) 57 | #print(srcs) 58 | #print(tgts) 59 | #print(titles) 60 | #import sys 61 | #sys.exit() 62 | """ 63 | with open(args.txtpath, "r") as f: 64 | for rawline in f.readlines(): 65 | print(rawline) 66 | title, src, tgt = rawline.strip().split("|") 67 | titles.append(title) 68 | srcs.append(src) 69 | tgts.append(tgt) 70 | """ 71 | 72 | print("Synthesizing...") 73 | with torch.no_grad(): 74 | for line in tqdm(zip(titles, srcs, tgts)): 75 | title, src, tgt = line 76 | srcname,tgtname=title.split("to") 77 | # tgt 78 | wav_tgt, _ = librosa.load(tgt, sr=hps.data.sampling_rate) 79 | sf.write(os.path.join(args.outdir, f"{tgtname}.wav"), wav_tgt, hps.data.sampling_rate) 80 | #wav_tgt, _ = librosa.effects.trim(wav_tgt, top_db=20) 81 | 82 | wav_tgt = torch.from_numpy(wav_tgt).unsqueeze(0).cuda() 83 | mel_tgt = mel_spectrogram_torch( 84 | wav_tgt, 85 | hps.data.filter_length, 86 | hps.data.n_mel_channels, 87 | hps.data.sampling_rate, 88 | hps.data.hop_length, 89 | hps.data.win_length, 90 | hps.data.mel_fmin, 91 | hps.data.mel_fmax 92 | ) 93 | # src 94 | audio, sr = librosa.load(src, sr=hps.data.sampling_rate) 95 | import numpy as np 96 | audio_result_sum=np.float32(np.zeros(len(audio))) 97 | #audio, sr = librosa.load(src_wav,sr=None) 98 | audiolen=audio.shape[0] 99 | print(audiolen) 100 | src_wav_wavs=[] 101 | num=int(audiolen/(sr*20)) 102 | print(num) 103 | whisper = load_model(os.path.join("whisper_pretrain", "medium.pt")) 104 | for i in range(0,num+1): 105 | 106 | #print(i*20*sr,(i*20*sr+25*sr)) 107 | tmp=audio[i*20*sr:(i*20*sr+25*sr)] 108 | sf.write(os.path.join(args.outdir, f"{srcname}_{i}.wav"), tmp, hps.data.sampling_rate) 109 | 110 | 111 | c=pred_ppg_c(whisper,os.path.join(args.outdir, f"{srcname}_{i}.wav"))#torch.from_numpy(np.load(c_filename)) 112 | c=torch.from_numpy(c) 113 | c=c.transpose(1,0) 114 | c=c.unsqueeze(0) 115 | 116 | #print(c.size(),mel_tgt.size()) 117 | audio_result = net_g.infer(c.cuda(), mel=mel_tgt) 118 | audio_result = audio_result[0][0].data.cpu().float().numpy() 119 | audio_result_sum[i*20*sr:(i*20*sr+audio_result.shape[0])]=audio_result 120 | #print(audio_result.dtype) 121 | #print(audio_result_sum.dtype) 122 | """ 123 | if args.use_timestamp: 124 | timestamp = time.strftime("%m-%d_%H-%M", time.localtime()) 125 | write(os.path.join(args.outdir, "{}.wav".format(timestamp+"_"+title)), hps.data.sampling_rate, audio_result) 126 | else: 127 | write(os.path.join(args.outdir, f"{title}_{i}.wav"), hps.data.sampling_rate, audio_result) 128 | """ 129 | write(os.path.join(args.outdir, f"{title}_sum.wav"), hps.data.sampling_rate, audio_result_sum) 130 | --------------------------------------------------------------------------------