├── LICENSE
├── README.md
├── attentions.py
├── commons.py
├── configs
    └── quickvc.json
├── convert.py
├── convert.txt
├── data_utils_new_new.py
├── dataset
    ├── encode.py
    ├── test.txt
    └── train.txt
├── logs
    └── quickvc
    │   └── config.json
├── losses.py
├── mel_processing.py
├── models.py
├── modules.py
├── output
    └── quickvc
    │   ├── title1.wav
    │   ├── title10.wav
    │   ├── title11.wav
    │   ├── title12.wav
    │   ├── title2.wav
    │   ├── title3.wav
    │   ├── title4.wav
    │   ├── title5.wav
    │   ├── title6.wav
    │   ├── title7.wav
    │   ├── title8.wav
    │   └── title9.wav
├── pqmf.py
├── qvcfinalwhite.png
├── stft.py
├── stft_loss.py
├── test_data
    ├── 3752-4944-0027.wav
    ├── 4280-185518-0004.wav
    ├── 4703-59231-0001.wav
    ├── 4821-27466-0000.wav
    ├── 4955-28244-0007.wav
    ├── LJ001-0001.wav
    ├── LJ032-0032.wav
    ├── LJ045-0246.wav
    ├── VO_ZH_Barbara_Hello.wav
    ├── p225_001.wav
    ├── p226_005.wav
    ├── p229_003.wav
    ├── p246_198.wav
    ├── p252_003.wav
    ├── p312_177.wav
    ├── p318_140.wav
    ├── p334_304.wav
    ├── p351_003.wav
    └── p374_004.wav
├── train.py
├── transforms.py
└── utils.py


/LICENSE:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright (c) 2023 quickvc
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # QuickVC
 2 | This repository contains the open source code, audio samples and pretrained models of my paper: QuickVC: Any-to-many Voice Conversion Using Inverse Short-time Fourier Transform for Faster Conversion
 3 | ## [Demo Page](https://quickvc.github.io/quickvc-demo)
 4 | <img src="qvcfinalwhite.png" width="100%">
 5 | 
 6 | ## [Pretrained Model](https://drive.google.com/drive/folders/1DF6RgIHHkn2aoyyUMt4_hPitKSc2YR9d?usp=share_link)
 7 | Put pretrained model into logs/quickvc
 8 | 
 9 | ## Inference with pretrained model
10 | ```python
11 | python convert.py
12 | ```
13 | You can change convert.txt to select the target and source
14 | ## Preprocess
15 | 1. Hubert-Soft
16 | ```python
17 | cd dataset
18 | python encode.py soft dataset/VCTK-16K dataset/VCTK-16K
19 | ```
20 | 2. Spectrogram resize data augumentation, please refer to [FreeVC](https://github.com/OlaWod/FreeVC).
21 | 
22 | ## Train
23 | 
24 | ```python
25 | python train.py
26 | ```
27 | 
28 | If you want to change the config and model name, change:
29 | ```python
30 | parser.add_argument('-c', '--config', type=str, default="./configs/quickvc.json",help='JSON file for configuration')
31 | parser.add_argument('-m', '--model', type=str,default="quickvc",help='Model name')
32 | ```                   
33 | in utils.py
34 | 
35 | In order to use the sr during training, change [this part](https://github.com/quickvc/QuickVC-VoiceConversion/blob/277118de9c81d1689e16be8a43408eda4223553d/data_utils_new_new.py#L70) to
36 | ```python
37 | i = random.randint(68,92)
38 | c_filename = filename.replace(".wav", f"_{i}.npy")
39 | ```    
40 | ## References
41 | If you have any question about the decoder, refer to [MS-ISTFT-VITS](https://github.com/MasayaKawamura/MB-iSTFT-VITS).
42 | 
43 | If you have any question about the Hubert-soft, refer to [Soft-VC](https://github.com/bshall/hubert).
44 | 
45 | If you have any question about the data augumentation, refer to [FreeVC](https://github.com/OlaWod/FreeVC).
46 | ## If you meet any problem, welcome to contact with me.
47 | 


--------------------------------------------------------------------------------
/attentions.py:
--------------------------------------------------------------------------------
  1 | import copy
  2 | import math
  3 | import numpy as np
  4 | import torch
  5 | from torch import nn
  6 | from torch.nn import functional as F
  7 | 
  8 | import commons
  9 | import modules
 10 | from modules import LayerNorm
 11 |    
 12 | 
 13 | class Encoder(nn.Module):
 14 |   def __init__(self, hidden_channels, filter_channels, n_heads, n_layers, kernel_size=1, p_dropout=0., window_size=4, **kwargs):
 15 |     super().__init__()
 16 |     self.hidden_channels = hidden_channels
 17 |     self.filter_channels = filter_channels
 18 |     self.n_heads = n_heads
 19 |     self.n_layers = n_layers
 20 |     self.kernel_size = kernel_size
 21 |     self.p_dropout = p_dropout
 22 |     self.window_size = window_size
 23 | 
 24 |     self.drop = nn.Dropout(p_dropout)
 25 |     self.attn_layers = nn.ModuleList()
 26 |     self.norm_layers_1 = nn.ModuleList()
 27 |     self.ffn_layers = nn.ModuleList()
 28 |     self.norm_layers_2 = nn.ModuleList()
 29 |     for i in range(self.n_layers):
 30 |       self.attn_layers.append(MultiHeadAttention(hidden_channels, hidden_channels, n_heads, p_dropout=p_dropout, window_size=window_size))
 31 |       self.norm_layers_1.append(LayerNorm(hidden_channels))
 32 |       self.ffn_layers.append(FFN(hidden_channels, hidden_channels, filter_channels, kernel_size, p_dropout=p_dropout))
 33 |       self.norm_layers_2.append(LayerNorm(hidden_channels))
 34 | 
 35 |   def forward(self, x, x_mask):
 36 |     attn_mask = x_mask.unsqueeze(2) * x_mask.unsqueeze(-1)
 37 |     x = x * x_mask
 38 |     for i in range(self.n_layers):
 39 |       y = self.attn_layers[i](x, x, attn_mask)
 40 |       y = self.drop(y)
 41 |       x = self.norm_layers_1[i](x + y)
 42 | 
 43 |       y = self.ffn_layers[i](x, x_mask)
 44 |       y = self.drop(y)
 45 |       x = self.norm_layers_2[i](x + y)
 46 |     x = x * x_mask
 47 |     return x
 48 | 
 49 | 
 50 | class Decoder(nn.Module):
 51 |   def __init__(self, hidden_channels, filter_channels, n_heads, n_layers, kernel_size=1, p_dropout=0., proximal_bias=False, proximal_init=True, **kwargs):
 52 |     super().__init__()
 53 |     self.hidden_channels = hidden_channels
 54 |     self.filter_channels = filter_channels
 55 |     self.n_heads = n_heads
 56 |     self.n_layers = n_layers
 57 |     self.kernel_size = kernel_size
 58 |     self.p_dropout = p_dropout
 59 |     self.proximal_bias = proximal_bias
 60 |     self.proximal_init = proximal_init
 61 | 
 62 |     self.drop = nn.Dropout(p_dropout)
 63 |     self.self_attn_layers = nn.ModuleList()
 64 |     self.norm_layers_0 = nn.ModuleList()
 65 |     self.encdec_attn_layers = nn.ModuleList()
 66 |     self.norm_layers_1 = nn.ModuleList()
 67 |     self.ffn_layers = nn.ModuleList()
 68 |     self.norm_layers_2 = nn.ModuleList()
 69 |     for i in range(self.n_layers):
 70 |       self.self_attn_layers.append(MultiHeadAttention(hidden_channels, hidden_channels, n_heads, p_dropout=p_dropout, proximal_bias=proximal_bias, proximal_init=proximal_init))
 71 |       self.norm_layers_0.append(LayerNorm(hidden_channels))
 72 |       self.encdec_attn_layers.append(MultiHeadAttention(hidden_channels, hidden_channels, n_heads, p_dropout=p_dropout))
 73 |       self.norm_layers_1.append(LayerNorm(hidden_channels))
 74 |       self.ffn_layers.append(FFN(hidden_channels, hidden_channels, filter_channels, kernel_size, p_dropout=p_dropout, causal=True))
 75 |       self.norm_layers_2.append(LayerNorm(hidden_channels))
 76 | 
 77 |   def forward(self, x, x_mask, h, h_mask):
 78 |     """
 79 |     x: decoder input
 80 |     h: encoder output
 81 |     """
 82 |     self_attn_mask = commons.subsequent_mask(x_mask.size(2)).to(device=x.device, dtype=x.dtype)
 83 |     encdec_attn_mask = h_mask.unsqueeze(2) * x_mask.unsqueeze(-1)
 84 |     x = x * x_mask
 85 |     for i in range(self.n_layers):
 86 |       y = self.self_attn_layers[i](x, x, self_attn_mask)
 87 |       y = self.drop(y)
 88 |       x = self.norm_layers_0[i](x + y)
 89 | 
 90 |       y = self.encdec_attn_layers[i](x, h, encdec_attn_mask)
 91 |       y = self.drop(y)
 92 |       x = self.norm_layers_1[i](x + y)
 93 |       
 94 |       y = self.ffn_layers[i](x, x_mask)
 95 |       y = self.drop(y)
 96 |       x = self.norm_layers_2[i](x + y)
 97 |     x = x * x_mask
 98 |     return x
 99 | 
100 | 
101 | class MultiHeadAttention(nn.Module):
102 |   def __init__(self, channels, out_channels, n_heads, p_dropout=0., window_size=None, heads_share=True, block_length=None, proximal_bias=False, proximal_init=False):
103 |     super().__init__()
104 |     assert channels % n_heads == 0
105 | 
106 |     self.channels = channels
107 |     self.out_channels = out_channels
108 |     self.n_heads = n_heads
109 |     self.p_dropout = p_dropout
110 |     self.window_size = window_size
111 |     self.heads_share = heads_share
112 |     self.block_length = block_length
113 |     self.proximal_bias = proximal_bias
114 |     self.proximal_init = proximal_init
115 |     self.attn = None
116 | 
117 |     self.k_channels = channels // n_heads
118 |     self.conv_q = nn.Conv1d(channels, channels, 1)
119 |     self.conv_k = nn.Conv1d(channels, channels, 1)
120 |     self.conv_v = nn.Conv1d(channels, channels, 1)
121 |     self.conv_o = nn.Conv1d(channels, out_channels, 1)
122 |     self.drop = nn.Dropout(p_dropout)
123 | 
124 |     if window_size is not None:
125 |       n_heads_rel = 1 if heads_share else n_heads
126 |       rel_stddev = self.k_channels**-0.5
127 |       self.emb_rel_k = nn.Parameter(torch.randn(n_heads_rel, window_size * 2 + 1, self.k_channels) * rel_stddev)
128 |       self.emb_rel_v = nn.Parameter(torch.randn(n_heads_rel, window_size * 2 + 1, self.k_channels) * rel_stddev)
129 | 
130 |     nn.init.xavier_uniform_(self.conv_q.weight)
131 |     nn.init.xavier_uniform_(self.conv_k.weight)
132 |     nn.init.xavier_uniform_(self.conv_v.weight)
133 |     if proximal_init:
134 |       with torch.no_grad():
135 |         self.conv_k.weight.copy_(self.conv_q.weight)
136 |         self.conv_k.bias.copy_(self.conv_q.bias)
137 |       
138 |   def forward(self, x, c, attn_mask=None):
139 |     q = self.conv_q(x)
140 |     k = self.conv_k(c)
141 |     v = self.conv_v(c)
142 |     
143 |     x, self.attn = self.attention(q, k, v, mask=attn_mask)
144 | 
145 |     x = self.conv_o(x)
146 |     return x
147 | 
148 |   def attention(self, query, key, value, mask=None):
149 |     # reshape [b, d, t] -> [b, n_h, t, d_k]
150 |     b, d, t_s, t_t = (*key.size(), query.size(2))
151 |     query = query.view(b, self.n_heads, self.k_channels, t_t).transpose(2, 3)
152 |     key = key.view(b, self.n_heads, self.k_channels, t_s).transpose(2, 3)
153 |     value = value.view(b, self.n_heads, self.k_channels, t_s).transpose(2, 3)
154 | 
155 |     scores = torch.matmul(query / math.sqrt(self.k_channels), key.transpose(-2, -1))
156 |     if self.window_size is not None:
157 |       assert t_s == t_t, "Relative attention is only available for self-attention."
158 |       key_relative_embeddings = self._get_relative_embeddings(self.emb_rel_k, t_s)
159 |       rel_logits = self._matmul_with_relative_keys(query /math.sqrt(self.k_channels), key_relative_embeddings)
160 |       scores_local = self._relative_position_to_absolute_position(rel_logits)
161 |       scores = scores + scores_local
162 |     if self.proximal_bias:
163 |       assert t_s == t_t, "Proximal bias is only available for self-attention."
164 |       scores = scores + self._attention_bias_proximal(t_s).to(device=scores.device, dtype=scores.dtype)
165 |     if mask is not None:
166 |       scores = scores.masked_fill(mask == 0, -1e4)
167 |       if self.block_length is not None:
168 |         assert t_s == t_t, "Local attention is only available for self-attention."
169 |         block_mask = torch.ones_like(scores).triu(-self.block_length).tril(self.block_length)
170 |         scores = scores.masked_fill(block_mask == 0, -1e4)
171 |     p_attn = F.softmax(scores, dim=-1) # [b, n_h, t_t, t_s]
172 |     p_attn = self.drop(p_attn)
173 |     output = torch.matmul(p_attn, value)
174 |     if self.window_size is not None:
175 |       relative_weights = self._absolute_position_to_relative_position(p_attn)
176 |       value_relative_embeddings = self._get_relative_embeddings(self.emb_rel_v, t_s)
177 |       output = output + self._matmul_with_relative_values(relative_weights, value_relative_embeddings)
178 |     output = output.transpose(2, 3).contiguous().view(b, d, t_t) # [b, n_h, t_t, d_k] -> [b, d, t_t]
179 |     return output, p_attn
180 | 
181 |   def _matmul_with_relative_values(self, x, y):
182 |     """
183 |     x: [b, h, l, m]
184 |     y: [h or 1, m, d]
185 |     ret: [b, h, l, d]
186 |     """
187 |     ret = torch.matmul(x, y.unsqueeze(0))
188 |     return ret
189 | 
190 |   def _matmul_with_relative_keys(self, x, y):
191 |     """
192 |     x: [b, h, l, d]
193 |     y: [h or 1, m, d]
194 |     ret: [b, h, l, m]
195 |     """
196 |     ret = torch.matmul(x, y.unsqueeze(0).transpose(-2, -1))
197 |     return ret
198 | 
199 |   def _get_relative_embeddings(self, relative_embeddings, length):
200 |     max_relative_position = 2 * self.window_size + 1
201 |     # Pad first before slice to avoid using cond ops.
202 |     pad_length = max(length - (self.window_size + 1), 0)
203 |     slice_start_position = max((self.window_size + 1) - length, 0)
204 |     slice_end_position = slice_start_position + 2 * length - 1
205 |     if pad_length > 0:
206 |       padded_relative_embeddings = F.pad(
207 |           relative_embeddings,
208 |           commons.convert_pad_shape([[0, 0], [pad_length, pad_length], [0, 0]]))
209 |     else:
210 |       padded_relative_embeddings = relative_embeddings
211 |     used_relative_embeddings = padded_relative_embeddings[:,slice_start_position:slice_end_position]
212 |     return used_relative_embeddings
213 | 
214 |   def _relative_position_to_absolute_position(self, x):
215 |     """
216 |     x: [b, h, l, 2*l-1]
217 |     ret: [b, h, l, l]
218 |     """
219 |     batch, heads, length, _ = x.size()
220 |     # Concat columns of pad to shift from relative to absolute indexing.
221 |     x = F.pad(x, commons.convert_pad_shape([[0,0],[0,0],[0,0],[0,1]]))
222 | 
223 |     # Concat extra elements so to add up to shape (len+1, 2*len-1).
224 |     x_flat = x.view([batch, heads, length * 2 * length])
225 |     x_flat = F.pad(x_flat, commons.convert_pad_shape([[0,0],[0,0],[0,length-1]]))
226 | 
227 |     # Reshape and slice out the padded elements.
228 |     x_final = x_flat.view([batch, heads, length+1, 2*length-1])[:, :, :length, length-1:]
229 |     return x_final
230 | 
231 |   def _absolute_position_to_relative_position(self, x):
232 |     """
233 |     x: [b, h, l, l]
234 |     ret: [b, h, l, 2*l-1]
235 |     """
236 |     batch, heads, length, _ = x.size()
237 |     # padd along column
238 |     x = F.pad(x, commons.convert_pad_shape([[0, 0], [0, 0], [0, 0], [0, length-1]]))
239 |     x_flat = x.view([batch, heads, length**2 + length*(length -1)])
240 |     # add 0's in the beginning that will skew the elements after reshape
241 |     x_flat = F.pad(x_flat, commons.convert_pad_shape([[0, 0], [0, 0], [length, 0]]))
242 |     x_final = x_flat.view([batch, heads, length, 2*length])[:,:,:,1:]
243 |     return x_final
244 | 
245 |   def _attention_bias_proximal(self, length):
246 |     """Bias for self-attention to encourage attention to close positions.
247 |     Args:
248 |       length: an integer scalar.
249 |     Returns:
250 |       a Tensor with shape [1, 1, length, length]
251 |     """
252 |     r = torch.arange(length, dtype=torch.float32)
253 |     diff = torch.unsqueeze(r, 0) - torch.unsqueeze(r, 1)
254 |     return torch.unsqueeze(torch.unsqueeze(-torch.log1p(torch.abs(diff)), 0), 0)
255 | 
256 | 
257 | class FFN(nn.Module):
258 |   def __init__(self, in_channels, out_channels, filter_channels, kernel_size, p_dropout=0., activation=None, causal=False):
259 |     super().__init__()
260 |     self.in_channels = in_channels
261 |     self.out_channels = out_channels
262 |     self.filter_channels = filter_channels
263 |     self.kernel_size = kernel_size
264 |     self.p_dropout = p_dropout
265 |     self.activation = activation
266 |     self.causal = causal
267 | 
268 |     if causal:
269 |       self.padding = self._causal_padding
270 |     else:
271 |       self.padding = self._same_padding
272 | 
273 |     self.conv_1 = nn.Conv1d(in_channels, filter_channels, kernel_size)
274 |     self.conv_2 = nn.Conv1d(filter_channels, out_channels, kernel_size)
275 |     self.drop = nn.Dropout(p_dropout)
276 | 
277 |   def forward(self, x, x_mask):
278 |     x = self.conv_1(self.padding(x * x_mask))
279 |     if self.activation == "gelu":
280 |       x = x * torch.sigmoid(1.702 * x)
281 |     else:
282 |       x = torch.relu(x)
283 |     x = self.drop(x)
284 |     x = self.conv_2(self.padding(x * x_mask))
285 |     return x * x_mask
286 |   
287 |   def _causal_padding(self, x):
288 |     if self.kernel_size == 1:
289 |       return x
290 |     pad_l = self.kernel_size - 1
291 |     pad_r = 0
292 |     padding = [[0, 0], [0, 0], [pad_l, pad_r]]
293 |     x = F.pad(x, commons.convert_pad_shape(padding))
294 |     return x
295 | 
296 |   def _same_padding(self, x):
297 |     if self.kernel_size == 1:
298 |       return x
299 |     pad_l = (self.kernel_size - 1) // 2
300 |     pad_r = self.kernel_size // 2
301 |     padding = [[0, 0], [0, 0], [pad_l, pad_r]]
302 |     x = F.pad(x, commons.convert_pad_shape(padding))
303 |     return x
304 | 


--------------------------------------------------------------------------------
/commons.py:
--------------------------------------------------------------------------------
  1 | import math
  2 | import numpy as np
  3 | import torch
  4 | from torch import nn
  5 | from torch.nn import functional as F
  6 | 
  7 | 
  8 | def init_weights(m, mean=0.0, std=0.01):
  9 |   classname = m.__class__.__name__
 10 |   if classname.find("Conv") != -1:
 11 |     m.weight.data.normal_(mean, std)
 12 | 
 13 | 
 14 | def get_padding(kernel_size, dilation=1):
 15 |   return int((kernel_size*dilation - dilation)/2)
 16 | 
 17 | 
 18 | def convert_pad_shape(pad_shape):
 19 |   l = pad_shape[::-1]
 20 |   pad_shape = [item for sublist in l for item in sublist]
 21 |   return pad_shape
 22 | 
 23 | 
 24 | def intersperse(lst, item):
 25 |   result = [item] * (len(lst) * 2 + 1)
 26 |   result[1::2] = lst
 27 |   return result
 28 | 
 29 | 
 30 | def kl_divergence(m_p, logs_p, m_q, logs_q):
 31 |   """KL(P||Q)"""
 32 |   kl = (logs_q - logs_p) - 0.5
 33 |   kl += 0.5 * (torch.exp(2. * logs_p) + ((m_p - m_q)**2)) * torch.exp(-2. * logs_q)
 34 |   return kl
 35 | 
 36 | 
 37 | def rand_gumbel(shape):
 38 |   """Sample from the Gumbel distribution, protect from overflows."""
 39 |   uniform_samples = torch.rand(shape) * 0.99998 + 0.00001
 40 |   return -torch.log(-torch.log(uniform_samples))
 41 | 
 42 | 
 43 | def rand_gumbel_like(x):
 44 |   g = rand_gumbel(x.size()).to(dtype=x.dtype, device=x.device)
 45 |   return g
 46 | 
 47 | 
 48 | def slice_segments(x, ids_str, segment_size=4):
 49 |   ret = torch.zeros_like(x[:, :, :segment_size])
 50 |   for i in range(x.size(0)):
 51 |     idx_str = ids_str[i]
 52 |     idx_end = idx_str + segment_size
 53 |     ret[i] = x[i, :, idx_str:idx_end]
 54 |   return ret
 55 | 
 56 | 
 57 | def rand_slice_segments(x, x_lengths=None, segment_size=4):
 58 |   b, d, t = x.size()
 59 |   if x_lengths is None:
 60 |     x_lengths = t
 61 |   ids_str_max = x_lengths - segment_size + 1
 62 |   ids_str = (torch.rand([b]).to(device=x.device) * ids_str_max).to(dtype=torch.long)
 63 |   ret = slice_segments(x, ids_str, segment_size)
 64 |   return ret, ids_str
 65 | 
 66 | def rand_spec_segments(x, x_lengths=None, segment_size=4):
 67 |   b, d, t = x.size()
 68 |   if x_lengths is None:
 69 |     x_lengths = t
 70 |   ids_str_max = x_lengths - segment_size
 71 |   ids_str = (torch.rand([b]).to(device=x.device) * ids_str_max).to(dtype=torch.long)
 72 |   ret = slice_segments(x, ids_str, segment_size)
 73 |   return ret, ids_str
 74 | 
 75 | 
 76 | def get_timing_signal_1d(
 77 |     length, channels, min_timescale=1.0, max_timescale=1.0e4):
 78 |   position = torch.arange(length, dtype=torch.float)
 79 |   num_timescales = channels // 2
 80 |   log_timescale_increment = (
 81 |       math.log(float(max_timescale) / float(min_timescale)) /
 82 |       (num_timescales - 1))
 83 |   inv_timescales = min_timescale * torch.exp(
 84 |       torch.arange(num_timescales, dtype=torch.float) * -log_timescale_increment)
 85 |   scaled_time = position.unsqueeze(0) * inv_timescales.unsqueeze(1)
 86 |   signal = torch.cat([torch.sin(scaled_time), torch.cos(scaled_time)], 0)
 87 |   signal = F.pad(signal, [0, 0, 0, channels % 2])
 88 |   signal = signal.view(1, channels, length)
 89 |   return signal
 90 | 
 91 | 
 92 | def add_timing_signal_1d(x, min_timescale=1.0, max_timescale=1.0e4):
 93 |   b, channels, length = x.size()
 94 |   signal = get_timing_signal_1d(length, channels, min_timescale, max_timescale)
 95 |   return x + signal.to(dtype=x.dtype, device=x.device)
 96 | 
 97 | 
 98 | def cat_timing_signal_1d(x, min_timescale=1.0, max_timescale=1.0e4, axis=1):
 99 |   b, channels, length = x.size()
100 |   signal = get_timing_signal_1d(length, channels, min_timescale, max_timescale)
101 |   return torch.cat([x, signal.to(dtype=x.dtype, device=x.device)], axis)
102 | 
103 | 
104 | def subsequent_mask(length):
105 |   mask = torch.tril(torch.ones(length, length)).unsqueeze(0).unsqueeze(0)
106 |   return mask
107 | 
108 | 
109 | @torch.jit.script
110 | def fused_add_tanh_sigmoid_multiply(input_a, input_b, n_channels):
111 |   n_channels_int = n_channels[0]
112 |   in_act = input_a + input_b
113 |   t_act = torch.tanh(in_act[:, :n_channels_int, :])
114 |   s_act = torch.sigmoid(in_act[:, n_channels_int:, :])
115 |   acts = t_act * s_act
116 |   return acts
117 | 
118 | 
119 | def convert_pad_shape(pad_shape):
120 |   l = pad_shape[::-1]
121 |   pad_shape = [item for sublist in l for item in sublist]
122 |   return pad_shape
123 | 
124 | 
125 | def shift_1d(x):
126 |   x = F.pad(x, convert_pad_shape([[0, 0], [0, 0], [1, 0]]))[:, :, :-1]
127 |   return x
128 | 
129 | 
130 | def sequence_mask(length, max_length=None):
131 |   if max_length is None:
132 |     max_length = length.max()
133 |   x = torch.arange(max_length, dtype=length.dtype, device=length.device)
134 |   return x.unsqueeze(0) < length.unsqueeze(1)
135 | 
136 | 
137 | def generate_path(duration, mask):
138 |   """
139 |   duration: [b, 1, t_x]
140 |   mask: [b, 1, t_y, t_x]
141 |   """
142 |   device = duration.device
143 |   
144 |   b, _, t_y, t_x = mask.shape
145 |   cum_duration = torch.cumsum(duration, -1)
146 |   
147 |   cum_duration_flat = cum_duration.view(b * t_x)
148 |   path = sequence_mask(cum_duration_flat, t_y).to(mask.dtype)
149 |   path = path.view(b, t_x, t_y)
150 |   path = path - F.pad(path, convert_pad_shape([[0, 0], [1, 0], [0, 0]]))[:, :-1]
151 |   path = path.unsqueeze(1).transpose(2,3) * mask
152 |   return path
153 | 
154 | 
155 | def clip_grad_value_(parameters, clip_value, norm_type=2):
156 |   if isinstance(parameters, torch.Tensor):
157 |     parameters = [parameters]
158 |   parameters = list(filter(lambda p: p.grad is not None, parameters))
159 |   norm_type = float(norm_type)
160 |   if clip_value is not None:
161 |     clip_value = float(clip_value)
162 | 
163 |   total_norm = 0
164 |   for p in parameters:
165 |     param_norm = p.grad.data.norm(norm_type)
166 |     total_norm += param_norm.item() ** norm_type
167 |     if clip_value is not None:
168 |       p.grad.data.clamp_(min=-clip_value, max=clip_value)
169 |   total_norm = total_norm ** (1. / norm_type)
170 |   return total_norm
171 | 


--------------------------------------------------------------------------------
/configs/quickvc.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "train": {
 3 |     "log_interval": 20,
 4 |     "eval_interval": 5000,
 5 |     "seed": 1234,
 6 |     "epochs": 20000,
 7 |     "learning_rate": 2e-4,
 8 |     "betas": [0.8, 0.99],
 9 |     "eps": 1e-9,
10 |     "batch_size": 64,
11 |     "fp16_run": false,
12 |     "lr_decay": 0.999875,
13 |     "segment_size": 10240,
14 |     "init_lr_ratio": 1,
15 |     "warmup_epochs": 0,
16 |     "c_mel": 45,
17 |     "c_kl": 1.0,
18 |     "use_sr": true,
19 |     "max_speclen": 512,
20 |     "port": "8002",
21 |     "fft_sizes": [384, 683, 171],
22 |     "hop_sizes": [30, 60, 10],
23 |     "win_lengths": [150, 300, 60],
24 |     "window": "hann_window"  
25 |   },
26 |   "data": {
27 |     "training_files":"./dataset/train.txt",
28 |     "validation_files":"./dataset/test.txt",
29 |     "text_cleaners":["english_cleaners2"],
30 |     "max_wav_value": 32768.0,
31 |     "sampling_rate": 16000,
32 |     "filter_length": 1280,
33 |     "hop_length": 320,
34 |     "win_length": 1280,
35 |     "n_mel_channels": 80,
36 |     "mel_fmin": 0.0,
37 |     "mel_fmax": null,
38 |     "add_blank": true,
39 |     "n_speakers": 0,
40 |     "cleaned_text": true
41 |   },
42 |   "model": {
43 |     "ms_istft_vits": true,
44 |     "mb_istft_vits": false,
45 |     "istft_vits": false,
46 |     "subbands": 4,
47 |     "gen_istft_n_fft": 16,
48 |     "gen_istft_hop_size": 4,
49 |     "inter_channels": 192,
50 |     "hidden_channels": 192,
51 |     "filter_channels": 768,
52 |     "n_heads": 2,
53 |     "n_layers": 6,
54 |     "kernel_size": 3,
55 |     "p_dropout": 0.1,
56 |     "resblock": "1",
57 |     "resblock_kernel_sizes": [3,7,11],
58 |     "resblock_dilation_sizes": [[1,3,5], [1,3,5], [1,3,5]],
59 |     "upsample_rates": [5,4],
60 |     "upsample_initial_channel": 512,
61 |     "upsample_kernel_sizes": [16,16],
62 |     "n_layers_q": 3,
63 |     "use_spectral_norm": false,
64 |     "gin_channels": 256,
65 |     "use_sdp": false,
66 |     "ssl_dim": 1024,
67 |     "use_spk": false
68 |   }
69 | 
70 | }
71 | 


--------------------------------------------------------------------------------
/convert.py:
--------------------------------------------------------------------------------
 1 | import os
 2 | import argparse
 3 | import torch
 4 | import librosa
 5 | import time
 6 | from scipy.io.wavfile import write
 7 | from tqdm import tqdm
 8 | 
 9 | import utils
10 | from models import SynthesizerTrn
11 | from mel_processing import mel_spectrogram_torch
12 | import logging
13 | logging.getLogger('numba').setLevel(logging.WARNING)
14 | import torch.autograd.profiler as profiler
15 | 
16 | if __name__ == "__main__":
17 |     parser = argparse.ArgumentParser()
18 |     parser.add_argument("--hpfile", type=str, default="logs/quickvc/config.json", help="path to json config file")
19 |     parser.add_argument("--ptfile", type=str, default="logs/quickvc/quickvc.pth", help="path to pth file")
20 |     parser.add_argument("--txtpath", type=str, default="convert.txt", help="path to txt file")
21 |     parser.add_argument("--outdir", type=str, default="output/quickvc", help="path to output dir")
22 |     parser.add_argument("--use_timestamp", default=False, action="store_true")
23 |     args = parser.parse_args()
24 |     
25 |     os.makedirs(args.outdir, exist_ok=True)
26 |     hps = utils.get_hparams_from_file(args.hpfile)
27 | 
28 |     print("Loading model...")
29 |     net_g = SynthesizerTrn(
30 |         hps.data.filter_length // 2 + 1,
31 |         hps.train.segment_size // hps.data.hop_length,
32 |         **hps.model).cuda()
33 |     _ = net_g.eval()
34 |     total = sum([param.nelement() for param in net_g.parameters()])
35 |  
36 |     print("Number of parameter: %.2fM" % (total/1e6))
37 |     print("Loading checkpoint...")
38 |     _ = utils.load_checkpoint(args.ptfile, net_g, None)
39 | 
40 |     print(f"Loading hubert_soft checkpoint")
41 |     hubert_soft = torch.hub.load("bshall/hubert:main", f"hubert_soft").cuda()
42 |     print("Loaded soft hubert.")
43 |     
44 |     print("Processing text...")
45 |     titles, srcs, tgts = [], [], []
46 |     with open(args.txtpath, "r") as f:
47 |         for rawline in f.readlines():
48 |             title, src, tgt = rawline.strip().split("|")
49 |             titles.append(title)
50 |             srcs.append(src)
51 |             tgts.append(tgt)
52 | 
53 |     print("Synthesizing...")
54 | 
55 |     with torch.no_grad():
56 |         for line in tqdm(zip(titles, srcs, tgts)):
57 |             title, src, tgt = line
58 |             # tgt
59 |             wav_tgt, _ = librosa.load(tgt, sr=hps.data.sampling_rate)
60 |             wav_tgt, _ = librosa.effects.trim(wav_tgt, top_db=20)
61 |             wav_tgt = torch.from_numpy(wav_tgt).unsqueeze(0).cuda()
62 |             mel_tgt = mel_spectrogram_torch(
63 |                 wav_tgt, 
64 |                 hps.data.filter_length,
65 |                 hps.data.n_mel_channels,
66 |                 hps.data.sampling_rate,
67 |                 hps.data.hop_length,
68 |                 hps.data.win_length,
69 |                 hps.data.mel_fmin,
70 |                 hps.data.mel_fmax
71 |             )
72 |             # src
73 |             wav_src, _ = librosa.load(src, sr=hps.data.sampling_rate)
74 |             wav_src = torch.from_numpy(wav_src).unsqueeze(0).unsqueeze(0).cuda()
75 |             print(wav_src.size())
76 |             #long running
77 |             #do something other
78 |             c = hubert_soft.units(wav_src)
79 | 
80 |             
81 |             
82 |             c=c.transpose(2,1)
83 |             #print(c.size())
84 |             audio = net_g.infer(c, mel=mel_tgt)
85 |          
86 |             audio = audio[0][0].data.cpu().float().numpy()
87 |             if args.use_timestamp:
88 |                 timestamp = time.strftime("%m-%d_%H-%M", time.localtime())
89 |                 write(os.path.join(args.outdir, "{}.wav".format(timestamp+"_"+title)), hps.data.sampling_rate, audio)
90 |             else:
91 |                 write(os.path.join(args.outdir, f"{title}.wav"), hps.data.sampling_rate, audio)
92 |             


--------------------------------------------------------------------------------
/convert.txt:
--------------------------------------------------------------------------------
 1 | title1|test_data/3752-4944-0027.wav|test_data/p225_001.wav
 2 | title2|test_data/p229_003.wav|test_data/p225_001.wav
 3 | title3|test_data/p252_003.wav|test_data/p225_001.wav
 4 | title4|test_data/p229_003.wav|test_data/p252_003.wav
 5 | title5|test_data/p225_001.wav|test_data/p252_003.wav
 6 | title6|test_data/LJ045-0246.wav|test_data/p229_003.wav
 7 | title7|test_data/LJ045-0246.wav|test_data/p252_003.wav
 8 | title8|test_data/LJ001-0001.wav|test_data/p252_003.wav
 9 | title9|test_data/LJ001-0001.wav|test_data/p229_003.wav
10 | title10|test_data/p226_005.wav|test_data/p252_003.wav
11 | title11|test_data/p225_001.wav|test_data/p226_005.wav
12 | title12|test_data/LJ001-0001.wav|test_data/LJ045-0246.wav


--------------------------------------------------------------------------------
/data_utils_new_new.py:
--------------------------------------------------------------------------------
  1 | import time
  2 | import os
  3 | import random
  4 | import numpy as np
  5 | import torch
  6 | import torch.utils.data
  7 | 
  8 | import commons 
  9 | from mel_processing import spectrogram_torch, spec_to_mel_torch
 10 | from utils import load_wav_to_torch, load_filepaths_and_text, transform
 11 | #import h5py
 12 | 
 13 | 
 14 | """Multi speaker version"""
 15 | class TextAudioSpeakerLoader(torch.utils.data.Dataset):
 16 |     """
 17 |         1) loads audio, speaker_id, text pairs
 18 |         2) normalizes text and converts them to sequences of integers
 19 |         3) computes spectrograms from audio files.
 20 |     """
 21 |     def __init__(self, audiopaths, hparams):
 22 |         self.audiopaths = load_filepaths_and_text(audiopaths)
 23 |         self.max_wav_value = hparams.data.max_wav_value
 24 |         self.sampling_rate = hparams.data.sampling_rate
 25 |         self.filter_length  = hparams.data.filter_length
 26 |         self.hop_length     = hparams.data.hop_length
 27 |         self.win_length     = hparams.data.win_length
 28 |         self.sampling_rate  = hparams.data.sampling_rate
 29 |         self.use_sr = hparams.train.use_sr
 30 |         self.use_spk = hparams.model.use_spk
 31 |         self.spec_len = hparams.train.max_speclen
 32 | 
 33 |         random.seed(1243)
 34 |         random.shuffle(self.audiopaths)
 35 |         self._filter()
 36 | 
 37 |     def _filter(self):
 38 |         """
 39 |         Filter text & store spec lengths
 40 |         """
 41 |         # Store spectrogram lengths for Bucketing
 42 |         # wav_length ~= file_size / (wav_channels * Bytes per dim) = file_size / (1 * 2)
 43 |         # spec_length = wav_length // hop_length
 44 | 
 45 |         lengths = []
 46 |         for audiopath in self.audiopaths:
 47 |             lengths.append(os.path.getsize(audiopath[0]) // (2 * self.hop_length))
 48 |         self.lengths = lengths
 49 |     def get_audio(self, filename):
 50 |         audio, sampling_rate = load_wav_to_torch(filename)
 51 |         if sampling_rate != self.sampling_rate:
 52 |             raise ValueError("{} SR doesn't match target {} SR".format(
 53 |                 sampling_rate, self.sampling_rate))
 54 |         audio_norm = audio / self.max_wav_value
 55 |         audio_norm = audio_norm.unsqueeze(0)
 56 |         
 57 |         spec_filename = filename.replace(".wav", ".spec.pt")
 58 | 
 59 |         if os.path.exists(spec_filename):
 60 |             spec = torch.load(spec_filename)
 61 |         else:
 62 |             spec = spectrogram_torch(audio_norm, self.filter_length,
 63 |                 self.sampling_rate, self.hop_length, self.win_length,
 64 |                 center=False)
 65 |             spec = torch.squeeze(spec, 0)
 66 |             torch.save(spec, spec_filename)
 67 |             
 68 |         
 69 | 
 70 |         #i = 80#random.randint(68,92)
 71 |   
 72 |         c_filename = filename.replace(".wav", f".npy")
 73 |         c = np.load(c_filename)#.squeeze(0)
 74 |         c=c.transpose(1,0)
 75 |         c = torch.FloatTensor(c)#.squeeze(0)
 76 |         
 77 |         return c, spec, audio_norm
 78 | 
 79 |     def __getitem__(self, index):
 80 |         return self.get_audio(self.audiopaths[index][0])
 81 | 
 82 |     def __len__(self):
 83 |         return len(self.audiopaths)
 84 | class TextAudioSpeakerCollate():
 85 |     """ Zero-pads model inputs and targets
 86 |     """
 87 |     def __init__(self, hps):
 88 |         self.hps = hps
 89 |         self.use_sr = hps.train.use_sr
 90 |         self.use_spk = hps.model.use_spk
 91 | 
 92 |     def __call__(self, batch):
 93 |         """Collate's training batch from normalized text, audio and speaker identities
 94 |         PARAMS
 95 |         ------
 96 |         batch: [text_normalized, spec_normalized, wav_normalized, sid]
 97 |         """
 98 |         # Right zero-pad all one-hot text sequences to max input length
 99 |         _, ids_sorted_decreasing = torch.sort(
100 |             torch.LongTensor([x[0].size(1) for x in batch]),
101 |             dim=0, descending=True)
102 |         max_c_len = max([x[0].size(1) for x in batch])
103 |         max_spec_len = max([x[1].size(1) for x in batch])
104 |         max_wav_len = max([x[2].size(1) for x in batch])
105 |         #print(max_spec_len,max_c_len)
106 |         spec_lengths = torch.LongTensor(len(batch))
107 |         wav_lengths = torch.LongTensor(len(batch))
108 |         if self.use_spk:
109 |             spks = torch.FloatTensor(len(batch), batch[0][3].size(0))
110 |         else:
111 |             spks = None
112 | 
113 |         c_padded = torch.FloatTensor(len(batch), batch[0][0].size(0), max_spec_len)
114 |         spec_padded = torch.FloatTensor(len(batch), batch[0][1].size(0), max_spec_len)
115 |         wav_padded = torch.FloatTensor(len(batch), 1, max_wav_len)
116 |         c_padded.zero_()
117 |         spec_padded.zero_()
118 |         wav_padded.zero_()
119 | 
120 |         for i in range(len(ids_sorted_decreasing)):
121 |             row = batch[ids_sorted_decreasing[i]]
122 | 
123 |             c = row[0]
124 |             c_padded[i, :, :c.size(1)] = c
125 | 
126 |             spec = row[1]
127 |             spec_padded[i, :, :spec.size(1)] = spec
128 |             spec_lengths[i] = spec.size(1)
129 | 
130 |             wav = row[2]
131 |             wav_padded[i, :, :wav.size(1)] = wav
132 |             wav_lengths[i] = wav.size(1)
133 | 
134 |             if self.use_spk:
135 |                 spks[i] = row[3]
136 |         #"""
137 |         spec_seglen = spec_lengths[-1] if spec_lengths[-1] < self.hps.train.max_speclen + 1 else self.hps.train.max_speclen + 1
138 |         #print(spec_seglen)
139 |         wav_seglen = spec_seglen * self.hps.data.hop_length 
140 |         #print(spec_padded.size(), spec_lengths, spec_seglen)
141 |         spec_padded, ids_slice = commons.rand_spec_segments(spec_padded, spec_lengths, spec_seglen)
142 |         wav_padded = commons.slice_segments(wav_padded, ids_slice * self.hps.data.hop_length, wav_seglen)
143 | 
144 |         c_padded = commons.slice_segments(c_padded, ids_slice, spec_seglen)[:,:,:-1]
145 | 
146 |         spec_padded = spec_padded[:,:,:-1]
147 |         wav_padded = wav_padded[:,:,:-self.hps.data.hop_length]
148 |         #"""
149 |         if self.use_spk:
150 |           return c_padded, spec_padded, wav_padded, spks
151 |         else:
152 |           return c_padded, spec_padded, wav_padded
153 | 
154 | 
155 | class DistributedBucketSampler(torch.utils.data.distributed.DistributedSampler):
156 |     """
157 |     Maintain similar input lengths in a batch.
158 |     Length groups are specified by boundaries.
159 |     Ex) boundaries = [b1, b2, b3] -> any batch is included either {x | b1 < length(x) <=b2} or {x | b2 < length(x) <= b3}.
160 |   
161 |     It removes samples which are not included in the boundaries.
162 |     Ex) boundaries = [b1, b2, b3] -> any x s.t. length(x) <= b1 or length(x) > b3 are discarded.
163 |     """
164 |     def __init__(self, dataset, batch_size, boundaries, num_replicas=None, rank=None, shuffle=True):
165 |         super().__init__(dataset, num_replicas=num_replicas, rank=rank, shuffle=shuffle)
166 |         self.lengths = dataset.lengths
167 |         self.batch_size = batch_size
168 |         self.boundaries = boundaries
169 | 
170 |         self.buckets, self.num_samples_per_bucket = self._create_buckets()
171 |         self.total_size = sum(self.num_samples_per_bucket)
172 |         self.num_samples = self.total_size // self.num_replicas
173 | 
174 |     def _create_buckets(self):
175 |         buckets = [[] for _ in range(len(self.boundaries) - 1)]
176 |         for i in range(len(self.lengths)):
177 |             length = self.lengths[i]
178 |             idx_bucket = self._bisect(length)
179 |             if idx_bucket != -1:
180 |                 buckets[idx_bucket].append(i)
181 | 
182 |         for i in range(len(buckets) - 1, 0, -1):
183 |             if len(buckets[i]) == 0:
184 |                 buckets.pop(i)
185 |                 self.boundaries.pop(i+1)
186 | 
187 |         num_samples_per_bucket = []
188 |         for i in range(len(buckets)):
189 |             len_bucket = len(buckets[i])
190 |             total_batch_size = self.num_replicas * self.batch_size
191 |             rem = (total_batch_size - (len_bucket % total_batch_size)) % total_batch_size
192 |             num_samples_per_bucket.append(len_bucket + rem)
193 |         return buckets, num_samples_per_bucket
194 | 
195 |     def __iter__(self):
196 |       # deterministically shuffle based on epoch
197 |       g = torch.Generator()
198 |       g.manual_seed(self.epoch)
199 | 
200 |       indices = []
201 |       if self.shuffle:
202 |           for bucket in self.buckets:
203 |               indices.append(torch.randperm(len(bucket), generator=g).tolist())
204 |       else:
205 |           for bucket in self.buckets:
206 |               indices.append(list(range(len(bucket))))
207 | 
208 |       batches = []
209 |       for i in range(len(self.buckets)):
210 |           bucket = self.buckets[i]
211 |           len_bucket = len(bucket)
212 |           ids_bucket = indices[i]
213 |           num_samples_bucket = self.num_samples_per_bucket[i]
214 | 
215 |           # add extra samples to make it evenly divisible
216 |           rem = num_samples_bucket - len_bucket
217 |           ids_bucket = ids_bucket + ids_bucket * (rem // len_bucket) + ids_bucket[:(rem % len_bucket)]
218 | 
219 |           # subsample
220 |           ids_bucket = ids_bucket[self.rank::self.num_replicas]
221 | 
222 |           # batching
223 |           for j in range(len(ids_bucket) // self.batch_size):
224 |               batch = [bucket[idx] for idx in ids_bucket[j*self.batch_size:(j+1)*self.batch_size]]
225 |               batches.append(batch)
226 | 
227 |       if self.shuffle:
228 |           batch_ids = torch.randperm(len(batches), generator=g).tolist()
229 |           batches = [batches[i] for i in batch_ids]
230 |       self.batches = batches
231 | 
232 |       assert len(self.batches) * self.batch_size == self.num_samples
233 |       return iter(self.batches)
234 | 
235 |     def _bisect(self, x, lo=0, hi=None):
236 |       if hi is None:
237 |           hi = len(self.boundaries) - 1
238 | 
239 |       if hi > lo:
240 |           mid = (hi + lo) // 2
241 |           if self.boundaries[mid] < x and x <= self.boundaries[mid+1]:
242 |               return mid
243 |           elif x <= self.boundaries[mid]:
244 |               return self._bisect(x, lo, mid)
245 |           else:
246 |               return self._bisect(x, mid + 1, hi)
247 |       else:
248 |           return -1
249 | 
250 |     def __len__(self):
251 |         return self.num_samples // self.batch_size
252 | import utils
253 | from torch.utils.data import DataLoader
254 | if __name__ == "__main__":
255 |     hps = utils.get_hparams()
256 |     train_dataset = TextAudioSpeakerLoader(hps.data.training_files, hps)
257 |     train_sampler = DistributedBucketSampler(
258 |         train_dataset,
259 |         hps.train.batch_size,
260 |         [32,70,100,200,300,400,500,600,700,800,900,1000],
261 |         num_replicas=1,
262 |         rank=0,
263 |         shuffle=True)
264 |     collate_fn = TextAudioSpeakerCollate(hps)
265 |     train_loader = DataLoader(train_dataset, num_workers=1, shuffle=False, pin_memory=True,
266 |         collate_fn=collate_fn, batch_sampler=train_sampler)
267 | 
268 |     for batch_idx, (c, spec, y) in enumerate(train_loader):
269 |         print(c.size(), spec.size(), y.size())
270 |         #print(batch_idx, c, spec, y)
271 |         #break
272 | 


--------------------------------------------------------------------------------
/dataset/encode.py:
--------------------------------------------------------------------------------
 1 | import argparse
 2 | import logging
 3 | import numpy as np
 4 | from pathlib import Path
 5 | from tqdm import tqdm
 6 | import os
 7 | 
 8 | import torch
 9 | import torchaudio
10 | from torchaudio.functional import resample
11 | 
12 | 
13 | def encode_dataset(args):
14 |     print(f"Loading hubert checkpoint")
15 |     hubert = torch.hub.load("bshall/hubert:main", f"hubert_soft").cuda().eval()
16 | 
17 |     print(f"Encoding dataset at {args.in_dir}")
18 |     for in_path in tqdm(list(args.in_dir.rglob(f"*{args.extension}"))):
19 |         out_path = args.out_dir / in_path.relative_to(args.in_dir)
20 |         out_path.parent.mkdir(parents=True, exist_ok=True)
21 |         if True:#not os.path.exists(out_path.with_suffix(".npy")):
22 |             wav, sr = torchaudio.load(in_path)
23 |             wav = resample(wav, sr, 16000)
24 |             wav = wav.unsqueeze(0).cuda()
25 | 
26 |             with torch.inference_mode():
27 |                 units = hubert.units(wav)
28 | 
29 |             
30 |             np.save(out_path.with_suffix(".npy"), units.squeeze().cpu().numpy())
31 | 
32 | 
33 | if __name__ == "__main__":
34 |     parser = argparse.ArgumentParser(description="Encode an audio dataset.")
35 |     parser.add_argument(
36 |         "model",
37 |         help="available models (HuBERT-Soft or HuBERT-Discrete)",
38 |         choices=["soft", "soft"],
39 |     )
40 |     parser.add_argument(
41 |         "in_dir",
42 |         metavar="in-dir",
43 |         help="path to the dataset directory.",
44 |         type=Path,
45 |     )
46 |     parser.add_argument(
47 |         "out_dir",
48 |         metavar="out-dir",
49 |         help="path to the output directory.",
50 |         type=Path,
51 |     )
52 |     parser.add_argument(
53 |         "--extension",
54 |         help="extension of the audio files (defaults to .flac).",
55 |         default=".wav",
56 |         type=str,
57 |     )
58 |     args = parser.parse_args()
59 |     encode_dataset(args)


--------------------------------------------------------------------------------
/logs/quickvc/config.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "train": {
 3 |     "log_interval": 20,
 4 |     "eval_interval": 5000,
 5 |     "seed": 1234,
 6 |     "epochs": 20000,
 7 |     "learning_rate": 2e-4,
 8 |     "betas": [0.8, 0.99],
 9 |     "eps": 1e-9,
10 |     "batch_size": 64,
11 |     "fp16_run": false,
12 |     "lr_decay": 0.999875,
13 |     "segment_size": 10240,
14 |     "init_lr_ratio": 1,
15 |     "warmup_epochs": 0,
16 |     "c_mel": 45,
17 |     "c_kl": 1.0,
18 |     "use_sr": true,
19 |     "max_speclen": 512,
20 |     "port": "8002",
21 |     "fft_sizes": [384, 683, 171],
22 |     "hop_sizes": [30, 60, 10],
23 |     "win_lengths": [150, 300, 60],
24 |     "window": "hann_window"  
25 |   },
26 |   "data": {
27 |     "training_files":"./dataset/train.txt",
28 |     "validation_files":"./dataset/test.txt",
29 |     "text_cleaners":["english_cleaners2"],
30 |     "max_wav_value": 32768.0,
31 |     "sampling_rate": 16000,
32 |     "filter_length": 1280,
33 |     "hop_length": 320,
34 |     "win_length": 1280,
35 |     "n_mel_channels": 80,
36 |     "mel_fmin": 0.0,
37 |     "mel_fmax": null,
38 |     "add_blank": true,
39 |     "n_speakers": 0,
40 |     "cleaned_text": true
41 |   },
42 |   "model": {
43 |     "ms_istft_vits": true,
44 |     "mb_istft_vits": false,
45 |     "istft_vits": false,
46 |     "subbands": 4,
47 |     "gen_istft_n_fft": 16,
48 |     "gen_istft_hop_size": 4,
49 |     "inter_channels": 192,
50 |     "hidden_channels": 192,
51 |     "filter_channels": 768,
52 |     "n_heads": 2,
53 |     "n_layers": 6,
54 |     "kernel_size": 3,
55 |     "p_dropout": 0.1,
56 |     "resblock": "1",
57 |     "resblock_kernel_sizes": [3,7,11],
58 |     "resblock_dilation_sizes": [[1,3,5], [1,3,5], [1,3,5]],
59 |     "upsample_rates": [5,4],
60 |     "upsample_initial_channel": 512,
61 |     "upsample_kernel_sizes": [16,16],
62 |     "n_layers_q": 3,
63 |     "use_spectral_norm": false,
64 |     "gin_channels": 256,
65 |     "use_sdp": false,
66 |     "ssl_dim": 1024,
67 |     "use_spk": false
68 |   }
69 | 
70 | }
71 | 


--------------------------------------------------------------------------------
/losses.py:
--------------------------------------------------------------------------------
 1 | import torch 
 2 | from torch.nn import functional as F
 3 | from stft_loss import MultiResolutionSTFTLoss
 4 | 
 5 | 
 6 | import commons
 7 | 
 8 | 
 9 | def feature_loss(fmap_r, fmap_g):
10 |   loss = 0
11 |   for dr, dg in zip(fmap_r, fmap_g):
12 |     for rl, gl in zip(dr, dg):
13 |       rl = rl.float().detach()
14 |       gl = gl.float()
15 |       loss += torch.mean(torch.abs(rl - gl))
16 | 
17 |   return loss * 2 
18 | 
19 | 
20 | def discriminator_loss(disc_real_outputs, disc_generated_outputs):
21 |   loss = 0
22 |   r_losses = []
23 |   g_losses = []
24 |   for dr, dg in zip(disc_real_outputs, disc_generated_outputs):
25 |     dr = dr.float()
26 |     dg = dg.float()
27 |     r_loss = torch.mean((1-dr)**2)
28 |     g_loss = torch.mean(dg**2)
29 |     loss += (r_loss + g_loss)
30 |     r_losses.append(r_loss.item())
31 |     g_losses.append(g_loss.item())
32 | 
33 |   return loss, r_losses, g_losses
34 | 
35 | 
36 | def generator_loss(disc_outputs):
37 |   loss = 0
38 |   gen_losses = []
39 |   for dg in disc_outputs:
40 |     dg = dg.float()
41 |     l = torch.mean((1-dg)**2)
42 |     gen_losses.append(l)
43 |     loss += l
44 | 
45 |   return loss, gen_losses
46 | 
47 | 
48 | def kl_loss(z_p, logs_q, m_p, logs_p, z_mask):
49 |   """
50 |   z_p, logs_q: [b, h, t_t]
51 |   m_p, logs_p: [b, h, t_t]
52 |   """
53 |   z_p = z_p.float()
54 |   logs_q = logs_q.float()
55 |   m_p = m_p.float()
56 |   logs_p = logs_p.float()
57 |   z_mask = z_mask.float()
58 | 
59 |   kl = logs_p - logs_q - 0.5
60 |   kl += 0.5 * ((z_p - m_p)**2) * torch.exp(-2. * logs_p)
61 |   kl = torch.sum(kl * z_mask)
62 |   l = kl / torch.sum(z_mask)
63 |   return l
64 | 
65 | def subband_stft_loss(h, y_mb, y_hat_mb):
66 |   sub_stft_loss = MultiResolutionSTFTLoss(h.train.fft_sizes, h.train.hop_sizes, h.train.win_lengths)
67 |   y_mb =  y_mb.view(-1, y_mb.size(2))
68 |   y_hat_mb = y_hat_mb.view(-1, y_hat_mb.size(2))
69 |   sub_sc_loss, sub_mag_loss = sub_stft_loss(y_hat_mb[:, :y_mb.size(-1)], y_mb)
70 |   return sub_sc_loss+sub_mag_loss
71 | 
72 | 


--------------------------------------------------------------------------------
/mel_processing.py:
--------------------------------------------------------------------------------
  1 | import math
  2 | import os
  3 | import random
  4 | import torch
  5 | from torch import nn
  6 | import torch.nn.functional as F
  7 | import torch.utils.data
  8 | import numpy as np
  9 | import librosa
 10 | import librosa.util as librosa_util
 11 | from librosa.util import normalize, pad_center, tiny
 12 | from scipy.signal import get_window
 13 | from scipy.io.wavfile import read
 14 | from librosa.filters import mel as librosa_mel_fn
 15 | 
 16 | MAX_WAV_VALUE = 32768.0
 17 | 
 18 | 
 19 | def dynamic_range_compression_torch(x, C=1, clip_val=1e-5):
 20 |     """
 21 |     PARAMS
 22 |     ------
 23 |     C: compression factor
 24 |     """
 25 |     return torch.log(torch.clamp(x, min=clip_val) * C)
 26 | 
 27 | 
 28 | def dynamic_range_decompression_torch(x, C=1):
 29 |     """
 30 |     PARAMS
 31 |     ------
 32 |     C: compression factor used to compress
 33 |     """
 34 |     return torch.exp(x) / C
 35 | 
 36 | 
 37 | def spectral_normalize_torch(magnitudes):
 38 |     output = dynamic_range_compression_torch(magnitudes)
 39 |     return output
 40 | 
 41 | 
 42 | def spectral_de_normalize_torch(magnitudes):
 43 |     output = dynamic_range_decompression_torch(magnitudes)
 44 |     return output
 45 | 
 46 | 
 47 | mel_basis = {}
 48 | hann_window = {}
 49 | 
 50 | 
 51 | def spectrogram_torch(y, n_fft, sampling_rate, hop_size, win_size, center=False):
 52 |     if torch.min(y) < -1.:
 53 |         print('min value is ', torch.min(y))
 54 |     if torch.max(y) > 1.:
 55 |         print('max value is ', torch.max(y))
 56 | 
 57 |     global hann_window
 58 |     dtype_device = str(y.dtype) + '_' + str(y.device)
 59 |     wnsize_dtype_device = str(win_size) + '_' + dtype_device
 60 |     if wnsize_dtype_device not in hann_window:
 61 |         hann_window[wnsize_dtype_device] = torch.hann_window(win_size).to(dtype=y.dtype, device=y.device)
 62 | 
 63 |     y = torch.nn.functional.pad(y.unsqueeze(1), (int((n_fft-hop_size)/2), int((n_fft-hop_size)/2)), mode='reflect')
 64 |     y = y.squeeze(1)
 65 | 
 66 |     spec = torch.stft(y, n_fft, hop_length=hop_size, win_length=win_size, window=hann_window[wnsize_dtype_device],
 67 |                       center=center, pad_mode='reflect', normalized=False, onesided=True)
 68 | 
 69 |     spec = torch.sqrt(spec.pow(2).sum(-1) + 1e-6)
 70 |     return spec
 71 | 
 72 | 
 73 | def spec_to_mel_torch(spec, n_fft, num_mels, sampling_rate, fmin, fmax):
 74 |     global mel_basis
 75 |     dtype_device = str(spec.dtype) + '_' + str(spec.device)
 76 |     fmax_dtype_device = str(fmax) + '_' + dtype_device
 77 |     if fmax_dtype_device not in mel_basis:
 78 |         mel = librosa_mel_fn(sampling_rate, n_fft, num_mels, fmin, fmax)
 79 |         mel_basis[fmax_dtype_device] = torch.from_numpy(mel).to(dtype=spec.dtype, device=spec.device)
 80 |     spec = torch.matmul(mel_basis[fmax_dtype_device], spec)
 81 |     spec = spectral_normalize_torch(spec)
 82 |     return spec
 83 | 
 84 | 
 85 | def mel_spectrogram_torch(y, n_fft, num_mels, sampling_rate, hop_size, win_size, fmin, fmax, center=False):
 86 |     if torch.min(y) < -1.:
 87 |         print('min value is ', torch.min(y))
 88 |     if torch.max(y) > 1.:
 89 |         print('max value is ', torch.max(y))
 90 | 
 91 |     global mel_basis, hann_window
 92 |     dtype_device = str(y.dtype) + '_' + str(y.device)
 93 |     fmax_dtype_device = str(fmax) + '_' + dtype_device
 94 |     wnsize_dtype_device = str(win_size) + '_' + dtype_device
 95 |     if fmax_dtype_device not in mel_basis:
 96 |         mel = librosa_mel_fn(sampling_rate, n_fft, num_mels, fmin, fmax)
 97 |         mel_basis[fmax_dtype_device] = torch.from_numpy(mel).to(dtype=y.dtype, device=y.device)
 98 |     if wnsize_dtype_device not in hann_window:
 99 |         hann_window[wnsize_dtype_device] = torch.hann_window(win_size).to(dtype=y.dtype, device=y.device)
100 | 
101 |     y = torch.nn.functional.pad(y.unsqueeze(1), (int((n_fft-hop_size)/2), int((n_fft-hop_size)/2)), mode='reflect')
102 |     y = y.squeeze(1)
103 | 
104 |     spec = torch.stft(y, n_fft, hop_length=hop_size, win_length=win_size, window=hann_window[wnsize_dtype_device],
105 |                       center=center, pad_mode='reflect', normalized=False, onesided=True)
106 | 
107 |     spec = torch.sqrt(spec.pow(2).sum(-1) + 1e-6)
108 | 
109 |     spec = torch.matmul(mel_basis[fmax_dtype_device], spec)
110 |     spec = spectral_normalize_torch(spec)
111 | 
112 |     return spec
113 | 


--------------------------------------------------------------------------------
/models.py:
--------------------------------------------------------------------------------
  1 | import copy
  2 | import math
  3 | import torch
  4 | from torch import nn
  5 | from torch.nn import functional as F
  6 | 
  7 | import commons
  8 | import modules
  9 | import attentions
 10 | 
 11 | from torch.nn import Conv1d, ConvTranspose1d, AvgPool1d, Conv2d
 12 | from torch.nn.utils import weight_norm, remove_weight_norm, spectral_norm
 13 | from commons import init_weights, get_padding
 14 | from pqmf import PQMF
 15 | from stft import TorchSTFT
 16 | import math
 17 | 
 18 | 
 19 | class StochasticDurationPredictor(nn.Module):
 20 |   def __init__(self, in_channels, filter_channels, kernel_size, p_dropout, n_flows=4, gin_channels=0):
 21 |     super().__init__()
 22 |     filter_channels = in_channels # it needs to be removed from future version.
 23 |     self.in_channels = in_channels
 24 |     self.filter_channels = filter_channels
 25 |     self.kernel_size = kernel_size
 26 |     self.p_dropout = p_dropout
 27 |     self.n_flows = n_flows
 28 |     self.gin_channels = gin_channels
 29 | 
 30 |     self.log_flow = modules.Log()
 31 |     self.flows = nn.ModuleList()
 32 |     self.flows.append(modules.ElementwiseAffine(2))
 33 |     for i in range(n_flows):
 34 |       self.flows.append(modules.ConvFlow(2, filter_channels, kernel_size, n_layers=3))
 35 |       self.flows.append(modules.Flip())
 36 | 
 37 |     self.post_pre = nn.Conv1d(1, filter_channels, 1)
 38 |     self.post_proj = nn.Conv1d(filter_channels, filter_channels, 1)
 39 |     self.post_convs = modules.DDSConv(filter_channels, kernel_size, n_layers=3, p_dropout=p_dropout)
 40 |     self.post_flows = nn.ModuleList()
 41 |     self.post_flows.append(modules.ElementwiseAffine(2))
 42 |     for i in range(4):
 43 |       self.post_flows.append(modules.ConvFlow(2, filter_channels, kernel_size, n_layers=3))
 44 |       self.post_flows.append(modules.Flip())
 45 | 
 46 |     self.pre = nn.Conv1d(in_channels, filter_channels, 1)
 47 |     self.proj = nn.Conv1d(filter_channels, filter_channels, 1)
 48 |     self.convs = modules.DDSConv(filter_channels, kernel_size, n_layers=3, p_dropout=p_dropout)
 49 |     if gin_channels != 0:
 50 |       self.cond = nn.Conv1d(gin_channels, filter_channels, 1)
 51 | 
 52 |   def forward(self, x, x_mask, w=None, g=None, reverse=False, noise_scale=1.0):
 53 |     x = torch.detach(x)
 54 |     x = self.pre(x)
 55 |     if g is not None:
 56 |       g = torch.detach(g)
 57 |       x = x + self.cond(g)
 58 |     x = self.convs(x, x_mask)
 59 |     x = self.proj(x) * x_mask
 60 | 
 61 |     if not reverse:
 62 |       flows = self.flows
 63 |       assert w is not None
 64 | 
 65 |       logdet_tot_q = 0 
 66 |       h_w = self.post_pre(w)
 67 |       h_w = self.post_convs(h_w, x_mask)
 68 |       h_w = self.post_proj(h_w) * x_mask
 69 |       e_q = torch.randn(w.size(0), 2, w.size(2)).to(device=x.device, dtype=x.dtype) * x_mask
 70 |       z_q = e_q
 71 |       for flow in self.post_flows:
 72 |         z_q, logdet_q = flow(z_q, x_mask, g=(x + h_w))
 73 |         logdet_tot_q += logdet_q
 74 |       z_u, z1 = torch.split(z_q, [1, 1], 1) 
 75 |       u = torch.sigmoid(z_u) * x_mask
 76 |       z0 = (w - u) * x_mask
 77 |       logdet_tot_q += torch.sum((F.logsigmoid(z_u) + F.logsigmoid(-z_u)) * x_mask, [1,2])
 78 |       logq = torch.sum(-0.5 * (math.log(2*math.pi) + (e_q**2)) * x_mask, [1,2]) - logdet_tot_q
 79 | 
 80 |       logdet_tot = 0
 81 |       z0, logdet = self.log_flow(z0, x_mask)
 82 |       logdet_tot += logdet
 83 |       z = torch.cat([z0, z1], 1)
 84 |       for flow in flows:
 85 |         z, logdet = flow(z, x_mask, g=x, reverse=reverse)
 86 |         logdet_tot = logdet_tot + logdet
 87 |       nll = torch.sum(0.5 * (math.log(2*math.pi) + (z**2)) * x_mask, [1,2]) - logdet_tot
 88 |       return nll + logq # [b]
 89 |     else:
 90 |       flows = list(reversed(self.flows))
 91 |       flows = flows[:-2] + [flows[-1]] # remove a useless vflow
 92 |       z = torch.randn(x.size(0), 2, x.size(2)).to(device=x.device, dtype=x.dtype) * noise_scale
 93 |       for flow in flows:
 94 |         z = flow(z, x_mask, g=x, reverse=reverse)
 95 |       z0, z1 = torch.split(z, [1, 1], 1)
 96 |       logw = z0
 97 |       return logw
 98 | 
 99 | 
100 | class DurationPredictor(nn.Module):
101 |   def __init__(self, in_channels, filter_channels, kernel_size, p_dropout, gin_channels=0):
102 |     super().__init__()
103 | 
104 |     self.in_channels = in_channels
105 |     self.filter_channels = filter_channels
106 |     self.kernel_size = kernel_size
107 |     self.p_dropout = p_dropout
108 |     self.gin_channels = gin_channels
109 | 
110 |     self.drop = nn.Dropout(p_dropout)
111 |     self.conv_1 = nn.Conv1d(in_channels, filter_channels, kernel_size, padding=kernel_size//2)
112 |     self.norm_1 = modules.LayerNorm(filter_channels)
113 |     self.conv_2 = nn.Conv1d(filter_channels, filter_channels, kernel_size, padding=kernel_size//2)
114 |     self.norm_2 = modules.LayerNorm(filter_channels)
115 |     self.proj = nn.Conv1d(filter_channels, 1, 1)
116 | 
117 |     if gin_channels != 0:
118 |       self.cond = nn.Conv1d(gin_channels, in_channels, 1)
119 | 
120 |   def forward(self, x, x_mask, g=None):
121 |     x = torch.detach(x)
122 |     if g is not None:
123 |       g = torch.detach(g)
124 |       x = x + self.cond(g)
125 |     x = self.conv_1(x * x_mask)
126 |     x = torch.relu(x)
127 |     x = self.norm_1(x)
128 |     x = self.drop(x)
129 |     x = self.conv_2(x * x_mask)
130 |     x = torch.relu(x)
131 |     x = self.norm_2(x)
132 |     x = self.drop(x)
133 |     x = self.proj(x * x_mask)
134 |     return x * x_mask
135 | 
136 | 
137 | class TextEncoder(nn.Module):
138 |   def __init__(self,
139 |       n_vocab,
140 |       out_channels,
141 |       hidden_channels,
142 |       filter_channels,
143 |       n_heads,
144 |       n_layers,
145 |       kernel_size,
146 |       p_dropout):
147 |     super().__init__()
148 |     self.n_vocab = n_vocab
149 |     self.out_channels = out_channels
150 |     self.hidden_channels = hidden_channels
151 |     self.filter_channels = filter_channels
152 |     self.n_heads = n_heads
153 |     self.n_layers = n_layers
154 |     self.kernel_size = kernel_size
155 |     self.p_dropout = p_dropout
156 | 
157 | 
158 |     self.encoder = attentions.Encoder(
159 |       hidden_channels,
160 |       filter_channels,
161 |       n_heads,
162 |       n_layers,
163 |       kernel_size,
164 |       p_dropout)
165 |     self.proj= nn.Conv1d(hidden_channels, out_channels * 2, 1)
166 | 
167 |   def forward(self, x, x_lengths):
168 |     x = torch.transpose(x, 1, -1) # [b, h, t]
169 |     x_mask = torch.unsqueeze(commons.sequence_mask(x_lengths, x.size(2)), 1).to(x.dtype)
170 | 
171 |     x = self.encoder(x * x_mask, x_mask)
172 |     stats = self.proj(x) * x_mask
173 | 
174 |     m, logs = torch.split(stats, self.out_channels, dim=1)
175 |     return x, m, logs, x_mask
176 | 
177 | class Encoder(nn.Module):
178 |   def __init__(self,
179 |       in_channels,
180 |       out_channels,
181 |       hidden_channels,
182 |       kernel_size,
183 |       dilation_rate,
184 |       n_layers,
185 |       gin_channels=0):
186 |     super().__init__()
187 |     self.in_channels = in_channels
188 |     self.out_channels = out_channels
189 |     self.hidden_channels = hidden_channels
190 |     self.kernel_size = kernel_size
191 |     self.dilation_rate = dilation_rate
192 |     self.n_layers = n_layers
193 |     self.gin_channels = gin_channels
194 | 
195 |     self.pre = nn.Conv1d(in_channels, hidden_channels, 1)
196 |     self.enc = modules.WN(hidden_channels, kernel_size, dilation_rate, n_layers, gin_channels=gin_channels)
197 |     self.proj = nn.Conv1d(hidden_channels, out_channels * 2, 1)
198 | 
199 |   def forward(self, x, x_lengths, g=None):
200 |     x_mask = torch.unsqueeze(commons.sequence_mask(x_lengths, x.size(2)), 1).to(x.dtype)
201 |     x = self.pre(x) * x_mask
202 |     x = self.enc(x, x_mask, g=g)
203 |     stats = self.proj(x) * x_mask
204 |     m, logs = torch.split(stats, self.out_channels, dim=1)
205 |     z = (m + torch.randn_like(m) * torch.exp(logs)) * x_mask
206 |     return z, m, logs, x_mask
207 | 
208 | 
209 | class ResidualCouplingBlock(nn.Module):
210 |   def __init__(self,
211 |       channels,
212 |       hidden_channels,
213 |       kernel_size,
214 |       dilation_rate,
215 |       n_layers,
216 |       n_flows=4,
217 |       gin_channels=0):
218 |     super().__init__()
219 |     self.channels = channels
220 |     self.hidden_channels = hidden_channels
221 |     self.kernel_size = kernel_size
222 |     self.dilation_rate = dilation_rate
223 |     self.n_layers = n_layers
224 |     self.n_flows = n_flows
225 |     self.gin_channels = gin_channels
226 | 
227 |     self.flows = nn.ModuleList()
228 |     for i in range(n_flows):
229 |       self.flows.append(modules.ResidualCouplingLayer(channels, hidden_channels, kernel_size, dilation_rate, n_layers, gin_channels=gin_channels, mean_only=True))
230 |       self.flows.append(modules.Flip())
231 | 
232 |   def forward(self, x, x_mask, g=None, reverse=False):
233 |     if not reverse:
234 |       for flow in self.flows:
235 |         x, _ = flow(x, x_mask, g=g, reverse=reverse)
236 |     else:
237 |       for flow in reversed(self.flows):
238 |         x = flow(x, x_mask, g=g, reverse=reverse)
239 |     return x
240 | 
241 | 
242 | class PosteriorEncoder(nn.Module):
243 |   def __init__(self,
244 |       in_channels,
245 |       out_channels,
246 |       hidden_channels,
247 |       kernel_size,
248 |       dilation_rate,
249 |       n_layers,
250 |       gin_channels=0):
251 |     super().__init__()
252 |     self.in_channels = in_channels
253 |     self.out_channels = out_channels
254 |     self.hidden_channels = hidden_channels
255 |     self.kernel_size = kernel_size
256 |     self.dilation_rate = dilation_rate
257 |     self.n_layers = n_layers
258 |     self.gin_channels = gin_channels
259 | 
260 |     self.pre = nn.Conv1d(in_channels, hidden_channels, 1)
261 |     self.enc = modules.WN(hidden_channels, kernel_size, dilation_rate, n_layers, gin_channels=gin_channels)
262 |     self.proj = nn.Conv1d(hidden_channels, out_channels * 2, 1)
263 | 
264 |   def forward(self, x, x_lengths, g=None):
265 |     x_mask = torch.unsqueeze(commons.sequence_mask(x_lengths, x.size(2)), 1).to(x.dtype)
266 |     x = self.pre(x) * x_mask
267 |     x = self.enc(x, x_mask, g=g)
268 |     stats = self.proj(x) * x_mask
269 |     m, logs = torch.split(stats, self.out_channels, dim=1)
270 |     z = (m + torch.randn_like(m) * torch.exp(logs)) * x_mask
271 |     return z, m, logs, x_mask
272 | 
273 | class iSTFT_Generator(torch.nn.Module):
274 |     def __init__(self, initial_channel, resblock, resblock_kernel_sizes, resblock_dilation_sizes, upsample_rates, upsample_initial_channel, upsample_kernel_sizes, gen_istft_n_fft, gen_istft_hop_size, gin_channels=0):
275 |         super(iSTFT_Generator, self).__init__()
276 |         # self.h = h
277 |         self.gen_istft_n_fft = gen_istft_n_fft
278 |         self.gen_istft_hop_size = gen_istft_hop_size
279 | 
280 |         self.num_kernels = len(resblock_kernel_sizes)
281 |         self.num_upsamples = len(upsample_rates)
282 |         self.conv_pre = weight_norm(Conv1d(initial_channel, upsample_initial_channel, 7, 1, padding=3))
283 |         resblock = modules.ResBlock1 if resblock == '1' else modules.ResBlock2
284 | 
285 |         self.ups = nn.ModuleList()
286 |         for i, (u, k) in enumerate(zip(upsample_rates, upsample_kernel_sizes)):
287 |             self.ups.append(weight_norm(
288 |                 ConvTranspose1d(upsample_initial_channel//(2**i), upsample_initial_channel//(2**(i+1)),
289 |                                 k, u, padding=(k-u)//2)))
290 | 
291 |         self.resblocks = nn.ModuleList()
292 |         for i in range(len(self.ups)):
293 |             ch = upsample_initial_channel//(2**(i+1))
294 |             for j, (k, d) in enumerate(zip(resblock_kernel_sizes, resblock_dilation_sizes)):
295 |                 self.resblocks.append(resblock(ch, k, d))
296 | 
297 |         self.post_n_fft = self.gen_istft_n_fft
298 |         self.conv_post = weight_norm(Conv1d(ch, self.post_n_fft + 2, 7, 1, padding=3))
299 |         self.ups.apply(init_weights)
300 |         self.conv_post.apply(init_weights)
301 |         self.reflection_pad = torch.nn.ReflectionPad1d((1, 0))
302 |         self.cond = nn.Conv1d(256, 512, 1)
303 |         self.stft = TorchSTFT(filter_length=self.gen_istft_n_fft, hop_length=self.gen_istft_hop_size, win_length=self.gen_istft_n_fft)
304 |     def forward(self, x, g=None):
305 |         
306 |         x = self.conv_pre(x)
307 |         x = x + self.cond(g)
308 |         for i in range(self.num_upsamples):
309 |             x = F.leaky_relu(x, modules.LRELU_SLOPE)
310 |             x = self.ups[i](x)
311 |             xs = None
312 |             for j in range(self.num_kernels):
313 |                 if xs is None:
314 |                     xs = self.resblocks[i*self.num_kernels+j](x)
315 |                 else:
316 |                     xs += self.resblocks[i*self.num_kernels+j](x)
317 |             x = xs / self.num_kernels
318 |         x = F.leaky_relu(x)
319 |         x = self.reflection_pad(x)
320 |         x = self.conv_post(x)
321 |         spec = torch.exp(x[:,:self.post_n_fft // 2 + 1, :])
322 |         phase = math.pi*torch.sin(x[:, self.post_n_fft // 2 + 1:, :])
323 |         out = self.stft.inverse(spec, phase).to(x.device)
324 |         return out, None
325 | 
326 |     def remove_weight_norm(self):
327 |         print('Removing weight norm...')
328 |         for l in self.ups:
329 |             remove_weight_norm(l)
330 |         for l in self.resblocks:
331 |             l.remove_weight_norm()
332 |         remove_weight_norm(self.conv_pre)
333 |         remove_weight_norm(self.conv_post)
334 | 
335 | 
336 | class Multiband_iSTFT_Generator(torch.nn.Module):
337 |     def __init__(self, initial_channel, resblock, resblock_kernel_sizes, resblock_dilation_sizes, upsample_rates, upsample_initial_channel, upsample_kernel_sizes, gen_istft_n_fft, gen_istft_hop_size, subbands, gin_channels=0):
338 |         super(Multiband_iSTFT_Generator, self).__init__()
339 |         # self.h = h
340 |         self.subbands = subbands
341 |         self.num_kernels = len(resblock_kernel_sizes)
342 |         self.num_upsamples = len(upsample_rates)
343 |         self.conv_pre = weight_norm(Conv1d(initial_channel, upsample_initial_channel, 7, 1, padding=3))
344 |         resblock = modules.ResBlock1 if resblock == '1' else modules.ResBlock2
345 | 
346 |         self.ups = nn.ModuleList()
347 |         for i, (u, k) in enumerate(zip(upsample_rates, upsample_kernel_sizes)):
348 |             self.ups.append(weight_norm(
349 |                 ConvTranspose1d(upsample_initial_channel//(2**i), upsample_initial_channel//(2**(i+1)),
350 |                                 k, u, padding=(k-u+1-i)//2,output_padding=1-i)))
351 | 
352 |         self.resblocks = nn.ModuleList()
353 |         for i in range(len(self.ups)):
354 |             ch = upsample_initial_channel//(2**(i+1))
355 |             for j, (k, d) in enumerate(zip(resblock_kernel_sizes, resblock_dilation_sizes)):
356 |                 self.resblocks.append(resblock(ch, k, d))
357 | 
358 |         self.post_n_fft = gen_istft_n_fft
359 |         self.ups.apply(init_weights)
360 |         self.reflection_pad = torch.nn.ReflectionPad1d((1, 0))
361 |         self.reshape_pixelshuffle = []
362 |  
363 |         self.subband_conv_post = weight_norm(Conv1d(ch, self.subbands*(self.post_n_fft + 2), 7, 1, padding=3))
364 |         
365 |         self.subband_conv_post.apply(init_weights)
366 |         self.cond = nn.Conv1d(256, 512, 1)
367 |         self.gen_istft_n_fft = gen_istft_n_fft
368 |         self.gen_istft_hop_size = gen_istft_hop_size
369 | 
370 | 
371 |     def forward(self, x, g=None):
372 |       
373 |       stft = TorchSTFT(filter_length=self.gen_istft_n_fft, hop_length=self.gen_istft_hop_size, win_length=self.gen_istft_n_fft).to(x.device)
374 |       #print(x.device)
375 |       pqmf = PQMF(x.device)
376 |       
377 |       x = self.conv_pre(x)#[B, ch, length]
378 |       x = x + self.cond(g)  
379 |       for i in range(self.num_upsamples):
380 |           x = F.leaky_relu(x, modules.LRELU_SLOPE)
381 |           x = self.ups[i](x)
382 |           
383 |           
384 |           xs = None
385 |           for j in range(self.num_kernels):
386 |               if xs is None:
387 |                   xs = self.resblocks[i*self.num_kernels+j](x)
388 |               else:
389 |                   xs += self.resblocks[i*self.num_kernels+j](x)
390 |           x = xs / self.num_kernels
391 |           
392 |       x = F.leaky_relu(x)
393 |       x = self.reflection_pad(x)
394 |       x = self.subband_conv_post(x)
395 |       x = torch.reshape(x, (x.shape[0], self.subbands, x.shape[1]//self.subbands, x.shape[-1]))
396 | 
397 |       spec = torch.exp(x[:,:,:self.post_n_fft // 2 + 1, :])
398 |       phase = math.pi*torch.sin(x[:,:, self.post_n_fft // 2 + 1:, :])
399 | 
400 |       y_mb_hat = stft.inverse(torch.reshape(spec, (spec.shape[0]*self.subbands, self.gen_istft_n_fft // 2 + 1, spec.shape[-1])), torch.reshape(phase, (phase.shape[0]*self.subbands, self.gen_istft_n_fft // 2 + 1, phase.shape[-1])))
401 |       y_mb_hat = torch.reshape(y_mb_hat, (x.shape[0], self.subbands, 1, y_mb_hat.shape[-1]))
402 |       y_mb_hat = y_mb_hat.squeeze(-2)
403 | 
404 |       y_g_hat = pqmf.synthesis(y_mb_hat)
405 | 
406 |       return y_g_hat, y_mb_hat
407 | 
408 |     def remove_weight_norm(self):
409 |       print('Removing weight norm...')
410 |       for l in self.ups:
411 |           remove_weight_norm(l)
412 |       for l in self.resblocks:
413 |           l.remove_weight_norm()
414 | 
415 | 
416 | class Multistream_iSTFT_Generator(torch.nn.Module):
417 |     def __init__(self, initial_channel, resblock, resblock_kernel_sizes, resblock_dilation_sizes, upsample_rates, upsample_initial_channel, upsample_kernel_sizes, gen_istft_n_fft, gen_istft_hop_size, subbands, gin_channels=0):
418 |         super(Multistream_iSTFT_Generator, self).__init__()
419 |         # self.h = h
420 |         self.subbands = subbands
421 |         self.num_kernels = len(resblock_kernel_sizes)
422 |         self.num_upsamples = len(upsample_rates)
423 |         self.conv_pre = weight_norm(Conv1d(initial_channel, upsample_initial_channel, 7, 1, padding=3))
424 |         resblock = modules.ResBlock1 if resblock == '1' else modules.ResBlock2
425 | 
426 |         self.ups = nn.ModuleList()
427 |         for i, (u, k) in enumerate(zip(upsample_rates, upsample_kernel_sizes)):
428 |             self.ups.append(weight_norm(
429 |                 ConvTranspose1d(upsample_initial_channel//(2**i), upsample_initial_channel//(2**(i+1)),
430 |                                 k, u, padding=(k-u+1-i)//2,output_padding=1-i)))#这里k和u不是成倍数的关系，对最终结果很有可能是有影响的，会有checkerboard artifacts的现象
431 | 
432 |         self.resblocks = nn.ModuleList()
433 |         for i in range(len(self.ups)):
434 |             ch = upsample_initial_channel//(2**(i+1))
435 |             for j, (k, d) in enumerate(zip(resblock_kernel_sizes, resblock_dilation_sizes)):
436 |                 self.resblocks.append(resblock(ch, k, d))
437 | 
438 |         self.post_n_fft = gen_istft_n_fft
439 |         self.ups.apply(init_weights)
440 |         self.reflection_pad = torch.nn.ReflectionPad1d((1, 0))
441 |         self.reshape_pixelshuffle = []
442 |  
443 |         self.subband_conv_post = weight_norm(Conv1d(ch, self.subbands*(self.post_n_fft + 2), 7, 1, padding=3))
444 |         
445 |         self.subband_conv_post.apply(init_weights)
446 |         
447 |         self.gen_istft_n_fft = gen_istft_n_fft
448 |         self.gen_istft_hop_size = gen_istft_hop_size
449 | 
450 |         updown_filter = torch.zeros((self.subbands, self.subbands, self.subbands)).float()
451 |         for k in range(self.subbands):
452 |             updown_filter[k, k, 0] = 1.0
453 |         self.register_buffer("updown_filter", updown_filter)
454 |         self.multistream_conv_post = weight_norm(Conv1d(4, 1, kernel_size=63, bias=False, padding=get_padding(63, 1)))
455 |         self.multistream_conv_post.apply(init_weights)
456 |         self.cond = nn.Conv1d(256, 512, 1)
457 | 
458 | 
459 |     def forward(self, x, g=None):
460 |       stft = TorchSTFT(filter_length=self.gen_istft_n_fft, hop_length=self.gen_istft_hop_size, win_length=self.gen_istft_n_fft).to(x.device)
461 |       # pqmf = PQMF(x.device)
462 | 
463 |       x = self.conv_pre(x)#[B, ch, length]
464 |       #print(x.size(),g.size())
465 |       x = x + self.cond(g) # g [b, 256, 1] => cond(g) [b, 512, 1] 
466 |       
467 |       for i in range(self.num_upsamples):
468 | 
469 |           #print(x.size(),g.size())
470 |           x = F.leaky_relu(x, modules.LRELU_SLOPE)
471 |           #print(x.size(),g.size())
472 |           x = self.ups[i](x)
473 |           
474 |           #print(x.size(),g.size())
475 |           xs = None
476 |           for j in range(self.num_kernels):
477 |               if xs is None:
478 |                   xs = self.resblocks[i*self.num_kernels+j](x)
479 |               else:
480 |                   xs += self.resblocks[i*self.num_kernels+j](x)
481 |           x = xs / self.num_kernels
482 |       #print(x.size(),g.size())    
483 |       x = F.leaky_relu(x)
484 |       x = self.reflection_pad(x)
485 |       x = self.subband_conv_post(x)
486 |       x = torch.reshape(x, (x.shape[0], self.subbands, x.shape[1]//self.subbands, x.shape[-1]))
487 |       #print(x.size(),g.size())
488 |       spec = torch.exp(x[:,:,:self.post_n_fft // 2 + 1, :])
489 |       phase = math.pi*torch.sin(x[:,:, self.post_n_fft // 2 + 1:, :])
490 |       #print(spec.size(),phase.size())
491 |       y_mb_hat = stft.inverse(torch.reshape(spec, (spec.shape[0]*self.subbands, self.gen_istft_n_fft // 2 + 1, spec.shape[-1])), torch.reshape(phase, (phase.shape[0]*self.subbands, self.gen_istft_n_fft // 2 + 1, phase.shape[-1])))
492 |       #print(y_mb_hat.size())
493 |       y_mb_hat = torch.reshape(y_mb_hat, (x.shape[0], self.subbands, 1, y_mb_hat.shape[-1]))
494 |       #print(y_mb_hat.size())
495 |       y_mb_hat = y_mb_hat.squeeze(-2)
496 |       #print(y_mb_hat.size())
497 |       y_mb_hat = F.conv_transpose1d(y_mb_hat, self.updown_filter* self.subbands, stride=self.subbands)#.cuda(x.device) * self.subbands, stride=self.subbands)
498 |       #print(y_mb_hat.size())
499 |       y_g_hat = self.multistream_conv_post(y_mb_hat)
500 |       #print(y_g_hat.size(),y_mb_hat.size())
501 |       return y_g_hat, y_mb_hat
502 | 
503 |     def remove_weight_norm(self):
504 |       print('Removing weight norm...')
505 |       for l in self.ups:
506 |           remove_weight_norm(l)
507 |       for l in self.resblocks:
508 |           l.remove_weight_norm()
509 | 
510 | 
511 | class DiscriminatorP(torch.nn.Module):
512 |     def __init__(self, period, kernel_size=5, stride=3, use_spectral_norm=False):
513 |         super(DiscriminatorP, self).__init__()
514 |         self.period = period
515 |         self.use_spectral_norm = use_spectral_norm
516 |         norm_f = weight_norm if use_spectral_norm == False else spectral_norm
517 |         self.convs = nn.ModuleList([
518 |             norm_f(Conv2d(1, 32, (kernel_size, 1), (stride, 1), padding=(get_padding(kernel_size, 1), 0))),
519 |             norm_f(Conv2d(32, 128, (kernel_size, 1), (stride, 1), padding=(get_padding(kernel_size, 1), 0))),
520 |             norm_f(Conv2d(128, 512, (kernel_size, 1), (stride, 1), padding=(get_padding(kernel_size, 1), 0))),
521 |             norm_f(Conv2d(512, 1024, (kernel_size, 1), (stride, 1), padding=(get_padding(kernel_size, 1), 0))),
522 |             norm_f(Conv2d(1024, 1024, (kernel_size, 1), 1, padding=(get_padding(kernel_size, 1), 0))),
523 |         ])
524 |         self.conv_post = norm_f(Conv2d(1024, 1, (3, 1), 1, padding=(1, 0)))
525 | 
526 |     def forward(self, x):
527 |         fmap = []
528 | 
529 |         # 1d to 2d
530 |         b, c, t = x.shape
531 |         if t % self.period != 0: # pad first
532 |             n_pad = self.period - (t % self.period)
533 |             x = F.pad(x, (0, n_pad), "reflect")
534 |             t = t + n_pad
535 |         x = x.view(b, c, t // self.period, self.period)
536 | 
537 |         for l in self.convs:
538 |             x = l(x)
539 |             x = F.leaky_relu(x, modules.LRELU_SLOPE)
540 |             fmap.append(x)
541 |         x = self.conv_post(x)
542 |         fmap.append(x)
543 |         x = torch.flatten(x, 1, -1)
544 | 
545 |         return x, fmap
546 | 
547 | 
548 | class DiscriminatorS(torch.nn.Module):
549 |     def __init__(self, use_spectral_norm=False):
550 |         super(DiscriminatorS, self).__init__()
551 |         norm_f = weight_norm if use_spectral_norm == False else spectral_norm
552 |         self.convs = nn.ModuleList([
553 |             norm_f(Conv1d(1, 16, 15, 1, padding=7)),
554 |             norm_f(Conv1d(16, 64, 41, 4, groups=4, padding=20)),
555 |             norm_f(Conv1d(64, 256, 41, 4, groups=16, padding=20)),
556 |             norm_f(Conv1d(256, 1024, 41, 4, groups=64, padding=20)),
557 |             norm_f(Conv1d(1024, 1024, 41, 4, groups=256, padding=20)),
558 |             norm_f(Conv1d(1024, 1024, 5, 1, padding=2)),
559 |         ])
560 |         self.conv_post = norm_f(Conv1d(1024, 1, 3, 1, padding=1))
561 | 
562 |     def forward(self, x):
563 |         fmap = []
564 | 
565 |         for l in self.convs:
566 |             x = l(x)
567 |             x = F.leaky_relu(x, modules.LRELU_SLOPE)
568 |             fmap.append(x)
569 |         x = self.conv_post(x)
570 |         fmap.append(x)
571 |         x = torch.flatten(x, 1, -1)
572 | 
573 |         return x, fmap
574 | 
575 | 
576 | class MultiPeriodDiscriminator(torch.nn.Module):
577 |     def __init__(self, use_spectral_norm=False):
578 |         super(MultiPeriodDiscriminator, self).__init__()
579 |         periods = [2,3,5,7,11]
580 | 
581 |         discs = [DiscriminatorS(use_spectral_norm=use_spectral_norm)]
582 |         discs = discs + [DiscriminatorP(i, use_spectral_norm=use_spectral_norm) for i in periods]
583 |         self.discriminators = nn.ModuleList(discs)
584 | 
585 |     def forward(self, y, y_hat):
586 | 
587 |         y_d_rs = []
588 |         y_d_gs = []
589 |         fmap_rs = []
590 |         fmap_gs = []
591 |         for i, d in enumerate(self.discriminators):
592 |             y_d_r, fmap_r = d(y)
593 |             y_d_g, fmap_g = d(y_hat)
594 |             y_d_rs.append(y_d_r)
595 |             y_d_gs.append(y_d_g)
596 |             fmap_rs.append(fmap_r)
597 |             fmap_gs.append(fmap_g)
598 | 
599 |         return y_d_rs, y_d_gs, fmap_rs, fmap_gs
600 | 
601 | class SpeakerEncoder(torch.nn.Module):
602 |     def __init__(self, mel_n_channels=80, model_num_layers=3, model_hidden_size=256, model_embedding_size=256):
603 |         super(SpeakerEncoder, self).__init__()
604 |         self.lstm = nn.LSTM(mel_n_channels, model_hidden_size, model_num_layers, batch_first=True)
605 |         self.linear = nn.Linear(model_hidden_size, model_embedding_size)
606 |         self.relu = nn.ReLU()
607 | 
608 |     def forward(self, mels):
609 |         self.lstm.flatten_parameters()
610 |         _, (hidden, _) = self.lstm(mels)
611 |         embeds_raw = self.relu(self.linear(hidden[-1]))
612 |         return embeds_raw / torch.norm(embeds_raw, dim=1, keepdim=True)
613 |         
614 |     def compute_partial_slices(self, total_frames, partial_frames, partial_hop):
615 |         mel_slices = []
616 |         for i in range(0, total_frames-partial_frames, partial_hop):
617 |             mel_range = torch.arange(i, i+partial_frames)
618 |             mel_slices.append(mel_range)
619 |             
620 |         return mel_slices
621 |     
622 |     def embed_utterance(self, mel, partial_frames=128, partial_hop=64):
623 |         mel_len = mel.size(1)
624 |         last_mel = mel[:,-partial_frames:]
625 |         
626 |         if mel_len > partial_frames:
627 |             mel_slices = self.compute_partial_slices(mel_len, partial_frames, partial_hop)
628 |             mels = list(mel[:,s] for s in mel_slices)
629 |             mels.append(last_mel)
630 |             mels = torch.stack(tuple(mels), 0).squeeze(1)
631 |         
632 |             with torch.no_grad():
633 |                 partial_embeds = self(mels)
634 |             embed = torch.mean(partial_embeds, axis=0).unsqueeze(0)
635 |             #embed = embed / torch.linalg.norm(embed, 2)
636 |         else:
637 |             with torch.no_grad():
638 |                 embed = self(last_mel)
639 |         
640 |         return embed
641 | 
642 | class SynthesizerTrn(nn.Module):
643 |   """
644 |   Synthesizer for Training
645 |   """
646 | 
647 |   def __init__(self, 
648 |     spec_channels,
649 |     segment_size,
650 |     inter_channels,
651 |     hidden_channels,
652 |     filter_channels,
653 |     n_heads,
654 |     n_layers,
655 |     kernel_size,
656 |     p_dropout,
657 |     resblock, 
658 |     resblock_kernel_sizes, 
659 |     resblock_dilation_sizes, 
660 |     upsample_rates, 
661 |     upsample_initial_channel, 
662 |     upsample_kernel_sizes,
663 |     gen_istft_n_fft,
664 |     gen_istft_hop_size,
665 |     n_speakers=0,
666 |     gin_channels=0,
667 |     use_sdp=False,
668 |     ms_istft_vits=False,
669 |     mb_istft_vits = False,
670 |     subbands = False,
671 |     istft_vits=False,
672 |     **kwargs):
673 | 
674 |     super().__init__()
675 |     self.spec_channels = spec_channels
676 |     self.inter_channels = inter_channels
677 |     self.hidden_channels = hidden_channels
678 |     self.filter_channels = filter_channels
679 |     self.n_heads = n_heads
680 |     self.n_layers = n_layers
681 |     self.kernel_size = kernel_size
682 |     self.p_dropout = p_dropout
683 |     self.resblock = resblock
684 |     self.resblock_kernel_sizes = resblock_kernel_sizes
685 |     self.resblock_dilation_sizes = resblock_dilation_sizes
686 |     self.upsample_rates = upsample_rates
687 |     self.upsample_initial_channel = upsample_initial_channel
688 |     self.upsample_kernel_sizes = upsample_kernel_sizes
689 |     self.segment_size = segment_size
690 |     self.n_speakers = n_speakers
691 |     self.gin_channels = gin_channels
692 |     self.ms_istft_vits = ms_istft_vits
693 |     self.mb_istft_vits = mb_istft_vits
694 |     self.istft_vits = istft_vits
695 | 
696 |     self.use_sdp = use_sdp
697 | 
698 |     self.enc_p = PosteriorEncoder(256, inter_channels, hidden_channels, 5, 1, 16)#768, inter_channels, hidden_channels, 5, 1, 16)
699 |     
700 |     if mb_istft_vits == True:
701 |       print('Mutli-band iSTFT VITS')
702 |       self.dec = Multiband_iSTFT_Generator(inter_channels, resblock, resblock_kernel_sizes, resblock_dilation_sizes, upsample_rates, upsample_initial_channel, upsample_kernel_sizes, gen_istft_n_fft, gen_istft_hop_size, subbands, gin_channels=gin_channels)
703 |     elif ms_istft_vits == True:
704 |       print('Mutli-stream iSTFT VITS')
705 |       self.dec = Multistream_iSTFT_Generator(inter_channels, resblock, resblock_kernel_sizes, resblock_dilation_sizes, upsample_rates, upsample_initial_channel, upsample_kernel_sizes, gen_istft_n_fft, gen_istft_hop_size, subbands, gin_channels=gin_channels)
706 |     elif istft_vits == True:
707 |       print('iSTFT-VITS')
708 |       self.dec = iSTFT_Generator(inter_channels, resblock, resblock_kernel_sizes, resblock_dilation_sizes, upsample_rates, upsample_initial_channel, upsample_kernel_sizes, gen_istft_n_fft, gen_istft_hop_size, gin_channels=gin_channels)
709 |     else:
710 |       print('Decoder Error in json file')
711 | 
712 |     self.enc_q = PosteriorEncoder(spec_channels, inter_channels, hidden_channels, 5, 1, 16, gin_channels=gin_channels)
713 |     self.flow = ResidualCouplingBlock(inter_channels, hidden_channels, 5, 1, 4, gin_channels=gin_channels)
714 | 
715 |     self.enc_spk = SpeakerEncoder(model_hidden_size=gin_channels, model_embedding_size=gin_channels)
716 | 
717 |   def forward(self, c, spec, g=None, mel=None, c_lengths=None, spec_lengths=None):
718 |     if c_lengths == None:
719 |       c_lengths = (torch.ones(c.size(0)) * c.size(-1)).to(c.device)
720 |     if spec_lengths == None:
721 |       spec_lengths = (torch.ones(spec.size(0)) * spec.size(-1)).to(spec.device)
722 |       
723 |     g = self.enc_spk(mel.transpose(1,2))
724 |     g = g.unsqueeze(-1)
725 |       
726 |     _, m_p, logs_p, _ = self.enc_p(c, c_lengths)
727 |     z, m_q, logs_q, spec_mask = self.enc_q(spec, spec_lengths, g=g) 
728 |     z_p = self.flow(z, spec_mask, g=g)
729 | 
730 |     z_slice, ids_slice = commons.rand_slice_segments(z, spec_lengths, self.segment_size)
731 |     o, o_mb = self.dec(z_slice, g=g)
732 |     
733 |     return o, o_mb, ids_slice, spec_mask, (z, z_p, m_p, logs_p, m_q, logs_q)
734 | 
735 |   def infer(self, c, g=None, mel=None, c_lengths=None):
736 |     if c_lengths == None:
737 |       c_lengths = (torch.ones(c.size(0)) * c.size(-1)).to(c.device)
738 |     g = self.enc_spk.embed_utterance(mel.transpose(1,2))
739 |     g = g.unsqueeze(-1)
740 | 
741 |     z_p, m_p, logs_p, c_mask = self.enc_p(c, c_lengths)
742 |     z = self.flow(z_p, c_mask, g=g, reverse=True)
743 |     o,o_mb = self.dec(z * c_mask, g=g)
744 |     
745 |     return o
746 | 


--------------------------------------------------------------------------------
/modules.py:
--------------------------------------------------------------------------------
  1 | import copy
  2 | import math
  3 | import numpy as np
  4 | import scipy
  5 | import torch
  6 | from torch import nn
  7 | from torch.nn import functional as F
  8 | 
  9 | from torch.nn import Conv1d, ConvTranspose1d, AvgPool1d, Conv2d
 10 | from torch.nn.utils import weight_norm, remove_weight_norm
 11 | 
 12 | import commons
 13 | from commons import init_weights, get_padding
 14 | from transforms import piecewise_rational_quadratic_transform
 15 | 
 16 | 
 17 | LRELU_SLOPE = 0.1
 18 | 
 19 | 
 20 | class LayerNorm(nn.Module):
 21 |   def __init__(self, channels, eps=1e-5):
 22 |     super().__init__()
 23 |     self.channels = channels
 24 |     self.eps = eps
 25 | 
 26 |     self.gamma = nn.Parameter(torch.ones(channels))
 27 |     self.beta = nn.Parameter(torch.zeros(channels))
 28 | 
 29 |   def forward(self, x):
 30 |     x = x.transpose(1, -1)
 31 |     x = F.layer_norm(x, (self.channels,), self.gamma, self.beta, self.eps)
 32 |     return x.transpose(1, -1)
 33 | 
 34 |  
 35 | class ConvReluNorm(nn.Module):
 36 |   def __init__(self, in_channels, hidden_channels, out_channels, kernel_size, n_layers, p_dropout):
 37 |     super().__init__()
 38 |     self.in_channels = in_channels
 39 |     self.hidden_channels = hidden_channels
 40 |     self.out_channels = out_channels
 41 |     self.kernel_size = kernel_size
 42 |     self.n_layers = n_layers
 43 |     self.p_dropout = p_dropout
 44 |     assert n_layers > 1, "Number of layers should be larger than 0."
 45 | 
 46 |     self.conv_layers = nn.ModuleList()
 47 |     self.norm_layers = nn.ModuleList()
 48 |     self.conv_layers.append(nn.Conv1d(in_channels, hidden_channels, kernel_size, padding=kernel_size//2))
 49 |     self.norm_layers.append(LayerNorm(hidden_channels))
 50 |     self.relu_drop = nn.Sequential(
 51 |         nn.ReLU(),
 52 |         nn.Dropout(p_dropout))
 53 |     for _ in range(n_layers-1):
 54 |       self.conv_layers.append(nn.Conv1d(hidden_channels, hidden_channels, kernel_size, padding=kernel_size//2))
 55 |       self.norm_layers.append(LayerNorm(hidden_channels))
 56 |     self.proj = nn.Conv1d(hidden_channels, out_channels, 1)
 57 |     self.proj.weight.data.zero_()
 58 |     self.proj.bias.data.zero_()
 59 | 
 60 |   def forward(self, x, x_mask):
 61 |     x_org = x
 62 |     for i in range(self.n_layers):
 63 |       x = self.conv_layers[i](x * x_mask)
 64 |       x = self.norm_layers[i](x)
 65 |       x = self.relu_drop(x)
 66 |     x = x_org + self.proj(x)
 67 |     return x * x_mask
 68 | 
 69 | 
 70 | class DDSConv(nn.Module):
 71 |   """
 72 |   Dialted and Depth-Separable Convolution
 73 |   """
 74 |   def __init__(self, channels, kernel_size, n_layers, p_dropout=0.):
 75 |     super().__init__()
 76 |     self.channels = channels
 77 |     self.kernel_size = kernel_size
 78 |     self.n_layers = n_layers
 79 |     self.p_dropout = p_dropout
 80 | 
 81 |     self.drop = nn.Dropout(p_dropout)
 82 |     self.convs_sep = nn.ModuleList()
 83 |     self.convs_1x1 = nn.ModuleList()
 84 |     self.norms_1 = nn.ModuleList()
 85 |     self.norms_2 = nn.ModuleList()
 86 |     for i in range(n_layers):
 87 |       dilation = kernel_size ** i
 88 |       padding = (kernel_size * dilation - dilation) // 2
 89 |       self.convs_sep.append(nn.Conv1d(channels, channels, kernel_size, 
 90 |           groups=channels, dilation=dilation, padding=padding
 91 |       ))
 92 |       self.convs_1x1.append(nn.Conv1d(channels, channels, 1))
 93 |       self.norms_1.append(LayerNorm(channels))
 94 |       self.norms_2.append(LayerNorm(channels))
 95 | 
 96 |   def forward(self, x, x_mask, g=None):
 97 |     if g is not None:
 98 |       x = x + g
 99 |     for i in range(self.n_layers):
100 |       y = self.convs_sep[i](x * x_mask)
101 |       y = self.norms_1[i](y)
102 |       y = F.gelu(y)
103 |       y = self.convs_1x1[i](y)
104 |       y = self.norms_2[i](y)
105 |       y = F.gelu(y)
106 |       y = self.drop(y)
107 |       x = x + y
108 |     return x * x_mask
109 | 
110 | 
111 | class WN(torch.nn.Module):
112 |   def __init__(self, hidden_channels, kernel_size, dilation_rate, n_layers, gin_channels=0, p_dropout=0):
113 |     super(WN, self).__init__()
114 |     assert(kernel_size % 2 == 1)
115 |     self.hidden_channels =hidden_channels
116 |     self.kernel_size = kernel_size,
117 |     self.dilation_rate = dilation_rate
118 |     self.n_layers = n_layers
119 |     self.gin_channels = gin_channels
120 |     self.p_dropout = p_dropout
121 | 
122 |     self.in_layers = torch.nn.ModuleList()
123 |     self.res_skip_layers = torch.nn.ModuleList()
124 |     self.drop = nn.Dropout(p_dropout)
125 | 
126 |     if gin_channels != 0:
127 |       cond_layer = torch.nn.Conv1d(gin_channels, 2*hidden_channels*n_layers, 1)
128 |       self.cond_layer = torch.nn.utils.weight_norm(cond_layer, name='weight')
129 | 
130 |     for i in range(n_layers):
131 |       dilation = dilation_rate ** i
132 |       padding = int((kernel_size * dilation - dilation) / 2)
133 |       in_layer = torch.nn.Conv1d(hidden_channels, 2*hidden_channels, kernel_size,
134 |                                  dilation=dilation, padding=padding)
135 |       in_layer = torch.nn.utils.weight_norm(in_layer, name='weight')
136 |       self.in_layers.append(in_layer)
137 | 
138 |       # last one is not necessary
139 |       if i < n_layers - 1:
140 |         res_skip_channels = 2 * hidden_channels
141 |       else:
142 |         res_skip_channels = hidden_channels
143 | 
144 |       res_skip_layer = torch.nn.Conv1d(hidden_channels, res_skip_channels, 1)
145 |       res_skip_layer = torch.nn.utils.weight_norm(res_skip_layer, name='weight')
146 |       self.res_skip_layers.append(res_skip_layer)
147 | 
148 |   def forward(self, x, x_mask, g=None, **kwargs):
149 |     output = torch.zeros_like(x)
150 |     n_channels_tensor = torch.IntTensor([self.hidden_channels])
151 | 
152 |     if g is not None:
153 |       g = self.cond_layer(g)
154 | 
155 |     for i in range(self.n_layers):
156 |       x_in = self.in_layers[i](x)
157 |       if g is not None:
158 |         cond_offset = i * 2 * self.hidden_channels
159 |         g_l = g[:,cond_offset:cond_offset+2*self.hidden_channels,:]
160 |       else:
161 |         g_l = torch.zeros_like(x_in)
162 | 
163 |       acts = commons.fused_add_tanh_sigmoid_multiply(
164 |           x_in,
165 |           g_l,
166 |           n_channels_tensor)
167 |       acts = self.drop(acts)
168 | 
169 |       res_skip_acts = self.res_skip_layers[i](acts)
170 |       if i < self.n_layers - 1:
171 |         res_acts = res_skip_acts[:,:self.hidden_channels,:]
172 |         x = (x + res_acts) * x_mask
173 |         output = output + res_skip_acts[:,self.hidden_channels:,:]
174 |       else:
175 |         output = output + res_skip_acts
176 |     return output * x_mask
177 | 
178 |   def remove_weight_norm(self):
179 |     if self.gin_channels != 0:
180 |       torch.nn.utils.remove_weight_norm(self.cond_layer)
181 |     for l in self.in_layers:
182 |       torch.nn.utils.remove_weight_norm(l)
183 |     for l in self.res_skip_layers:
184 |      torch.nn.utils.remove_weight_norm(l)
185 | 
186 | 
187 | class ResBlock1(torch.nn.Module):
188 |     def __init__(self, channels, kernel_size=3, dilation=(1, 3, 5)):
189 |         super(ResBlock1, self).__init__()
190 |         self.convs1 = nn.ModuleList([
191 |             weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[0],
192 |                                padding=get_padding(kernel_size, dilation[0]))),
193 |             weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[1],
194 |                                padding=get_padding(kernel_size, dilation[1]))),
195 |             weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[2],
196 |                                padding=get_padding(kernel_size, dilation[2])))
197 |         ])
198 |         self.convs1.apply(init_weights)
199 | 
200 |         self.convs2 = nn.ModuleList([
201 |             weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=1,
202 |                                padding=get_padding(kernel_size, 1))),
203 |             weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=1,
204 |                                padding=get_padding(kernel_size, 1))),
205 |             weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=1,
206 |                                padding=get_padding(kernel_size, 1)))
207 |         ])
208 |         self.convs2.apply(init_weights)
209 | 
210 |     def forward(self, x, x_mask=None):
211 |         for c1, c2 in zip(self.convs1, self.convs2):
212 |             xt = F.leaky_relu(x, LRELU_SLOPE)
213 |             if x_mask is not None:
214 |                 xt = xt * x_mask
215 |             xt = c1(xt)
216 |             xt = F.leaky_relu(xt, LRELU_SLOPE)
217 |             #print(xt.size())
218 |             if x_mask is not None:
219 |                 xt = xt * x_mask
220 |             xt = c2(xt)
221 |             #print(xt.size())
222 |             x = xt + x
223 |         if x_mask is not None:
224 |             x = x * x_mask
225 |         return x
226 | 
227 |     def remove_weight_norm(self):
228 |         for l in self.convs1:
229 |             remove_weight_norm(l)
230 |         for l in self.convs2:
231 |             remove_weight_norm(l)
232 | 
233 | 
234 | class ResBlock2(torch.nn.Module):
235 |     def __init__(self, channels, kernel_size=3, dilation=(1, 3)):
236 |         super(ResBlock2, self).__init__()
237 |         self.convs = nn.ModuleList([
238 |             weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[0],
239 |                                padding=get_padding(kernel_size, dilation[0]))),
240 |             weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[1],
241 |                                padding=get_padding(kernel_size, dilation[1])))
242 |         ])
243 |         self.convs.apply(init_weights)
244 | 
245 |     def forward(self, x, x_mask=None):
246 |         for c in self.convs:
247 |             xt = F.leaky_relu(x, LRELU_SLOPE)
248 |             if x_mask is not None:
249 |                 xt = xt * x_mask
250 |             xt = c(xt)
251 |             x = xt + x
252 |         if x_mask is not None:
253 |             x = x * x_mask
254 |         return x
255 | 
256 |     def remove_weight_norm(self):
257 |         for l in self.convs:
258 |             remove_weight_norm(l)
259 | 
260 | 
261 | class Log(nn.Module):
262 |   def forward(self, x, x_mask, reverse=False, **kwargs):
263 |     if not reverse:
264 |       y = torch.log(torch.clamp_min(x, 1e-5)) * x_mask
265 |       logdet = torch.sum(-y, [1, 2])
266 |       return y, logdet
267 |     else:
268 |       x = torch.exp(x) * x_mask
269 |       return x
270 |     
271 | 
272 | class Flip(nn.Module):
273 |   def forward(self, x, *args, reverse=False, **kwargs):
274 |     x = torch.flip(x, [1])
275 |     if not reverse:
276 |       logdet = torch.zeros(x.size(0)).to(dtype=x.dtype, device=x.device)
277 |       return x, logdet
278 |     else:
279 |       return x
280 | 
281 | 
282 | class ElementwiseAffine(nn.Module):
283 |   def __init__(self, channels):
284 |     super().__init__()
285 |     self.channels = channels
286 |     self.m = nn.Parameter(torch.zeros(channels,1))
287 |     self.logs = nn.Parameter(torch.zeros(channels,1))
288 | 
289 |   def forward(self, x, x_mask, reverse=False, **kwargs):
290 |     if not reverse:
291 |       y = self.m + torch.exp(self.logs) * x
292 |       y = y * x_mask
293 |       logdet = torch.sum(self.logs * x_mask, [1,2])
294 |       return y, logdet
295 |     else:
296 |       x = (x - self.m) * torch.exp(-self.logs) * x_mask
297 |       return x
298 | 
299 | 
300 | class ResidualCouplingLayer(nn.Module):
301 |   def __init__(self,
302 |       channels,
303 |       hidden_channels,
304 |       kernel_size,
305 |       dilation_rate,
306 |       n_layers,
307 |       p_dropout=0,
308 |       gin_channels=0,
309 |       mean_only=False):
310 |     assert channels % 2 == 0, "channels should be divisible by 2"
311 |     super().__init__()
312 |     self.channels = channels
313 |     self.hidden_channels = hidden_channels
314 |     self.kernel_size = kernel_size
315 |     self.dilation_rate = dilation_rate
316 |     self.n_layers = n_layers
317 |     self.half_channels = channels // 2
318 |     self.mean_only = mean_only
319 | 
320 |     self.pre = nn.Conv1d(self.half_channels, hidden_channels, 1)
321 |     self.enc = WN(hidden_channels, kernel_size, dilation_rate, n_layers, p_dropout=p_dropout, gin_channels=gin_channels)
322 |     self.post = nn.Conv1d(hidden_channels, self.half_channels * (2 - mean_only), 1)
323 |     self.post.weight.data.zero_()
324 |     self.post.bias.data.zero_()
325 | 
326 |   def forward(self, x, x_mask, g=None, reverse=False):
327 |     x0, x1 = torch.split(x, [self.half_channels]*2, 1)
328 |     h = self.pre(x0) * x_mask
329 |     h = self.enc(h, x_mask, g=g)
330 |     stats = self.post(h) * x_mask
331 |     if not self.mean_only:
332 |       m, logs = torch.split(stats, [self.half_channels]*2, 1)
333 |     else:
334 |       m = stats
335 |       logs = torch.zeros_like(m)
336 | 
337 |     if not reverse:
338 |       x1 = m + x1 * torch.exp(logs) * x_mask
339 |       x = torch.cat([x0, x1], 1)
340 |       logdet = torch.sum(logs, [1,2])
341 |       return x, logdet
342 |     else:
343 |       x1 = (x1 - m) * torch.exp(-logs) * x_mask
344 |       x = torch.cat([x0, x1], 1)
345 |       return x
346 | 
347 | 
348 | class ConvFlow(nn.Module):
349 |   def __init__(self, in_channels, filter_channels, kernel_size, n_layers, num_bins=10, tail_bound=5.0):
350 |     super().__init__()
351 |     self.in_channels = in_channels
352 |     self.filter_channels = filter_channels
353 |     self.kernel_size = kernel_size
354 |     self.n_layers = n_layers
355 |     self.num_bins = num_bins
356 |     self.tail_bound = tail_bound
357 |     self.half_channels = in_channels // 2
358 | 
359 |     self.pre = nn.Conv1d(self.half_channels, filter_channels, 1)
360 |     self.convs = DDSConv(filter_channels, kernel_size, n_layers, p_dropout=0.)
361 |     self.proj = nn.Conv1d(filter_channels, self.half_channels * (num_bins * 3 - 1), 1)
362 |     self.proj.weight.data.zero_()
363 |     self.proj.bias.data.zero_()
364 | 
365 |   def forward(self, x, x_mask, g=None, reverse=False):
366 |     x0, x1 = torch.split(x, [self.half_channels]*2, 1)
367 |     h = self.pre(x0)
368 |     h = self.convs(h, x_mask, g=g)
369 |     h = self.proj(h) * x_mask
370 | 
371 |     b, c, t = x0.shape
372 |     h = h.reshape(b, c, -1, t).permute(0, 1, 3, 2) # [b, cx?, t] -> [b, c, t, ?]
373 | 
374 |     unnormalized_widths = h[..., :self.num_bins] / math.sqrt(self.filter_channels)
375 |     unnormalized_heights = h[..., self.num_bins:2*self.num_bins] / math.sqrt(self.filter_channels)
376 |     unnormalized_derivatives = h[..., 2 * self.num_bins:]
377 | 
378 |     x1, logabsdet = piecewise_rational_quadratic_transform(x1,
379 |         unnormalized_widths,
380 |         unnormalized_heights,
381 |         unnormalized_derivatives,
382 |         inverse=reverse,
383 |         tails='linear',
384 |         tail_bound=self.tail_bound
385 |     )
386 | 
387 |     x = torch.cat([x0, x1], 1) * x_mask
388 |     logdet = torch.sum(logabsdet * x_mask, [1,2])
389 |     if not reverse:
390 |         return x, logdet
391 |     else:
392 |         return x
393 | 


--------------------------------------------------------------------------------
/output/quickvc/title1.wav:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/quickvc/QuickVC-VoiceConversion/2fe4d6486485e9fcae2d20ece899e8f8d2c2ed96/output/quickvc/title1.wav


--------------------------------------------------------------------------------
/output/quickvc/title10.wav:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/quickvc/QuickVC-VoiceConversion/2fe4d6486485e9fcae2d20ece899e8f8d2c2ed96/output/quickvc/title10.wav


--------------------------------------------------------------------------------
/output/quickvc/title11.wav:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/quickvc/QuickVC-VoiceConversion/2fe4d6486485e9fcae2d20ece899e8f8d2c2ed96/output/quickvc/title11.wav


--------------------------------------------------------------------------------
/output/quickvc/title12.wav:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/quickvc/QuickVC-VoiceConversion/2fe4d6486485e9fcae2d20ece899e8f8d2c2ed96/output/quickvc/title12.wav


--------------------------------------------------------------------------------
/output/quickvc/title2.wav:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/quickvc/QuickVC-VoiceConversion/2fe4d6486485e9fcae2d20ece899e8f8d2c2ed96/output/quickvc/title2.wav


--------------------------------------------------------------------------------
/output/quickvc/title3.wav:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/quickvc/QuickVC-VoiceConversion/2fe4d6486485e9fcae2d20ece899e8f8d2c2ed96/output/quickvc/title3.wav


--------------------------------------------------------------------------------
/output/quickvc/title4.wav:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/quickvc/QuickVC-VoiceConversion/2fe4d6486485e9fcae2d20ece899e8f8d2c2ed96/output/quickvc/title4.wav


--------------------------------------------------------------------------------
/output/quickvc/title5.wav:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/quickvc/QuickVC-VoiceConversion/2fe4d6486485e9fcae2d20ece899e8f8d2c2ed96/output/quickvc/title5.wav


--------------------------------------------------------------------------------
/output/quickvc/title6.wav:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/quickvc/QuickVC-VoiceConversion/2fe4d6486485e9fcae2d20ece899e8f8d2c2ed96/output/quickvc/title6.wav


--------------------------------------------------------------------------------
/output/quickvc/title7.wav:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/quickvc/QuickVC-VoiceConversion/2fe4d6486485e9fcae2d20ece899e8f8d2c2ed96/output/quickvc/title7.wav


--------------------------------------------------------------------------------
/output/quickvc/title8.wav:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/quickvc/QuickVC-VoiceConversion/2fe4d6486485e9fcae2d20ece899e8f8d2c2ed96/output/quickvc/title8.wav


--------------------------------------------------------------------------------
/output/quickvc/title9.wav:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/quickvc/QuickVC-VoiceConversion/2fe4d6486485e9fcae2d20ece899e8f8d2c2ed96/output/quickvc/title9.wav


--------------------------------------------------------------------------------
/pqmf.py:
--------------------------------------------------------------------------------
  1 | # -*- coding: utf-8 -*-
  2 | 
  3 | # Copyright 2020 Tomoki Hayashi
  4 | #  MIT License (https://opensource.org/licenses/MIT)
  5 | 
  6 | """Pseudo QMF modules."""
  7 | 
  8 | import numpy as np
  9 | import torch
 10 | import torch.nn.functional as F
 11 | 
 12 | from scipy.signal import kaiser
 13 | 
 14 | 
 15 | def design_prototype_filter(taps=62, cutoff_ratio=0.15, beta=9.0):
 16 |     """Design prototype filter for PQMF.
 17 |     This method is based on `A Kaiser window approach for the design of prototype
 18 |     filters of cosine modulated filterbanks`_.
 19 |     Args:
 20 |         taps (int): The number of filter taps.
 21 |         cutoff_ratio (float): Cut-off frequency ratio.
 22 |         beta (float): Beta coefficient for kaiser window.
 23 |     Returns:
 24 |         ndarray: Impluse response of prototype filter (taps + 1,).
 25 |     .. _`A Kaiser window approach for the design of prototype filters of cosine modulated filterbanks`:
 26 |         https://ieeexplore.ieee.org/abstract/document/681427
 27 |     """
 28 |     # check the arguments are valid
 29 |     assert taps % 2 == 0, "The number of taps mush be even number."
 30 |     assert 0.0 < cutoff_ratio < 1.0, "Cutoff ratio must be > 0.0 and < 1.0."
 31 | 
 32 |     # make initial filter
 33 |     omega_c = np.pi * cutoff_ratio
 34 |     with np.errstate(invalid='ignore'):
 35 |         h_i = np.sin(omega_c * (np.arange(taps + 1) - 0.5 * taps)) \
 36 |             / (np.pi * (np.arange(taps + 1) - 0.5 * taps))
 37 |     h_i[taps // 2] = np.cos(0) * cutoff_ratio  # fix nan due to indeterminate form
 38 | 
 39 |     # apply kaiser window
 40 |     w = kaiser(taps + 1, beta)
 41 |     h = h_i * w
 42 | 
 43 |     return h
 44 | 
 45 | 
 46 | class PQMF(torch.nn.Module):
 47 |     """PQMF module.
 48 |     This module is based on `Near-perfect-reconstruction pseudo-QMF banks`_.
 49 |     .. _`Near-perfect-reconstruction pseudo-QMF banks`:
 50 |         https://ieeexplore.ieee.org/document/258122
 51 |     """
 52 | 
 53 |     def __init__(self, device, subbands=4, taps=62, cutoff_ratio=0.15, beta=9.0):
 54 |         """Initilize PQMF module.
 55 |         Args:
 56 |             subbands (int): The number of subbands.
 57 |             taps (int): The number of filter taps.
 58 |             cutoff_ratio (float): Cut-off frequency ratio.
 59 |             beta (float): Beta coefficient for kaiser window.
 60 |         """
 61 |         super(PQMF, self).__init__()
 62 | 
 63 |         # define filter coefficient
 64 |         h_proto = design_prototype_filter(taps, cutoff_ratio, beta)
 65 |         h_analysis = np.zeros((subbands, len(h_proto)))
 66 |         h_synthesis = np.zeros((subbands, len(h_proto)))
 67 |         for k in range(subbands):
 68 |             h_analysis[k] = 2 * h_proto * np.cos(
 69 |                 (2 * k + 1) * (np.pi / (2 * subbands)) *
 70 |                 (np.arange(taps + 1) - ((taps - 1) / 2)) +
 71 |                 (-1) ** k * np.pi / 4)
 72 |             h_synthesis[k] = 2 * h_proto * np.cos(
 73 |                 (2 * k + 1) * (np.pi / (2 * subbands)) *
 74 |                 (np.arange(taps + 1) - ((taps - 1) / 2)) -
 75 |                 (-1) ** k * np.pi / 4)
 76 | 
 77 |         # convert to tensor
 78 |         analysis_filter = torch.from_numpy(h_analysis).float().unsqueeze(1).cuda(device)
 79 |         synthesis_filter = torch.from_numpy(h_synthesis).float().unsqueeze(0).cuda(device)
 80 | 
 81 |         # register coefficients as beffer
 82 |         self.register_buffer("analysis_filter", analysis_filter)
 83 |         self.register_buffer("synthesis_filter", synthesis_filter)
 84 | 
 85 |         # filter for downsampling & upsampling
 86 |         updown_filter = torch.zeros((subbands, subbands, subbands)).float().cuda(device)
 87 |         for k in range(subbands):
 88 |             updown_filter[k, k, 0] = 1.0
 89 |         self.register_buffer("updown_filter", updown_filter)
 90 |         self.subbands = subbands
 91 | 
 92 |         # keep padding info
 93 |         self.pad_fn = torch.nn.ConstantPad1d(taps // 2, 0.0)
 94 | 
 95 |     def analysis(self, x):
 96 |         """Analysis with PQMF.
 97 |         Args:
 98 |             x (Tensor): Input tensor (B, 1, T).
 99 |         Returns:
100 |             Tensor: Output tensor (B, subbands, T // subbands).
101 |         """
102 |         x = F.conv1d(self.pad_fn(x), self.analysis_filter)
103 |         return F.conv1d(x, self.updown_filter, stride=self.subbands)
104 | 
105 |     def synthesis(self, x):
106 |         """Synthesis with PQMF.
107 |         Args:
108 |             x (Tensor): Input tensor (B, subbands, T // subbands).
109 |         Returns:
110 |             Tensor: Output tensor (B, 1, T).
111 |         """
112 |         # NOTE(kan-bayashi): Power will be dreased so here multipy by # subbands.
113 |         #   Not sure this is the correct way, it is better to check again.
114 |         # TODO(kan-bayashi): Understand the reconstruction procedure
115 |         x = F.conv_transpose1d(x, self.updown_filter * self.subbands, stride=self.subbands)
116 |         return F.conv1d(self.pad_fn(x), self.synthesis_filter)


--------------------------------------------------------------------------------
/qvcfinalwhite.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/quickvc/QuickVC-VoiceConversion/2fe4d6486485e9fcae2d20ece899e8f8d2c2ed96/qvcfinalwhite.png


--------------------------------------------------------------------------------
/stft.py:
--------------------------------------------------------------------------------
  1 | """
  2 | BSD 3-Clause License
  3 | Copyright (c) 2017, Prem Seetharaman
  4 | All rights reserved.
  5 | * Redistribution and use in source and binary forms, with or without
  6 |   modification, are permitted provided that the following conditions are met:
  7 | * Redistributions of source code must retain the above copyright notice,
  8 |   this list of conditions and the following disclaimer.
  9 | * Redistributions in binary form must reproduce the above copyright notice, this
 10 |   list of conditions and the following disclaimer in the
 11 |   documentation and/or other materials provided with the distribution.
 12 | * Neither the name of the copyright holder nor the names of its
 13 |   contributors may be used to endorse or promote products derived from this
 14 |   software without specific prior written permission.
 15 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
 16 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 17 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 18 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
 19 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 20 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 21 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
 22 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 23 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 24 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 25 | """
 26 | 
 27 | import torch
 28 | import numpy as np
 29 | import torch.nn.functional as F
 30 | from torch.autograd import Variable
 31 | from scipy.signal import get_window
 32 | from librosa.util import pad_center, tiny
 33 | import librosa.util as librosa_util
 34 | 
 35 | def window_sumsquare(window, n_frames, hop_length=200, win_length=800,
 36 |                      n_fft=800, dtype=np.float32, norm=None):
 37 |     """
 38 |     # from librosa 0.6
 39 |     Compute the sum-square envelope of a window function at a given hop length.
 40 |     This is used to estimate modulation effects induced by windowing
 41 |     observations in short-time fourier transforms.
 42 |     Parameters
 43 |     ----------
 44 |     window : string, tuple, number, callable, or list-like
 45 |         Window specification, as in `get_window`
 46 |     n_frames : int > 0
 47 |         The number of analysis frames
 48 |     hop_length : int > 0
 49 |         The number of samples to advance between frames
 50 |     win_length : [optional]
 51 |         The length of the window function.  By default, this matches `n_fft`.
 52 |     n_fft : int > 0
 53 |         The length of each analysis frame.
 54 |     dtype : np.dtype
 55 |         The data type of the output
 56 |     Returns
 57 |     -------
 58 |     wss : np.ndarray, shape=`(n_fft + hop_length * (n_frames - 1))`
 59 |         The sum-squared envelope of the window function
 60 |     """
 61 |     if win_length is None:
 62 |         win_length = n_fft
 63 | 
 64 |     n = n_fft + hop_length * (n_frames - 1)
 65 |     x = np.zeros(n, dtype=dtype)
 66 | 
 67 |     # Compute the squared window at the desired length
 68 |     win_sq = get_window(window, win_length, fftbins=True)
 69 |     win_sq = librosa_util.normalize(win_sq, norm=norm)**2
 70 |     win_sq = librosa_util.pad_center(win_sq, n_fft)
 71 | 
 72 |     # Fill the envelope
 73 |     for i in range(n_frames):
 74 |         sample = i * hop_length
 75 |         x[sample:min(n, sample + n_fft)] += win_sq[:max(0, min(n_fft, n - sample))]
 76 |     return x
 77 | 
 78 | 
 79 | class STFT(torch.nn.Module):
 80 |     """adapted from Prem Seetharaman's https://github.com/pseeth/pytorch-stft"""
 81 |     def __init__(self, filter_length=800, hop_length=200, win_length=800,
 82 |                  window='hann'):
 83 |         super(STFT, self).__init__()
 84 |         self.filter_length = filter_length
 85 |         self.hop_length = hop_length
 86 |         self.win_length = win_length
 87 |         self.window = window
 88 |         self.forward_transform = None
 89 |         scale = self.filter_length / self.hop_length
 90 |         fourier_basis = np.fft.fft(np.eye(self.filter_length))
 91 | 
 92 |         cutoff = int((self.filter_length / 2 + 1))
 93 |         fourier_basis = np.vstack([np.real(fourier_basis[:cutoff, :]),
 94 |                                    np.imag(fourier_basis[:cutoff, :])])
 95 | 
 96 |         forward_basis = torch.FloatTensor(fourier_basis[:, None, :])
 97 |         inverse_basis = torch.FloatTensor(
 98 |             np.linalg.pinv(scale * fourier_basis).T[:, None, :])
 99 | 
100 |         if window is not None:
101 |             assert(filter_length >= win_length)
102 |             # get window and zero center pad it to filter_length
103 |             fft_window = get_window(window, win_length, fftbins=True)
104 |             fft_window = pad_center(fft_window, filter_length)
105 |             fft_window = torch.from_numpy(fft_window).float()
106 | 
107 |             # window the bases
108 |             forward_basis *= fft_window
109 |             inverse_basis *= fft_window
110 | 
111 |         self.register_buffer('forward_basis', forward_basis.float())
112 |         self.register_buffer('inverse_basis', inverse_basis.float())
113 | 
114 |     def transform(self, input_data):
115 |         num_batches = input_data.size(0)
116 |         num_samples = input_data.size(1)
117 | 
118 |         self.num_samples = num_samples
119 | 
120 |         # similar to librosa, reflect-pad the input
121 |         input_data = input_data.view(num_batches, 1, num_samples)
122 |         input_data = F.pad(
123 |             input_data.unsqueeze(1),
124 |             (int(self.filter_length / 2), int(self.filter_length / 2), 0, 0),
125 |             mode='reflect')
126 |         input_data = input_data.squeeze(1)
127 | 
128 |         forward_transform = F.conv1d(
129 |             input_data,
130 |             Variable(self.forward_basis, requires_grad=False),
131 |             stride=self.hop_length,
132 |             padding=0)
133 | 
134 |         cutoff = int((self.filter_length / 2) + 1)
135 |         real_part = forward_transform[:, :cutoff, :]
136 |         imag_part = forward_transform[:, cutoff:, :]
137 | 
138 |         magnitude = torch.sqrt(real_part**2 + imag_part**2)
139 |         phase = torch.autograd.Variable(
140 |             torch.atan2(imag_part.data, real_part.data))
141 | 
142 |         return magnitude, phase
143 | 
144 |     def inverse(self, magnitude, phase):
145 |         recombine_magnitude_phase = torch.cat(
146 |             [magnitude*torch.cos(phase), magnitude*torch.sin(phase)], dim=1)
147 | 
148 |         inverse_transform = F.conv_transpose1d(
149 |             recombine_magnitude_phase,
150 |             Variable(self.inverse_basis, requires_grad=False),
151 |             stride=self.hop_length,
152 |             padding=0)
153 | 
154 |         if self.window is not None:
155 |             window_sum = window_sumsquare(
156 |                 self.window, magnitude.size(-1), hop_length=self.hop_length,
157 |                 win_length=self.win_length, n_fft=self.filter_length,
158 |                 dtype=np.float32)
159 |             # remove modulation effects
160 |             approx_nonzero_indices = torch.from_numpy(
161 |                 np.where(window_sum > tiny(window_sum))[0])
162 |             window_sum = torch.autograd.Variable(
163 |                 torch.from_numpy(window_sum), requires_grad=False)
164 |             window_sum = window_sum.to(inverse_transform.device()) if magnitude.is_cuda else window_sum
165 |             inverse_transform[:, :, approx_nonzero_indices] /= window_sum[approx_nonzero_indices]
166 | 
167 |             # scale by hop ratio
168 |             inverse_transform *= float(self.filter_length) / self.hop_length
169 | 
170 |         inverse_transform = inverse_transform[:, :, int(self.filter_length/2):]
171 |         inverse_transform = inverse_transform[:, :, :-int(self.filter_length/2):]
172 | 
173 |         return inverse_transform
174 | 
175 |     def forward(self, input_data):
176 |         self.magnitude, self.phase = self.transform(input_data)
177 |         reconstruction = self.inverse(self.magnitude, self.phase)
178 |         return reconstruction
179 | 
180 | 
181 | class TorchSTFT(torch.nn.Module):
182 |     def __init__(self, filter_length=800, hop_length=200, win_length=800, window='hann'):
183 |         super().__init__()
184 |         self.filter_length = filter_length
185 |         self.hop_length = hop_length
186 |         self.win_length = win_length
187 |         self.window = torch.from_numpy(get_window(window, win_length, fftbins=True).astype(np.float32))
188 | 
189 |     def transform(self, input_data):
190 |         forward_transform = torch.stft(
191 |             input_data,
192 |             self.filter_length, self.hop_length, self.win_length, window=self.window,
193 |             return_complex=True)
194 | 
195 |         return torch.abs(forward_transform), torch.angle(forward_transform)
196 | 
197 |     def inverse(self, magnitude, phase):
198 |         inverse_transform = torch.istft(
199 |             magnitude * torch.exp(phase * 1j),
200 |             self.filter_length, self.hop_length, self.win_length, window=self.window.to(magnitude.device))
201 | 
202 |         return inverse_transform.unsqueeze(-2)  # unsqueeze to stay consistent with conv_transpose1d implementation
203 | 
204 |     def forward(self, input_data):
205 |         self.magnitude, self.phase = self.transform(input_data)
206 |         reconstruction = self.inverse(self.magnitude, self.phase)
207 |         return reconstruction
208 | 
209 | 
210 | 


--------------------------------------------------------------------------------
/stft_loss.py:
--------------------------------------------------------------------------------
  1 | # -*- coding: utf-8 -*-
  2 | 
  3 | # Copyright 2019 Tomoki Hayashi
  4 | #  MIT License (https://opensource.org/licenses/MIT)
  5 | 
  6 | """STFT-based Loss modules."""
  7 | 
  8 | import torch
  9 | import torch.nn.functional as F
 10 | 
 11 | 
 12 | def stft(x, fft_size, hop_size, win_length, window):
 13 |     """Perform STFT and convert to magnitude spectrogram.
 14 |     Args:
 15 |         x (Tensor): Input signal tensor (B, T).
 16 |         fft_size (int): FFT size.
 17 |         hop_size (int): Hop size.
 18 |         win_length (int): Window length.
 19 |         window (str): Window function type.
 20 |     Returns:
 21 |         Tensor: Magnitude spectrogram (B, #frames, fft_size // 2 + 1).
 22 |     """
 23 |     x_stft = torch.stft(x, fft_size, hop_size, win_length, window.to(x.device))
 24 |     real = x_stft[..., 0]
 25 |     imag = x_stft[..., 1]
 26 | 
 27 |     # NOTE(kan-bayashi): clamp is needed to avoid nan or inf
 28 |     return torch.sqrt(torch.clamp(real ** 2 + imag ** 2, min=1e-7)).transpose(2, 1)
 29 | 
 30 | 
 31 | class SpectralConvergengeLoss(torch.nn.Module):
 32 |     """Spectral convergence loss module."""
 33 | 
 34 |     def __init__(self):
 35 |         """Initilize spectral convergence loss module."""
 36 |         super(SpectralConvergengeLoss, self).__init__()
 37 | 
 38 |     def forward(self, x_mag, y_mag):
 39 |         """Calculate forward propagation.
 40 |         Args:
 41 |             x_mag (Tensor): Magnitude spectrogram of predicted signal (B, #frames, #freq_bins).
 42 |             y_mag (Tensor): Magnitude spectrogram of groundtruth signal (B, #frames, #freq_bins).
 43 |         Returns:
 44 |             Tensor: Spectral convergence loss value.
 45 |         """
 46 |         return torch.norm(y_mag - x_mag, p="fro") / torch.norm(y_mag, p="fro")
 47 | 
 48 | 
 49 | class LogSTFTMagnitudeLoss(torch.nn.Module):
 50 |     """Log STFT magnitude loss module."""
 51 | 
 52 |     def __init__(self):
 53 |         """Initilize los STFT magnitude loss module."""
 54 |         super(LogSTFTMagnitudeLoss, self).__init__()
 55 | 
 56 |     def forward(self, x_mag, y_mag):
 57 |         """Calculate forward propagation.
 58 |         Args:
 59 |             x_mag (Tensor): Magnitude spectrogram of predicted signal (B, #frames, #freq_bins).
 60 |             y_mag (Tensor): Magnitude spectrogram of groundtruth signal (B, #frames, #freq_bins).
 61 |         Returns:
 62 |             Tensor: Log STFT magnitude loss value.
 63 |         """
 64 |         return F.l1_loss(torch.log(y_mag), torch.log(x_mag))
 65 | 
 66 | 
 67 | class STFTLoss(torch.nn.Module):
 68 |     """STFT loss module."""
 69 | 
 70 |     def __init__(self, fft_size=1024, shift_size=120, win_length=600, window="hann_window"):
 71 |         """Initialize STFT loss module."""
 72 |         super(STFTLoss, self).__init__()
 73 |         self.fft_size = fft_size
 74 |         self.shift_size = shift_size
 75 |         self.win_length = win_length
 76 |         self.window = getattr(torch, window)(win_length)
 77 |         self.spectral_convergenge_loss = SpectralConvergengeLoss()
 78 |         self.log_stft_magnitude_loss = LogSTFTMagnitudeLoss()
 79 | 
 80 |     def forward(self, x, y):
 81 |         """Calculate forward propagation.
 82 |         Args:
 83 |             x (Tensor): Predicted signal (B, T).
 84 |             y (Tensor): Groundtruth signal (B, T).
 85 |         Returns:
 86 |             Tensor: Spectral convergence loss value.
 87 |             Tensor: Log STFT magnitude loss value.
 88 |         """
 89 |         x_mag = stft(x, self.fft_size, self.shift_size, self.win_length, self.window)
 90 |         y_mag = stft(y, self.fft_size, self.shift_size, self.win_length, self.window)
 91 |         sc_loss = self.spectral_convergenge_loss(x_mag, y_mag)
 92 |         mag_loss = self.log_stft_magnitude_loss(x_mag, y_mag)
 93 | 
 94 |         return sc_loss, mag_loss
 95 | 
 96 | 
 97 | class MultiResolutionSTFTLoss(torch.nn.Module):
 98 |     """Multi resolution STFT loss module."""
 99 | 
100 |     def __init__(self,
101 |                  fft_sizes=[1024, 2048, 512],
102 |                  hop_sizes=[120, 240, 50],
103 |                  win_lengths=[600, 1200, 240],
104 |                  window="hann_window"):
105 |         """Initialize Multi resolution STFT loss module.
106 |         Args:
107 |             fft_sizes (list): List of FFT sizes.
108 |             hop_sizes (list): List of hop sizes.
109 |             win_lengths (list): List of window lengths.
110 |             window (str): Window function type.
111 |         """
112 |         super(MultiResolutionSTFTLoss, self).__init__()
113 |         assert len(fft_sizes) == len(hop_sizes) == len(win_lengths)
114 |         self.stft_losses = torch.nn.ModuleList()
115 |         for fs, ss, wl in zip(fft_sizes, hop_sizes, win_lengths):
116 |             self.stft_losses += [STFTLoss(fs, ss, wl, window)]
117 | 
118 |     def forward(self, x, y):
119 |         """Calculate forward propagation.
120 |         Args:
121 |             x (Tensor): Predicted signal (B, T).
122 |             y (Tensor): Groundtruth signal (B, T).
123 |         Returns:
124 |             Tensor: Multi resolution spectral convergence loss value.
125 |             Tensor: Multi resolution log STFT magnitude loss value.
126 |         """
127 |         sc_loss = 0.0
128 |         mag_loss = 0.0
129 |         for f in self.stft_losses:
130 |             sc_l, mag_l = f(x, y)
131 |             sc_loss += sc_l
132 |             mag_loss += mag_l
133 |         sc_loss /= len(self.stft_losses)
134 |         mag_loss /= len(self.stft_losses)
135 | 
136 |         return sc_loss, mag_loss


--------------------------------------------------------------------------------
/test_data/3752-4944-0027.wav:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/quickvc/QuickVC-VoiceConversion/2fe4d6486485e9fcae2d20ece899e8f8d2c2ed96/test_data/3752-4944-0027.wav


--------------------------------------------------------------------------------
/test_data/4280-185518-0004.wav:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/quickvc/QuickVC-VoiceConversion/2fe4d6486485e9fcae2d20ece899e8f8d2c2ed96/test_data/4280-185518-0004.wav


--------------------------------------------------------------------------------
/test_data/4703-59231-0001.wav:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/quickvc/QuickVC-VoiceConversion/2fe4d6486485e9fcae2d20ece899e8f8d2c2ed96/test_data/4703-59231-0001.wav


--------------------------------------------------------------------------------
/test_data/4821-27466-0000.wav:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/quickvc/QuickVC-VoiceConversion/2fe4d6486485e9fcae2d20ece899e8f8d2c2ed96/test_data/4821-27466-0000.wav


--------------------------------------------------------------------------------
/test_data/4955-28244-0007.wav:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/quickvc/QuickVC-VoiceConversion/2fe4d6486485e9fcae2d20ece899e8f8d2c2ed96/test_data/4955-28244-0007.wav


--------------------------------------------------------------------------------
/test_data/LJ001-0001.wav:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/quickvc/QuickVC-VoiceConversion/2fe4d6486485e9fcae2d20ece899e8f8d2c2ed96/test_data/LJ001-0001.wav


--------------------------------------------------------------------------------
/test_data/LJ032-0032.wav:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/quickvc/QuickVC-VoiceConversion/2fe4d6486485e9fcae2d20ece899e8f8d2c2ed96/test_data/LJ032-0032.wav


--------------------------------------------------------------------------------
/test_data/LJ045-0246.wav:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/quickvc/QuickVC-VoiceConversion/2fe4d6486485e9fcae2d20ece899e8f8d2c2ed96/test_data/LJ045-0246.wav


--------------------------------------------------------------------------------
/test_data/VO_ZH_Barbara_Hello.wav:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/quickvc/QuickVC-VoiceConversion/2fe4d6486485e9fcae2d20ece899e8f8d2c2ed96/test_data/VO_ZH_Barbara_Hello.wav


--------------------------------------------------------------------------------
/test_data/p225_001.wav:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/quickvc/QuickVC-VoiceConversion/2fe4d6486485e9fcae2d20ece899e8f8d2c2ed96/test_data/p225_001.wav


--------------------------------------------------------------------------------
/test_data/p226_005.wav:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/quickvc/QuickVC-VoiceConversion/2fe4d6486485e9fcae2d20ece899e8f8d2c2ed96/test_data/p226_005.wav


--------------------------------------------------------------------------------
/test_data/p229_003.wav:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/quickvc/QuickVC-VoiceConversion/2fe4d6486485e9fcae2d20ece899e8f8d2c2ed96/test_data/p229_003.wav


--------------------------------------------------------------------------------
/test_data/p246_198.wav:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/quickvc/QuickVC-VoiceConversion/2fe4d6486485e9fcae2d20ece899e8f8d2c2ed96/test_data/p246_198.wav


--------------------------------------------------------------------------------
/test_data/p252_003.wav:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/quickvc/QuickVC-VoiceConversion/2fe4d6486485e9fcae2d20ece899e8f8d2c2ed96/test_data/p252_003.wav


--------------------------------------------------------------------------------
/test_data/p312_177.wav:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/quickvc/QuickVC-VoiceConversion/2fe4d6486485e9fcae2d20ece899e8f8d2c2ed96/test_data/p312_177.wav


--------------------------------------------------------------------------------
/test_data/p318_140.wav:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/quickvc/QuickVC-VoiceConversion/2fe4d6486485e9fcae2d20ece899e8f8d2c2ed96/test_data/p318_140.wav


--------------------------------------------------------------------------------
/test_data/p334_304.wav:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/quickvc/QuickVC-VoiceConversion/2fe4d6486485e9fcae2d20ece899e8f8d2c2ed96/test_data/p334_304.wav


--------------------------------------------------------------------------------
/test_data/p351_003.wav:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/quickvc/QuickVC-VoiceConversion/2fe4d6486485e9fcae2d20ece899e8f8d2c2ed96/test_data/p351_003.wav


--------------------------------------------------------------------------------
/test_data/p374_004.wav:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/quickvc/QuickVC-VoiceConversion/2fe4d6486485e9fcae2d20ece899e8f8d2c2ed96/test_data/p374_004.wav


--------------------------------------------------------------------------------
/train.py:
--------------------------------------------------------------------------------
  1 | import os
  2 | import json
  3 | import argparse
  4 | import itertools
  5 | import math
  6 | import torch
  7 | from torch import nn, optim
  8 | from torch.nn import functional as F
  9 | from torch.utils.data import DataLoader
 10 | from torch.utils.tensorboard import SummaryWriter
 11 | import torch.multiprocessing as mp
 12 | import torch.distributed as dist
 13 | from torch.nn.parallel import DistributedDataParallel as DDP
 14 | from torch.cuda.amp import autocast, GradScaler
 15 | from pqmf import PQMF
 16 | 
 17 | import commons
 18 | import utils
 19 | 
 20 | from data_utils_new_new import (
 21 |   TextAudioSpeakerLoader,
 22 |   TextAudioSpeakerCollate,
 23 |   DistributedBucketSampler
 24 | )
 25 | from models import (
 26 |   SynthesizerTrn,
 27 |   MultiPeriodDiscriminator,
 28 | )
 29 | from losses import (
 30 |   generator_loss,
 31 |   discriminator_loss,
 32 |   feature_loss,
 33 |   kl_loss,
 34 |   subband_stft_loss
 35 | )
 36 | from mel_processing import mel_spectrogram_torch, spec_to_mel_torch
 37 | #from text.symbols import symbols
 38 | 
 39 | torch.autograd.set_detect_anomaly(True)
 40 | torch.backends.cudnn.benchmark = True
 41 | global_step = 0
 42 | 
 43 | 
 44 | def main():
 45 |   """Assume Single Node Multi GPUs Training Only"""
 46 |   assert torch.cuda.is_available(), "CPU training is not allowed."
 47 | 
 48 |   n_gpus = torch.cuda.device_count()
 49 |   os.environ['MASTER_ADDR'] = 'localhost'
 50 |   os.environ['MASTER_PORT'] = '65520'
 51 | #   n_gpus = 1
 52 | 
 53 |   hps = utils.get_hparams()
 54 |   mp.spawn(run, nprocs=n_gpus, args=(n_gpus, hps,))
 55 | 
 56 | 
 57 | def run(rank, n_gpus, hps):
 58 |   global global_step
 59 |   if rank == 0:
 60 |     logger = utils.get_logger(hps.model_dir)
 61 |     logger.info(hps)
 62 |     utils.check_git_hash(hps.model_dir)
 63 |     writer = SummaryWriter(log_dir=hps.model_dir)
 64 |     writer_eval = SummaryWriter(log_dir=os.path.join(hps.model_dir, "eval"))
 65 | 
 66 |   dist.init_process_group(backend='nccl', init_method='env://', world_size=n_gpus, rank=rank)
 67 |   torch.manual_seed(hps.train.seed)
 68 |   torch.cuda.set_device(rank)
 69 | 
 70 | 
 71 |   train_dataset = TextAudioSpeakerLoader(hps.data.training_files, hps)
 72 |   train_sampler = DistributedBucketSampler(
 73 |       train_dataset,
 74 |       hps.train.batch_size,
 75 |       [32,40,50,60,70,80,90,100,110,120,160,200,230,260,300,350,400,450,500,600,700,800,900,1000],
 76 |       num_replicas=n_gpus,
 77 |       rank=rank,
 78 |       shuffle=True)
 79 |   collate_fn = TextAudioSpeakerCollate(hps)
 80 |   train_loader = DataLoader(train_dataset, num_workers=8, shuffle=False, pin_memory=True,
 81 |       collate_fn=collate_fn, batch_sampler=train_sampler)
 82 |   if rank == 0:
 83 |     eval_dataset = TextAudioSpeakerLoader(hps.data.validation_files, hps)
 84 |     eval_loader = DataLoader(eval_dataset, num_workers=8, shuffle=True,
 85 |         batch_size=1, pin_memory=False,
 86 |         drop_last=False)
 87 |   
 88 |   net_g = SynthesizerTrn(
 89 |       hps.data.filter_length // 2 + 1,
 90 |       hps.train.segment_size // hps.data.hop_length,
 91 |       **hps.model).cuda(rank)
 92 |   net_d = MultiPeriodDiscriminator(hps.model.use_spectral_norm).cuda(rank)
 93 |   optim_g = torch.optim.AdamW(
 94 |       net_g.parameters(), 
 95 |       hps.train.learning_rate, 
 96 |       betas=hps.train.betas, 
 97 |       eps=hps.train.eps)
 98 |   optim_d = torch.optim.AdamW(
 99 |       net_d.parameters(),
100 |       hps.train.learning_rate, 
101 |       betas=hps.train.betas, 
102 |       eps=hps.train.eps)
103 |   net_g = DDP(net_g, device_ids=[rank])
104 |   net_d = DDP(net_d, device_ids=[rank])
105 | 
106 |   try:
107 |     _, _, _, epoch_str = utils.load_checkpoint(utils.latest_checkpoint_path(hps.model_dir, "G_*.pth"), net_g, optim_g)
108 |     _, _, _, epoch_str = utils.load_checkpoint(utils.latest_checkpoint_path(hps.model_dir, "D_*.pth"), net_d, optim_d)
109 |     global_step = (epoch_str - 1) * len(train_loader)
110 |   except:
111 |     epoch_str = 1
112 |     global_step = 0
113 | 
114 |   scheduler_g = torch.optim.lr_scheduler.ExponentialLR(optim_g, gamma=hps.train.lr_decay, last_epoch=epoch_str-2)
115 |   scheduler_d = torch.optim.lr_scheduler.ExponentialLR(optim_d, gamma=hps.train.lr_decay, last_epoch=epoch_str-2)
116 | 
117 |   scaler = GradScaler(enabled=hps.train.fp16_run)
118 | 
119 |   for epoch in range(epoch_str, hps.train.epochs + 1):
120 |     if rank==0:
121 |       train_and_evaluate(rank, epoch, hps, [net_g, net_d], [optim_g, optim_d], [scheduler_g, scheduler_d], scaler, [train_loader, eval_loader], logger, [writer, writer_eval])
122 |     else:
123 |       train_and_evaluate(rank, epoch, hps, [net_g, net_d], [optim_g, optim_d], [scheduler_g, scheduler_d], scaler, [train_loader, None], None, None)
124 |     scheduler_g.step()
125 |     scheduler_d.step()
126 | 
127 | 
128 | 
129 | def train_and_evaluate(rank, epoch, hps, nets, optims, schedulers, scaler, loaders, logger, writers):
130 |   net_g, net_d = nets
131 |   optim_g, optim_d = optims
132 |   scheduler_g, scheduler_d = schedulers
133 |   train_loader, eval_loader = loaders
134 |   if writers is not None:
135 |     writer, writer_eval = writers
136 |   tmp=0
137 |   tmp1=1000000000
138 |   #train_loader.batch_sampler.set_epoch(epoch)
139 |   global global_step
140 | 
141 |   net_g.train()
142 |   net_d.train()
143 |   for batch_idx, (c, spec, y) in enumerate(train_loader):
144 |     g = None
145 |     spec, y = spec.cuda(rank, non_blocking=True), y.cuda(rank, non_blocking=True)
146 |     
147 |     c = c.cuda(rank, non_blocking=True)
148 |     mel = spec_to_mel_torch(
149 |           spec, 
150 |           hps.data.filter_length, 
151 |           hps.data.n_mel_channels, 
152 |           hps.data.sampling_rate,
153 |           hps.data.mel_fmin, 
154 |           hps.data.mel_fmax)
155 | 
156 | 
157 |     with autocast(enabled=hps.train.fp16_run):
158 |       #print(c.size())
159 |       y_hat, y_hat_mb, ids_slice, z_mask,\
160 |       (z, z_p, m_p, logs_p, m_q, logs_q) = net_g(c, spec, g=g, mel=mel)
161 |  
162 |       mel = spec_to_mel_torch(
163 |           spec, 
164 |           hps.data.filter_length, 
165 |           hps.data.n_mel_channels, 
166 |           hps.data.sampling_rate,
167 |           hps.data.mel_fmin, 
168 |           hps.data.mel_fmax)
169 |       y_mel = commons.slice_segments(mel, ids_slice, hps.train.segment_size // hps.data.hop_length)
170 |       y_hat_mel = mel_spectrogram_torch(
171 |           y_hat.squeeze(1), 
172 |           hps.data.filter_length, 
173 |           hps.data.n_mel_channels, 
174 |           hps.data.sampling_rate, 
175 |           hps.data.hop_length, 
176 |           hps.data.win_length, 
177 |           hps.data.mel_fmin, 
178 |           hps.data.mel_fmax
179 |       )
180 |       tmp=max(tmp,y.size()[2])
181 |       tmp1=min(tmp1,y.size()[2])
182 |       y = commons.slice_segments(y, ids_slice * hps.data.hop_length, hps.train.segment_size) # slice 
183 | 
184 |       y_d_hat_r, y_d_hat_g, _, _ = net_d(y, y_hat.detach())
185 |       with autocast(enabled=False):
186 |         loss_disc, losses_disc_r, losses_disc_g = discriminator_loss(y_d_hat_r, y_d_hat_g)
187 |         loss_disc_all = loss_disc
188 |     optim_d.zero_grad()
189 |     scaler.scale(loss_disc_all).backward()
190 |     scaler.unscale_(optim_d)
191 |     grad_norm_d = commons.clip_grad_value_(net_d.parameters(), None)
192 |     scaler.step(optim_d)
193 | 
194 |     
195 | 
196 | 
197 |     with autocast(enabled=hps.train.fp16_run):
198 |       # Generator
199 |       y_d_hat_r, y_d_hat_g, fmap_r, fmap_g = net_d(y, y_hat)
200 |       with autocast(enabled=False):
201 |         #loss_dur = torch.sum(l_length.float())
202 |         loss_mel = F.l1_loss(y_mel, y_hat_mel) * hps.train.c_mel
203 |         loss_kl = kl_loss(z_p, logs_q, m_p, logs_p, z_mask) * hps.train.c_kl
204 | 
205 |         loss_fm = feature_loss(fmap_r, fmap_g)
206 |         loss_gen, losses_gen = generator_loss(y_d_hat_g)
207 |         
208 |         if hps.model.mb_istft_vits == True:
209 |           pqmf = PQMF(y.device)
210 |           y_mb = pqmf.analysis(y)
211 |           loss_subband = subband_stft_loss(hps, y_mb, y_hat_mb)
212 |         else:
213 |           loss_subband = torch.tensor(0.0)
214 | 
215 |         loss_gen_all = loss_gen + loss_fm + loss_mel + loss_kl + loss_subband#+ loss_dur 
216 | 
217 |     optim_g.zero_grad()
218 |     scaler.scale(loss_gen_all).backward()
219 |     scaler.unscale_(optim_g)
220 |     grad_norm_g = commons.clip_grad_value_(net_g.parameters(), None)
221 |     scaler.step(optim_g)
222 |     scaler.update()
223 | 
224 |     if rank==0:
225 |       if global_step % hps.train.log_interval == 0:
226 |         lr = optim_g.param_groups[0]['lr']
227 |         losses = [loss_disc, loss_gen, loss_fm, loss_mel, loss_kl, loss_subband]
228 |         logger.info('Train Epoch: {} [{:.0f}%]'.format(
229 |           epoch,
230 |           100. * batch_idx / len(train_loader)))
231 |         logger.info([x.item() for x in losses] + [global_step, lr])
232 |         
233 |         scalar_dict = {"loss/g/total": loss_gen_all, "loss/d/total": loss_disc_all, "learning_rate": lr, "grad_norm_d": grad_norm_d, "grad_norm_g": grad_norm_g}
234 |         scalar_dict.update({"loss/g/fm": loss_fm, "loss/g/mel": loss_mel,  "loss/g/kl": loss_kl, "loss/g/subband": loss_subband})
235 | 
236 |         scalar_dict.update({"loss/g/{}".format(i): v for i, v in enumerate(losses_gen)})
237 |         scalar_dict.update({"loss/d_r/{}".format(i): v for i, v in enumerate(losses_disc_r)})
238 |         scalar_dict.update({"loss/d_g/{}".format(i): v for i, v in enumerate(losses_disc_g)})
239 |         image_dict = { 
240 |             "slice/mel_org": utils.plot_spectrogram_to_numpy(y_mel[0].data.cpu().numpy()),
241 |             "slice/mel_gen": utils.plot_spectrogram_to_numpy(y_hat_mel[0].data.cpu().numpy()), 
242 |             "all/mel": utils.plot_spectrogram_to_numpy(mel[0].data.cpu().numpy()),
243 |             #"all/attn": utils.plot_alignment_to_numpy(attn[0,0].data.cpu().numpy())
244 |         }
245 |         utils.summarize(
246 |           writer=writer,
247 |           global_step=global_step, 
248 |           images=image_dict,
249 |           scalars=scalar_dict)
250 | 
251 |       if global_step % hps.train.eval_interval == 0:
252 |         evaluate(hps, net_g, eval_loader, writer_eval)
253 |         utils.save_checkpoint(net_g, optim_g, hps.train.learning_rate, epoch, os.path.join(hps.model_dir, "G_{}.pth".format(global_step)))
254 |         utils.save_checkpoint(net_d, optim_d, hps.train.learning_rate, epoch, os.path.join(hps.model_dir, "D_{}.pth".format(global_step)))
255 |     global_step += 1
256 | 
257 |   
258 |   if rank == 0:
259 |     logger.info('====> Epoch: {}'.format(epoch))
260 |   print(tmp,tmp1)
261 |     
262 | 
263 |  
264 | def evaluate(hps, generator, eval_loader, writer_eval):
265 |     generator.eval()
266 |     with torch.no_grad():
267 |       for batch_idx, (c, spec, y) in enumerate(eval_loader):
268 |         g = None
269 |         spec, y = spec[:1].cuda(0), y[:1].cuda(0)
270 |         c = c[:1].cuda(0)
271 | 
272 |         break
273 |       mel = spec_to_mel_torch(
274 |         spec, 
275 |         hps.data.filter_length, 
276 |         hps.data.n_mel_channels, 
277 |         hps.data.sampling_rate,
278 |         hps.data.mel_fmin, 
279 |         hps.data.mel_fmax)
280 |       #y_hat, y_hat_mb, attn, mask, *_ = generator.module.infer(x, x_lengths, max_len=1000)
281 |       #y_hat_lengths = mask.sum([1,2]).long() * hps.data.hop_length
282 |       y_hat = generator.module.infer(c, g=g, mel=mel)
283 |       mel = spec_to_mel_torch(
284 |           spec, 
285 |           hps.data.filter_length, 
286 |           hps.data.n_mel_channels, 
287 |           hps.data.sampling_rate,
288 |           hps.data.mel_fmin, 
289 |           hps.data.mel_fmax)
290 |       y_hat_mel = mel_spectrogram_torch(
291 |         y_hat.squeeze(1).float(),
292 |         hps.data.filter_length,
293 |         hps.data.n_mel_channels,
294 |         hps.data.sampling_rate,
295 |         hps.data.hop_length,
296 |         hps.data.win_length,
297 |         hps.data.mel_fmin,
298 |         hps.data.mel_fmax
299 |       )
300 |     image_dict = {
301 |       "gen/mel": utils.plot_spectrogram_to_numpy(y_hat_mel[0].cpu().numpy()),
302 |       "gt/mel": utils.plot_spectrogram_to_numpy(mel[0].cpu().numpy())
303 |     }
304 |     audio_dict = {
305 |       "gen/audio": y_hat[0],
306 |       "gt/audio": y[0]
307 |     }
308 | 
309 |     import torchaudio
310 |     y_gt=y*32768
311 |     print(y_hat.size())
312 |     torchaudio.save("temp_result/vctkms_new_tem_result_{}.wav".format(global_step),y_hat[0, :, :].cpu(),16000)
313 |     torchaudio.save("temp_result/vctkms_new_tem_result_gt_{}.wav".format(global_step),y[0, :, :].cpu(),16000)
314 |     #torchaudio.save("tem_result_gt32768_{}.wav".format(global_step),y_gt[0, :, :].cpu(),16000)
315 | 
316 |     utils.summarize(
317 |       writer=writer_eval,
318 |       global_step=global_step, 
319 |       images=image_dict,
320 |       audios=audio_dict,
321 |       audio_sampling_rate=hps.data.sampling_rate
322 |     )
323 |     generator.train()
324 | 
325 |                            
326 | if __name__ == "__main__":
327 |   os.environ[
328 |         "TORCH_DISTRIBUTED_DEBUG"
329 |     ] = "DETAIL"
330 |   main()
331 | 


--------------------------------------------------------------------------------
/transforms.py:
--------------------------------------------------------------------------------
  1 | import torch
  2 | from torch.nn import functional as F
  3 | 
  4 | import numpy as np
  5 | 
  6 | 
  7 | DEFAULT_MIN_BIN_WIDTH = 1e-3
  8 | DEFAULT_MIN_BIN_HEIGHT = 1e-3
  9 | DEFAULT_MIN_DERIVATIVE = 1e-3
 10 | 
 11 | 
 12 | def piecewise_rational_quadratic_transform(inputs, 
 13 |                                            unnormalized_widths,
 14 |                                            unnormalized_heights,
 15 |                                            unnormalized_derivatives,
 16 |                                            inverse=False,
 17 |                                            tails=None, 
 18 |                                            tail_bound=1.,
 19 |                                            min_bin_width=DEFAULT_MIN_BIN_WIDTH,
 20 |                                            min_bin_height=DEFAULT_MIN_BIN_HEIGHT,
 21 |                                            min_derivative=DEFAULT_MIN_DERIVATIVE):
 22 | 
 23 |     if tails is None:
 24 |         spline_fn = rational_quadratic_spline
 25 |         spline_kwargs = {}
 26 |     else:
 27 |         spline_fn = unconstrained_rational_quadratic_spline
 28 |         spline_kwargs = {
 29 |             'tails': tails,
 30 |             'tail_bound': tail_bound
 31 |         }
 32 | 
 33 |     outputs, logabsdet = spline_fn(
 34 |             inputs=inputs,
 35 |             unnormalized_widths=unnormalized_widths,
 36 |             unnormalized_heights=unnormalized_heights,
 37 |             unnormalized_derivatives=unnormalized_derivatives,
 38 |             inverse=inverse,
 39 |             min_bin_width=min_bin_width,
 40 |             min_bin_height=min_bin_height,
 41 |             min_derivative=min_derivative,
 42 |             **spline_kwargs
 43 |     )
 44 |     return outputs, logabsdet
 45 | 
 46 | 
 47 | def searchsorted(bin_locations, inputs, eps=1e-6):
 48 |     bin_locations[..., -1] += eps
 49 |     return torch.sum(
 50 |         inputs[..., None] >= bin_locations,
 51 |         dim=-1
 52 |     ) - 1
 53 | 
 54 | 
 55 | def unconstrained_rational_quadratic_spline(inputs,
 56 |                                             unnormalized_widths,
 57 |                                             unnormalized_heights,
 58 |                                             unnormalized_derivatives,
 59 |                                             inverse=False,
 60 |                                             tails='linear',
 61 |                                             tail_bound=1.,
 62 |                                             min_bin_width=DEFAULT_MIN_BIN_WIDTH,
 63 |                                             min_bin_height=DEFAULT_MIN_BIN_HEIGHT,
 64 |                                             min_derivative=DEFAULT_MIN_DERIVATIVE):
 65 |     inside_interval_mask = (inputs >= -tail_bound) & (inputs <= tail_bound)
 66 |     outside_interval_mask = ~inside_interval_mask
 67 | 
 68 |     outputs = torch.zeros_like(inputs)
 69 |     logabsdet = torch.zeros_like(inputs)
 70 | 
 71 |     if tails == 'linear':
 72 |         unnormalized_derivatives = F.pad(unnormalized_derivatives, pad=(1, 1))
 73 |         constant = np.log(np.exp(1 - min_derivative) - 1)
 74 |         unnormalized_derivatives[..., 0] = constant
 75 |         unnormalized_derivatives[..., -1] = constant
 76 | 
 77 |         outputs[outside_interval_mask] = inputs[outside_interval_mask]
 78 |         logabsdet[outside_interval_mask] = 0
 79 |     else:
 80 |         raise RuntimeError('{} tails are not implemented.'.format(tails))
 81 | 
 82 |     outputs[inside_interval_mask], logabsdet[inside_interval_mask] = rational_quadratic_spline(
 83 |         inputs=inputs[inside_interval_mask],
 84 |         unnormalized_widths=unnormalized_widths[inside_interval_mask, :],
 85 |         unnormalized_heights=unnormalized_heights[inside_interval_mask, :],
 86 |         unnormalized_derivatives=unnormalized_derivatives[inside_interval_mask, :],
 87 |         inverse=inverse,
 88 |         left=-tail_bound, right=tail_bound, bottom=-tail_bound, top=tail_bound,
 89 |         min_bin_width=min_bin_width,
 90 |         min_bin_height=min_bin_height,
 91 |         min_derivative=min_derivative
 92 |     )
 93 | 
 94 |     return outputs, logabsdet
 95 | 
 96 | def rational_quadratic_spline(inputs,
 97 |                               unnormalized_widths,
 98 |                               unnormalized_heights,
 99 |                               unnormalized_derivatives,
100 |                               inverse=False,
101 |                               left=0., right=1., bottom=0., top=1.,
102 |                               min_bin_width=DEFAULT_MIN_BIN_WIDTH,
103 |                               min_bin_height=DEFAULT_MIN_BIN_HEIGHT,
104 |                               min_derivative=DEFAULT_MIN_DERIVATIVE):
105 |     if torch.min(inputs) < left or torch.max(inputs) > right:
106 |         raise ValueError('Input to a transform is not within its domain')
107 | 
108 |     num_bins = unnormalized_widths.shape[-1]
109 | 
110 |     if min_bin_width * num_bins > 1.0:
111 |         raise ValueError('Minimal bin width too large for the number of bins')
112 |     if min_bin_height * num_bins > 1.0:
113 |         raise ValueError('Minimal bin height too large for the number of bins')
114 | 
115 |     widths = F.softmax(unnormalized_widths, dim=-1)
116 |     widths = min_bin_width + (1 - min_bin_width * num_bins) * widths
117 |     cumwidths = torch.cumsum(widths, dim=-1)
118 |     cumwidths = F.pad(cumwidths, pad=(1, 0), mode='constant', value=0.0)
119 |     cumwidths = (right - left) * cumwidths + left
120 |     cumwidths[..., 0] = left
121 |     cumwidths[..., -1] = right
122 |     widths = cumwidths[..., 1:] - cumwidths[..., :-1]
123 | 
124 |     derivatives = min_derivative + F.softplus(unnormalized_derivatives)
125 | 
126 |     heights = F.softmax(unnormalized_heights, dim=-1)
127 |     heights = min_bin_height + (1 - min_bin_height * num_bins) * heights
128 |     cumheights = torch.cumsum(heights, dim=-1)
129 |     cumheights = F.pad(cumheights, pad=(1, 0), mode='constant', value=0.0)
130 |     cumheights = (top - bottom) * cumheights + bottom
131 |     cumheights[..., 0] = bottom
132 |     cumheights[..., -1] = top
133 |     heights = cumheights[..., 1:] - cumheights[..., :-1]
134 | 
135 |     if inverse:
136 |         bin_idx = searchsorted(cumheights, inputs)[..., None]
137 |     else:
138 |         bin_idx = searchsorted(cumwidths, inputs)[..., None]
139 | 
140 |     input_cumwidths = cumwidths.gather(-1, bin_idx)[..., 0]
141 |     input_bin_widths = widths.gather(-1, bin_idx)[..., 0]
142 | 
143 |     input_cumheights = cumheights.gather(-1, bin_idx)[..., 0]
144 |     delta = heights / widths
145 |     input_delta = delta.gather(-1, bin_idx)[..., 0]
146 | 
147 |     input_derivatives = derivatives.gather(-1, bin_idx)[..., 0]
148 |     input_derivatives_plus_one = derivatives[..., 1:].gather(-1, bin_idx)[..., 0]
149 | 
150 |     input_heights = heights.gather(-1, bin_idx)[..., 0]
151 | 
152 |     if inverse:
153 |         a = (((inputs - input_cumheights) * (input_derivatives
154 |                                              + input_derivatives_plus_one
155 |                                              - 2 * input_delta)
156 |               + input_heights * (input_delta - input_derivatives)))
157 |         b = (input_heights * input_derivatives
158 |              - (inputs - input_cumheights) * (input_derivatives
159 |                                               + input_derivatives_plus_one
160 |                                               - 2 * input_delta))
161 |         c = - input_delta * (inputs - input_cumheights)
162 | 
163 |         discriminant = b.pow(2) - 4 * a * c
164 |         assert (discriminant >= 0).all()
165 | 
166 |         root = (2 * c) / (-b - torch.sqrt(discriminant))
167 |         outputs = root * input_bin_widths + input_cumwidths
168 | 
169 |         theta_one_minus_theta = root * (1 - root)
170 |         denominator = input_delta + ((input_derivatives + input_derivatives_plus_one - 2 * input_delta)
171 |                                      * theta_one_minus_theta)
172 |         derivative_numerator = input_delta.pow(2) * (input_derivatives_plus_one * root.pow(2)
173 |                                                      + 2 * input_delta * theta_one_minus_theta
174 |                                                      + input_derivatives * (1 - root).pow(2))
175 |         logabsdet = torch.log(derivative_numerator) - 2 * torch.log(denominator)
176 | 
177 |         return outputs, -logabsdet
178 |     else:
179 |         theta = (inputs - input_cumwidths) / input_bin_widths
180 |         theta_one_minus_theta = theta * (1 - theta)
181 | 
182 |         numerator = input_heights * (input_delta * theta.pow(2)
183 |                                      + input_derivatives * theta_one_minus_theta)
184 |         denominator = input_delta + ((input_derivatives + input_derivatives_plus_one - 2 * input_delta)
185 |                                      * theta_one_minus_theta)
186 |         outputs = input_cumheights + numerator / denominator
187 | 
188 |         derivative_numerator = input_delta.pow(2) * (input_derivatives_plus_one * theta.pow(2)
189 |                                                      + 2 * input_delta * theta_one_minus_theta
190 |                                                      + input_derivatives * (1 - theta).pow(2))
191 |         logabsdet = torch.log(derivative_numerator) - 2 * torch.log(denominator)
192 | 
193 |         return outputs, logabsdet
194 | 


--------------------------------------------------------------------------------
/utils.py:
--------------------------------------------------------------------------------
  1 | import os
  2 | import glob
  3 | import sys
  4 | import argparse
  5 | import logging
  6 | import json
  7 | import subprocess
  8 | import numpy as np
  9 | from scipy.io.wavfile import read
 10 | import torch
 11 | import torchvision
 12 | MATPLOTLIB_FLAG = False
 13 | 
 14 | logging.basicConfig(stream=sys.stdout, level=logging.WARNING)
 15 | logger = logging
 16 | 
 17 | 
 18 | def load_checkpoint(checkpoint_path, model, optimizer=None):
 19 |   assert os.path.isfile(checkpoint_path)
 20 |   checkpoint_dict = torch.load(checkpoint_path, map_location='cpu')
 21 |   iteration = checkpoint_dict['iteration']
 22 |   learning_rate = checkpoint_dict['learning_rate']
 23 |   if optimizer is not None:
 24 |     optimizer.load_state_dict(checkpoint_dict['optimizer'])
 25 |   saved_state_dict = checkpoint_dict['model']
 26 |   if hasattr(model, 'module'):
 27 |     state_dict = model.module.state_dict()
 28 |   else:
 29 |     state_dict = model.state_dict()
 30 |   new_state_dict= {}
 31 |   for k, v in state_dict.items():
 32 |     try:
 33 |       new_state_dict[k] = saved_state_dict[k]
 34 |     except:
 35 |       logger.info("%s is not in the checkpoint" % k)
 36 |       new_state_dict[k] = v
 37 |   if hasattr(model, 'module'):
 38 |     model.module.load_state_dict(new_state_dict)
 39 |   else:
 40 |     model.load_state_dict(new_state_dict)
 41 |   logger.info("Loaded checkpoint '{}' (iteration {})" .format(
 42 |     checkpoint_path, iteration))
 43 |   return model, optimizer, learning_rate, iteration
 44 | 
 45 | 
 46 | def save_checkpoint(model, optimizer, learning_rate, iteration, checkpoint_path):
 47 |   logger.info("Saving model and optimizer state at iteration {} to {}".format(
 48 |     iteration, checkpoint_path))
 49 |   if hasattr(model, 'module'):
 50 |     state_dict = model.module.state_dict()
 51 |   else:
 52 |     state_dict = model.state_dict()
 53 |   torch.save({'model': state_dict,
 54 |               'iteration': iteration,
 55 |               'optimizer': optimizer.state_dict(),
 56 |               'learning_rate': learning_rate}, checkpoint_path)
 57 | 
 58 | 
 59 | def summarize(writer, global_step, scalars={}, histograms={}, images={}, audios={}, audio_sampling_rate=22050):
 60 |   for k, v in scalars.items():
 61 |     writer.add_scalar(k, v, global_step)
 62 |   for k, v in histograms.items():
 63 |     writer.add_histogram(k, v, global_step)
 64 |   for k, v in images.items():
 65 |     writer.add_image(k, v, global_step, dataformats='HWC')
 66 |   for k, v in audios.items():
 67 |     writer.add_audio(k, v, global_step, audio_sampling_rate)
 68 | 
 69 | 
 70 | def latest_checkpoint_path(dir_path, regex="G_*.pth"):
 71 |   f_list = glob.glob(os.path.join(dir_path, regex))
 72 |   f_list.sort(key=lambda f: int("".join(filter(str.isdigit, f))))
 73 |   x = f_list[-1]
 74 |   print(x)
 75 |   return x
 76 | 
 77 | 
 78 | def plot_spectrogram_to_numpy(spectrogram):
 79 |   global MATPLOTLIB_FLAG
 80 |   if not MATPLOTLIB_FLAG:
 81 |     import matplotlib
 82 |     matplotlib.use("Agg")
 83 |     MATPLOTLIB_FLAG = True
 84 |     mpl_logger = logging.getLogger('matplotlib')
 85 |     mpl_logger.setLevel(logging.WARNING)
 86 |   import matplotlib.pylab as plt
 87 |   import numpy as np
 88 |   
 89 |   fig, ax = plt.subplots(figsize=(10,2))
 90 |   im = ax.imshow(spectrogram, aspect="auto", origin="lower",
 91 |                   interpolation='none')
 92 |   plt.colorbar(im, ax=ax)
 93 |   plt.xlabel("Frames")
 94 |   plt.ylabel("Channels")
 95 |   plt.tight_layout()
 96 | 
 97 |   fig.canvas.draw()
 98 |   data = np.fromstring(fig.canvas.tostring_rgb(), dtype=np.uint8, sep='')
 99 |   data = data.reshape(fig.canvas.get_width_height()[::-1] + (3,))
100 |   plt.close()
101 |   return data
102 | 
103 | 
104 | def plot_alignment_to_numpy(alignment, info=None):
105 |   global MATPLOTLIB_FLAG
106 |   if not MATPLOTLIB_FLAG:
107 |     import matplotlib
108 |     matplotlib.use("Agg")
109 |     MATPLOTLIB_FLAG = True
110 |     mpl_logger = logging.getLogger('matplotlib')
111 |     mpl_logger.setLevel(logging.WARNING)
112 |   import matplotlib.pylab as plt
113 |   import numpy as np
114 | 
115 |   fig, ax = plt.subplots(figsize=(6, 4))
116 |   im = ax.imshow(alignment.transpose(), aspect='auto', origin='lower',
117 |                   interpolation='none')
118 |   fig.colorbar(im, ax=ax)
119 |   xlabel = 'Decoder timestep'
120 |   if info is not None:
121 |       xlabel += '\n\n' + info
122 |   plt.xlabel(xlabel)
123 |   plt.ylabel('Encoder timestep')
124 |   plt.tight_layout()
125 | 
126 |   fig.canvas.draw()
127 |   data = np.fromstring(fig.canvas.tostring_rgb(), dtype=np.uint8, sep='')
128 |   data = data.reshape(fig.canvas.get_width_height()[::-1] + (3,))
129 |   plt.close()
130 |   return data
131 | 
132 | 
133 | def load_wav_to_torch(full_path):
134 |   sampling_rate, data = read(full_path)
135 |   return torch.FloatTensor(data.astype(np.float32)), sampling_rate
136 | 
137 | 
138 | def load_filepaths_and_text(filename, split="|"):
139 |   with open(filename, encoding='utf-8') as f:
140 |     filepaths_and_text = [line.strip().split(split) for line in f]
141 |   return filepaths_and_text
142 | 
143 | 
144 | def get_hparams(init=True):
145 |   parser = argparse.ArgumentParser()
146 |   parser.add_argument('-c', '--config', type=str, default="./configs/quickvc.json",
147 |                       help='JSON file for configuration')
148 |   parser.add_argument('-m', '--model', type=str,default="quickvc",
149 |                       help='Model name')
150 |   
151 |   args = parser.parse_args()
152 |   model_dir = os.path.join("./logs", args.model)
153 | 
154 |   if not os.path.exists(model_dir):
155 |     os.makedirs(model_dir)
156 | 
157 |   config_path = args.config
158 |   config_save_path = os.path.join(model_dir, "config.json")
159 |   if init:
160 |     with open(config_path, "r") as f:
161 |       data = f.read()
162 |     with open(config_save_path, "w") as f:
163 |       f.write(data)
164 |   else:
165 |     with open(config_save_path, "r") as f:
166 |       data = f.read()
167 |   config = json.loads(data)
168 |   
169 |   hparams = HParams(**config)
170 |   hparams.model_dir = model_dir
171 |   return hparams
172 | 
173 | def transform(mel, height): # 68-92
174 |     #r = np.random.random()
175 |     #rate = r * 0.3 + 0.85 # 0.85-1.15
176 |     #height = int(mel.size(-2) * rate)
177 |     tgt = torchvision.transforms.functional.resize(mel, (height, mel.size(-1)))
178 |     if height >= mel.size(-2):
179 |         return tgt[:, :mel.size(-2), :]
180 |     else:
181 |         silence = tgt[:,-1:,:].repeat(1,mel.size(-2)-height,1) 
182 |         silence += torch.randn_like(silence) / 10
183 |         return torch.cat((tgt, silence), 1)
184 | 
185 | def get_hparams_from_dir(model_dir):
186 |   config_save_path = os.path.join(model_dir, "config.json")
187 |   with open(config_save_path, "r") as f:
188 |     data = f.read()
189 |   config = json.loads(data)
190 | 
191 |   hparams =HParams(**config)
192 |   hparams.model_dir = model_dir
193 |   return hparams
194 | 
195 | 
196 | def get_hparams_from_file(config_path):
197 |   with open(config_path, "r") as f:
198 |     data = f.read()
199 |   config = json.loads(data)
200 | 
201 |   hparams =HParams(**config)
202 |   return hparams
203 | 
204 | 
205 | def check_git_hash(model_dir):
206 |   source_dir = os.path.dirname(os.path.realpath(__file__))
207 |   if not os.path.exists(os.path.join(source_dir, ".git")):
208 |     logger.warn("{} is not a git repository, therefore hash value comparison will be ignored.".format(
209 |       source_dir
210 |     ))
211 |     return
212 | 
213 |   cur_hash = subprocess.getoutput("git rev-parse HEAD")
214 | 
215 |   path = os.path.join(model_dir, "githash")
216 |   if os.path.exists(path):
217 |     saved_hash = open(path).read()
218 |     if saved_hash != cur_hash:
219 |       logger.warn("git hash values are different. {}(saved) != {}(current)".format(
220 |         saved_hash[:8], cur_hash[:8]))
221 |   else:
222 |     open(path, "w").write(cur_hash)
223 | 
224 | 
225 | def get_logger(model_dir, filename="train.log"):
226 |   global logger
227 |   logger = logging.getLogger(os.path.basename(model_dir))
228 |   logger.setLevel(logging.DEBUG)
229 |   
230 |   formatter = logging.Formatter("%(asctime)s\t%(name)s\t%(levelname)s\t%(message)s")
231 |   if not os.path.exists(model_dir):
232 |     os.makedirs(model_dir)
233 |   h = logging.FileHandler(os.path.join(model_dir, filename))
234 |   h.setLevel(logging.DEBUG)
235 |   h.setFormatter(formatter)
236 |   logger.addHandler(h)
237 |   return logger
238 | 
239 | 
240 | class HParams():
241 |   def __init__(self, **kwargs):
242 |     for k, v in kwargs.items():
243 |       if type(v) == dict:
244 |         v = HParams(**v)
245 |       self[k] = v
246 |     
247 |   def keys(self):
248 |     return self.__dict__.keys()
249 | 
250 |   def items(self):
251 |     return self.__dict__.items()
252 | 
253 |   def values(self):
254 |     return self.__dict__.values()
255 | 
256 |   def __len__(self):
257 |     return len(self.__dict__)
258 | 
259 |   def __getitem__(self, key):
260 |     return getattr(self, key)
261 | 
262 |   def __setitem__(self, key, value):
263 |     return setattr(self, key, value)
264 | 
265 |   def __contains__(self, key):
266 |     return key in self.__dict__
267 | 
268 |   def __repr__(self):
269 |     return self.__dict__.__repr__()
270 | 


--------------------------------------------------------------------------------