├── .gitignore
├── LICENSE
├── README.md
├── attentions.py
├── commons.py
├── configs
    ├── libritts.json
    └── vctk_base.json
├── data_utils.py
├── filelists
    ├── libritts_train.txt
    └── libritts_val.txt
├── img
    └── Overview.jpg
├── inference.py
├── libritts.py
├── losses.py
├── mel_processing.py
├── models.py
├── modules.py
├── monotonic_align
    ├── __init__.py
    ├── core.pyx
    └── setup.py
├── prepare_wav.py
├── preprocess.py
├── requirements.txt
├── text
    ├── __init__.py
    ├── cleaners.py
    ├── cmudict.py
    ├── numbers.py
    └── symbols.py
├── train.py
├── train_ms.py
├── transforms.py
└── utils.py


/.gitignore:
--------------------------------------------------------------------------------
 1 | DUMMY1
 2 | DUMMY2
 3 | DUMMY3
 4 | logs
 5 | __pycache__
 6 | .ipynb_checkpoints
 7 | .*.swp
 8 | wav*
 9 | 
10 | build
11 | *.c
12 | monotonic_align/monotonic_align
13 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright (c) 2021 Jaehyeon Kim
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # TransferTTS (Zero-shot VITS) - PyTorch Implementation (-Ongoing-)
 2 | 
 3 | ## Note!!(09.23.)
 4 | In current, this is just a implementation of zero-shot system; Not the implementation of the first contribution of the paper: Transfer learning framework using wav2vec2.0. 
 5 | As the future work, the model equipped with complete implementations of the two contributions (zero-shot and transfer-learning) will be implemented in the follwoing [repository](https://github.com/hcy71o/SC-VITS).
 6 | Congratulations on being awarded the best paper in INTERSPEECH 2022.
 7 | 
 8 | # Overview
 9 | Unofficial PyTorch Implementation of [Transfer Learning Framework for Low-Resource Text-to-Speech using a Large-Scale Unlabeled Speech Corpus](https://arxiv.org/abs/2203.15447). Most of codes are based on [VITS](https://github.com/jaywalnut310/vits)
10 | 
11 | 0. MelStyleEncoder from [StyleSpeech](https://arxiv.org/abs/2106.03153) is used instead of the reference encoder.
12 | 1. Implementation of untranscribed data training is omitted.
13 | 2. [LibriTTS]((https://research.google/tools/datasets/libri-tts/)) dataset (train-clean-100 and train-clean-360) is used. Sampling rate is set to 22050Hz.
14 | 
15 | <p align="center">
16 |     <img src="img/Overview.jpg" width="80%">
17 | </p>
18 | 
19 | ## Pre-requisites (from VITS)
20 | 0. Python >= 3.6
21 | 0. Clone this repository
22 | 0. Install python requirements. Please refer [requirements.txt](requirements.txt)
23 |     1. You may need to install espeak first: `apt-get install espeak`
24 | 0. Build Monotonic Alignment Search and run preprocessing if you use your own datasets.
25 | ```sh
26 | # Cython-version Monotonoic Alignment Search
27 | cd monotonic_align
28 | python setup.py build_ext --inplace
29 | ```
30 | ## Preprocessing
31 |  
32 | Run 
33 | ```
34 | python prepare_wav.py --data_path [LibriTTS DATAPATH]
35 | ```
36 | for some preparations.
37 | 
38 | ## Training
39 | 
40 | Train your model with
41 | ```
42 | python train_ms.py -c configs/libritts.json -m libritts_base
43 | ```
44 | 
45 | ## Inference
46 | ```
47 | python inference.py --ref_audio [REF AUDIO PATH] --text [INPUT TEXT]
48 | ```
49 | 
50 | # References
51 | - [TransferTTS](https://arxiv.org/abs/2203.15447)
52 | - [VITS](https://arxiv.org/abs/2106.06103)
53 | - [Meta-StyleSpeech](https://arxiv.org/abs/2106.03153)
54 | 


--------------------------------------------------------------------------------
/attentions.py:
--------------------------------------------------------------------------------
  1 | import copy
  2 | import math
  3 | import numpy as np
  4 | import torch
  5 | from torch import nn
  6 | from torch.nn import functional as F
  7 | 
  8 | import commons
  9 | import modules
 10 | from modules import LayerNorm
 11 |    
 12 | 
 13 | class Encoder(nn.Module):
 14 |   def __init__(self, hidden_channels, filter_channels, n_heads, n_layers, kernel_size=1, p_dropout=0., window_size=4, **kwargs):
 15 |     super().__init__()
 16 |     self.hidden_channels = hidden_channels
 17 |     self.filter_channels = filter_channels
 18 |     self.n_heads = n_heads
 19 |     self.n_layers = n_layers
 20 |     self.kernel_size = kernel_size
 21 |     self.p_dropout = p_dropout
 22 |     self.window_size = window_size
 23 | 
 24 |     self.drop = nn.Dropout(p_dropout)
 25 |     self.attn_layers = nn.ModuleList()
 26 |     self.norm_layers_1 = nn.ModuleList()
 27 |     self.ffn_layers = nn.ModuleList()
 28 |     self.norm_layers_2 = nn.ModuleList()
 29 |     for i in range(self.n_layers):
 30 |       self.attn_layers.append(MultiHeadAttention(hidden_channels, hidden_channels, n_heads, p_dropout=p_dropout, window_size=window_size))
 31 |       self.norm_layers_1.append(LayerNorm(hidden_channels))
 32 |       self.ffn_layers.append(FFN(hidden_channels, hidden_channels, filter_channels, kernel_size, p_dropout=p_dropout))
 33 |       self.norm_layers_2.append(LayerNorm(hidden_channels))
 34 | 
 35 |   def forward(self, x, x_mask):
 36 |     attn_mask = x_mask.unsqueeze(2) * x_mask.unsqueeze(-1)
 37 |     x = x * x_mask
 38 |     for i in range(self.n_layers):
 39 |       y = self.attn_layers[i](x, x, attn_mask)
 40 |       y = self.drop(y)
 41 |       x = self.norm_layers_1[i](x + y)
 42 | 
 43 |       y = self.ffn_layers[i](x, x_mask)
 44 |       y = self.drop(y)
 45 |       x = self.norm_layers_2[i](x + y)
 46 |     x = x * x_mask
 47 |     return x
 48 | 
 49 | 
 50 | class Decoder(nn.Module):
 51 |   def __init__(self, hidden_channels, filter_channels, n_heads, n_layers, kernel_size=1, p_dropout=0., proximal_bias=False, proximal_init=True, **kwargs):
 52 |     super().__init__()
 53 |     self.hidden_channels = hidden_channels
 54 |     self.filter_channels = filter_channels
 55 |     self.n_heads = n_heads
 56 |     self.n_layers = n_layers
 57 |     self.kernel_size = kernel_size
 58 |     self.p_dropout = p_dropout
 59 |     self.proximal_bias = proximal_bias
 60 |     self.proximal_init = proximal_init
 61 | 
 62 |     self.drop = nn.Dropout(p_dropout)
 63 |     self.self_attn_layers = nn.ModuleList()
 64 |     self.norm_layers_0 = nn.ModuleList()
 65 |     self.encdec_attn_layers = nn.ModuleList()
 66 |     self.norm_layers_1 = nn.ModuleList()
 67 |     self.ffn_layers = nn.ModuleList()
 68 |     self.norm_layers_2 = nn.ModuleList()
 69 |     for i in range(self.n_layers):
 70 |       self.self_attn_layers.append(MultiHeadAttention(hidden_channels, hidden_channels, n_heads, p_dropout=p_dropout, proximal_bias=proximal_bias, proximal_init=proximal_init))
 71 |       self.norm_layers_0.append(LayerNorm(hidden_channels))
 72 |       self.encdec_attn_layers.append(MultiHeadAttention(hidden_channels, hidden_channels, n_heads, p_dropout=p_dropout))
 73 |       self.norm_layers_1.append(LayerNorm(hidden_channels))
 74 |       self.ffn_layers.append(FFN(hidden_channels, hidden_channels, filter_channels, kernel_size, p_dropout=p_dropout, causal=True))
 75 |       self.norm_layers_2.append(LayerNorm(hidden_channels))
 76 | 
 77 |   def forward(self, x, x_mask, h, h_mask):
 78 |     """
 79 |     x: decoder input
 80 |     h: encoder output
 81 |     """
 82 |     self_attn_mask = commons.subsequent_mask(x_mask.size(2)).to(device=x.device, dtype=x.dtype)
 83 |     encdec_attn_mask = h_mask.unsqueeze(2) * x_mask.unsqueeze(-1)
 84 |     x = x * x_mask
 85 |     for i in range(self.n_layers):
 86 |       y = self.self_attn_layers[i](x, x, self_attn_mask)
 87 |       y = self.drop(y)
 88 |       x = self.norm_layers_0[i](x + y)
 89 | 
 90 |       y = self.encdec_attn_layers[i](x, h, encdec_attn_mask)
 91 |       y = self.drop(y)
 92 |       x = self.norm_layers_1[i](x + y)
 93 |       
 94 |       y = self.ffn_layers[i](x, x_mask)
 95 |       y = self.drop(y)
 96 |       x = self.norm_layers_2[i](x + y)
 97 |     x = x * x_mask
 98 |     return x
 99 | 
100 | 
101 | class MultiHeadAttention(nn.Module):
102 |   def __init__(self, channels, out_channels, n_heads, p_dropout=0., window_size=None, heads_share=True, block_length=None, proximal_bias=False, proximal_init=False):
103 |     super().__init__()
104 |     assert channels % n_heads == 0
105 | 
106 |     self.channels = channels
107 |     self.out_channels = out_channels
108 |     self.n_heads = n_heads
109 |     self.p_dropout = p_dropout
110 |     self.window_size = window_size
111 |     self.heads_share = heads_share
112 |     self.block_length = block_length
113 |     self.proximal_bias = proximal_bias
114 |     self.proximal_init = proximal_init
115 |     self.attn = None
116 | 
117 |     self.k_channels = channels // n_heads
118 |     self.conv_q = nn.Conv1d(channels, channels, 1)
119 |     self.conv_k = nn.Conv1d(channels, channels, 1)
120 |     self.conv_v = nn.Conv1d(channels, channels, 1)
121 |     self.conv_o = nn.Conv1d(channels, out_channels, 1)
122 |     self.drop = nn.Dropout(p_dropout)
123 | 
124 |     if window_size is not None:
125 |       n_heads_rel = 1 if heads_share else n_heads
126 |       rel_stddev = self.k_channels**-0.5
127 |       self.emb_rel_k = nn.Parameter(torch.randn(n_heads_rel, window_size * 2 + 1, self.k_channels) * rel_stddev)
128 |       self.emb_rel_v = nn.Parameter(torch.randn(n_heads_rel, window_size * 2 + 1, self.k_channels) * rel_stddev)
129 | 
130 |     nn.init.xavier_uniform_(self.conv_q.weight)
131 |     nn.init.xavier_uniform_(self.conv_k.weight)
132 |     nn.init.xavier_uniform_(self.conv_v.weight)
133 |     if proximal_init:
134 |       with torch.no_grad():
135 |         self.conv_k.weight.copy_(self.conv_q.weight)
136 |         self.conv_k.bias.copy_(self.conv_q.bias)
137 |       
138 |   def forward(self, x, c, attn_mask=None):
139 |     q = self.conv_q(x)
140 |     k = self.conv_k(c)
141 |     v = self.conv_v(c)
142 |     
143 |     x, self.attn = self.attention(q, k, v, mask=attn_mask)
144 | 
145 |     x = self.conv_o(x)
146 |     return x
147 | 
148 |   def attention(self, query, key, value, mask=None):
149 |     # reshape [b, d, t] -> [b, n_h, t, d_k]
150 |     b, d, t_s, t_t = (*key.size(), query.size(2))
151 |     query = query.view(b, self.n_heads, self.k_channels, t_t).transpose(2, 3)
152 |     key = key.view(b, self.n_heads, self.k_channels, t_s).transpose(2, 3)
153 |     value = value.view(b, self.n_heads, self.k_channels, t_s).transpose(2, 3)
154 | 
155 |     scores = torch.matmul(query / math.sqrt(self.k_channels), key.transpose(-2, -1))
156 |     if self.window_size is not None:
157 |       assert t_s == t_t, "Relative attention is only available for self-attention."
158 |       key_relative_embeddings = self._get_relative_embeddings(self.emb_rel_k, t_s)
159 |       rel_logits = self._matmul_with_relative_keys(query /math.sqrt(self.k_channels), key_relative_embeddings)
160 |       scores_local = self._relative_position_to_absolute_position(rel_logits)
161 |       scores = scores + scores_local
162 |     if self.proximal_bias:
163 |       assert t_s == t_t, "Proximal bias is only available for self-attention."
164 |       scores = scores + self._attention_bias_proximal(t_s).to(device=scores.device, dtype=scores.dtype)
165 |     if mask is not None:
166 |       scores = scores.masked_fill(mask == 0, -1e4)
167 |       if self.block_length is not None:
168 |         assert t_s == t_t, "Local attention is only available for self-attention."
169 |         block_mask = torch.ones_like(scores).triu(-self.block_length).tril(self.block_length)
170 |         scores = scores.masked_fill(block_mask == 0, -1e4)
171 |     p_attn = F.softmax(scores, dim=-1) # [b, n_h, t_t, t_s]
172 |     p_attn = self.drop(p_attn)
173 |     output = torch.matmul(p_attn, value)
174 |     if self.window_size is not None:
175 |       relative_weights = self._absolute_position_to_relative_position(p_attn)
176 |       value_relative_embeddings = self._get_relative_embeddings(self.emb_rel_v, t_s)
177 |       output = output + self._matmul_with_relative_values(relative_weights, value_relative_embeddings)
178 |     output = output.transpose(2, 3).contiguous().view(b, d, t_t) # [b, n_h, t_t, d_k] -> [b, d, t_t]
179 |     return output, p_attn
180 | 
181 |   def _matmul_with_relative_values(self, x, y):
182 |     """
183 |     x: [b, h, l, m]
184 |     y: [h or 1, m, d]
185 |     ret: [b, h, l, d]
186 |     """
187 |     ret = torch.matmul(x, y.unsqueeze(0))
188 |     return ret
189 | 
190 |   def _matmul_with_relative_keys(self, x, y):
191 |     """
192 |     x: [b, h, l, d]
193 |     y: [h or 1, m, d]
194 |     ret: [b, h, l, m]
195 |     """
196 |     ret = torch.matmul(x, y.unsqueeze(0).transpose(-2, -1))
197 |     return ret
198 | 
199 |   def _get_relative_embeddings(self, relative_embeddings, length):
200 |     max_relative_position = 2 * self.window_size + 1
201 |     # Pad first before slice to avoid using cond ops.
202 |     pad_length = max(length - (self.window_size + 1), 0)
203 |     slice_start_position = max((self.window_size + 1) - length, 0)
204 |     slice_end_position = slice_start_position + 2 * length - 1
205 |     if pad_length > 0:
206 |       padded_relative_embeddings = F.pad(
207 |           relative_embeddings,
208 |           commons.convert_pad_shape([[0, 0], [pad_length, pad_length], [0, 0]]))
209 |     else:
210 |       padded_relative_embeddings = relative_embeddings
211 |     used_relative_embeddings = padded_relative_embeddings[:,slice_start_position:slice_end_position]
212 |     return used_relative_embeddings
213 | 
214 |   def _relative_position_to_absolute_position(self, x):
215 |     """
216 |     x: [b, h, l, 2*l-1]
217 |     ret: [b, h, l, l]
218 |     """
219 |     batch, heads, length, _ = x.size()
220 |     # Concat columns of pad to shift from relative to absolute indexing.
221 |     x = F.pad(x, commons.convert_pad_shape([[0,0],[0,0],[0,0],[0,1]]))
222 | 
223 |     # Concat extra elements so to add up to shape (len+1, 2*len-1).
224 |     x_flat = x.view([batch, heads, length * 2 * length])
225 |     x_flat = F.pad(x_flat, commons.convert_pad_shape([[0,0],[0,0],[0,length-1]]))
226 | 
227 |     # Reshape and slice out the padded elements.
228 |     x_final = x_flat.view([batch, heads, length+1, 2*length-1])[:, :, :length, length-1:]
229 |     return x_final
230 | 
231 |   def _absolute_position_to_relative_position(self, x):
232 |     """
233 |     x: [b, h, l, l]
234 |     ret: [b, h, l, 2*l-1]
235 |     """
236 |     batch, heads, length, _ = x.size()
237 |     # padd along column
238 |     x = F.pad(x, commons.convert_pad_shape([[0, 0], [0, 0], [0, 0], [0, length-1]]))
239 |     x_flat = x.view([batch, heads, length**2 + length*(length -1)])
240 |     # add 0's in the beginning that will skew the elements after reshape
241 |     x_flat = F.pad(x_flat, commons.convert_pad_shape([[0, 0], [0, 0], [length, 0]]))
242 |     x_final = x_flat.view([batch, heads, length, 2*length])[:,:,:,1:]
243 |     return x_final
244 | 
245 |   def _attention_bias_proximal(self, length):
246 |     """Bias for self-attention to encourage attention to close positions.
247 |     Args:
248 |       length: an integer scalar.
249 |     Returns:
250 |       a Tensor with shape [1, 1, length, length]
251 |     """
252 |     r = torch.arange(length, dtype=torch.float32)
253 |     diff = torch.unsqueeze(r, 0) - torch.unsqueeze(r, 1)
254 |     return torch.unsqueeze(torch.unsqueeze(-torch.log1p(torch.abs(diff)), 0), 0)
255 | 
256 | 
257 | class FFN(nn.Module):
258 |   def __init__(self, in_channels, out_channels, filter_channels, kernel_size, p_dropout=0., activation=None, causal=False):
259 |     super().__init__()
260 |     self.in_channels = in_channels
261 |     self.out_channels = out_channels
262 |     self.filter_channels = filter_channels
263 |     self.kernel_size = kernel_size
264 |     self.p_dropout = p_dropout
265 |     self.activation = activation
266 |     self.causal = causal
267 | 
268 |     if causal:
269 |       self.padding = self._causal_padding
270 |     else:
271 |       self.padding = self._same_padding
272 | 
273 |     self.conv_1 = nn.Conv1d(in_channels, filter_channels, kernel_size)
274 |     self.conv_2 = nn.Conv1d(filter_channels, out_channels, kernel_size)
275 |     self.drop = nn.Dropout(p_dropout)
276 | 
277 |   def forward(self, x, x_mask):
278 |     x = self.conv_1(self.padding(x * x_mask))
279 |     if self.activation == "gelu":
280 |       x = x * torch.sigmoid(1.702 * x)
281 |     else:
282 |       x = torch.relu(x)
283 |     x = self.drop(x)
284 |     x = self.conv_2(self.padding(x * x_mask))
285 |     return x * x_mask
286 |   
287 |   def _causal_padding(self, x):
288 |     if self.kernel_size == 1:
289 |       return x
290 |     pad_l = self.kernel_size - 1
291 |     pad_r = 0
292 |     padding = [[0, 0], [0, 0], [pad_l, pad_r]]
293 |     x = F.pad(x, commons.convert_pad_shape(padding))
294 |     return x
295 | 
296 |   def _same_padding(self, x):
297 |     if self.kernel_size == 1:
298 |       return x
299 |     pad_l = (self.kernel_size - 1) // 2
300 |     pad_r = self.kernel_size // 2
301 |     padding = [[0, 0], [0, 0], [pad_l, pad_r]]
302 |     x = F.pad(x, commons.convert_pad_shape(padding))
303 |     return x
304 | 


--------------------------------------------------------------------------------
/commons.py:
--------------------------------------------------------------------------------
  1 | import math
  2 | import numpy as np
  3 | import torch
  4 | from torch import nn
  5 | from torch.nn import functional as F
  6 | 
  7 | 
  8 | def init_weights(m, mean=0.0, std=0.01):
  9 |   classname = m.__class__.__name__
 10 |   if classname.find("Conv") != -1:
 11 |     m.weight.data.normal_(mean, std)
 12 | 
 13 | 
 14 | def get_padding(kernel_size, dilation=1):
 15 |   return int((kernel_size*dilation - dilation)/2)
 16 | 
 17 | 
 18 | def convert_pad_shape(pad_shape):
 19 |   l = pad_shape[::-1]
 20 |   pad_shape = [item for sublist in l for item in sublist]
 21 |   return pad_shape
 22 | 
 23 | 
 24 | def intersperse(lst, item):
 25 |   result = [item] * (len(lst) * 2 + 1)
 26 |   result[1::2] = lst
 27 |   return result
 28 | 
 29 | 
 30 | def kl_divergence(m_p, logs_p, m_q, logs_q):
 31 |   """KL(P||Q)"""
 32 |   kl = (logs_q - logs_p) - 0.5
 33 |   kl += 0.5 * (torch.exp(2. * logs_p) + ((m_p - m_q)**2)) * torch.exp(-2. * logs_q)
 34 |   return kl
 35 | 
 36 | 
 37 | def rand_gumbel(shape):
 38 |   """Sample from the Gumbel distribution, protect from overflows."""
 39 |   uniform_samples = torch.rand(shape) * 0.99998 + 0.00001
 40 |   return -torch.log(-torch.log(uniform_samples))
 41 | 
 42 | 
 43 | def rand_gumbel_like(x):
 44 |   g = rand_gumbel(x.size()).to(dtype=x.dtype, device=x.device)
 45 |   return g
 46 | 
 47 | 
 48 | def slice_segments(x, ids_str, segment_size=4):
 49 |   ret = torch.zeros_like(x[:, :, :segment_size])
 50 |   for i in range(x.size(0)):
 51 |     idx_str = ids_str[i]
 52 |     idx_end = idx_str + segment_size
 53 |     ret[i] = x[i, :, idx_str:idx_end]
 54 |   return ret
 55 | 
 56 | 
 57 | def rand_slice_segments(x, x_lengths=None, segment_size=4):
 58 |   b, d, t = x.size()
 59 |   if x_lengths is None:
 60 |     x_lengths = t
 61 |   ids_str_max = x_lengths - segment_size + 1
 62 |   ids_str = (torch.rand([b]).to(device=x.device) * ids_str_max).to(dtype=torch.long)
 63 |   ret = slice_segments(x, ids_str, segment_size)
 64 |   return ret, ids_str
 65 | 
 66 | 
 67 | def get_timing_signal_1d(
 68 |     length, channels, min_timescale=1.0, max_timescale=1.0e4):
 69 |   position = torch.arange(length, dtype=torch.float)
 70 |   num_timescales = channels // 2
 71 |   log_timescale_increment = (
 72 |       math.log(float(max_timescale) / float(min_timescale)) /
 73 |       (num_timescales - 1))
 74 |   inv_timescales = min_timescale * torch.exp(
 75 |       torch.arange(num_timescales, dtype=torch.float) * -log_timescale_increment)
 76 |   scaled_time = position.unsqueeze(0) * inv_timescales.unsqueeze(1)
 77 |   signal = torch.cat([torch.sin(scaled_time), torch.cos(scaled_time)], 0)
 78 |   signal = F.pad(signal, [0, 0, 0, channels % 2])
 79 |   signal = signal.view(1, channels, length)
 80 |   return signal
 81 | 
 82 | 
 83 | def add_timing_signal_1d(x, min_timescale=1.0, max_timescale=1.0e4):
 84 |   b, channels, length = x.size()
 85 |   signal = get_timing_signal_1d(length, channels, min_timescale, max_timescale)
 86 |   return x + signal.to(dtype=x.dtype, device=x.device)
 87 | 
 88 | 
 89 | def cat_timing_signal_1d(x, min_timescale=1.0, max_timescale=1.0e4, axis=1):
 90 |   b, channels, length = x.size()
 91 |   signal = get_timing_signal_1d(length, channels, min_timescale, max_timescale)
 92 |   return torch.cat([x, signal.to(dtype=x.dtype, device=x.device)], axis)
 93 | 
 94 | 
 95 | def subsequent_mask(length):
 96 |   mask = torch.tril(torch.ones(length, length)).unsqueeze(0).unsqueeze(0)
 97 |   return mask
 98 | 
 99 | 
100 | @torch.jit.script
101 | def fused_add_tanh_sigmoid_multiply(input_a, input_b, n_channels):
102 |   n_channels_int = n_channels[0]
103 |   in_act = input_a + input_b
104 |   t_act = torch.tanh(in_act[:, :n_channels_int, :])
105 |   s_act = torch.sigmoid(in_act[:, n_channels_int:, :])
106 |   acts = t_act * s_act
107 |   return acts
108 | 
109 | 
110 | def convert_pad_shape(pad_shape):
111 |   l = pad_shape[::-1]
112 |   pad_shape = [item for sublist in l for item in sublist]
113 |   return pad_shape
114 | 
115 | 
116 | def shift_1d(x):
117 |   x = F.pad(x, convert_pad_shape([[0, 0], [0, 0], [1, 0]]))[:, :, :-1]
118 |   return x
119 | 
120 | 
121 | def sequence_mask(length, max_length=None):
122 |   if max_length is None:
123 |     max_length = length.max()
124 |   x = torch.arange(max_length, dtype=length.dtype, device=length.device)
125 |   return x.unsqueeze(0) < length.unsqueeze(1)
126 | 
127 | 
128 | def generate_path(duration, mask):
129 |   """
130 |   duration: [b, 1, t_x]
131 |   mask: [b, 1, t_y, t_x]
132 |   """
133 |   device = duration.device
134 |   
135 |   b, _, t_y, t_x = mask.shape
136 |   cum_duration = torch.cumsum(duration, -1)
137 |   
138 |   cum_duration_flat = cum_duration.view(b * t_x)
139 |   path = sequence_mask(cum_duration_flat, t_y).to(mask.dtype)
140 |   path = path.view(b, t_x, t_y)
141 |   path = path - F.pad(path, convert_pad_shape([[0, 0], [1, 0], [0, 0]]))[:, :-1]
142 |   path = path.unsqueeze(1).transpose(2,3) * mask
143 |   return path
144 | 
145 | 
146 | def clip_grad_value_(parameters, clip_value, norm_type=2):
147 |   if isinstance(parameters, torch.Tensor):
148 |     parameters = [parameters]
149 |   parameters = list(filter(lambda p: p.grad is not None, parameters))
150 |   norm_type = float(norm_type)
151 |   if clip_value is not None:
152 |     clip_value = float(clip_value)
153 | 
154 |   total_norm = 0
155 |   for p in parameters:
156 |     param_norm = p.grad.data.norm(norm_type)
157 |     total_norm += param_norm.item() ** norm_type
158 |     if clip_value is not None:
159 |       p.grad.data.clamp_(min=-clip_value, max=clip_value)
160 |   total_norm = total_norm ** (1. / norm_type)
161 |   return total_norm
162 | 


--------------------------------------------------------------------------------
/configs/libritts.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "train": {
 3 |     "log_interval": 2000,
 4 |     "eval_interval": 10000,
 5 |     "seed": 1234,
 6 |     "epochs": 10000,
 7 |     "learning_rate": 2e-4,
 8 |     "betas": [0.8, 0.99],
 9 |     "eps": 1e-9,
10 |     "batch_size": 8,
11 |     "fp16_run": false,
12 |     "lr_decay": 0.9999,
13 |     "segment_size": 8192,
14 |     "init_lr_ratio": 1,
15 |     "warmup_epochs": 0,
16 |     "c_mel": 45,
17 |     "c_kl": 1.0
18 |   },
19 |   "data": {
20 |     "data_path":"/home/hcy71/DATA/preprocessed_data/LibriTTS",
21 |     "training_files":"filelists/libritts_train.txt",
22 |     "validation_files":"filelists/libritts_val.txt",
23 |     "text_cleaners":["english_cleaners2"],
24 |     "max_wav_value": 32768.0,
25 |     "sampling_rate": 22050,
26 |     "filter_length": 1024,
27 |     "hop_length": 256,
28 |     "win_length": 1024,
29 |     "n_mel_channels": 80,
30 |     "mel_fmin": 0.0,
31 |     "mel_fmax": null,
32 |     "add_blank": true,
33 |     "n_speakers": 1147,
34 |     "cleaned_text": true
35 |   },
36 |   "model": {
37 |     "inter_channels": 192,
38 |     "hidden_channels": 192,
39 |     "filter_channels": 768,
40 |     "n_heads": 2,
41 |     "n_layers": 6,
42 |     "kernel_size": 3,
43 |     "p_dropout": 0.1,
44 |     "resblock": "1",
45 |     "resblock_kernel_sizes": [3,7,11],
46 |     "resblock_dilation_sizes": [[1,3,5], [1,3,5], [1,3,5]],
47 |     "upsample_rates": [8,8,2,2],
48 |     "upsample_initial_channel": 512,
49 |     "upsample_kernel_sizes": [16,16,4,4],
50 |     "n_layers_q": 3,
51 |     "use_spectral_norm": false,
52 |     "gin_channels": 256
53 |   }
54 | }
55 | 


--------------------------------------------------------------------------------
/configs/vctk_base.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "train": {
 3 |     "log_interval": 200,
 4 |     "eval_interval": 1000,
 5 |     "seed": 1234,
 6 |     "epochs": 10000,
 7 |     "learning_rate": 2e-4,
 8 |     "betas": [0.8, 0.99],
 9 |     "eps": 1e-9,
10 |     "batch_size": 64,
11 |     "fp16_run": true,
12 |     "lr_decay": 0.999875,
13 |     "segment_size": 8192,
14 |     "init_lr_ratio": 1,
15 |     "warmup_epochs": 0,
16 |     "c_mel": 45,
17 |     "c_kl": 1.0
18 |   },
19 |   "data": {
20 |     "training_files":"filelists/vctk_audio_sid_text_train_filelist.txt.cleaned",
21 |     "validation_files":"filelists/vctk_audio_sid_text_val_filelist.txt.cleaned",
22 |     "text_cleaners":["english_cleaners2"],
23 |     "max_wav_value": 32768.0,
24 |     "sampling_rate": 22050,
25 |     "filter_length": 1024,
26 |     "hop_length": 256,
27 |     "win_length": 1024,
28 |     "n_mel_channels": 80,
29 |     "mel_fmin": 0.0,
30 |     "mel_fmax": null,
31 |     "add_blank": true,
32 |     "n_speakers": 109,
33 |     "cleaned_text": true
34 |   },
35 |   "model": {
36 |     "inter_channels": 192,
37 |     "hidden_channels": 192,
38 |     "filter_channels": 768,
39 |     "n_heads": 2,
40 |     "n_layers": 6,
41 |     "kernel_size": 3,
42 |     "p_dropout": 0.1,
43 |     "resblock": "1",
44 |     "resblock_kernel_sizes": [3,7,11],
45 |     "resblock_dilation_sizes": [[1,3,5], [1,3,5], [1,3,5]],
46 |     "upsample_rates": [8,8,2,2],
47 |     "upsample_initial_channel": 512,
48 |     "upsample_kernel_sizes": [16,16,4,4],
49 |     "n_layers_q": 3,
50 |     "use_spectral_norm": false,
51 |     "gin_channels": 256
52 |   }
53 | }
54 | 


--------------------------------------------------------------------------------
/data_utils.py:
--------------------------------------------------------------------------------
  1 | import time
  2 | import os
  3 | import random
  4 | import numpy as np
  5 | import torch
  6 | import torch.utils.data
  7 | import librosa
  8 | 
  9 | import commons 
 10 | from mel_processing import spectrogram_torch
 11 | from utils import load_wav_to_torch, load_filepaths_and_text
 12 | from text import text_to_sequence
 13 | 
 14 | 
 15 | class TextAudioLoader(torch.utils.data.Dataset):
 16 |     """
 17 |         1) loads audio, text pairs
 18 |         2) normalizes text and converts them to sequences of integers
 19 |         3) computes spectrograms from audio files.
 20 |     """
 21 |     def __init__(self, audiopaths_and_text, hparams):
 22 |         self.audiopaths_and_text = load_filepaths_and_text(audiopaths_and_text)
 23 |         self.text_cleaners  = hparams.text_cleaners
 24 |         self.max_wav_value  = hparams.max_wav_value
 25 |         self.sampling_rate  = hparams.sampling_rate
 26 |         self.filter_length  = hparams.filter_length 
 27 |         self.hop_length     = hparams.hop_length 
 28 |         self.win_length     = hparams.win_length
 29 |         self.sampling_rate  = hparams.sampling_rate 
 30 | 
 31 |         self.cleaned_text = getattr(hparams, "cleaned_text", False)
 32 | 
 33 |         self.add_blank = hparams.add_blank
 34 |         self.min_text_len = getattr(hparams, "min_text_len", 1)
 35 |         self.max_text_len = getattr(hparams, "max_text_len", 190)
 36 |         random.seed(1234)
 37 |         random.shuffle(self.audiopaths_and_text)
 38 |         self._filter()
 39 | 
 40 | 
 41 |     def _filter(self):
 42 |         """
 43 |         Filter text & store spec lengths
 44 |         """
 45 |         # Store spectrogram lengths for Bucketing
 46 |         # wav_length ~= file_size / (wav_channels * Bytes per dim) = file_size / (1 * 2)
 47 |         # spec_length = wav_length // hop_length
 48 | 
 49 |         audiopaths_and_text_new = []
 50 |         lengths = []
 51 |         for audiopath, text in self.audiopaths_and_text:
 52 |             if self.min_text_len <= len(text) and len(text) <= self.max_text_len:
 53 |                 audiopaths_and_text_new.append([audiopath, text])
 54 |                 lengths.append(os.path.getsize(audiopath) // (2 * self.hop_length))
 55 |         self.audiopaths_and_text = audiopaths_and_text_new
 56 |         self.lengths = lengths
 57 | 
 58 |     def get_audio_text_pair(self, audiopath_and_text):
 59 |         # separate filename and text
 60 |         audiopath, text = audiopath_and_text[0], audiopath_and_text[1]
 61 |         text = self.get_text(text)
 62 |         spec, wav = self.get_audio(audiopath)
 63 |         return (text, spec, wav)
 64 | 
 65 |     def get_audio(self, filename):
 66 |         audio, sampling_rate = load_wav_to_torch(filename)
 67 |         if sampling_rate != self.sampling_rate:
 68 |             raise ValueError("{} {} SR doesn't match target {} SR".format(
 69 |                 sampling_rate, self.sampling_rate))
 70 |         audio_norm = audio / self.max_wav_value
 71 |         audio_norm = audio_norm.unsqueeze(0)
 72 |         spec_filename = filename.replace(".wav", ".spec.pt")
 73 |         if os.path.exists(spec_filename):
 74 |             spec = torch.load(spec_filename)
 75 |         else:
 76 |             spec = spectrogram_torch(audio_norm, self.filter_length,
 77 |                 self.sampling_rate, self.hop_length, self.win_length,
 78 |                 center=False)
 79 |             spec = torch.squeeze(spec, 0)
 80 |             torch.save(spec, spec_filename)
 81 |         return spec, audio_norm
 82 | 
 83 |     def get_text(self, text):
 84 |         if self.cleaned_text:
 85 |             text_norm = cleaned_text_to_sequence(text)
 86 |         else:
 87 |             text_norm = text_to_sequence(text, self.text_cleaners)
 88 |         if self.add_blank:
 89 |             text_norm = commons.intersperse(text_norm, 0)
 90 |         text_norm = torch.LongTensor(text_norm)
 91 |         return text_norm
 92 | 
 93 |     def __getitem__(self, index):
 94 |         return self.get_audio_text_pair(self.audiopaths_and_text[index])
 95 | 
 96 |     def __len__(self):
 97 |         return len(self.audiopaths_and_text)
 98 | 
 99 | 
100 | class TextAudioCollate():
101 |     """ Zero-pads model inputs and targets
102 |     """
103 |     def __init__(self, return_ids=False):
104 |         self.return_ids = return_ids
105 | 
106 |     def __call__(self, batch):
107 |         """Collate's training batch from normalized text and aduio
108 |         PARAMS
109 |         ------
110 |         batch: [text_normalized, spec_normalized, wav_normalized]
111 |         """
112 |         # Right zero-pad all one-hot text sequences to max input length
113 |         _, ids_sorted_decreasing = torch.sort(
114 |             torch.LongTensor([x[1].size(1) for x in batch]),
115 |             dim=0, descending=True)
116 | 
117 |         max_text_len = max([len(x[0]) for x in batch])
118 |         max_spec_len = max([x[1].size(1) for x in batch])
119 |         max_wav_len = max([x[2].size(1) for x in batch])
120 | 
121 |         text_lengths = torch.LongTensor(len(batch))
122 |         spec_lengths = torch.LongTensor(len(batch))
123 |         wav_lengths = torch.LongTensor(len(batch))
124 | 
125 |         text_padded = torch.LongTensor(len(batch), max_text_len)
126 |         spec_padded = torch.FloatTensor(len(batch), batch[0][1].size(0), max_spec_len)
127 |         wav_padded = torch.FloatTensor(len(batch), 1, max_wav_len)
128 |         text_padded.zero_()
129 |         spec_padded.zero_()
130 |         wav_padded.zero_()
131 |         for i in range(len(ids_sorted_decreasing)):
132 |             row = batch[ids_sorted_decreasing[i]]
133 | 
134 |             text = row[0]
135 |             text_padded[i, :text.size(0)] = text
136 |             text_lengths[i] = text.size(0)
137 | 
138 |             spec = row[1]
139 |             spec_padded[i, :, :spec.size(1)] = spec
140 |             spec_lengths[i] = spec.size(1)
141 | 
142 |             wav = row[2]
143 |             wav_padded[i, :, :wav.size(1)] = wav
144 |             wav_lengths[i] = wav.size(1)
145 | 
146 |         if self.return_ids:
147 |             return text_padded, text_lengths, spec_padded, spec_lengths, wav_padded, wav_lengths, ids_sorted_decreasing
148 |         return text_padded, text_lengths, spec_padded, spec_lengths, wav_padded, wav_lengths
149 | 
150 | 
151 | """Multi speaker version"""
152 | class TextAudioSpeakerLoader(torch.utils.data.Dataset):
153 |     """
154 |         1) loads audio, speaker_id, text pairs
155 |         2) normalizes text and converts them to sequences of integers
156 |         3) computes spectrograms from audio files.
157 |     """
158 |     def __init__(self, audiopaths_sid_text, hparams):
159 |         self.audiopaths_sid_text = load_filepaths_and_text(audiopaths_sid_text)
160 |         self.text_cleaners = hparams.text_cleaners
161 |         self.max_wav_value = hparams.max_wav_value
162 |         self.sampling_rate = hparams.sampling_rate
163 |         self.filter_length  = hparams.filter_length
164 |         self.hop_length     = hparams.hop_length
165 |         self.win_length     = hparams.win_length
166 |         self.sampling_rate  = hparams.sampling_rate
167 |         self.data_path = hparams.data_path
168 |         sids = []
169 |         for info in self.audiopaths_sid_text:
170 |             sids.append(info[2])
171 |         self.sid_dict = self.create_speaker_table(sids)
172 | 
173 |         self.cleaned_text = getattr(hparams, "cleaned_text", False)
174 | 
175 |         self.add_blank = hparams.add_blank
176 |         self.min_text_len = getattr(hparams, "min_text_len", 1)
177 |         self.max_text_len = getattr(hparams, "max_text_len", 190)
178 |         
179 | 
180 |         random.seed(1234)
181 |         random.shuffle(self.audiopaths_sid_text)
182 |         self._filter()
183 | 
184 |     def _filter(self):
185 |         """
186 |         Filter text & store spec lengths
187 |         """
188 |         # Store spectrogram lengths for Bucketing
189 |         # wav_length ~= file_size / (wav_channels * Bytes per dim) = file_size / (1 * 2)
190 |         # spec_length = wav_length // hop_length
191 | 
192 |         audiopaths_sid_text_new = []
193 |         lengths = []
194 |         for audiopath, text, sid in self.audiopaths_sid_text:
195 |             audiopath = os.path.join(self.data_path, 'wav{}'.format(self.sampling_rate//1000), sid, '{}.wav'.format(audiopath))
196 |             if self.min_text_len <= len(text) and len(text) <= self.max_text_len:
197 |                 audiopaths_sid_text_new.append([audiopath, text, sid])
198 |                 lengths.append(os.path.getsize(audiopath) // (2 * self.hop_length))
199 |         self.audiopaths_sid_text = audiopaths_sid_text_new
200 |         self.lengths = lengths
201 | 
202 |     def get_audio_text_speaker_pair(self, audiopath_sid_text):
203 |         # separate filename, speaker_id and text
204 |         audiopath, text, sid = audiopath_sid_text[0], audiopath_sid_text[1], audiopath_sid_text[2]
205 | 
206 |         # raise ValueError('sotp')
207 |         text = self.get_text(text)
208 |         spec, wav = self.get_audio(audiopath, sid)
209 |         sid = self.get_sid(sid)
210 |         return (text, spec, wav, sid)
211 | 
212 |     def get_audio(self, filename, sid):
213 |         # filename = os.path.join(self.data_path, 'wav{}'.format(self.sampling_rate//1000), sid, '{}.wav'.format(filename))
214 |         filename = os.path.join(self.data_path, 'wav{}'.format(self.sampling_rate//1000), sid, filename)
215 |         audio, _ = librosa.load(filename, sr=None)
216 |         audio = torch.from_numpy(audio)
217 |         audio = audio.unsqueeze(0)
218 |         spec_filename = filename.replace(".wav", ".spec.pt")
219 |         if os.path.exists(spec_filename):
220 |             spec = torch.load(spec_filename)
221 |         else:
222 |             spec = spectrogram_torch(audio, self.filter_length,
223 |                 self.sampling_rate, self.hop_length, self.win_length,
224 |                 center=False)
225 |             spec = torch.squeeze(spec, 0)
226 |             torch.save(spec, spec_filename)
227 |         return spec, audio
228 |     
229 |     def get_text(self, text):
230 |         # if self.cleaned_text:
231 |         #     text_norm = cleaned_text_to_sequence(text)
232 |         # else:
233 |         #     text_norm = text_to_sequence(text, [])
234 |         text_norm = text_to_sequence(text, [])
235 |         if self.add_blank:
236 |             text_norm = commons.intersperse(text_norm, 0)
237 |         text_norm = torch.LongTensor(text_norm)
238 |         return text_norm
239 | 
240 |     def get_sid(self, sid):
241 |         sid = self.sid_dict[sid]
242 |         sid = torch.LongTensor([int(sid)])
243 |         return sid
244 | 
245 |     def create_speaker_table(self, sids):
246 |         speaker_ids = np.sort(np.unique(sids))
247 |         d = {speaker_ids[i]: i for i in range(len(speaker_ids))}
248 |         return d
249 | 
250 |     def __getitem__(self, index):
251 |         return self.get_audio_text_speaker_pair(self.audiopaths_sid_text[index])
252 | 
253 |     def __len__(self):
254 |         return len(self.audiopaths_sid_text)
255 | 
256 | 
257 | class TextAudioSpeakerCollate():
258 |     """ Zero-pads model inputs and targets
259 |     """
260 |     def __init__(self, return_ids=False):
261 |         self.return_ids = return_ids
262 | 
263 |     def __call__(self, batch):
264 |         """Collate's training batch from normalized text, audio and speaker identities
265 |         PARAMS
266 |         ------
267 |         batch: [text_normalized, spec_normalized, wav_normalized, sid]
268 |         """
269 |         # Right zero-pad all one-hot text sequences to max input length
270 |         _, ids_sorted_decreasing = torch.sort(
271 |             torch.LongTensor([x[1].size(1) for x in batch]),
272 |             dim=0, descending=True)
273 | 
274 |         max_text_len = max([len(x[0]) for x in batch])
275 |         max_spec_len = max([x[1].size(1) for x in batch])
276 |         max_wav_len = max([x[2].size(1) for x in batch])
277 | 
278 |         text_lengths = torch.LongTensor(len(batch))
279 |         spec_lengths = torch.LongTensor(len(batch))
280 |         wav_lengths = torch.LongTensor(len(batch))
281 |         sid = torch.LongTensor(len(batch))
282 | 
283 |         text_padded = torch.LongTensor(len(batch), max_text_len)
284 |         spec_padded = torch.FloatTensor(len(batch), batch[0][1].size(0), max_spec_len)
285 |         wav_padded = torch.FloatTensor(len(batch), 1, max_wav_len)
286 |         text_padded.zero_()
287 |         spec_padded.zero_()
288 |         wav_padded.zero_()
289 |         for i in range(len(ids_sorted_decreasing)):
290 |             row = batch[ids_sorted_decreasing[i]]
291 | 
292 |             text = row[0]
293 |             text_padded[i, :text.size(0)] = text
294 |             text_lengths[i] = text.size(0)
295 | 
296 |             spec = row[1]
297 |             spec_padded[i, :, :spec.size(1)] = spec
298 |             spec_lengths[i] = spec.size(1)
299 | 
300 |             wav = row[2]
301 |             wav_padded[i, :, :wav.size(1)] = wav
302 |             wav_lengths[i] = wav.size(1)
303 | 
304 |             sid[i] = row[3]
305 | 
306 |         if self.return_ids:
307 |             return text_padded, text_lengths, spec_padded, spec_lengths, wav_padded, wav_lengths, sid, ids_sorted_decreasing
308 |         return text_padded, text_lengths, spec_padded, spec_lengths, wav_padded, wav_lengths, sid
309 | 
310 | 
311 | class DistributedBucketSampler(torch.utils.data.distributed.DistributedSampler):
312 |     """
313 |     Maintain similar input lengths in a batch.
314 |     Length groups are specified by boundaries.
315 |     Ex) boundaries = [b1, b2, b3] -> any batch is included either {x | b1 < length(x) <=b2} or {x | b2 < length(x) <= b3}.
316 |   
317 |     It removes samples which are not included in the boundaries.
318 |     Ex) boundaries = [b1, b2, b3] -> any x s.t. length(x) <= b1 or length(x) > b3 are discarded.
319 |     """
320 |     def __init__(self, dataset, batch_size, boundaries, num_replicas=None, rank=None, shuffle=True):
321 |         super().__init__(dataset, num_replicas=num_replicas, rank=rank, shuffle=shuffle)
322 |         self.lengths = dataset.lengths
323 |         self.batch_size = batch_size
324 |         self.boundaries = boundaries
325 |   
326 |         self.buckets, self.num_samples_per_bucket = self._create_buckets()
327 |         self.total_size = sum(self.num_samples_per_bucket)
328 |         self.num_samples = self.total_size // self.num_replicas
329 |   
330 |     def _create_buckets(self):
331 |         buckets = [[] for _ in range(len(self.boundaries) - 1)]
332 |         for i in range(len(self.lengths)):
333 |             length = self.lengths[i]
334 |             idx_bucket = self._bisect(length)
335 |             if idx_bucket != -1:
336 |                 buckets[idx_bucket].append(i)
337 |   
338 |         for i in range(len(buckets) - 1, 0, -1):
339 |             if len(buckets[i]) == 0:
340 |                 buckets.pop(i)
341 |                 self.boundaries.pop(i+1)
342 |   
343 |         num_samples_per_bucket = []
344 |         for i in range(len(buckets)):
345 |             len_bucket = len(buckets[i])
346 |             total_batch_size = self.num_replicas * self.batch_size
347 |             rem = (total_batch_size - (len_bucket % total_batch_size)) % total_batch_size
348 |             num_samples_per_bucket.append(len_bucket + rem)
349 |         return buckets, num_samples_per_bucket
350 |   
351 |     def __iter__(self):
352 |       # deterministically shuffle based on epoch
353 |       g = torch.Generator()
354 |       g.manual_seed(self.epoch)
355 |   
356 |       indices = []
357 |       if self.shuffle:
358 |           for bucket in self.buckets:
359 |               indices.append(torch.randperm(len(bucket), generator=g).tolist())
360 |       else:
361 |           for bucket in self.buckets:
362 |               indices.append(list(range(len(bucket))))
363 |   
364 |       batches = []
365 |       for i in range(len(self.buckets)):
366 |           bucket = self.buckets[i]
367 |           len_bucket = len(bucket)
368 |           ids_bucket = indices[i]
369 |           num_samples_bucket = self.num_samples_per_bucket[i]
370 |   
371 |           # add extra samples to make it evenly divisible
372 |           rem = num_samples_bucket - len_bucket
373 |           ids_bucket = ids_bucket + ids_bucket * (rem // len_bucket) + ids_bucket[:(rem % len_bucket)]
374 |   
375 |           # subsample
376 |           ids_bucket = ids_bucket[self.rank::self.num_replicas]
377 |   
378 |           # batching
379 |           for j in range(len(ids_bucket) // self.batch_size):
380 |               batch = [bucket[idx] for idx in ids_bucket[j*self.batch_size:(j+1)*self.batch_size]]
381 |               batches.append(batch)
382 |   
383 |       if self.shuffle:
384 |           batch_ids = torch.randperm(len(batches), generator=g).tolist()
385 |           batches = [batches[i] for i in batch_ids]
386 |       self.batches = batches
387 |   
388 |       assert len(self.batches) * self.batch_size == self.num_samples
389 |       return iter(self.batches)
390 |   
391 |     def _bisect(self, x, lo=0, hi=None):
392 |       if hi is None:
393 |           hi = len(self.boundaries) - 1
394 |   
395 |       if hi > lo:
396 |           mid = (hi + lo) // 2
397 |           if self.boundaries[mid] < x and x <= self.boundaries[mid+1]:
398 |               return mid
399 |           elif x <= self.boundaries[mid]:
400 |               return self._bisect(x, lo, mid)
401 |           else:
402 |               return self._bisect(x, mid + 1, hi)
403 |       else:
404 |           return -1
405 | 
406 |     def __len__(self):
407 |         return self.num_samples // self.batch_size
408 | 


--------------------------------------------------------------------------------
/img/Overview.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/hcy71o/TransferTTS/bdee6576a034e02da2ac049fb65c198be89b375a/img/Overview.jpg


--------------------------------------------------------------------------------
/inference.py:
--------------------------------------------------------------------------------
 1 | import os
 2 | import json
 3 | import math
 4 | import torch
 5 | from torch import nn
 6 | from torch.nn import functional as F
 7 | import librosa
 8 | import argparse
 9 | 
10 | import commons
11 | import utils
12 | from data_utils import TextAudioLoader, TextAudioCollate, TextAudioSpeakerLoader, TextAudioSpeakerCollate
13 | from models import SynthesizerTrn
14 | from text.symbols import symbols
15 | from text import text_to_sequence
16 | from mel_processing import spectrogram_torch, spec_to_mel_torch
17 | 
18 | from scipy.io.wavfile import write
19 | 
20 | 
21 | def get_text(text, hps):
22 |     text_norm = text_to_sequence(text, [])
23 |     if hps.data.add_blank:
24 |         text_norm = commons.intersperse(text_norm, 0)
25 |     text_norm = torch.LongTensor(text_norm)
26 |     return text_norm
27 | 
28 | def main(args):
29 | 
30 |     hps = utils.get_hparams_from_file(args.config)
31 | 
32 |     net_g = SynthesizerTrn(
33 |         len(symbols),
34 |         hps.data.filter_length // 2 + 1,
35 |         hps.train.segment_size // hps.data.hop_length,
36 |         # n_speakers=hps.data.n_speakers, #* Few-shot
37 |         n_speakers=0, #* Zero-shot
38 |         **hps.model).cuda()
39 |     
40 |     _ = net_g.eval()
41 |     _ = utils.load_checkpoint(args.checkpoint_path, net_g, None)
42 |     
43 |     audio, _ = librosa.load(args.ref_audio, sr=hps.data.sampling_rate)
44 |     audio = torch.from_numpy(audio)
45 |     audio = audio.unsqueeze(0)
46 |     spec = spectrogram_torch(audio, hps.data.filter_length,
47 |     hps.data.sampling_rate, hps.data.hop_length, hps.data.win_length,
48 |     center=False)
49 |     
50 |     spec = torch.squeeze(spec, 0)
51 |     mel = spec_to_mel_torch(
52 |         spec, 
53 |         hps.data.filter_length, 
54 |         hps.data.n_mel_channels, 
55 |         hps.data.sampling_rate,
56 |         hps.data.mel_fmin, 
57 |         hps.data.mel_fmax)
58 |     
59 |     os.makedirs(args.save_path, exist_ok=True)
60 |     
61 |     stn_tst = get_text(args.text, hps)
62 |     
63 |     with torch.no_grad():
64 |         x_tst = stn_tst.cuda().unsqueeze(0)
65 |         x_tst_lengths = torch.LongTensor([stn_tst.size(0)]).cuda()
66 |         sid = torch.LongTensor([4]).cuda()
67 |         mel = mel.cuda()
68 |         audio_gen = net_g.infer(x_tst, x_tst_lengths, mel.unsqueeze(0), sid=None, noise_scale=.667, noise_scale_w=0.8, length_scale=1)[0][0,0].data.cpu().float().numpy()
69 |         output_file = os.path.join(args.save_path, '{}.wav'.format(args.text[:10]))
70 |         ref_file = os.path.join(args.save_path, 'ref_of_{}.wav'.format(args.text[:10]))
71 |        
72 |         write(output_file, hps.data.sampling_rate, audio_gen)
73 |         write(ref_file, hps.data.sampling_rate, audio[0].cpu().float().numpy())
74 |             
75 |         # audio = y_g_hat.squeeze()
76 |         # audio = audio * MAX_WAV_VALUE
77 |         # audio = audio.cpu().numpy().astype('int16')
78 |         
79 | if __name__ == "__main__":
80 |     parser = argparse.ArgumentParser()
81 |     parser.add_argument("--checkpoint_path", type=str,
82 |         default="logs/libritts_base/G_319000.pth")
83 |     parser.add_argument('--config', default='configs/libritts.json')
84 |     parser.add_argument("--save_path", type=str, default='wav_results/')
85 |     parser.add_argument("--ref_audio", type=str, required=True,
86 |         help="path to an reference speech audio sample")
87 |     parser.add_argument("--text", type=str,
88 |         help="raw text to synthesize", default = 'in being comparatively modern.')
89 |     
90 |     args = parser.parse_args()
91 |     
92 |     main(args)


--------------------------------------------------------------------------------
/libritts.py:
--------------------------------------------------------------------------------
 1 | from multiprocessing.sharedctypes import Value
 2 | from text import _clean_text
 3 | import numpy as np
 4 | import librosa
 5 | import os
 6 | from pathlib import Path
 7 | from scipy.io.wavfile import write
 8 | from joblib import Parallel, delayed
 9 | from scipy.interpolate import interp1d
10 | import json
11 | 
12 | 
13 | def write_single(output_folder, wav_fname, resample_rate, top_db=None):
14 |     data, sample_rate = librosa.load(wav_fname, sr=None)
15 |     # trim audio
16 |     if top_db is not None:
17 |         trimmed, _ = librosa.effects.trim(data, top_db=top_db)
18 |     else:
19 |         trimmed = data
20 |     # resample audio
21 |     resampled = librosa.resample(trimmed, sample_rate, resample_rate)
22 |     y = (resampled * 32767.0).astype(np.int16)
23 |     wav_fname = wav_fname.split('/')[-1]
24 |     target_wav_fname = os.path.join(output_folder, wav_fname)
25 | 
26 |     if not os.path.exists(output_folder):
27 |         os.makedirs(output_folder, exist_ok=True)
28 | 
29 |     write(target_wav_fname, resample_rate, y)
30 | 
31 |     return y.shape[0] / float(resample_rate)
32 | 
33 | 
34 | def prepare_align_and_resample(data_dir, sr):
35 |     wav_foder_names = ['train-clean-100', 'train-clean-360']
36 |     wavs = []
37 |     for wav_folder in wav_foder_names:
38 |         wav_folder = os.path.join(data_dir, wav_folder)
39 |         wav_fname_list = [str(f) for f in list(Path(wav_folder).rglob('*.wav'))]
40 | 
41 |         output_wavs_folder_name = 'wav{}'.format(sr//1000)
42 |         output_wavs_folder = os.path.join(data_dir, output_wavs_folder_name)
43 |         if not os.path.exists(output_wavs_folder):
44 |             os.mkdir(output_wavs_folder)
45 | 
46 |         for wav_fname in wav_fname_list:
47 |             _sid = wav_fname.split('/')[-3]
48 |             output_folder = os.path.join(output_wavs_folder, _sid)
49 |             wavs.append((output_folder, wav_fname))
50 | 
51 |     lengths = Parallel(n_jobs=10, verbose=1)(
52 |         delayed(write_single)(wav[0], wav[1], sr) for wav in wavs
53 |     )


--------------------------------------------------------------------------------
/losses.py:
--------------------------------------------------------------------------------
 1 | import torch 
 2 | from torch.nn import functional as F
 3 | 
 4 | import commons
 5 | 
 6 | 
 7 | def feature_loss(fmap_r, fmap_g):
 8 |   loss = 0
 9 |   for dr, dg in zip(fmap_r, fmap_g):
10 |     for rl, gl in zip(dr, dg):
11 |       rl = rl.float().detach()
12 |       gl = gl.float()
13 |       loss += torch.mean(torch.abs(rl - gl))
14 | 
15 |   return loss * 2 
16 | 
17 | 
18 | def discriminator_loss(disc_real_outputs, disc_generated_outputs):
19 |   loss = 0
20 |   r_losses = []
21 |   g_losses = []
22 |   for dr, dg in zip(disc_real_outputs, disc_generated_outputs):
23 |     dr = dr.float()
24 |     dg = dg.float()
25 |     r_loss = torch.mean((1-dr)**2)
26 |     g_loss = torch.mean(dg**2)
27 |     loss += (r_loss + g_loss)
28 |     r_losses.append(r_loss.item())
29 |     g_losses.append(g_loss.item())
30 | 
31 |   return loss, r_losses, g_losses
32 | 
33 | 
34 | def generator_loss(disc_outputs):
35 |   loss = 0
36 |   gen_losses = []
37 |   for dg in disc_outputs:
38 |     dg = dg.float()
39 |     l = torch.mean((1-dg)**2)
40 |     gen_losses.append(l)
41 |     loss += l
42 | 
43 |   return loss, gen_losses
44 | 
45 | 
46 | def kl_loss(z_p, logs_q, m_p, logs_p, z_mask):
47 |   """
48 |   z_p, logs_q: [b, h, t_t]
49 |   m_p, logs_p: [b, h, t_t]
50 |   """
51 |   z_p = z_p.float()
52 |   logs_q = logs_q.float()
53 |   m_p = m_p.float()
54 |   logs_p = logs_p.float()
55 |   z_mask = z_mask.float()
56 | 
57 |   kl = logs_p - logs_q - 0.5
58 |   kl += 0.5 * ((z_p - m_p)**2) * torch.exp(-2. * logs_p)
59 |   kl = torch.sum(kl * z_mask)
60 |   l = kl / torch.sum(z_mask)
61 |   return l
62 | 


--------------------------------------------------------------------------------
/mel_processing.py:
--------------------------------------------------------------------------------
  1 | import math
  2 | import os
  3 | import random
  4 | import torch
  5 | from torch import nn
  6 | import torch.nn.functional as F
  7 | import torch.utils.data
  8 | import numpy as np
  9 | import librosa
 10 | import librosa.util as librosa_util
 11 | from librosa.util import normalize, pad_center, tiny
 12 | from scipy.signal import get_window
 13 | from scipy.io.wavfile import read
 14 | from librosa.filters import mel as librosa_mel_fn
 15 | 
 16 | MAX_WAV_VALUE = 32768.0
 17 | 
 18 | 
 19 | def dynamic_range_compression_torch(x, C=1, clip_val=1e-5):
 20 |     """
 21 |     PARAMS
 22 |     ------
 23 |     C: compression factor
 24 |     """
 25 |     return torch.log(torch.clamp(x, min=clip_val) * C)
 26 | 
 27 | 
 28 | def dynamic_range_decompression_torch(x, C=1):
 29 |     """
 30 |     PARAMS
 31 |     ------
 32 |     C: compression factor used to compress
 33 |     """
 34 |     return torch.exp(x) / C
 35 | 
 36 | 
 37 | def spectral_normalize_torch(magnitudes):
 38 |     output = dynamic_range_compression_torch(magnitudes)
 39 |     return output
 40 | 
 41 | 
 42 | def spectral_de_normalize_torch(magnitudes):
 43 |     output = dynamic_range_decompression_torch(magnitudes)
 44 |     return output
 45 | 
 46 | 
 47 | mel_basis = {}
 48 | hann_window = {}
 49 | 
 50 | 
 51 | def spectrogram_torch(y, n_fft, sampling_rate, hop_size, win_size, center=False):
 52 |     if torch.min(y) < -1.:
 53 |         print('min value is ', torch.min(y))
 54 |     if torch.max(y) > 1.:
 55 |         print('max value is ', torch.max(y))
 56 | 
 57 |     global hann_window
 58 |     dtype_device = str(y.dtype) + '_' + str(y.device)
 59 |     wnsize_dtype_device = str(win_size) + '_' + dtype_device
 60 |     if wnsize_dtype_device not in hann_window:
 61 |         hann_window[wnsize_dtype_device] = torch.hann_window(win_size).to(dtype=y.dtype, device=y.device)
 62 |     y = torch.nn.functional.pad(y.unsqueeze(1), (int((n_fft-hop_size)/2), int((n_fft-hop_size)/2)), mode='reflect')
 63 |     y = y.squeeze(1)
 64 | 
 65 |     spec = torch.stft(y, n_fft, hop_length=hop_size, win_length=win_size, window=hann_window[wnsize_dtype_device],
 66 |                       center=center, pad_mode='reflect', normalized=False, onesided=True)
 67 | 
 68 |     spec = torch.sqrt(spec.pow(2).sum(-1) + 1e-6)
 69 |     return spec
 70 | 
 71 | 
 72 | def spec_to_mel_torch(spec, n_fft, num_mels, sampling_rate, fmin, fmax):
 73 |     global mel_basis
 74 |     dtype_device = str(spec.dtype) + '_' + str(spec.device)
 75 |     fmax_dtype_device = str(fmax) + '_' + dtype_device
 76 |     if fmax_dtype_device not in mel_basis:
 77 |         mel = librosa_mel_fn(sampling_rate, n_fft, num_mels, fmin, fmax)
 78 |         mel_basis[fmax_dtype_device] = torch.from_numpy(mel).to(dtype=spec.dtype, device=spec.device)
 79 |     spec = torch.matmul(mel_basis[fmax_dtype_device], spec)
 80 |     spec = spectral_normalize_torch(spec)
 81 | 
 82 |     return spec
 83 | 
 84 | 
 85 | def mel_spectrogram_torch(y, n_fft, num_mels, sampling_rate, hop_size, win_size, fmin, fmax, center=False):
 86 |     if torch.min(y) < -1.:
 87 |         print('min value is ', torch.min(y))
 88 |     if torch.max(y) > 1.:
 89 |         print('max value is ', torch.max(y))
 90 | 
 91 |     global mel_basis, hann_window
 92 |     dtype_device = str(y.dtype) + '_' + str(y.device)
 93 |     fmax_dtype_device = str(fmax) + '_' + dtype_device
 94 |     wnsize_dtype_device = str(win_size) + '_' + dtype_device
 95 |     if fmax_dtype_device not in mel_basis:
 96 |         mel = librosa_mel_fn(sampling_rate, n_fft, num_mels, fmin, fmax)
 97 |         mel_basis[fmax_dtype_device] = torch.from_numpy(mel).to(dtype=y.dtype, device=y.device)
 98 |     if wnsize_dtype_device not in hann_window:
 99 |         hann_window[wnsize_dtype_device] = torch.hann_window(win_size).to(dtype=y.dtype, device=y.device)
100 | 
101 |     y = torch.nn.functional.pad(y.unsqueeze(1), (int((n_fft-hop_size)/2), int((n_fft-hop_size)/2)), mode='reflect')
102 |     y = y.squeeze(1)
103 | 
104 |     spec = torch.stft(y, n_fft, hop_length=hop_size, win_length=win_size, window=hann_window[wnsize_dtype_device],
105 |                       center=center, pad_mode='reflect', normalized=False, onesided=True)
106 | 
107 |     spec = torch.sqrt(spec.pow(2).sum(-1) + 1e-6)
108 | 
109 |     spec = torch.matmul(mel_basis[fmax_dtype_device], spec)
110 |     spec = spectral_normalize_torch(spec)
111 | 
112 |     return spec
113 | 


--------------------------------------------------------------------------------
/models.py:
--------------------------------------------------------------------------------
  1 | import copy
  2 | import math
  3 | import torch
  4 | from torch import nn
  5 | from torch.nn import functional as F
  6 | 
  7 | import commons
  8 | import modules
  9 | import attentions
 10 | import monotonic_align
 11 | 
 12 | from torch.nn import Conv1d, ConvTranspose1d, AvgPool1d, Conv2d
 13 | from torch.nn.utils import weight_norm, remove_weight_norm, spectral_norm
 14 | from commons import init_weights, get_padding
 15 | 
 16 | class StochasticDurationPredictor(nn.Module):
 17 |   def __init__(self, in_channels, filter_channels, kernel_size, p_dropout, n_flows=4, gin_channels=0):
 18 |     super().__init__()
 19 |     filter_channels = in_channels # it needs to be removed from future version.
 20 |     self.in_channels = in_channels
 21 |     self.filter_channels = filter_channels
 22 |     self.kernel_size = kernel_size
 23 |     self.p_dropout = p_dropout
 24 |     self.n_flows = n_flows
 25 |     self.gin_channels = gin_channels
 26 | 
 27 |     self.log_flow = modules.Log()
 28 |     self.flows = nn.ModuleList()
 29 |     self.flows.append(modules.ElementwiseAffine(2))
 30 |     for i in range(n_flows):
 31 |       self.flows.append(modules.ConvFlow(2, filter_channels, kernel_size, n_layers=3))
 32 |       self.flows.append(modules.Flip())
 33 | 
 34 |     self.post_pre = nn.Conv1d(1, filter_channels, 1)
 35 |     self.post_proj = nn.Conv1d(filter_channels, filter_channels, 1)
 36 |     self.post_convs = modules.DDSConv(filter_channels, kernel_size, n_layers=3, p_dropout=p_dropout)
 37 |     self.post_flows = nn.ModuleList()
 38 |     self.post_flows.append(modules.ElementwiseAffine(2))
 39 |     for i in range(4):
 40 |       self.post_flows.append(modules.ConvFlow(2, filter_channels, kernel_size, n_layers=3))
 41 |       self.post_flows.append(modules.Flip())
 42 | 
 43 |     self.pre = nn.Conv1d(in_channels, filter_channels, 1)
 44 |     self.proj = nn.Conv1d(filter_channels, filter_channels, 1)
 45 |     self.convs = modules.DDSConv(filter_channels, kernel_size, n_layers=3, p_dropout=p_dropout)
 46 |     # if gin_channels != 0:
 47 |     #   self.cond = nn.Conv1d(gin_channels, filter_channels, 1)
 48 | 
 49 |   def forward(self, x, x_mask, w=None, g=None, reverse=False, noise_scale=1.0):
 50 |     x = torch.detach(x)
 51 |     x = self.pre(x)
 52 |     # if g is not None:
 53 |     #   g = torch.detach(g)
 54 |     #   x = x + self.cond(g)
 55 |     x = self.convs(x, x_mask)
 56 |     x = self.proj(x) * x_mask
 57 | 
 58 |     if not reverse:
 59 |       flows = self.flows
 60 |       assert w is not None
 61 | 
 62 |       logdet_tot_q = 0 
 63 |       h_w = self.post_pre(w)
 64 |       h_w = self.post_convs(h_w, x_mask)
 65 |       h_w = self.post_proj(h_w) * x_mask
 66 |       e_q = torch.randn(w.size(0), 2, w.size(2)).to(device=x.device, dtype=x.dtype) * x_mask
 67 |       z_q = e_q
 68 |       for flow in self.post_flows:
 69 |         z_q, logdet_q = flow(z_q, x_mask, g=(x + h_w))
 70 |         logdet_tot_q += logdet_q
 71 |       z_u, z1 = torch.split(z_q, [1, 1], 1) 
 72 |       u = torch.sigmoid(z_u) * x_mask
 73 |       z0 = (w - u) * x_mask
 74 |       logdet_tot_q += torch.sum((F.logsigmoid(z_u) + F.logsigmoid(-z_u)) * x_mask, [1,2])
 75 |       logq = torch.sum(-0.5 * (math.log(2*math.pi) + (e_q**2)) * x_mask, [1,2]) - logdet_tot_q
 76 | 
 77 |       logdet_tot = 0
 78 |       z0, logdet = self.log_flow(z0, x_mask)
 79 |       logdet_tot += logdet
 80 |       z = torch.cat([z0, z1], 1)
 81 |       for flow in flows:
 82 |         z, logdet = flow(z, x_mask, g=x, reverse=reverse)
 83 |         logdet_tot = logdet_tot + logdet
 84 |       nll = torch.sum(0.5 * (math.log(2*math.pi) + (z**2)) * x_mask, [1,2]) - logdet_tot
 85 |       return nll + logq # [b]
 86 |     else:
 87 |       flows = list(reversed(self.flows))
 88 |       flows = flows[:-2] + [flows[-1]] # remove a useless vflow
 89 |       z = torch.randn(x.size(0), 2, x.size(2)).to(device=x.device, dtype=x.dtype) * noise_scale
 90 |       for flow in flows:
 91 |         z = flow(z, x_mask, g=x, reverse=reverse)
 92 |       z0, z1 = torch.split(z, [1, 1], 1)
 93 |       logw = z0
 94 |       return logw
 95 | 
 96 | 
 97 | class DurationPredictor(nn.Module):
 98 |   def __init__(self, in_channels, filter_channels, kernel_size, p_dropout, gin_channels=0):
 99 |     super().__init__()
100 | 
101 |     self.in_channels = in_channels
102 |     self.filter_channels = filter_channels
103 |     self.kernel_size = kernel_size
104 |     self.p_dropout = p_dropout
105 |     self.gin_channels = gin_channels
106 | 
107 |     self.drop = nn.Dropout(p_dropout)
108 |     self.conv_1 = nn.Conv1d(in_channels, filter_channels, kernel_size, padding=kernel_size//2)
109 |     self.norm_1 = modules.LayerNorm(filter_channels)
110 |     self.conv_2 = nn.Conv1d(filter_channels, filter_channels, kernel_size, padding=kernel_size//2)
111 |     self.norm_2 = modules.LayerNorm(filter_channels)
112 |     self.proj = nn.Conv1d(filter_channels, 1, 1)
113 | 
114 |     # if gin_channels != 0:
115 |     #   self.cond = nn.Conv1d(gin_channels, in_channels, 1)
116 | 
117 |   def forward(self, x, x_mask, g=None):
118 |     x = torch.detach(x)
119 |     if g is not None:
120 |       g = torch.detach(g)
121 |       x = x + self.cond(g)
122 |     x = self.conv_1(x * x_mask)
123 |     x = torch.relu(x)
124 |     x = self.norm_1(x)
125 |     x = self.drop(x)
126 |     x = self.conv_2(x * x_mask)
127 |     x = torch.relu(x)
128 |     x = self.norm_2(x)
129 |     x = self.drop(x)
130 |     x = self.proj(x * x_mask)
131 |     return x * x_mask
132 | 
133 | 
134 | class TextEncoder(nn.Module):
135 |   def __init__(self,
136 |       n_vocab,
137 |       out_channels,
138 |       hidden_channels,
139 |       filter_channels,
140 |       n_heads,
141 |       n_layers,
142 |       kernel_size,
143 |       p_dropout):
144 |     super().__init__()
145 |     self.n_vocab = n_vocab
146 |     self.out_channels = out_channels
147 |     self.hidden_channels = hidden_channels
148 |     self.filter_channels = filter_channels
149 |     self.n_heads = n_heads
150 |     self.n_layers = n_layers
151 |     self.kernel_size = kernel_size
152 |     self.p_dropout = p_dropout
153 | 
154 |     self.emb = nn.Embedding(n_vocab, hidden_channels)
155 |     nn.init.normal_(self.emb.weight, 0.0, hidden_channels**-0.5)
156 | 
157 |     self.encoder = attentions.Encoder(
158 |       hidden_channels,
159 |       filter_channels,
160 |       n_heads,
161 |       n_layers,
162 |       kernel_size,
163 |       p_dropout)
164 |     self.proj= nn.Conv1d(hidden_channels, out_channels * 2, 1)
165 | 
166 |   def forward(self, x, x_lengths):
167 |     x = self.emb(x) * math.sqrt(self.hidden_channels) # [b, t, h]
168 |     x = torch.transpose(x, 1, -1) # [b, h, t]
169 |     x_mask = torch.unsqueeze(commons.sequence_mask(x_lengths, x.size(2)), 1).to(x.dtype)
170 | 
171 |     x = self.encoder(x * x_mask, x_mask)
172 |     stats = self.proj(x) * x_mask
173 | 
174 |     m, logs = torch.split(stats, self.out_channels, dim=1)
175 |     return x, m, logs, x_mask
176 | 
177 | 
178 | class ResidualCouplingBlock(nn.Module):
179 |   def __init__(self,
180 |       channels,
181 |       hidden_channels,
182 |       kernel_size,
183 |       dilation_rate,
184 |       n_layers,
185 |       n_flows=4,
186 |       gin_channels=0):
187 |     super().__init__()
188 |     self.channels = channels
189 |     self.hidden_channels = hidden_channels
190 |     self.kernel_size = kernel_size
191 |     self.dilation_rate = dilation_rate
192 |     self.n_layers = n_layers
193 |     self.n_flows = n_flows
194 |     self.gin_channels = gin_channels
195 | 
196 |     self.flows = nn.ModuleList()
197 |     for i in range(n_flows):
198 |       self.flows.append(modules.ResidualCouplingLayer(channels, hidden_channels, kernel_size, dilation_rate, n_layers, gin_channels=gin_channels, mean_only=True))
199 |       self.flows.append(modules.Flip())
200 | 
201 |   def forward(self, x, x_mask, g=None, reverse=False):
202 |     if not reverse:
203 |       for flow in self.flows:
204 |         x, _ = flow(x, x_mask, g=g, reverse=reverse)
205 |     else:
206 |       for flow in reversed(self.flows):
207 |         x = flow(x, x_mask, g=g, reverse=reverse)
208 |     return x
209 | 
210 | 
211 | class PosteriorEncoder(nn.Module):
212 |   def __init__(self,
213 |       in_channels,
214 |       out_channels,
215 |       hidden_channels,
216 |       kernel_size,
217 |       dilation_rate,
218 |       n_layers,
219 |       gin_channels=0):
220 |     super().__init__()
221 |     self.in_channels = in_channels
222 |     self.out_channels = out_channels
223 |     self.hidden_channels = hidden_channels
224 |     self.kernel_size = kernel_size
225 |     self.dilation_rate = dilation_rate
226 |     self.n_layers = n_layers
227 |     self.gin_channels = gin_channels
228 | 
229 |     self.pre = nn.Conv1d(in_channels, hidden_channels, 1)
230 |     self.enc = modules.WN(hidden_channels, kernel_size, dilation_rate, n_layers, gin_channels=gin_channels)
231 |     self.proj = nn.Conv1d(hidden_channels, out_channels * 2, 1)
232 | 
233 |   def forward(self, x, x_lengths, g=None):
234 |     x_mask = torch.unsqueeze(commons.sequence_mask(x_lengths, x.size(2)), 1).to(x.dtype)
235 |     x = self.pre(x)
236 |     x = x * x_mask
237 |     x = self.enc(x, x_mask, g=g)
238 |     stats = self.proj(x) * x_mask
239 |     m, logs = torch.split(stats, self.out_channels, dim=1)
240 |     z = (m + torch.randn_like(m) * torch.exp(logs)) * x_mask
241 |     return z, m, logs, x_mask
242 | 
243 | 
244 | class Generator(torch.nn.Module):
245 |     def __init__(self, initial_channel, resblock, resblock_kernel_sizes, resblock_dilation_sizes, upsample_rates, upsample_initial_channel, upsample_kernel_sizes, gin_channels=0):
246 |         super(Generator, self).__init__()
247 |         self.num_kernels = len(resblock_kernel_sizes)
248 |         self.num_upsamples = len(upsample_rates)
249 |         self.conv_pre = Conv1d(initial_channel, upsample_initial_channel, 7, 1, padding=3)
250 |         resblock = modules.ResBlock1 if resblock == '1' else modules.ResBlock2
251 | 
252 |         self.ups = nn.ModuleList()
253 |         for i, (u, k) in enumerate(zip(upsample_rates, upsample_kernel_sizes)):
254 |             self.ups.append(weight_norm(
255 |                 ConvTranspose1d(upsample_initial_channel//(2**i), upsample_initial_channel//(2**(i+1)),
256 |                                 k, u, padding=(k-u)//2)))
257 | 
258 |         self.resblocks = nn.ModuleList()
259 |         for i in range(len(self.ups)):
260 |             ch = upsample_initial_channel//(2**(i+1))
261 |             for j, (k, d) in enumerate(zip(resblock_kernel_sizes, resblock_dilation_sizes)):
262 |                 self.resblocks.append(resblock(ch, k, d))
263 | 
264 |         self.conv_post = Conv1d(ch, 1, 7, 1, padding=3, bias=False)
265 |         self.ups.apply(init_weights)
266 | 
267 |         # if gin_channels != 0:
268 |         #     self.cond = nn.Conv1d(gin_channels, upsample_initial_channel, 1)
269 | 
270 |     def forward(self, x, g=None):
271 |         x = self.conv_pre(x)
272 |         if g is not None:
273 |           x = x + self.cond(g)
274 | 
275 |         for i in range(self.num_upsamples):
276 |             x = F.leaky_relu(x, modules.LRELU_SLOPE)
277 |             x = self.ups[i](x)
278 |             xs = None
279 |             for j in range(self.num_kernels):
280 |                 if xs is None:
281 |                     xs = self.resblocks[i*self.num_kernels+j](x)
282 |                 else:
283 |                     xs += self.resblocks[i*self.num_kernels+j](x)
284 |             x = xs / self.num_kernels
285 |         x = F.leaky_relu(x)
286 |         x = self.conv_post(x)
287 |         x = torch.tanh(x)
288 | 
289 |         return x
290 | 
291 |     def remove_weight_norm(self):
292 |         print('Removing weight norm...')
293 |         for l in self.ups:
294 |             remove_weight_norm(l)
295 |         for l in self.resblocks:
296 |             l.remove_weight_norm()
297 | 
298 | 
299 | class DiscriminatorP(torch.nn.Module):
300 |     def __init__(self, period, kernel_size=5, stride=3, use_spectral_norm=False):
301 |         super(DiscriminatorP, self).__init__()
302 |         self.period = period
303 |         self.use_spectral_norm = use_spectral_norm
304 |         norm_f = weight_norm if use_spectral_norm == False else spectral_norm
305 |         self.convs = nn.ModuleList([
306 |             norm_f(Conv2d(1, 32, (kernel_size, 1), (stride, 1), padding=(get_padding(kernel_size, 1), 0))),
307 |             norm_f(Conv2d(32, 128, (kernel_size, 1), (stride, 1), padding=(get_padding(kernel_size, 1), 0))),
308 |             norm_f(Conv2d(128, 512, (kernel_size, 1), (stride, 1), padding=(get_padding(kernel_size, 1), 0))),
309 |             norm_f(Conv2d(512, 1024, (kernel_size, 1), (stride, 1), padding=(get_padding(kernel_size, 1), 0))),
310 |             norm_f(Conv2d(1024, 1024, (kernel_size, 1), 1, padding=(get_padding(kernel_size, 1), 0))),
311 |         ])
312 |         self.conv_post = norm_f(Conv2d(1024, 1, (3, 1), 1, padding=(1, 0)))
313 | 
314 |     def forward(self, x):
315 |         fmap = []
316 | 
317 |         # 1d to 2d
318 |         b, c, t = x.shape
319 |         if t % self.period != 0: # pad first
320 |             n_pad = self.period - (t % self.period)
321 |             x = F.pad(x, (0, n_pad), "reflect")
322 |             t = t + n_pad
323 |         x = x.view(b, c, t // self.period, self.period)
324 | 
325 |         for l in self.convs:
326 |             x = l(x)
327 |             x = F.leaky_relu(x, modules.LRELU_SLOPE)
328 |             fmap.append(x)
329 |         x = self.conv_post(x)
330 |         fmap.append(x)
331 |         x = torch.flatten(x, 1, -1)
332 | 
333 |         return x, fmap
334 | 
335 | 
336 | class DiscriminatorS(torch.nn.Module):
337 |     def __init__(self, use_spectral_norm=False):
338 |         super(DiscriminatorS, self).__init__()
339 |         norm_f = weight_norm if use_spectral_norm == False else spectral_norm
340 |         self.convs = nn.ModuleList([
341 |             norm_f(Conv1d(1, 16, 15, 1, padding=7)),
342 |             norm_f(Conv1d(16, 64, 41, 4, groups=4, padding=20)),
343 |             norm_f(Conv1d(64, 256, 41, 4, groups=16, padding=20)),
344 |             norm_f(Conv1d(256, 1024, 41, 4, groups=64, padding=20)),
345 |             norm_f(Conv1d(1024, 1024, 41, 4, groups=256, padding=20)),
346 |             norm_f(Conv1d(1024, 1024, 5, 1, padding=2)),
347 |         ])
348 |         self.conv_post = norm_f(Conv1d(1024, 1, 3, 1, padding=1))
349 | 
350 |     def forward(self, x):
351 |         fmap = []
352 | 
353 |         for l in self.convs:
354 |             x = l(x)
355 |             x = F.leaky_relu(x, modules.LRELU_SLOPE)
356 |             fmap.append(x)
357 |         x = self.conv_post(x)
358 |         fmap.append(x)
359 |         x = torch.flatten(x, 1, -1)
360 | 
361 |         return x, fmap
362 | 
363 | 
364 | class MultiPeriodDiscriminator(torch.nn.Module):
365 |     def __init__(self, use_spectral_norm=False):
366 |         super(MultiPeriodDiscriminator, self).__init__()
367 |         periods = [2,3,5,7,11]
368 | 
369 |         discs = [DiscriminatorS(use_spectral_norm=use_spectral_norm)]
370 |         discs = discs + [DiscriminatorP(i, use_spectral_norm=use_spectral_norm) for i in periods]
371 |         self.discriminators = nn.ModuleList(discs)
372 | 
373 |     def forward(self, y, y_hat):
374 |         y_d_rs = []
375 |         y_d_gs = []
376 |         fmap_rs = []
377 |         fmap_gs = []
378 |         for i, d in enumerate(self.discriminators):
379 |             y_d_r, fmap_r = d(y)
380 |             y_d_g, fmap_g = d(y_hat)
381 |             y_d_rs.append(y_d_r)
382 |             y_d_gs.append(y_d_g)
383 |             fmap_rs.append(fmap_r)
384 |             fmap_gs.append(fmap_g)
385 | 
386 |         return y_d_rs, y_d_gs, fmap_rs, fmap_gs
387 | 
388 | 
389 | 
390 | class SynthesizerTrn(nn.Module):
391 |   """
392 |   Synthesizer for Training
393 |   """
394 | 
395 |   def __init__(self, 
396 |     n_vocab,
397 |     spec_channels,
398 |     segment_size,
399 |     inter_channels,
400 |     hidden_channels,
401 |     filter_channels,
402 |     n_heads,
403 |     n_layers,
404 |     kernel_size,
405 |     p_dropout,
406 |     resblock, 
407 |     resblock_kernel_sizes, 
408 |     resblock_dilation_sizes, 
409 |     upsample_rates, 
410 |     upsample_initial_channel, 
411 |     upsample_kernel_sizes,
412 |     n_speakers=0,
413 |     gin_channels=0,
414 |     use_sdp=True,
415 |     **kwargs):
416 | 
417 |     super().__init__()
418 |     self.n_vocab = n_vocab
419 |     self.spec_channels = spec_channels
420 |     self.inter_channels = inter_channels
421 |     self.hidden_channels = hidden_channels
422 |     self.filter_channels = filter_channels
423 |     self.n_heads = n_heads
424 |     self.n_layers = n_layers
425 |     self.kernel_size = kernel_size
426 |     self.p_dropout = p_dropout
427 |     self.resblock = resblock
428 |     self.resblock_kernel_sizes = resblock_kernel_sizes
429 |     self.resblock_dilation_sizes = resblock_dilation_sizes
430 |     self.upsample_rates = upsample_rates
431 |     self.upsample_initial_channel = upsample_initial_channel
432 |     self.upsample_kernel_sizes = upsample_kernel_sizes
433 |     self.segment_size = segment_size
434 |     self.n_speakers = n_speakers
435 |     self.gin_channels = gin_channels
436 | 
437 |     self.use_sdp = use_sdp
438 | 
439 |     self.enc_p = TextEncoder(n_vocab,
440 |         inter_channels,
441 |         hidden_channels,
442 |         filter_channels,
443 |         n_heads,
444 |         n_layers,
445 |         kernel_size,
446 |         p_dropout)
447 |     self.dec = Generator(inter_channels, resblock, resblock_kernel_sizes, resblock_dilation_sizes, upsample_rates, upsample_initial_channel, upsample_kernel_sizes, gin_channels=gin_channels)
448 |     self.enc_q = PosteriorEncoder(spec_channels, inter_channels, hidden_channels, 5, 1, 16, gin_channels=gin_channels)
449 |     self.flow = ResidualCouplingBlock(inter_channels, hidden_channels, 5, 1, 4, gin_channels=gin_channels)
450 |     self.style_encoder = MelStyleEncoder()
451 |     if use_sdp:
452 |       self.dp = StochasticDurationPredictor(hidden_channels, 192, 3, 0.5, 4, gin_channels=gin_channels)
453 |     else:
454 |       self.dp = DurationPredictor(hidden_channels, 256, 3, 0.5, gin_channels=gin_channels)
455 | 
456 |     if n_speakers > 1:
457 |       self.emb_g = nn.Embedding(n_speakers, gin_channels)
458 | 
459 |   def forward(self, x, x_lengths, mel, y, y_lengths, sid=None):
460 |     '''
461 |     set g = None for posterior enc, sdp(dp), vocoder except flow
462 |     '''
463 |     x, m_p, logs_p, x_mask = self.enc_p(x, x_lengths)
464 |     if self.n_speakers > 0:
465 |       g = self.emb_g(sid).unsqueeze(-1) # [b, h, 1]
466 |     else:
467 |       g = None
468 |     #* g: (8,256,1)
469 |     # z, m_q, logs_q, y_mask = self.enc_q(y, y_lengths, g=g)
470 |     z, m_q, logs_q, y_mask = self.enc_q(y, y_lengths, g=None)
471 |     #* y_mask:(8,1,262)
472 |     # z_p = self.flow(z, y_mask, g=g)
473 |     #* Zero-shot
474 |     style_vector = self.style_encoder(mel.transpose(1,2), (y_mask.int()==0).squeeze(1))
475 |     z_p = self.flow(z, y_mask, g=style_vector.unsqueeze(-1))
476 | 
477 |     with torch.no_grad():
478 |       # negative cross-entropy
479 |       s_p_sq_r = torch.exp(-2 * logs_p) # [b, d, t]
480 |       neg_cent1 = torch.sum(-0.5 * math.log(2 * math.pi) - logs_p, [1], keepdim=True) # [b, 1, t_s]
481 |       neg_cent2 = torch.matmul(-0.5 * (z_p ** 2).transpose(1, 2), s_p_sq_r) # [b, t_t, d] x [b, d, t_s] = [b, t_t, t_s]
482 |       neg_cent3 = torch.matmul(z_p.transpose(1, 2), (m_p * s_p_sq_r)) # [b, t_t, d] x [b, d, t_s] = [b, t_t, t_s]
483 |       neg_cent4 = torch.sum(-0.5 * (m_p ** 2) * s_p_sq_r, [1], keepdim=True) # [b, 1, t_s]
484 |       neg_cent = neg_cent1 + neg_cent2 + neg_cent3 + neg_cent4
485 | 
486 |       attn_mask = torch.unsqueeze(x_mask, 2) * torch.unsqueeze(y_mask, -1)
487 |       attn = monotonic_align.maximum_path(neg_cent, attn_mask.squeeze(1)).unsqueeze(1).detach()
488 | 
489 |     w = attn.sum(2)
490 |     if self.use_sdp:
491 |       # l_length = self.dp(x, x_mask, w, g=g)
492 |       l_length = self.dp(x, x_mask, w, g=None)
493 |       l_length = l_length / torch.sum(x_mask)
494 |     else:
495 |       logw_ = torch.log(w + 1e-6) * x_mask
496 |       # logw = self.dp(x, x_mask, g=g)
497 |       logw = self.dp(x, x_mask, g=None)
498 |       l_length = torch.sum((logw - logw_)**2, [1,2]) / torch.sum(x_mask) # for averaging 
499 | 
500 |     # expand prior
501 |     m_p = torch.matmul(attn.squeeze(1), m_p.transpose(1, 2)).transpose(1, 2)
502 |     logs_p = torch.matmul(attn.squeeze(1), logs_p.transpose(1, 2)).transpose(1, 2)
503 | 
504 |     z_slice, ids_slice = commons.rand_slice_segments(z, y_lengths, self.segment_size)
505 |     # o = self.dec(z_slice, g=g)
506 |     o = self.dec(z_slice, g=None)
507 |     return o, l_length, attn, ids_slice, x_mask, y_mask, (z, z_p, m_p, logs_p, m_q, logs_q)
508 | 
509 |   def infer(self, x, x_lengths, mel, mel_lengths = None, sid=None, noise_scale=1, length_scale=1, noise_scale_w=1., max_len=None):
510 |     x, m_p, logs_p, x_mask = self.enc_p(x, x_lengths)
511 |     if self.n_speakers > 0:
512 |       g = self.emb_g(sid).unsqueeze(-1) # [b, h, 1]
513 |     else:
514 |       g = None
515 | 
516 |     if self.use_sdp:
517 |       # logw = self.dp(x, x_mask, g=g, reverse=True, noise_scale=noise_scale_w)
518 |       logw = self.dp(x, x_mask, g=None, reverse=True, noise_scale=noise_scale_w)
519 |     else:
520 |       # logw = self.dp(x, x_mask, g=g)
521 |       logw = self.dp(x, x_mask, g=None)
522 |     w = torch.exp(logw) * x_mask * length_scale
523 |     w_ceil = torch.ceil(w)
524 |     y_lengths = torch.clamp_min(torch.sum(w_ceil, [1, 2]), 1).long()
525 |     y_mask = torch.unsqueeze(commons.sequence_mask(y_lengths, None), 1).to(x_mask.dtype)
526 |     
527 |     attn_mask = torch.unsqueeze(x_mask, 2) * torch.unsqueeze(y_mask, -1)
528 |     attn = commons.generate_path(w_ceil, attn_mask)
529 | 
530 |     m_p = torch.matmul(attn.squeeze(1), m_p.transpose(1, 2)).transpose(1, 2) # [b, t', t], [b, t, d] -> [b, d, t']
531 |     logs_p = torch.matmul(attn.squeeze(1), logs_p.transpose(1, 2)).transpose(1, 2) # [b, t', t], [b, t, d] -> [b, d, t']
532 | 
533 |     z_p = m_p + torch.randn_like(m_p) * torch.exp(logs_p) * noise_scale
534 |     #* used for mel style encoder
535 |     if mel_lengths is not None:
536 |       style_mask = torch.unsqueeze(commons.sequence_mask(mel_lengths, mel.size(2)), 1).to(x.dtype)
537 |       style_vector = self.style_encoder(mel.transpose(1,2), (style_mask.int()==0).squeeze(1))
538 |     else:
539 |       style_vector = self.style_encoder(mel.transpose(1,2), None)
540 |     # z = self.flow(z_p, y_mask, g=g, reverse=True)
541 |     z = self.flow(z_p, y_mask, g=style_vector.unsqueeze(-1), reverse=True)
542 |     # o = self.dec((z * y_mask)[:,:,:max_len], g=g)
543 |     o = self.dec((z * y_mask)[:,:,:max_len], g=None)
544 |     return o, attn, y_mask, (z, z_p, m_p, logs_p)
545 | 
546 |   def voice_conversion(self, y, y_lengths, sid_src, sid_tgt):
547 |     assert self.n_speakers > 0, "n_speakers have to be larger than 0."
548 |     g_src = self.emb_g(sid_src).unsqueeze(-1)
549 |     g_tgt = self.emb_g(sid_tgt).unsqueeze(-1)
550 |     z, m_q, logs_q, y_mask = self.enc_q(y, y_lengths, g=g_src)
551 |     z_p = self.flow(z, y_mask, g=g_src)
552 |     z_hat = self.flow(z_p, y_mask, g=g_tgt, reverse=True)
553 |     o_hat = self.dec(z_hat * y_mask, g=g_tgt)
554 |     return o_hat, y_mask, (z, z_p, z_hat)
555 | 
556 | class MelStyleEncoder(nn.Module):
557 |     ''' MelStyleEncoder '''
558 |     def __init__(self, n_mel_channels=80,
559 |           style_hidden=128,
560 |           style_vector_dim=256,
561 |           style_kernel_size=5,
562 |           style_head=2,
563 |           dropout=0.1):
564 |         super(MelStyleEncoder, self).__init__()
565 |         self.in_dim = n_mel_channels 
566 |         self.hidden_dim = style_hidden
567 |         self.out_dim = style_vector_dim
568 |         self.kernel_size = style_kernel_size
569 |         self.n_head = style_head
570 |         self.dropout = dropout
571 | 
572 |         self.spectral = nn.Sequential(
573 |             modules.LinearNorm(self.in_dim, self.hidden_dim),
574 |             modules.Mish(),
575 |             nn.Dropout(self.dropout),
576 |             modules.LinearNorm(self.hidden_dim, self.hidden_dim),
577 |             modules.Mish(),
578 |             nn.Dropout(self.dropout)
579 |         )
580 | 
581 |         self.temporal = nn.Sequential(
582 |             modules.Conv1dGLU(self.hidden_dim, self.hidden_dim, self.kernel_size, self.dropout),
583 |             modules.Conv1dGLU(self.hidden_dim, self.hidden_dim, self.kernel_size, self.dropout),
584 |         )
585 | 
586 |         self.slf_attn = modules.MultiHeadAttention(self.n_head, self.hidden_dim, 
587 |                                 self.hidden_dim//self.n_head, self.hidden_dim//self.n_head, self.dropout) 
588 | 
589 |         self.fc = modules.LinearNorm(self.hidden_dim, self.out_dim)
590 | 
591 |     def temporal_avg_pool(self, x, mask=None):
592 |         if mask is None:
593 |             out = torch.mean(x, dim=1)
594 |         else:
595 |             len_ = (~mask).sum(dim=1).unsqueeze(1)
596 |             x = x.masked_fill(mask.unsqueeze(-1), 0)
597 |             x = x.sum(dim=1)
598 |             out = torch.div(x, len_)
599 |         return out
600 | 
601 |     def forward(self, x, mask=None):
602 |         max_len = x.shape[1]
603 |         slf_attn_mask = mask.unsqueeze(1).expand(-1, max_len, -1) if mask is not None else None
604 |         
605 |         # spectral
606 |         x = self.spectral(x)
607 |         # temporal
608 |         x = x.transpose(1,2)
609 |         x = self.temporal(x)
610 |         x = x.transpose(1,2)
611 |         # self-attention
612 |         if mask is not None:
613 |             x = x.masked_fill(mask.unsqueeze(-1), 0)
614 |         x, _ = self.slf_attn(x, mask=slf_attn_mask)
615 |         # fc
616 |         x = self.fc(x)
617 |         # temoral average pooling
618 |         w = self.temporal_avg_pool(x, mask=mask)
619 | 
620 |         return w
621 | 


--------------------------------------------------------------------------------
/modules.py:
--------------------------------------------------------------------------------
  1 | import copy
  2 | import math
  3 | import numpy as np
  4 | import scipy
  5 | import torch
  6 | from torch import nn
  7 | from torch.nn import functional as F
  8 | 
  9 | from torch.nn import Conv1d, ConvTranspose1d, AvgPool1d, Conv2d
 10 | from torch.nn.utils import weight_norm, remove_weight_norm
 11 | 
 12 | import commons
 13 | from commons import init_weights, get_padding
 14 | from transforms import piecewise_rational_quadratic_transform
 15 | 
 16 | 
 17 | LRELU_SLOPE = 0.1
 18 | 
 19 | 
 20 | class LayerNorm(nn.Module):
 21 |   def __init__(self, channels, eps=1e-5):
 22 |     super().__init__()
 23 |     self.channels = channels
 24 |     self.eps = eps
 25 | 
 26 |     self.gamma = nn.Parameter(torch.ones(channels))
 27 |     self.beta = nn.Parameter(torch.zeros(channels))
 28 | 
 29 |   def forward(self, x):
 30 |     x = x.transpose(1, -1)
 31 |     x = F.layer_norm(x, (self.channels,), self.gamma, self.beta, self.eps)
 32 |     return x.transpose(1, -1)
 33 | 
 34 |  
 35 | class ConvReluNorm(nn.Module):
 36 |   def __init__(self, in_channels, hidden_channels, out_channels, kernel_size, n_layers, p_dropout):
 37 |     super().__init__()
 38 |     self.in_channels = in_channels
 39 |     self.hidden_channels = hidden_channels
 40 |     self.out_channels = out_channels
 41 |     self.kernel_size = kernel_size
 42 |     self.n_layers = n_layers
 43 |     self.p_dropout = p_dropout
 44 |     assert n_layers > 1, "Number of layers should be larger than 0."
 45 | 
 46 |     self.conv_layers = nn.ModuleList()
 47 |     self.norm_layers = nn.ModuleList()
 48 |     self.conv_layers.append(nn.Conv1d(in_channels, hidden_channels, kernel_size, padding=kernel_size//2))
 49 |     self.norm_layers.append(LayerNorm(hidden_channels))
 50 |     self.relu_drop = nn.Sequential(
 51 |         nn.ReLU(),
 52 |         nn.Dropout(p_dropout))
 53 |     for _ in range(n_layers-1):
 54 |       self.conv_layers.append(nn.Conv1d(hidden_channels, hidden_channels, kernel_size, padding=kernel_size//2))
 55 |       self.norm_layers.append(LayerNorm(hidden_channels))
 56 |     self.proj = nn.Conv1d(hidden_channels, out_channels, 1)
 57 |     self.proj.weight.data.zero_()
 58 |     self.proj.bias.data.zero_()
 59 | 
 60 |   def forward(self, x, x_mask):
 61 |     x_org = x
 62 |     for i in range(self.n_layers):
 63 |       x = self.conv_layers[i](x * x_mask)
 64 |       x = self.norm_layers[i](x)
 65 |       x = self.relu_drop(x)
 66 |     x = x_org + self.proj(x)
 67 |     return x * x_mask
 68 | 
 69 | 
 70 | class DDSConv(nn.Module):
 71 |   """
 72 |   Dialted and Depth-Separable Convolution
 73 |   """
 74 |   def __init__(self, channels, kernel_size, n_layers, p_dropout=0.):
 75 |     super().__init__()
 76 |     self.channels = channels
 77 |     self.kernel_size = kernel_size
 78 |     self.n_layers = n_layers
 79 |     self.p_dropout = p_dropout
 80 | 
 81 |     self.drop = nn.Dropout(p_dropout)
 82 |     self.convs_sep = nn.ModuleList()
 83 |     self.convs_1x1 = nn.ModuleList()
 84 |     self.norms_1 = nn.ModuleList()
 85 |     self.norms_2 = nn.ModuleList()
 86 |     for i in range(n_layers):
 87 |       dilation = kernel_size ** i
 88 |       padding = (kernel_size * dilation - dilation) // 2
 89 |       self.convs_sep.append(nn.Conv1d(channels, channels, kernel_size, 
 90 |           groups=channels, dilation=dilation, padding=padding
 91 |       ))
 92 |       self.convs_1x1.append(nn.Conv1d(channels, channels, 1))
 93 |       self.norms_1.append(LayerNorm(channels))
 94 |       self.norms_2.append(LayerNorm(channels))
 95 | 
 96 |   def forward(self, x, x_mask, g=None):
 97 |     if g is not None:
 98 |       x = x + g
 99 |     for i in range(self.n_layers):
100 |       y = self.convs_sep[i](x * x_mask)
101 |       y = self.norms_1[i](y)
102 |       y = F.gelu(y)
103 |       y = self.convs_1x1[i](y)
104 |       y = self.norms_2[i](y)
105 |       y = F.gelu(y)
106 |       y = self.drop(y)
107 |       x = x + y
108 |     return x * x_mask
109 | 
110 | 
111 | class WN(torch.nn.Module):
112 |   def __init__(self, hidden_channels, kernel_size, dilation_rate, n_layers, gin_channels=0, p_dropout=0):
113 |     super(WN, self).__init__()
114 |     assert(kernel_size % 2 == 1)
115 |     self.hidden_channels =hidden_channels
116 |     self.kernel_size = kernel_size,
117 |     self.dilation_rate = dilation_rate
118 |     self.n_layers = n_layers
119 |     self.gin_channels = gin_channels
120 |     self.p_dropout = p_dropout
121 | 
122 |     self.in_layers = torch.nn.ModuleList()
123 |     self.res_skip_layers = torch.nn.ModuleList()
124 |     self.drop = nn.Dropout(p_dropout)
125 | 
126 |     if gin_channels != 0:
127 |       cond_layer = torch.nn.Conv1d(gin_channels, 2*hidden_channels*n_layers, 1)
128 |       self.cond_layer = torch.nn.utils.weight_norm(cond_layer, name='weight')
129 | 
130 |     for i in range(n_layers):
131 |       dilation = dilation_rate ** i
132 |       padding = int((kernel_size * dilation - dilation) / 2)
133 |       in_layer = torch.nn.Conv1d(hidden_channels, 2*hidden_channels, kernel_size,
134 |                                  dilation=dilation, padding=padding)
135 |       in_layer = torch.nn.utils.weight_norm(in_layer, name='weight')
136 |       self.in_layers.append(in_layer)
137 | 
138 |       # last one is not necessary
139 |       if i < n_layers - 1:
140 |         res_skip_channels = 2 * hidden_channels
141 |       else:
142 |         res_skip_channels = hidden_channels
143 | 
144 |       res_skip_layer = torch.nn.Conv1d(hidden_channels, res_skip_channels, 1)
145 |       res_skip_layer = torch.nn.utils.weight_norm(res_skip_layer, name='weight')
146 |       self.res_skip_layers.append(res_skip_layer)
147 | 
148 |   def forward(self, x, x_mask, g=None, **kwargs):
149 |     output = torch.zeros_like(x)
150 |     n_channels_tensor = torch.IntTensor([self.hidden_channels])
151 | 
152 |     if g is not None:
153 |       g = self.cond_layer(g)
154 | 
155 |     for i in range(self.n_layers):
156 |       x_in = self.in_layers[i](x)
157 |       if g is not None:
158 |         cond_offset = i * 2 * self.hidden_channels
159 |         g_l = g[:,cond_offset:cond_offset+2*self.hidden_channels,:]
160 |       else:
161 |         g_l = torch.zeros_like(x_in)
162 | 
163 |       acts = commons.fused_add_tanh_sigmoid_multiply(
164 |           x_in,
165 |           g_l,
166 |           n_channels_tensor)
167 |       acts = self.drop(acts)
168 | 
169 |       res_skip_acts = self.res_skip_layers[i](acts)
170 |       if i < self.n_layers - 1:
171 |         res_acts = res_skip_acts[:,:self.hidden_channels,:]
172 |         x = (x + res_acts) * x_mask
173 |         output = output + res_skip_acts[:,self.hidden_channels:,:]
174 |       else:
175 |         output = output + res_skip_acts
176 |     return output * x_mask
177 | 
178 |   def remove_weight_norm(self):
179 |     if self.gin_channels != 0:
180 |       torch.nn.utils.remove_weight_norm(self.cond_layer)
181 |     for l in self.in_layers:
182 |       torch.nn.utils.remove_weight_norm(l)
183 |     for l in self.res_skip_layers:
184 |      torch.nn.utils.remove_weight_norm(l)
185 | 
186 | 
187 | class ResBlock1(torch.nn.Module):
188 |     def __init__(self, channels, kernel_size=3, dilation=(1, 3, 5)):
189 |         super(ResBlock1, self).__init__()
190 |         self.convs1 = nn.ModuleList([
191 |             weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[0],
192 |                                padding=get_padding(kernel_size, dilation[0]))),
193 |             weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[1],
194 |                                padding=get_padding(kernel_size, dilation[1]))),
195 |             weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[2],
196 |                                padding=get_padding(kernel_size, dilation[2])))
197 |         ])
198 |         self.convs1.apply(init_weights)
199 | 
200 |         self.convs2 = nn.ModuleList([
201 |             weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=1,
202 |                                padding=get_padding(kernel_size, 1))),
203 |             weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=1,
204 |                                padding=get_padding(kernel_size, 1))),
205 |             weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=1,
206 |                                padding=get_padding(kernel_size, 1)))
207 |         ])
208 |         self.convs2.apply(init_weights)
209 | 
210 |     def forward(self, x, x_mask=None):
211 |         for c1, c2 in zip(self.convs1, self.convs2):
212 |             xt = F.leaky_relu(x, LRELU_SLOPE)
213 |             if x_mask is not None:
214 |                 xt = xt * x_mask
215 |             xt = c1(xt)
216 |             xt = F.leaky_relu(xt, LRELU_SLOPE)
217 |             if x_mask is not None:
218 |                 xt = xt * x_mask
219 |             xt = c2(xt)
220 |             x = xt + x
221 |         if x_mask is not None:
222 |             x = x * x_mask
223 |         return x
224 | 
225 |     def remove_weight_norm(self):
226 |         for l in self.convs1:
227 |             remove_weight_norm(l)
228 |         for l in self.convs2:
229 |             remove_weight_norm(l)
230 | 
231 | 
232 | class ResBlock2(torch.nn.Module):
233 |     def __init__(self, channels, kernel_size=3, dilation=(1, 3)):
234 |         super(ResBlock2, self).__init__()
235 |         self.convs = nn.ModuleList([
236 |             weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[0],
237 |                                padding=get_padding(kernel_size, dilation[0]))),
238 |             weight_norm(Conv1d(channels, channels, kernel_size, 1, dilation=dilation[1],
239 |                                padding=get_padding(kernel_size, dilation[1])))
240 |         ])
241 |         self.convs.apply(init_weights)
242 | 
243 |     def forward(self, x, x_mask=None):
244 |         for c in self.convs:
245 |             xt = F.leaky_relu(x, LRELU_SLOPE)
246 |             if x_mask is not None:
247 |                 xt = xt * x_mask
248 |             xt = c(xt)
249 |             x = xt + x
250 |         if x_mask is not None:
251 |             x = x * x_mask
252 |         return x
253 | 
254 |     def remove_weight_norm(self):
255 |         for l in self.convs:
256 |             remove_weight_norm(l)
257 | 
258 | 
259 | class Log(nn.Module):
260 |   def forward(self, x, x_mask, reverse=False, **kwargs):
261 |     if not reverse:
262 |       y = torch.log(torch.clamp_min(x, 1e-5)) * x_mask
263 |       logdet = torch.sum(-y, [1, 2])
264 |       return y, logdet
265 |     else:
266 |       x = torch.exp(x) * x_mask
267 |       return x
268 |     
269 | 
270 | class Flip(nn.Module):
271 |   def forward(self, x, *args, reverse=False, **kwargs):
272 |     x = torch.flip(x, [1])
273 |     if not reverse:
274 |       logdet = torch.zeros(x.size(0)).to(dtype=x.dtype, device=x.device)
275 |       return x, logdet
276 |     else:
277 |       return x
278 | 
279 | 
280 | class ElementwiseAffine(nn.Module):
281 |   def __init__(self, channels):
282 |     super().__init__()
283 |     self.channels = channels
284 |     self.m = nn.Parameter(torch.zeros(channels,1))
285 |     self.logs = nn.Parameter(torch.zeros(channels,1))
286 | 
287 |   def forward(self, x, x_mask, reverse=False, **kwargs):
288 |     if not reverse:
289 |       y = self.m + torch.exp(self.logs) * x
290 |       y = y * x_mask
291 |       logdet = torch.sum(self.logs * x_mask, [1,2])
292 |       return y, logdet
293 |     else:
294 |       x = (x - self.m) * torch.exp(-self.logs) * x_mask
295 |       return x
296 | 
297 | 
298 | class ResidualCouplingLayer(nn.Module):
299 |   def __init__(self,
300 |       channels,
301 |       hidden_channels,
302 |       kernel_size,
303 |       dilation_rate,
304 |       n_layers,
305 |       p_dropout=0,
306 |       gin_channels=0,
307 |       mean_only=False):
308 |     assert channels % 2 == 0, "channels should be divisible by 2"
309 |     super().__init__()
310 |     self.channels = channels
311 |     self.hidden_channels = hidden_channels
312 |     self.kernel_size = kernel_size
313 |     self.dilation_rate = dilation_rate
314 |     self.n_layers = n_layers
315 |     self.half_channels = channels // 2
316 |     self.mean_only = mean_only
317 | 
318 |     self.pre = nn.Conv1d(self.half_channels, hidden_channels, 1)
319 |     self.enc = WN(hidden_channels, kernel_size, dilation_rate, n_layers, p_dropout=p_dropout, gin_channels=gin_channels)
320 |     self.post = nn.Conv1d(hidden_channels, self.half_channels * (2 - mean_only), 1)
321 |     self.post.weight.data.zero_()
322 |     self.post.bias.data.zero_()
323 | 
324 |   def forward(self, x, x_mask, g=None, reverse=False):
325 |     x0, x1 = torch.split(x, [self.half_channels]*2, 1)
326 |     h = self.pre(x0) * x_mask
327 |     h = self.enc(h, x_mask, g=g)
328 |     stats = self.post(h) * x_mask
329 |     if not self.mean_only:
330 |       m, logs = torch.split(stats, [self.half_channels]*2, 1)
331 |     else:
332 |       m = stats
333 |       logs = torch.zeros_like(m)
334 | 
335 |     if not reverse:
336 |       x1 = m + x1 * torch.exp(logs) * x_mask
337 |       x = torch.cat([x0, x1], 1)
338 |       logdet = torch.sum(logs, [1,2])
339 |       return x, logdet
340 |     else:
341 |       x1 = (x1 - m) * torch.exp(-logs) * x_mask
342 |       x = torch.cat([x0, x1], 1)
343 |       return x
344 | 
345 | 
346 | class ConvFlow(nn.Module):
347 |   def __init__(self, in_channels, filter_channels, kernel_size, n_layers, num_bins=10, tail_bound=5.0):
348 |     super().__init__()
349 |     self.in_channels = in_channels
350 |     self.filter_channels = filter_channels
351 |     self.kernel_size = kernel_size
352 |     self.n_layers = n_layers
353 |     self.num_bins = num_bins
354 |     self.tail_bound = tail_bound
355 |     self.half_channels = in_channels // 2
356 | 
357 |     self.pre = nn.Conv1d(self.half_channels, filter_channels, 1)
358 |     self.convs = DDSConv(filter_channels, kernel_size, n_layers, p_dropout=0.)
359 |     self.proj = nn.Conv1d(filter_channels, self.half_channels * (num_bins * 3 - 1), 1)
360 |     self.proj.weight.data.zero_()
361 |     self.proj.bias.data.zero_()
362 | 
363 |   def forward(self, x, x_mask, g=None, reverse=False):
364 |     x0, x1 = torch.split(x, [self.half_channels]*2, 1)
365 |     h = self.pre(x0)
366 |     h = self.convs(h, x_mask, g=g)
367 |     h = self.proj(h) * x_mask
368 | 
369 |     b, c, t = x0.shape
370 |     h = h.reshape(b, c, -1, t).permute(0, 1, 3, 2) # [b, cx?, t] -> [b, c, t, ?]
371 | 
372 |     unnormalized_widths = h[..., :self.num_bins] / math.sqrt(self.filter_channels)
373 |     unnormalized_heights = h[..., self.num_bins:2*self.num_bins] / math.sqrt(self.filter_channels)
374 |     unnormalized_derivatives = h[..., 2 * self.num_bins:]
375 | 
376 |     x1, logabsdet = piecewise_rational_quadratic_transform(x1,
377 |         unnormalized_widths,
378 |         unnormalized_heights,
379 |         unnormalized_derivatives,
380 |         inverse=reverse,
381 |         tails='linear',
382 |         tail_bound=self.tail_bound
383 |     )
384 | 
385 |     x = torch.cat([x0, x1], 1) * x_mask
386 |     logdet = torch.sum(logabsdet * x_mask, [1,2])
387 |     if not reverse:
388 |         return x, logdet
389 |     else:
390 |         return x
391 | 
392 | class LinearNorm(nn.Module):
393 |   def __init__(self,
394 |                 in_channels,
395 |                 out_channels,
396 |                 bias=True, 
397 |                 spectral_norm=False,
398 |                 ):
399 |       super(LinearNorm, self).__init__()
400 |       self.fc = nn.Linear(in_channels, out_channels, bias)
401 |       
402 |       if spectral_norm:
403 |           self.fc = nn.utils.spectral_norm(self.fc)
404 | 
405 |   def forward(self, input):
406 |       out = self.fc(input)
407 |       return out
408 |       
409 | class Mish(nn.Module):
410 |   def __init__(self):
411 |       super(Mish, self).__init__()
412 |   def forward(self, x):
413 |       return x * torch.tanh(F.softplus(x))
414 |     
415 | class Conv1dGLU(nn.Module):
416 |   '''
417 |   Conv1d + GLU(Gated Linear Unit) with residual connection.
418 |   For GLU refer to https://arxiv.org/abs/1612.08083 paper.
419 |   '''
420 |   def __init__(self, in_channels, out_channels, kernel_size, dropout):
421 |       super(Conv1dGLU, self).__init__()
422 |       self.out_channels = out_channels
423 |       self.conv1 = ConvNorm(in_channels, 2*out_channels, kernel_size=kernel_size)
424 |       self.dropout = nn.Dropout(dropout)
425 |           
426 |   def forward(self, x):
427 |       residual = x
428 |       x = self.conv1(x)
429 |       x1, x2 = torch.split(x, split_size_or_sections=self.out_channels, dim=1)
430 |       x = x1 * torch.sigmoid(x2)
431 |       x = residual + self.dropout(x)
432 |       return x
433 | 
434 | class ConvNorm(nn.Module):
435 |   def __init__(self,
436 |                 in_channels,
437 |                 out_channels,
438 |                 kernel_size=1,
439 |                 stride=1,
440 |                 padding=None,
441 |                 dilation=1,
442 |                 bias=True, 
443 |                 spectral_norm=False,
444 |                 ):
445 |       super(ConvNorm, self).__init__()
446 | 
447 |       if padding is None:
448 |           assert(kernel_size % 2 == 1)
449 |           padding = int(dilation * (kernel_size - 1) / 2)
450 | 
451 |       self.conv = torch.nn.Conv1d(in_channels,
452 |                                   out_channels,
453 |                                   kernel_size=kernel_size,
454 |                                   stride=stride,
455 |                                   padding=padding,
456 |                                   dilation=dilation,
457 |                                   bias=bias)
458 |       
459 |       if spectral_norm:
460 |           self.conv = nn.utils.spectral_norm(self.conv)
461 | 
462 |   def forward(self, input):
463 |       out = self.conv(input)
464 |       return out
465 | 
466 | class MultiHeadAttention(nn.Module):
467 |     ''' Multi-Head Attention module '''
468 |     def __init__(self, n_head, d_model, d_k, d_v, dropout=0., spectral_norm=False):
469 |         super().__init__()
470 | 
471 |         self.n_head = n_head
472 |         self.d_k = d_k
473 |         self.d_v = d_v
474 | 
475 |         self.w_qs = nn.Linear(d_model, n_head * d_k)
476 |         self.w_ks = nn.Linear(d_model, n_head * d_k)
477 |         self.w_vs = nn.Linear(d_model, n_head * d_v)
478 |         
479 |         self.attention = ScaledDotProductAttention(temperature=np.power(d_model, 0.5), dropout=dropout)
480 | 
481 |         self.fc = nn.Linear(n_head * d_v, d_model)
482 |         self.dropout = nn.Dropout(dropout)
483 | 
484 |         if spectral_norm:
485 |             self.w_qs = nn.utils.spectral_norm(self.w_qs)
486 |             self.w_ks = nn.utils.spectral_norm(self.w_ks)
487 |             self.w_vs = nn.utils.spectral_norm(self.w_vs)
488 |             self.fc = nn.utils.spectral_norm(self.fc)
489 | 
490 |     def forward(self, x, mask=None):
491 |         d_k, d_v, n_head = self.d_k, self.d_v, self.n_head
492 |         sz_b, len_x, _ = x.size()
493 | 
494 |         residual = x
495 | 
496 |         q = self.w_qs(x).view(sz_b, len_x, n_head, d_k)
497 |         k = self.w_ks(x).view(sz_b, len_x, n_head, d_k)
498 |         v = self.w_vs(x).view(sz_b, len_x, n_head, d_v)
499 |         q = q.permute(2, 0, 1, 3).contiguous().view(-1,
500 |                                                     len_x, d_k)  # (n*b) x lq x dk
501 |         k = k.permute(2, 0, 1, 3).contiguous().view(-1,
502 |                                                     len_x, d_k)  # (n*b) x lk x dk
503 |         v = v.permute(2, 0, 1, 3).contiguous().view(-1,
504 |                                                     len_x, d_v)  # (n*b) x lv x dv
505 | 
506 |         if mask is not None:
507 |             slf_mask = mask.repeat(n_head, 1, 1)  # (n*b) x .. x ..
508 |         else:
509 |             slf_mask = None
510 |         output, attn = self.attention(q, k, v, mask=slf_mask)
511 | 
512 |         output = output.view(n_head, sz_b, len_x, d_v)
513 |         output = output.permute(1, 2, 0, 3).contiguous().view(
514 |                         sz_b, len_x, -1)  # b x lq x (n*dv)
515 | 
516 |         output = self.fc(output)
517 | 
518 |         output = self.dropout(output) + residual
519 |         return output, attn
520 | 
521 | 
522 | class ScaledDotProductAttention(nn.Module):
523 |     ''' Scaled Dot-Product Attention '''
524 | 
525 |     def __init__(self, temperature, dropout):
526 |         super().__init__()
527 |         self.temperature = temperature
528 |         self.softmax = nn.Softmax(dim=2)
529 |         self.dropout = nn.Dropout(dropout)
530 | 
531 |     def forward(self, q, k, v, mask=None):
532 | 
533 |         attn = torch.bmm(q, k.transpose(1, 2))
534 |         attn = attn / self.temperature
535 | 
536 |         if mask is not None:
537 |             attn = attn.masked_fill(mask, -np.inf)
538 | 
539 |         attn = self.softmax(attn)
540 |         p_attn = self.dropout(attn)
541 | 
542 |         output = torch.bmm(p_attn, v)
543 |         return output, attn


--------------------------------------------------------------------------------
/monotonic_align/__init__.py:
--------------------------------------------------------------------------------
 1 | import numpy as np
 2 | import torch
 3 | from .monotonic_align.core import maximum_path_c
 4 | 
 5 | 
 6 | def maximum_path(neg_cent, mask):
 7 |   """ Cython optimized version.
 8 |   neg_cent: [b, t_t, t_s]
 9 |   mask: [b, t_t, t_s]
10 |   """
11 |   device = neg_cent.device
12 |   dtype = neg_cent.dtype
13 |   neg_cent = neg_cent.data.cpu().numpy().astype(np.float32)
14 |   path = np.zeros(neg_cent.shape, dtype=np.int32)
15 | 
16 |   t_t_max = mask.sum(1)[:, 0].data.cpu().numpy().astype(np.int32)
17 |   t_s_max = mask.sum(2)[:, 0].data.cpu().numpy().astype(np.int32)
18 |   maximum_path_c(path, neg_cent, t_t_max, t_s_max)
19 |   return torch.from_numpy(path).to(device=device, dtype=dtype)
20 | 


--------------------------------------------------------------------------------
/monotonic_align/core.pyx:
--------------------------------------------------------------------------------
 1 | cimport cython
 2 | from cython.parallel import prange
 3 | 
 4 | 
 5 | @cython.boundscheck(False)
 6 | @cython.wraparound(False)
 7 | cdef void maximum_path_each(int[:,::1] path, float[:,::1] value, int t_y, int t_x, float max_neg_val=-1e9) nogil:
 8 |   cdef int x
 9 |   cdef int y
10 |   cdef float v_prev
11 |   cdef float v_cur
12 |   cdef float tmp
13 |   cdef int index = t_x - 1
14 | 
15 |   for y in range(t_y):
16 |     for x in range(max(0, t_x + y - t_y), min(t_x, y + 1)):
17 |       if x == y:
18 |         v_cur = max_neg_val
19 |       else:
20 |         v_cur = value[y-1, x]
21 |       if x == 0:
22 |         if y == 0:
23 |           v_prev = 0.
24 |         else:
25 |           v_prev = max_neg_val
26 |       else:
27 |         v_prev = value[y-1, x-1]
28 |       value[y, x] += max(v_prev, v_cur)
29 | 
30 |   for y in range(t_y - 1, -1, -1):
31 |     path[y, index] = 1
32 |     if index != 0 and (index == y or value[y-1, index] < value[y-1, index-1]):
33 |       index = index - 1
34 | 
35 | 
36 | @cython.boundscheck(False)
37 | @cython.wraparound(False)
38 | cpdef void maximum_path_c(int[:,:,::1] paths, float[:,:,::1] values, int[::1] t_ys, int[::1] t_xs) nogil:
39 |   cdef int b = paths.shape[0]
40 |   cdef int i
41 |   for i in prange(b, nogil=True):
42 |     maximum_path_each(paths[i], values[i], t_ys[i], t_xs[i])
43 | 


--------------------------------------------------------------------------------
/monotonic_align/setup.py:
--------------------------------------------------------------------------------
 1 | from distutils.core import setup
 2 | from Cython.Build import cythonize
 3 | import numpy
 4 | 
 5 | setup(
 6 |   name = 'monotonic_align',
 7 |   ext_modules = cythonize("core.pyx"),
 8 |   include_dirs=[numpy.get_include()]
 9 | )
10 | 


--------------------------------------------------------------------------------
/prepare_wav.py:
--------------------------------------------------------------------------------
 1 | import argparse
 2 | import libritts
 3 | 
 4 | '''
 5 | resample to 22050Hz, trim
 6 | normalize txt
 7 | save them as /LibriTTS/wav22/NAME.wav, /LibriTTS/wav22/NAME.txt
 8 | '''
 9 | def main(data_path, sr):
10 |     libritts.prepare_align_and_resample(data_path, sr)
11 |     
12 | 
13 | if __name__ == "__main__":
14 |     parser = argparse.ArgumentParser()
15 |     parser.add_argument('--data_path', type=str, default='/home/hcy71/DATA/LibriTTS')
16 |     parser.add_argument('--resample_rate', '-sr', type=int, default=22050)
17 | 
18 |     args = parser.parse_args()
19 | 
20 |     main(args.data_path, args.resample_rate)
21 | 


--------------------------------------------------------------------------------
/preprocess.py:
--------------------------------------------------------------------------------
 1 | import argparse
 2 | import text
 3 | from utils import load_filepaths_and_text
 4 | 
 5 | if __name__ == '__main__':
 6 |   parser = argparse.ArgumentParser()
 7 |   parser.add_argument("--out_extension", default="cleaned")
 8 |   parser.add_argument("--text_index", default=1, type=int)
 9 |   parser.add_argument("--filelists", nargs="+", default=["filelists/ljs_audio_text_val_filelist.txt", "filelists/ljs_audio_text_test_filelist.txt"])
10 |   parser.add_argument("--text_cleaners", nargs="+", default=["english_cleaners2"])
11 | 
12 |   args = parser.parse_args()
13 |     
14 | 
15 |   for filelist in args.filelists:
16 |     print("START:", filelist)
17 |     filepaths_and_text = load_filepaths_and_text(filelist)
18 |     for i in range(len(filepaths_and_text)):
19 |       original_text = filepaths_and_text[i][args.text_index]
20 |       cleaned_text = text._clean_text(original_text, args.text_cleaners)
21 |       filepaths_and_text[i][args.text_index] = cleaned_text
22 | 
23 |     new_filelist = filelist + "." + args.out_extension
24 |     with open(new_filelist, "w", encoding="utf-8") as f:
25 |       f.writelines(["|".join(x) + "\n" for x in filepaths_and_text])
26 | 


--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
 1 | Cython==0.29.21
 2 | librosa==0.8.0
 3 | matplotlib==3.3.1
 4 | numpy==1.18.5
 5 | phonemizer==2.2.1
 6 | scipy==1.5.2
 7 | tensorboard==2.3.0
 8 | torch==1.6.0
 9 | torchvision==0.7.0
10 | Unidecode==1.1.1
11 | 


--------------------------------------------------------------------------------
/text/__init__.py:
--------------------------------------------------------------------------------
 1 | """ from https://github.com/keithito/tacotron """
 2 | import re
 3 | from text import cleaners
 4 | from text.symbols import symbols
 5 | 
 6 | 
 7 | # Mappings from symbol to numeric ID and vice versa:
 8 | _symbol_to_id = {s: i for i, s in enumerate(symbols)}
 9 | _id_to_symbol = {i: s for i, s in enumerate(symbols)}
10 | 
11 | # Regular expression matching text enclosed in curly braces:
12 | _curly_re = re.compile(r'(.*?)\{(.+?)\}(.*)')
13 | 
14 | 
15 | def text_to_sequence(text, cleaner_names):
16 |     '''Converts a string of text to a sequence of IDs corresponding to the symbols in the text.
17 | 
18 |       The text can optionally have ARPAbet sequences enclosed in curly braces embedded
19 |       in it. For example, "Turn left on {HH AW1 S S T AH0 N} Street."
20 | 
21 |       Args:
22 |         text: string to convert to a sequence
23 |         cleaner_names: names of the cleaner functions to run the text through
24 | 
25 |       Returns:
26 |         List of integers corresponding to the symbols in the text
27 |     '''
28 |     sequence = []
29 | 
30 |     # Check for curly braces and treat their contents as ARPAbet:
31 |     while len(text):
32 |         m = _curly_re.match(text)
33 |         
34 |         if not m:
35 |             sequence += _symbols_to_sequence(_clean_text(text, cleaner_names))
36 |             break
37 |         sequence += _symbols_to_sequence(
38 |             _clean_text(m.group(1), cleaner_names))
39 |         sequence += _arpabet_to_sequence(m.group(2))
40 |         text = m.group(3)
41 | 
42 |     return sequence
43 | 
44 | 
45 | def sequence_to_text(sequence):
46 |     '''Converts a sequence of IDs back to a string'''
47 |     result = ''
48 |     for symbol_id in sequence:
49 |         if symbol_id in _id_to_symbol:
50 |             s = _id_to_symbol[symbol_id]
51 |             # Enclose ARPAbet back in curly braces:
52 |             if len(s) > 1 and s[0] == '@':
53 |                 s = '{%s}' % s[1:]
54 |             result += s
55 |     return result.replace('}{', ' ')
56 | 
57 | 
58 | def _clean_text(text, cleaner_names):
59 |     for name in cleaner_names:
60 |         cleaner = getattr(cleaners, name)
61 |         if not cleaner:
62 |             raise Exception('Unknown cleaner: %s' % name)
63 |         text = cleaner(text)
64 |     return text
65 | 
66 | 
67 | def _symbols_to_sequence(symbols):
68 |     return [_symbol_to_id[s] for s in symbols if _should_keep_symbol(s)]
69 | 
70 | 
71 | def _arpabet_to_sequence(text):
72 |     return _symbols_to_sequence(['@' + s for s in text.split()])
73 | 
74 | 
75 | def _should_keep_symbol(s):
76 |     return s in _symbol_to_id and s is not '_' and s is not '~'
77 | 


--------------------------------------------------------------------------------
/text/cleaners.py:
--------------------------------------------------------------------------------
 1 | """ from https://github.com/keithito/tacotron """
 2 | 
 3 | '''
 4 | Cleaners are transformations that run over the input text at both training and eval time.
 5 | 
 6 | Cleaners can be selected by passing a comma-delimited list of cleaner names as the "cleaners"
 7 | hyperparameter. Some cleaners are English-specific. You'll typically want to use:
 8 |   1. "english_cleaners" for English text
 9 |   2. "transliteration_cleaners" for non-English text that can be transliterated to ASCII using
10 |      the Unidecode library (https://pypi.python.org/pypi/Unidecode)
11 |   3. "basic_cleaners" if you do not want to transliterate (in this case, you should also update
12 |      the symbols in symbols.py to match your data).
13 | '''
14 | 
15 | 
16 | # Regular expression matching whitespace:
17 | import re
18 | from unidecode import unidecode
19 | from .numbers import normalize_numbers
20 | _whitespace_re = re.compile(r'\s+')
21 | 
22 | # List of (regular expression, replacement) pairs for abbreviations:
23 | _abbreviations = [(re.compile('\\b%s\\.' % x[0], re.IGNORECASE), x[1]) for x in [
24 |     ('mrs', 'misess'),
25 |     ('mr', 'mister'),
26 |     ('dr', 'doctor'),
27 |     ('st', 'saint'),
28 |     ('co', 'company'),
29 |     ('jr', 'junior'),
30 |     ('maj', 'major'),
31 |     ('gen', 'general'),
32 |     ('drs', 'doctors'),
33 |     ('rev', 'reverend'),
34 |     ('lt', 'lieutenant'),
35 |     ('hon', 'honorable'),
36 |     ('sgt', 'sergeant'),
37 |     ('capt', 'captain'),
38 |     ('esq', 'esquire'),
39 |     ('ltd', 'limited'),
40 |     ('col', 'colonel'),
41 |     ('ft', 'fort'),
42 | ]]
43 | 
44 | 
45 | def expand_abbreviations(text):
46 |     for regex, replacement in _abbreviations:
47 |         text = re.sub(regex, replacement, text)
48 |     return text
49 | 
50 | 
51 | def expand_numbers(text):
52 |     return normalize_numbers(text)
53 | 
54 | 
55 | def lowercase(text):
56 |     return text.lower()
57 | 
58 | 
59 | def collapse_whitespace(text):
60 |     return re.sub(_whitespace_re, ' ', text)
61 | 
62 | 
63 | def convert_to_ascii(text):
64 |     return unidecode(text)
65 | 
66 | 
67 | def basic_cleaners(text):
68 |     '''Basic pipeline that lowercases and collapses whitespace without transliteration.'''
69 |     text = lowercase(text)
70 |     text = collapse_whitespace(text)
71 |     return text
72 | 
73 | 
74 | def transliteration_cleaners(text):
75 |     '''Pipeline for non-English text that transliterates to ASCII.'''
76 |     text = convert_to_ascii(text)
77 |     text = lowercase(text)
78 |     text = collapse_whitespace(text)
79 |     return text
80 | 
81 | 
82 | def english_cleaners(text):
83 |     '''Pipeline for English text, including number and abbreviation expansion.'''
84 |     text = convert_to_ascii(text)
85 |     text = lowercase(text)
86 |     text = expand_numbers(text)
87 |     text = expand_abbreviations(text)
88 |     text = collapse_whitespace(text)
89 |     return text
90 | 


--------------------------------------------------------------------------------
/text/cmudict.py:
--------------------------------------------------------------------------------
 1 | """ from https://github.com/keithito/tacotron """
 2 | 
 3 | import re
 4 | 
 5 | 
 6 | valid_symbols = [
 7 |     'AA', 'AA0', 'AA1', 'AA2', 'AE', 'AE0', 'AE1', 'AE2', 'AH', 'AH0', 'AH1', 'AH2',
 8 |     'AO', 'AO0', 'AO1', 'AO2', 'AW', 'AW0', 'AW1', 'AW2', 'AY', 'AY0', 'AY1', 'AY2',
 9 |     'B', 'CH', 'D', 'DH', 'EH', 'EH0', 'EH1', 'EH2', 'ER', 'ER0', 'ER1', 'ER2', 'EY',
10 |     'EY0', 'EY1', 'EY2', 'F', 'G', 'HH', 'IH', 'IH0', 'IH1', 'IH2', 'IY', 'IY0', 'IY1',
11 |     'IY2', 'JH', 'K', 'L', 'M', 'N', 'NG', 'OW', 'OW0', 'OW1', 'OW2', 'OY', 'OY0',
12 |     'OY1', 'OY2', 'P', 'R', 'S', 'SH', 'T', 'TH', 'UH', 'UH0', 'UH1', 'UH2', 'UW',
13 |     'UW0', 'UW1', 'UW2', 'V', 'W', 'Y', 'Z', 'ZH'
14 | ]
15 | 
16 | _valid_symbol_set = set(valid_symbols)
17 | 
18 | 
19 | class CMUDict:
20 |     '''Thin wrapper around CMUDict data. http://www.speech.cs.cmu.edu/cgi-bin/cmudict'''
21 | 
22 |     def __init__(self, file_or_path, keep_ambiguous=True):
23 |         if isinstance(file_or_path, str):
24 |             with open(file_or_path, encoding='latin-1') as f:
25 |                 entries = _parse_cmudict(f)
26 |         else:
27 |             entries = _parse_cmudict(file_or_path)
28 |         if not keep_ambiguous:
29 |             entries = {word: pron for word,
30 |                        pron in entries.items() if len(pron) == 1}
31 |         self._entries = entries
32 | 
33 |     def __len__(self):
34 |         return len(self._entries)
35 | 
36 |     def lookup(self, word):
37 |         '''Returns list of ARPAbet pronunciations of the given word.'''
38 |         return self._entries.get(word.upper())
39 | 
40 | 
41 | _alt_re = re.compile(r'\([0-9]+\)')
42 | 
43 | 
44 | def _parse_cmudict(file):
45 |     cmudict = {}
46 |     for line in file:
47 |         if len(line) and (line[0] >= 'A' and line[0] <= 'Z' or line[0] == "'"):
48 |             parts = line.split('  ')
49 |             word = re.sub(_alt_re, '', parts[0])
50 |             pronunciation = _get_pronunciation(parts[1])
51 |             if pronunciation:
52 |                 if word in cmudict:
53 |                     cmudict[word].append(pronunciation)
54 |                 else:
55 |                     cmudict[word] = [pronunciation]
56 |     return cmudict
57 | 
58 | 
59 | def _get_pronunciation(s):
60 |     parts = s.strip().split(' ')
61 |     for part in parts:
62 |         if part not in _valid_symbol_set:
63 |             return None
64 |     return ' '.join(parts)
65 | 


--------------------------------------------------------------------------------
/text/numbers.py:
--------------------------------------------------------------------------------
 1 | """ from https://github.com/keithito/tacotron """
 2 | 
 3 | import inflect
 4 | import re
 5 | 
 6 | 
 7 | _inflect = inflect.engine()
 8 | _comma_number_re = re.compile(r'([0-9][0-9\,]+[0-9])')
 9 | _decimal_number_re = re.compile(r'([0-9]+\.[0-9]+)')
10 | _pounds_re = re.compile(r'£([0-9\,]*[0-9]+)')
11 | _dollars_re = re.compile(r'\$([0-9\.\,]*[0-9]+)')
12 | _ordinal_re = re.compile(r'[0-9]+(st|nd|rd|th)')
13 | _number_re = re.compile(r'[0-9]+')
14 | 
15 | 
16 | def _remove_commas(m):
17 |     return m.group(1).replace(',', '')
18 | 
19 | 
20 | def _expand_decimal_point(m):
21 |     return m.group(1).replace('.', ' point ')
22 | 
23 | 
24 | def _expand_dollars(m):
25 |     match = m.group(1)
26 |     parts = match.split('.')
27 |     if len(parts) > 2:
28 |         return match + ' dollars'  # Unexpected format
29 |     dollars = int(parts[0]) if parts[0] else 0
30 |     cents = int(parts[1]) if len(parts) > 1 and parts[1] else 0
31 |     if dollars and cents:
32 |         dollar_unit = 'dollar' if dollars == 1 else 'dollars'
33 |         cent_unit = 'cent' if cents == 1 else 'cents'
34 |         return '%s %s, %s %s' % (dollars, dollar_unit, cents, cent_unit)
35 |     elif dollars:
36 |         dollar_unit = 'dollar' if dollars == 1 else 'dollars'
37 |         return '%s %s' % (dollars, dollar_unit)
38 |     elif cents:
39 |         cent_unit = 'cent' if cents == 1 else 'cents'
40 |         return '%s %s' % (cents, cent_unit)
41 |     else:
42 |         return 'zero dollars'
43 | 
44 | 
45 | def _expand_ordinal(m):
46 |     return _inflect.number_to_words(m.group(0))
47 | 
48 | 
49 | def _expand_number(m):
50 |     num = int(m.group(0))
51 |     if num > 1000 and num < 3000:
52 |         if num == 2000:
53 |             return 'two thousand'
54 |         elif num > 2000 and num < 2010:
55 |             return 'two thousand ' + _inflect.number_to_words(num % 100)
56 |         elif num % 100 == 0:
57 |             return _inflect.number_to_words(num // 100) + ' hundred'
58 |         else:
59 |             return _inflect.number_to_words(num, andword='', zero='oh', group=2).replace(', ', ' ')
60 |     else:
61 |         return _inflect.number_to_words(num, andword='')
62 | 
63 | 
64 | def normalize_numbers(text):
65 |     text = re.sub(_comma_number_re, _remove_commas, text)
66 |     text = re.sub(_pounds_re, r'\1 pounds', text)
67 |     text = re.sub(_dollars_re, _expand_dollars, text)
68 |     text = re.sub(_decimal_number_re, _expand_decimal_point, text)
69 |     text = re.sub(_ordinal_re, _expand_ordinal, text)
70 |     text = re.sub(_number_re, _expand_number, text)
71 |     return text
72 | 


--------------------------------------------------------------------------------
/text/symbols.py:
--------------------------------------------------------------------------------
 1 | """ from https://github.com/keithito/tacotron """
 2 | 
 3 | '''
 4 | Defines the set of symbols used in text input to the model.
 5 | 
 6 | The default is a set of ASCII characters that works well for English or text that has been run through Unidecode. For other data, you can modify _characters. See TRAINING_DATA.md for details. '''
 7 | 
 8 | from text import cmudict
 9 | _pad        = '_'
10 | _punctuation = '!\'(),.:;? '
11 | _special = '-'
12 | _letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'
13 | _silences = ['@sp', '@spn', '@sil']
14 | 
15 | # Prepend "@" to ARPAbet symbols to ensure uniqueness (some are the same as uppercase letters):
16 | _arpabet = ['@' + s for s in cmudict.valid_symbols]
17 | 
18 | # Export all symbols:
19 | symbols = [_pad] + list(_special) + list(_punctuation) + \
20 |         list(_letters) + _arpabet + _silences
21 | 


--------------------------------------------------------------------------------
/train.py:
--------------------------------------------------------------------------------
  1 | import os
  2 | import json
  3 | import argparse
  4 | import itertools
  5 | import math
  6 | import torch
  7 | from torch import nn, optim
  8 | from torch.nn import functional as F
  9 | from torch.utils.data import DataLoader
 10 | from torch.utils.tensorboard import SummaryWriter
 11 | import torch.multiprocessing as mp
 12 | import torch.distributed as dist
 13 | from torch.nn.parallel import DistributedDataParallel as DDP
 14 | from torch.cuda.amp import autocast, GradScaler
 15 | 
 16 | import commons
 17 | import utils
 18 | from data_utils import (
 19 |   TextAudioLoader,
 20 |   TextAudioCollate,
 21 |   DistributedBucketSampler
 22 | )
 23 | from models import (
 24 |   SynthesizerTrn,
 25 |   MultiPeriodDiscriminator,
 26 | )
 27 | from losses import (
 28 |   generator_loss,
 29 |   discriminator_loss,
 30 |   feature_loss,
 31 |   kl_loss
 32 | )
 33 | from mel_processing import mel_spectrogram_torch, spec_to_mel_torch
 34 | from text.symbols import symbols
 35 | 
 36 | 
 37 | torch.backends.cudnn.benchmark = True
 38 | global_step = 0
 39 | 
 40 | 
 41 | def main():
 42 |   """Assume Single Node Multi GPUs Training Only"""
 43 |   assert torch.cuda.is_available(), "CPU training is not allowed."
 44 | 
 45 |   n_gpus = torch.cuda.device_count()
 46 |   os.environ['MASTER_ADDR'] = 'localhost'
 47 |   os.environ['MASTER_PORT'] = '80000'
 48 | 
 49 |   hps = utils.get_hparams()
 50 |   mp.spawn(run, nprocs=n_gpus, args=(n_gpus, hps,))
 51 | 
 52 | 
 53 | def run(rank, n_gpus, hps):
 54 |   global global_step
 55 |   if rank == 0:
 56 |     logger = utils.get_logger(hps.model_dir)
 57 |     logger.info(hps)
 58 |     utils.check_git_hash(hps.model_dir)
 59 |     writer = SummaryWriter(log_dir=hps.model_dir)
 60 |     writer_eval = SummaryWriter(log_dir=os.path.join(hps.model_dir, "eval"))
 61 | 
 62 |   dist.init_process_group(backend='nccl', init_method='env://', world_size=n_gpus, rank=rank)
 63 |   torch.manual_seed(hps.train.seed)
 64 |   torch.cuda.set_device(rank)
 65 | 
 66 |   train_dataset = TextAudioLoader(hps.data.training_files, hps.data)
 67 |   train_sampler = DistributedBucketSampler(
 68 |       train_dataset,
 69 |       hps.train.batch_size,
 70 |       [32,300,400,500,600,700,800,900,1000],
 71 |       num_replicas=n_gpus,
 72 |       rank=rank,
 73 |       shuffle=True)
 74 |   collate_fn = TextAudioCollate()
 75 |   train_loader = DataLoader(train_dataset, num_workers=8, shuffle=False, pin_memory=True,
 76 |       collate_fn=collate_fn, batch_sampler=train_sampler)
 77 |   if rank == 0:
 78 |     eval_dataset = TextAudioLoader(hps.data.validation_files, hps.data)
 79 |     eval_loader = DataLoader(eval_dataset, num_workers=8, shuffle=False,
 80 |         batch_size=hps.train.batch_size, pin_memory=True,
 81 |         drop_last=False, collate_fn=collate_fn)
 82 | 
 83 |   net_g = SynthesizerTrn(
 84 |       len(symbols),
 85 |       hps.data.filter_length // 2 + 1,
 86 |       hps.train.segment_size // hps.data.hop_length,
 87 |       **hps.model).cuda(rank)
 88 |   net_d = MultiPeriodDiscriminator(hps.model.use_spectral_norm).cuda(rank)
 89 |   optim_g = torch.optim.AdamW(
 90 |       net_g.parameters(), 
 91 |       hps.train.learning_rate, 
 92 |       betas=hps.train.betas, 
 93 |       eps=hps.train.eps)
 94 |   optim_d = torch.optim.AdamW(
 95 |       net_d.parameters(),
 96 |       hps.train.learning_rate, 
 97 |       betas=hps.train.betas, 
 98 |       eps=hps.train.eps)
 99 |   net_g = DDP(net_g, device_ids=[rank])
100 |   net_d = DDP(net_d, device_ids=[rank])
101 | 
102 |   try:
103 |     _, _, _, epoch_str = utils.load_checkpoint(utils.latest_checkpoint_path(hps.model_dir, "G_*.pth"), net_g, optim_g)
104 |     _, _, _, epoch_str = utils.load_checkpoint(utils.latest_checkpoint_path(hps.model_dir, "D_*.pth"), net_d, optim_d)
105 |     global_step = (epoch_str - 1) * len(train_loader)
106 |   except:
107 |     epoch_str = 1
108 |     global_step = 0
109 | 
110 |   scheduler_g = torch.optim.lr_scheduler.ExponentialLR(optim_g, gamma=hps.train.lr_decay, last_epoch=epoch_str-2)
111 |   scheduler_d = torch.optim.lr_scheduler.ExponentialLR(optim_d, gamma=hps.train.lr_decay, last_epoch=epoch_str-2)
112 | 
113 |   scaler = GradScaler(enabled=hps.train.fp16_run)
114 | 
115 |   for epoch in range(epoch_str, hps.train.epochs + 1):
116 |     if rank==0:
117 |       train_and_evaluate(rank, epoch, hps, [net_g, net_d], [optim_g, optim_d], [scheduler_g, scheduler_d], scaler, [train_loader, eval_loader], logger, [writer, writer_eval])
118 |     else:
119 |       train_and_evaluate(rank, epoch, hps, [net_g, net_d], [optim_g, optim_d], [scheduler_g, scheduler_d], scaler, [train_loader, None], None, None)
120 |     scheduler_g.step()
121 |     scheduler_d.step()
122 | 
123 | 
124 | def train_and_evaluate(rank, epoch, hps, nets, optims, schedulers, scaler, loaders, logger, writers):
125 |   net_g, net_d = nets
126 |   optim_g, optim_d = optims
127 |   scheduler_g, scheduler_d = schedulers
128 |   train_loader, eval_loader = loaders
129 |   if writers is not None:
130 |     writer, writer_eval = writers
131 | 
132 |   train_loader.batch_sampler.set_epoch(epoch)
133 |   global global_step
134 | 
135 |   net_g.train()
136 |   net_d.train()
137 |   for batch_idx, (x, x_lengths, spec, spec_lengths, y, y_lengths) in enumerate(train_loader):
138 |     x, x_lengths = x.cuda(rank, non_blocking=True), x_lengths.cuda(rank, non_blocking=True)
139 |     spec, spec_lengths = spec.cuda(rank, non_blocking=True), spec_lengths.cuda(rank, non_blocking=True)
140 |     y, y_lengths = y.cuda(rank, non_blocking=True), y_lengths.cuda(rank, non_blocking=True)
141 | 
142 |     with autocast(enabled=hps.train.fp16_run):
143 |       y_hat, l_length, attn, ids_slice, x_mask, z_mask,\
144 |       (z, z_p, m_p, logs_p, m_q, logs_q) = net_g(x, x_lengths, spec, spec_lengths)
145 | 
146 |       mel = spec_to_mel_torch(
147 |           spec, 
148 |           hps.data.filter_length, 
149 |           hps.data.n_mel_channels, 
150 |           hps.data.sampling_rate,
151 |           hps.data.mel_fmin, 
152 |           hps.data.mel_fmax)
153 |       y_mel = commons.slice_segments(mel, ids_slice, hps.train.segment_size // hps.data.hop_length)
154 |       y_hat_mel = mel_spectrogram_torch(
155 |           y_hat.squeeze(1), 
156 |           hps.data.filter_length, 
157 |           hps.data.n_mel_channels, 
158 |           hps.data.sampling_rate, 
159 |           hps.data.hop_length, 
160 |           hps.data.win_length, 
161 |           hps.data.mel_fmin, 
162 |           hps.data.mel_fmax
163 |       )
164 | 
165 |       y = commons.slice_segments(y, ids_slice * hps.data.hop_length, hps.train.segment_size) # slice 
166 | 
167 |       # Discriminator
168 |       y_d_hat_r, y_d_hat_g, _, _ = net_d(y, y_hat.detach())
169 |       with autocast(enabled=False):
170 |         loss_disc, losses_disc_r, losses_disc_g = discriminator_loss(y_d_hat_r, y_d_hat_g)
171 |         loss_disc_all = loss_disc
172 |     optim_d.zero_grad()
173 |     scaler.scale(loss_disc_all).backward()
174 |     scaler.unscale_(optim_d)
175 |     grad_norm_d = commons.clip_grad_value_(net_d.parameters(), None)
176 |     scaler.step(optim_d)
177 | 
178 |     with autocast(enabled=hps.train.fp16_run):
179 |       # Generator
180 |       y_d_hat_r, y_d_hat_g, fmap_r, fmap_g = net_d(y, y_hat)
181 |       with autocast(enabled=False):
182 |         loss_dur = torch.sum(l_length.float())
183 |         loss_mel = F.l1_loss(y_mel, y_hat_mel) * hps.train.c_mel
184 |         loss_kl = kl_loss(z_p, logs_q, m_p, logs_p, z_mask) * hps.train.c_kl
185 | 
186 |         loss_fm = feature_loss(fmap_r, fmap_g)
187 |         loss_gen, losses_gen = generator_loss(y_d_hat_g)
188 |         loss_gen_all = loss_gen + loss_fm + loss_mel + loss_dur + loss_kl
189 |     optim_g.zero_grad()
190 |     scaler.scale(loss_gen_all).backward()
191 |     scaler.unscale_(optim_g)
192 |     grad_norm_g = commons.clip_grad_value_(net_g.parameters(), None)
193 |     scaler.step(optim_g)
194 |     scaler.update()
195 | 
196 |     if rank==0:
197 |       if global_step % hps.train.log_interval == 0:
198 |         lr = optim_g.param_groups[0]['lr']
199 |         losses = [loss_disc, loss_gen, loss_fm, loss_mel, loss_dur, loss_kl]
200 |         logger.info('Train Epoch: {} [{:.0f}%]'.format(
201 |           epoch,
202 |           100. * batch_idx / len(train_loader)))
203 |         logger.info([x.item() for x in losses] + [global_step, lr])
204 |         
205 |         scalar_dict = {"loss/g/total": loss_gen_all, "loss/d/total": loss_disc_all, "learning_rate": lr, "grad_norm_d": grad_norm_d, "grad_norm_g": grad_norm_g}
206 |         scalar_dict.update({"loss/g/fm": loss_fm, "loss/g/mel": loss_mel, "loss/g/dur": loss_dur, "loss/g/kl": loss_kl})
207 | 
208 |         scalar_dict.update({"loss/g/{}".format(i): v for i, v in enumerate(losses_gen)})
209 |         scalar_dict.update({"loss/d_r/{}".format(i): v for i, v in enumerate(losses_disc_r)})
210 |         scalar_dict.update({"loss/d_g/{}".format(i): v for i, v in enumerate(losses_disc_g)})
211 |         image_dict = { 
212 |             "slice/mel_org": utils.plot_spectrogram_to_numpy(y_mel[0].data.cpu().numpy()),
213 |             "slice/mel_gen": utils.plot_spectrogram_to_numpy(y_hat_mel[0].data.cpu().numpy()), 
214 |             "all/mel": utils.plot_spectrogram_to_numpy(mel[0].data.cpu().numpy()),
215 |             "all/attn": utils.plot_alignment_to_numpy(attn[0,0].data.cpu().numpy())
216 |         }
217 |         utils.summarize(
218 |           writer=writer,
219 |           global_step=global_step, 
220 |           images=image_dict,
221 |           scalars=scalar_dict)
222 | 
223 |       if global_step % hps.train.eval_interval == 0:
224 |         evaluate(hps, net_g, eval_loader, writer_eval)
225 |         utils.save_checkpoint(net_g, optim_g, hps.train.learning_rate, epoch, os.path.join(hps.model_dir, "G_{}.pth".format(global_step)))
226 |         utils.save_checkpoint(net_d, optim_d, hps.train.learning_rate, epoch, os.path.join(hps.model_dir, "D_{}.pth".format(global_step)))
227 |     global_step += 1
228 |   
229 |   if rank == 0:
230 |     logger.info('====> Epoch: {}'.format(epoch))
231 | 
232 |  
233 | def evaluate(hps, generator, eval_loader, writer_eval):
234 |     generator.eval()
235 |     with torch.no_grad():
236 |       for batch_idx, (x, x_lengths, spec, spec_lengths, y, y_lengths) in enumerate(eval_loader):
237 |         x, x_lengths = x.cuda(0), x_lengths.cuda(0)
238 |         spec, spec_lengths = spec.cuda(0), spec_lengths.cuda(0)
239 |         y, y_lengths = y.cuda(0), y_lengths.cuda(0)
240 | 
241 |         # remove else
242 |         x = x[:1]
243 |         x_lengths = x_lengths[:1]
244 |         spec = spec[:1]
245 |         spec_lengths = spec_lengths[:1]
246 |         y = y[:1]
247 |         y_lengths = y_lengths[:1]
248 |         break
249 |       y_hat, attn, mask, *_ = generator.module.infer(x, x_lengths, max_len=1000)
250 |       y_hat_lengths = mask.sum([1,2]).long() * hps.data.hop_length
251 | 
252 |       mel = spec_to_mel_torch(
253 |         spec, 
254 |         hps.data.filter_length, 
255 |         hps.data.n_mel_channels, 
256 |         hps.data.sampling_rate,
257 |         hps.data.mel_fmin, 
258 |         hps.data.mel_fmax)
259 |       y_hat_mel = mel_spectrogram_torch(
260 |         y_hat.squeeze(1).float(),
261 |         hps.data.filter_length,
262 |         hps.data.n_mel_channels,
263 |         hps.data.sampling_rate,
264 |         hps.data.hop_length,
265 |         hps.data.win_length,
266 |         hps.data.mel_fmin,
267 |         hps.data.mel_fmax
268 |       )
269 |     image_dict = {
270 |       "gen/mel": utils.plot_spectrogram_to_numpy(y_hat_mel[0].cpu().numpy())
271 |     }
272 |     audio_dict = {
273 |       "gen/audio": y_hat[0,:,:y_hat_lengths[0]]
274 |     }
275 |     if global_step == 0:
276 |       image_dict.update({"gt/mel": utils.plot_spectrogram_to_numpy(mel[0].cpu().numpy())})
277 |       audio_dict.update({"gt/audio": y[0,:,:y_lengths[0]]})
278 | 
279 |     utils.summarize(
280 |       writer=writer_eval,
281 |       global_step=global_step, 
282 |       images=image_dict,
283 |       audios=audio_dict,
284 |       audio_sampling_rate=hps.data.sampling_rate
285 |     )
286 |     generator.train()
287 | 
288 |                            
289 | if __name__ == "__main__":
290 |   main()
291 | 


--------------------------------------------------------------------------------
/train_ms.py:
--------------------------------------------------------------------------------
  1 | import os
  2 | import json
  3 | import argparse
  4 | import itertools
  5 | import math
  6 | import torch
  7 | from torch import nn, optim
  8 | from torch.nn import functional as F
  9 | from torch.utils.data import DataLoader
 10 | from torch.utils.tensorboard import SummaryWriter
 11 | import torch.multiprocessing as mp
 12 | import torch.distributed as dist
 13 | from torch.nn.parallel import DistributedDataParallel as DDP
 14 | from torch.cuda.amp import autocast, GradScaler
 15 | 
 16 | import commons
 17 | import utils
 18 | from data_utils import (
 19 |   TextAudioSpeakerLoader,
 20 |   TextAudioSpeakerCollate,
 21 |   DistributedBucketSampler
 22 | )
 23 | from models import (
 24 |   SynthesizerTrn,
 25 |   MultiPeriodDiscriminator,
 26 | )
 27 | from losses import (
 28 |   generator_loss,
 29 |   discriminator_loss,
 30 |   feature_loss,
 31 |   kl_loss
 32 | )
 33 | from mel_processing import mel_spectrogram_torch, spec_to_mel_torch
 34 | from text.symbols import symbols
 35 | 
 36 | 
 37 | torch.backends.cudnn.benchmark = True
 38 | global_step = 0
 39 | 
 40 | 
 41 | def main():
 42 |   """Assume Single Node Multi GPUs Training Only"""
 43 |   print(torch.cuda.is_available())
 44 |   assert torch.cuda.is_available(), "CPU training is not allowed."
 45 |   
 46 |   n_gpus = torch.cuda.device_count()
 47 |   os.environ['MASTER_ADDR'] = 'localhost'
 48 |   os.environ['MASTER_PORT'] = '8888'
 49 | 
 50 |   hps = utils.get_hparams()
 51 |   mp.spawn(run, nprocs=n_gpus, args=(n_gpus, hps,))
 52 | 
 53 | 
 54 | def run(rank, n_gpus, hps):
 55 |   global global_step
 56 |   if rank == 0:
 57 |     logger = utils.get_logger(hps.model_dir)
 58 |     logger.info(hps)
 59 |     utils.check_git_hash(hps.model_dir)
 60 |     writer = SummaryWriter(log_dir=hps.model_dir)
 61 |     writer_eval = SummaryWriter(log_dir=os.path.join(hps.model_dir, "eval"))
 62 | 
 63 |   dist.init_process_group(backend='nccl', init_method='env://', world_size=n_gpus, rank=rank)
 64 |   torch.manual_seed(hps.train.seed)
 65 |   torch.cuda.set_device(rank)
 66 | 
 67 |   train_dataset = TextAudioSpeakerLoader(hps.data.training_files, hps.data)
 68 |   train_sampler = DistributedBucketSampler(
 69 |       train_dataset,
 70 |       hps.train.batch_size,
 71 |       [32,300,400,500,600,700,800,900,1000],
 72 |       num_replicas=n_gpus,
 73 |       rank=rank,
 74 |       shuffle=True)
 75 |   collate_fn = TextAudioSpeakerCollate()
 76 |   train_loader = DataLoader(train_dataset, num_workers=8, shuffle=False, pin_memory=True,
 77 |       collate_fn=collate_fn, batch_sampler=train_sampler)
 78 |   if rank == 0:
 79 |     eval_dataset = TextAudioSpeakerLoader(hps.data.validation_files, hps.data)
 80 |     eval_loader = DataLoader(eval_dataset, num_workers=8, shuffle=False,
 81 |         batch_size=hps.train.batch_size, pin_memory=True,
 82 |         drop_last=False, collate_fn=collate_fn)
 83 | 
 84 |   net_g = SynthesizerTrn(
 85 |       len(symbols),
 86 |       hps.data.filter_length // 2 + 1,
 87 |       hps.train.segment_size // hps.data.hop_length,
 88 |       n_speakers=hps.data.n_speakers,
 89 |       **hps.model).cuda(rank)
 90 |   net_d = MultiPeriodDiscriminator(hps.model.use_spectral_norm).cuda(rank)
 91 |   optim_g = torch.optim.AdamW(
 92 |       net_g.parameters(), 
 93 |       hps.train.learning_rate, 
 94 |       betas=hps.train.betas, 
 95 |       eps=hps.train.eps)
 96 |   optim_d = torch.optim.AdamW(
 97 |       net_d.parameters(),
 98 |       hps.train.learning_rate, 
 99 |       betas=hps.train.betas, 
100 |       eps=hps.train.eps)
101 |   net_g = DDP(net_g, device_ids=[rank],find_unused_parameters=True)
102 |   net_d = DDP(net_d, device_ids=[rank],find_unused_parameters=True)
103 | 
104 |   try:
105 |     _, _, _, epoch_str = utils.load_checkpoint(utils.latest_checkpoint_path(hps.model_dir, "G_*.pth"), net_g, optim_g)
106 |     _, _, _, epoch_str = utils.load_checkpoint(utils.latest_checkpoint_path(hps.model_dir, "D_*.pth"), net_d, optim_d)
107 |     global_step = (epoch_str - 1) * len(train_loader)
108 |   except:
109 |     epoch_str = 1
110 |     global_step = 0
111 | 
112 |   scheduler_g = torch.optim.lr_scheduler.ExponentialLR(optim_g, gamma=hps.train.lr_decay, last_epoch=epoch_str-2)
113 |   scheduler_d = torch.optim.lr_scheduler.ExponentialLR(optim_d, gamma=hps.train.lr_decay, last_epoch=epoch_str-2)
114 | 
115 |   scaler = GradScaler(enabled=hps.train.fp16_run)
116 | 
117 |   for epoch in range(epoch_str, hps.train.epochs + 1):
118 |     if rank==0:
119 |       train_and_evaluate(rank, epoch, hps, [net_g, net_d], [optim_g, optim_d], [scheduler_g, scheduler_d], scaler, [train_loader, eval_loader], logger, [writer, writer_eval])
120 |     else:
121 |       train_and_evaluate(rank, epoch, hps, [net_g, net_d], [optim_g, optim_d], [scheduler_g, scheduler_d], scaler, [train_loader, None], None, None)
122 |     scheduler_g.step()
123 |     scheduler_d.step()
124 | 
125 | 
126 | def train_and_evaluate(rank, epoch, hps, nets, optims, schedulers, scaler, loaders, logger, writers):
127 |   net_g, net_d = nets
128 |   optim_g, optim_d = optims
129 |   scheduler_g, scheduler_d = schedulers
130 |   train_loader, eval_loader = loaders
131 |   if writers is not None:
132 |     writer, writer_eval = writers
133 | 
134 |   train_loader.batch_sampler.set_epoch(epoch)
135 |   global global_step
136 | 
137 |   net_g.train()
138 |   net_d.train()
139 |   for batch_idx, (x, x_lengths, spec, spec_lengths, y, y_lengths, speakers) in enumerate(train_loader):
140 |     x, x_lengths = x.cuda(rank, non_blocking=True), x_lengths.cuda(rank, non_blocking=True)
141 |     spec, spec_lengths = spec.cuda(rank, non_blocking=True), spec_lengths.cuda(rank, non_blocking=True)
142 |     y, y_lengths = y.cuda(rank, non_blocking=True), y_lengths.cuda(rank, non_blocking=True)
143 |     speakers = speakers.cuda(rank, non_blocking=True)
144 | 
145 |     with autocast(enabled=hps.train.fp16_run):
146 |     
147 |       mel = spec_to_mel_torch(
148 |           spec, 
149 |           hps.data.filter_length, 
150 |           hps.data.n_mel_channels, 
151 |           hps.data.sampling_rate,
152 |           hps.data.mel_fmin, 
153 |           hps.data.mel_fmax)
154 |       
155 |       y_hat, l_length, attn, ids_slice, x_mask, z_mask,\
156 |       (z, z_p, m_p, logs_p, m_q, logs_q) = net_g(x, x_lengths, mel, spec, spec_lengths, speakers)
157 | 
158 |       y_mel = commons.slice_segments(mel, ids_slice, hps.train.segment_size // hps.data.hop_length)
159 |       y_hat_mel = mel_spectrogram_torch(
160 |           y_hat.squeeze(1), 
161 |           hps.data.filter_length, 
162 |           hps.data.n_mel_channels, 
163 |           hps.data.sampling_rate, 
164 |           hps.data.hop_length, 
165 |           hps.data.win_length, 
166 |           hps.data.mel_fmin, 
167 |           hps.data.mel_fmax
168 |       )
169 | 
170 |       y = commons.slice_segments(y, ids_slice * hps.data.hop_length, hps.train.segment_size) # slice 
171 | 
172 |       # Discriminator
173 |       y_d_hat_r, y_d_hat_g, _, _ = net_d(y, y_hat.detach())
174 |       with autocast(enabled=False):
175 |         loss_disc, losses_disc_r, losses_disc_g = discriminator_loss(y_d_hat_r, y_d_hat_g)
176 |         loss_disc_all = loss_disc
177 |     optim_d.zero_grad()
178 |     scaler.scale(loss_disc_all).backward()
179 |     scaler.unscale_(optim_d)
180 |     grad_norm_d = commons.clip_grad_value_(net_d.parameters(), None)
181 |     scaler.step(optim_d)
182 | 
183 |     with autocast(enabled=hps.train.fp16_run):
184 |       # Generator
185 |       y_d_hat_r, y_d_hat_g, fmap_r, fmap_g = net_d(y, y_hat)
186 |       with autocast(enabled=False):
187 |         loss_dur = torch.sum(l_length.float())
188 |         loss_mel = F.l1_loss(y_mel, y_hat_mel) * hps.train.c_mel
189 |         loss_kl = kl_loss(z_p, logs_q, m_p, logs_p, z_mask) * hps.train.c_kl
190 | 
191 |         loss_fm = feature_loss(fmap_r, fmap_g)
192 |         loss_gen, losses_gen = generator_loss(y_d_hat_g)
193 |         loss_gen_all = loss_gen + loss_fm + loss_mel + loss_dur + loss_kl
194 |     optim_g.zero_grad()
195 |     scaler.scale(loss_gen_all).backward()
196 |     scaler.unscale_(optim_g)
197 |     grad_norm_g = commons.clip_grad_value_(net_g.parameters(), None)
198 |     scaler.step(optim_g)
199 |     scaler.update()
200 | 
201 |     if rank==0:
202 |       if global_step % hps.train.log_interval == 0:
203 |         lr = optim_g.param_groups[0]['lr']
204 |         losses = [loss_disc, loss_gen, loss_fm, loss_mel, loss_dur, loss_kl]
205 |         logger.info('Train Epoch: {} [{:.0f}%]'.format(
206 |           epoch,
207 |           100. * batch_idx / len(train_loader)))
208 |         logger.info([x.item() for x in losses] + [global_step, lr])
209 |         
210 |         scalar_dict = {"loss/g/total": loss_gen_all, "loss/d/total": loss_disc_all, "learning_rate": lr, "grad_norm_d": grad_norm_d, "grad_norm_g": grad_norm_g}
211 |         scalar_dict.update({"loss/g/fm": loss_fm, "loss/g/mel": loss_mel, "loss/g/dur": loss_dur, "loss/g/kl": loss_kl})
212 | 
213 |         scalar_dict.update({"loss/g/{}".format(i): v for i, v in enumerate(losses_gen)})
214 |         scalar_dict.update({"loss/d_r/{}".format(i): v for i, v in enumerate(losses_disc_r)})
215 |         scalar_dict.update({"loss/d_g/{}".format(i): v for i, v in enumerate(losses_disc_g)})
216 |         image_dict = { 
217 |             "slice/mel_org": utils.plot_spectrogram_to_numpy(y_mel[0].data.cpu().numpy()),
218 |             "slice/mel_gen": utils.plot_spectrogram_to_numpy(y_hat_mel[0].data.cpu().numpy()), 
219 |             "all/mel": utils.plot_spectrogram_to_numpy(mel[0].data.cpu().numpy()),
220 |             "all/attn": utils.plot_alignment_to_numpy(attn[0,0].data.cpu().numpy())
221 |         }
222 |         utils.summarize(
223 |           writer=writer,
224 |           global_step=global_step, 
225 |           images=image_dict,
226 |           scalars=scalar_dict)
227 | 
228 |       if global_step % hps.train.eval_interval == 0:
229 |         evaluate(hps, net_g, eval_loader, writer_eval)
230 |         utils.save_checkpoint(net_g, optim_g, hps.train.learning_rate, epoch, os.path.join(hps.model_dir, "G_{}.pth".format(global_step)))
231 |         utils.save_checkpoint(net_d, optim_d, hps.train.learning_rate, epoch, os.path.join(hps.model_dir, "D_{}.pth".format(global_step)))
232 |     global_step += 1
233 |   
234 |   if rank == 0:
235 |     logger.info('====> Epoch: {}'.format(epoch))
236 | 
237 |  
238 | def evaluate(hps, generator, eval_loader, writer_eval):
239 |     generator.eval()
240 |     with torch.no_grad():
241 |       for batch_idx, (x, x_lengths, spec, spec_lengths, y, y_lengths, speakers) in enumerate(eval_loader):
242 |         print(x[0])
243 |         x, x_lengths = x.cuda(0), x_lengths.cuda(0)
244 |         spec, spec_lengths = spec.cuda(0), spec_lengths.cuda(0)
245 |         y, y_lengths = y.cuda(0), y_lengths.cuda(0)
246 |         speakers = speakers.cuda(0)
247 | 
248 |         # remove else
249 |         x = x[:1]
250 |         x_lengths = x_lengths[:1]
251 |         spec = spec[:1]
252 |         spec_lengths = spec_lengths[:1]
253 |         y = y[:1]
254 |         y_lengths = y_lengths[:1]
255 |         speakers = speakers[:1]
256 |         break
257 |       mel = spec_to_mel_torch(
258 |         spec, 
259 |         hps.data.filter_length, 
260 |         hps.data.n_mel_channels, 
261 |         hps.data.sampling_rate,
262 |         hps.data.mel_fmin, 
263 |         hps.data.mel_fmax)
264 |       y_hat, attn, mask, *_ = generator.module.infer(x, x_lengths, mel, y_lengths, speakers, max_len=1000)
265 |       y_hat_lengths = mask.sum([1,2]).long() * hps.data.hop_length
266 | 
267 |       y_hat_mel = mel_spectrogram_torch(
268 |         y_hat.squeeze(1).float(),
269 |         hps.data.filter_length,
270 |         hps.data.n_mel_channels,
271 |         hps.data.sampling_rate,
272 |         hps.data.hop_length,
273 |         hps.data.win_length,
274 |         hps.data.mel_fmin,
275 |         hps.data.mel_fmax
276 |       )
277 |     image_dict = {
278 |       "gen/mel": utils.plot_spectrogram_to_numpy(y_hat_mel[0].cpu().numpy())
279 |     }
280 |     audio_dict = {
281 |       "gen/audio": y_hat[0,:,:y_hat_lengths[0]]
282 |     }
283 |     if global_step == 0:
284 |       image_dict.update({"gt/mel": utils.plot_spectrogram_to_numpy(mel[0].cpu().numpy())})
285 |       audio_dict.update({"gt/audio": y[0,:,:y_lengths[0]]})
286 | 
287 |     utils.summarize(
288 |       writer=writer_eval,
289 |       global_step=global_step, 
290 |       images=image_dict,
291 |       audios=audio_dict,
292 |       audio_sampling_rate=hps.data.sampling_rate
293 |     )
294 |     generator.train()
295 | 
296 |                            
297 | if __name__ == "__main__":
298 |   main()
299 | 


--------------------------------------------------------------------------------
/transforms.py:
--------------------------------------------------------------------------------
  1 | import torch
  2 | from torch.nn import functional as F
  3 | 
  4 | import numpy as np
  5 | 
  6 | 
  7 | DEFAULT_MIN_BIN_WIDTH = 1e-3
  8 | DEFAULT_MIN_BIN_HEIGHT = 1e-3
  9 | DEFAULT_MIN_DERIVATIVE = 1e-3
 10 | 
 11 | 
 12 | def piecewise_rational_quadratic_transform(inputs, 
 13 |                                            unnormalized_widths,
 14 |                                            unnormalized_heights,
 15 |                                            unnormalized_derivatives,
 16 |                                            inverse=False,
 17 |                                            tails=None, 
 18 |                                            tail_bound=1.,
 19 |                                            min_bin_width=DEFAULT_MIN_BIN_WIDTH,
 20 |                                            min_bin_height=DEFAULT_MIN_BIN_HEIGHT,
 21 |                                            min_derivative=DEFAULT_MIN_DERIVATIVE):
 22 | 
 23 |     if tails is None:
 24 |         spline_fn = rational_quadratic_spline
 25 |         spline_kwargs = {}
 26 |     else:
 27 |         spline_fn = unconstrained_rational_quadratic_spline
 28 |         spline_kwargs = {
 29 |             'tails': tails,
 30 |             'tail_bound': tail_bound
 31 |         }
 32 | 
 33 |     outputs, logabsdet = spline_fn(
 34 |             inputs=inputs,
 35 |             unnormalized_widths=unnormalized_widths,
 36 |             unnormalized_heights=unnormalized_heights,
 37 |             unnormalized_derivatives=unnormalized_derivatives,
 38 |             inverse=inverse,
 39 |             min_bin_width=min_bin_width,
 40 |             min_bin_height=min_bin_height,
 41 |             min_derivative=min_derivative,
 42 |             **spline_kwargs
 43 |     )
 44 |     return outputs, logabsdet
 45 | 
 46 | 
 47 | def searchsorted(bin_locations, inputs, eps=1e-6):
 48 |     bin_locations[..., -1] += eps
 49 |     return torch.sum(
 50 |         inputs[..., None] >= bin_locations,
 51 |         dim=-1
 52 |     ) - 1
 53 | 
 54 | 
 55 | def unconstrained_rational_quadratic_spline(inputs,
 56 |                                             unnormalized_widths,
 57 |                                             unnormalized_heights,
 58 |                                             unnormalized_derivatives,
 59 |                                             inverse=False,
 60 |                                             tails='linear',
 61 |                                             tail_bound=1.,
 62 |                                             min_bin_width=DEFAULT_MIN_BIN_WIDTH,
 63 |                                             min_bin_height=DEFAULT_MIN_BIN_HEIGHT,
 64 |                                             min_derivative=DEFAULT_MIN_DERIVATIVE):
 65 |     inside_interval_mask = (inputs >= -tail_bound) & (inputs <= tail_bound)
 66 |     outside_interval_mask = ~inside_interval_mask
 67 | 
 68 |     outputs = torch.zeros_like(inputs)
 69 |     logabsdet = torch.zeros_like(inputs)
 70 | 
 71 |     if tails == 'linear':
 72 |         unnormalized_derivatives = F.pad(unnormalized_derivatives, pad=(1, 1))
 73 |         constant = np.log(np.exp(1 - min_derivative) - 1)
 74 |         unnormalized_derivatives[..., 0] = constant
 75 |         unnormalized_derivatives[..., -1] = constant
 76 | 
 77 |         outputs[outside_interval_mask] = inputs[outside_interval_mask]
 78 |         logabsdet[outside_interval_mask] = 0
 79 |     else:
 80 |         raise RuntimeError('{} tails are not implemented.'.format(tails))
 81 | 
 82 |     outputs[inside_interval_mask], logabsdet[inside_interval_mask] = rational_quadratic_spline(
 83 |         inputs=inputs[inside_interval_mask],
 84 |         unnormalized_widths=unnormalized_widths[inside_interval_mask, :],
 85 |         unnormalized_heights=unnormalized_heights[inside_interval_mask, :],
 86 |         unnormalized_derivatives=unnormalized_derivatives[inside_interval_mask, :],
 87 |         inverse=inverse,
 88 |         left=-tail_bound, right=tail_bound, bottom=-tail_bound, top=tail_bound,
 89 |         min_bin_width=min_bin_width,
 90 |         min_bin_height=min_bin_height,
 91 |         min_derivative=min_derivative
 92 |     )
 93 | 
 94 |     return outputs, logabsdet
 95 | 
 96 | def rational_quadratic_spline(inputs,
 97 |                               unnormalized_widths,
 98 |                               unnormalized_heights,
 99 |                               unnormalized_derivatives,
100 |                               inverse=False,
101 |                               left=0., right=1., bottom=0., top=1.,
102 |                               min_bin_width=DEFAULT_MIN_BIN_WIDTH,
103 |                               min_bin_height=DEFAULT_MIN_BIN_HEIGHT,
104 |                               min_derivative=DEFAULT_MIN_DERIVATIVE):
105 |     if torch.min(inputs) < left or torch.max(inputs) > right:
106 |         raise ValueError('Input to a transform is not within its domain')
107 | 
108 |     num_bins = unnormalized_widths.shape[-1]
109 | 
110 |     if min_bin_width * num_bins > 1.0:
111 |         raise ValueError('Minimal bin width too large for the number of bins')
112 |     if min_bin_height * num_bins > 1.0:
113 |         raise ValueError('Minimal bin height too large for the number of bins')
114 | 
115 |     widths = F.softmax(unnormalized_widths, dim=-1)
116 |     widths = min_bin_width + (1 - min_bin_width * num_bins) * widths
117 |     cumwidths = torch.cumsum(widths, dim=-1)
118 |     cumwidths = F.pad(cumwidths, pad=(1, 0), mode='constant', value=0.0)
119 |     cumwidths = (right - left) * cumwidths + left
120 |     cumwidths[..., 0] = left
121 |     cumwidths[..., -1] = right
122 |     widths = cumwidths[..., 1:] - cumwidths[..., :-1]
123 | 
124 |     derivatives = min_derivative + F.softplus(unnormalized_derivatives)
125 | 
126 |     heights = F.softmax(unnormalized_heights, dim=-1)
127 |     heights = min_bin_height + (1 - min_bin_height * num_bins) * heights
128 |     cumheights = torch.cumsum(heights, dim=-1)
129 |     cumheights = F.pad(cumheights, pad=(1, 0), mode='constant', value=0.0)
130 |     cumheights = (top - bottom) * cumheights + bottom
131 |     cumheights[..., 0] = bottom
132 |     cumheights[..., -1] = top
133 |     heights = cumheights[..., 1:] - cumheights[..., :-1]
134 | 
135 |     if inverse:
136 |         bin_idx = searchsorted(cumheights, inputs)[..., None]
137 |     else:
138 |         bin_idx = searchsorted(cumwidths, inputs)[..., None]
139 | 
140 |     input_cumwidths = cumwidths.gather(-1, bin_idx)[..., 0]
141 |     input_bin_widths = widths.gather(-1, bin_idx)[..., 0]
142 | 
143 |     input_cumheights = cumheights.gather(-1, bin_idx)[..., 0]
144 |     delta = heights / widths
145 |     input_delta = delta.gather(-1, bin_idx)[..., 0]
146 | 
147 |     input_derivatives = derivatives.gather(-1, bin_idx)[..., 0]
148 |     input_derivatives_plus_one = derivatives[..., 1:].gather(-1, bin_idx)[..., 0]
149 | 
150 |     input_heights = heights.gather(-1, bin_idx)[..., 0]
151 | 
152 |     if inverse:
153 |         a = (((inputs - input_cumheights) * (input_derivatives
154 |                                              + input_derivatives_plus_one
155 |                                              - 2 * input_delta)
156 |               + input_heights * (input_delta - input_derivatives)))
157 |         b = (input_heights * input_derivatives
158 |              - (inputs - input_cumheights) * (input_derivatives
159 |                                               + input_derivatives_plus_one
160 |                                               - 2 * input_delta))
161 |         c = - input_delta * (inputs - input_cumheights)
162 | 
163 |         discriminant = b.pow(2) - 4 * a * c
164 |         assert (discriminant >= 0).all()
165 | 
166 |         root = (2 * c) / (-b - torch.sqrt(discriminant))
167 |         outputs = root * input_bin_widths + input_cumwidths
168 | 
169 |         theta_one_minus_theta = root * (1 - root)
170 |         denominator = input_delta + ((input_derivatives + input_derivatives_plus_one - 2 * input_delta)
171 |                                      * theta_one_minus_theta)
172 |         derivative_numerator = input_delta.pow(2) * (input_derivatives_plus_one * root.pow(2)
173 |                                                      + 2 * input_delta * theta_one_minus_theta
174 |                                                      + input_derivatives * (1 - root).pow(2))
175 |         logabsdet = torch.log(derivative_numerator) - 2 * torch.log(denominator)
176 | 
177 |         return outputs, -logabsdet
178 |     else:
179 |         theta = (inputs - input_cumwidths) / input_bin_widths
180 |         theta_one_minus_theta = theta * (1 - theta)
181 | 
182 |         numerator = input_heights * (input_delta * theta.pow(2)
183 |                                      + input_derivatives * theta_one_minus_theta)
184 |         denominator = input_delta + ((input_derivatives + input_derivatives_plus_one - 2 * input_delta)
185 |                                      * theta_one_minus_theta)
186 |         outputs = input_cumheights + numerator / denominator
187 | 
188 |         derivative_numerator = input_delta.pow(2) * (input_derivatives_plus_one * theta.pow(2)
189 |                                                      + 2 * input_delta * theta_one_minus_theta
190 |                                                      + input_derivatives * (1 - theta).pow(2))
191 |         logabsdet = torch.log(derivative_numerator) - 2 * torch.log(denominator)
192 | 
193 |         return outputs, logabsdet
194 | 


--------------------------------------------------------------------------------
/utils.py:
--------------------------------------------------------------------------------
  1 | import os
  2 | import glob
  3 | import sys
  4 | import argparse
  5 | import logging
  6 | import json
  7 | import subprocess
  8 | import numpy as np
  9 | from scipy.io.wavfile import read
 10 | import torch
 11 | 
 12 | MATPLOTLIB_FLAG = False
 13 | 
 14 | logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
 15 | logger = logging
 16 | 
 17 | 
 18 | def load_checkpoint(checkpoint_path, model, optimizer=None):
 19 |   assert os.path.isfile(checkpoint_path)
 20 |   checkpoint_dict = torch.load(checkpoint_path, map_location='cpu')
 21 |   iteration = checkpoint_dict['iteration']
 22 |   learning_rate = checkpoint_dict['learning_rate']
 23 |   if optimizer is not None:
 24 |     optimizer.load_state_dict(checkpoint_dict['optimizer'])
 25 |   saved_state_dict = checkpoint_dict['model']
 26 |   if hasattr(model, 'module'):
 27 |     state_dict = model.module.state_dict()
 28 |   else:
 29 |     state_dict = model.state_dict()
 30 |   new_state_dict= {}
 31 |   for k, v in state_dict.items():
 32 |     try:
 33 |       new_state_dict[k] = saved_state_dict[k]
 34 |     except:
 35 |       logger.info("%s is not in the checkpoint" % k)
 36 |       new_state_dict[k] = v
 37 |   if hasattr(model, 'module'):
 38 |     model.module.load_state_dict(new_state_dict)
 39 |   else:
 40 |     model.load_state_dict(new_state_dict)
 41 |   logger.info("Loaded checkpoint '{}' (iteration {})" .format(
 42 |     checkpoint_path, iteration))
 43 |   return model, optimizer, learning_rate, iteration
 44 | 
 45 | 
 46 | def save_checkpoint(model, optimizer, learning_rate, iteration, checkpoint_path):
 47 |   logger.info("Saving model and optimizer state at iteration {} to {}".format(
 48 |     iteration, checkpoint_path))
 49 |   if hasattr(model, 'module'):
 50 |     state_dict = model.module.state_dict()
 51 |   else:
 52 |     state_dict = model.state_dict()
 53 |   torch.save({'model': state_dict,
 54 |               'iteration': iteration,
 55 |               'optimizer': optimizer.state_dict(),
 56 |               'learning_rate': learning_rate}, checkpoint_path)
 57 | 
 58 | 
 59 | def summarize(writer, global_step, scalars={}, histograms={}, images={}, audios={}, audio_sampling_rate=22050):
 60 |   for k, v in scalars.items():
 61 |     writer.add_scalar(k, v, global_step)
 62 |   for k, v in histograms.items():
 63 |     writer.add_histogram(k, v, global_step)
 64 |   for k, v in images.items():
 65 |     writer.add_image(k, v, global_step, dataformats='HWC')
 66 |   for k, v in audios.items():
 67 |     writer.add_audio(k, v, global_step, audio_sampling_rate)
 68 | 
 69 | 
 70 | def latest_checkpoint_path(dir_path, regex="G_*.pth"):
 71 |   f_list = glob.glob(os.path.join(dir_path, regex))
 72 |   f_list.sort(key=lambda f: int("".join(filter(str.isdigit, f))))
 73 |   x = f_list[-1]
 74 |   print(x)
 75 |   return x
 76 | 
 77 | 
 78 | def plot_spectrogram_to_numpy(spectrogram):
 79 |   global MATPLOTLIB_FLAG
 80 |   if not MATPLOTLIB_FLAG:
 81 |     import matplotlib
 82 |     matplotlib.use("Agg")
 83 |     MATPLOTLIB_FLAG = True
 84 |     mpl_logger = logging.getLogger('matplotlib')
 85 |     mpl_logger.setLevel(logging.WARNING)
 86 |   import matplotlib.pylab as plt
 87 |   import numpy as np
 88 |   
 89 |   fig, ax = plt.subplots(figsize=(10,2))
 90 |   im = ax.imshow(spectrogram, aspect="auto", origin="lower",
 91 |                   interpolation='none')
 92 |   plt.colorbar(im, ax=ax)
 93 |   plt.xlabel("Frames")
 94 |   plt.ylabel("Channels")
 95 |   plt.tight_layout()
 96 | 
 97 |   fig.canvas.draw()
 98 |   data = np.fromstring(fig.canvas.tostring_rgb(), dtype=np.uint8, sep='')
 99 |   data = data.reshape(fig.canvas.get_width_height()[::-1] + (3,))
100 |   plt.close()
101 |   return data
102 | 
103 | 
104 | def plot_alignment_to_numpy(alignment, info=None):
105 |   global MATPLOTLIB_FLAG
106 |   if not MATPLOTLIB_FLAG:
107 |     import matplotlib
108 |     matplotlib.use("Agg")
109 |     MATPLOTLIB_FLAG = True
110 |     mpl_logger = logging.getLogger('matplotlib')
111 |     mpl_logger.setLevel(logging.WARNING)
112 |   import matplotlib.pylab as plt
113 |   import numpy as np
114 | 
115 |   fig, ax = plt.subplots(figsize=(6, 4))
116 |   im = ax.imshow(alignment.transpose(), aspect='auto', origin='lower',
117 |                   interpolation='none')
118 |   fig.colorbar(im, ax=ax)
119 |   xlabel = 'Decoder timestep'
120 |   if info is not None:
121 |       xlabel += '\n\n' + info
122 |   plt.xlabel(xlabel)
123 |   plt.ylabel('Encoder timestep')
124 |   plt.tight_layout()
125 | 
126 |   fig.canvas.draw()
127 |   data = np.fromstring(fig.canvas.tostring_rgb(), dtype=np.uint8, sep='')
128 |   data = data.reshape(fig.canvas.get_width_height()[::-1] + (3,))
129 |   plt.close()
130 |   return data
131 | 
132 | 
133 | def load_wav_to_torch(full_path):
134 |   sampling_rate, data = read(full_path)
135 |   return torch.FloatTensor(data.astype(np.float32)), sampling_rate
136 | 
137 | 
138 | def load_filepaths_and_text(filename, split="|"):
139 |   with open(filename, encoding='utf-8') as f:
140 |     filepaths_and_text = [line.strip().split(split) for line in f]
141 |   return filepaths_and_text
142 | 
143 | 
144 | def get_hparams(init=True):
145 |   parser = argparse.ArgumentParser()
146 |   parser.add_argument('-c', '--config', type=str, default="./configs/base.json",
147 |                       help='JSON file for configuration')
148 |   parser.add_argument('-m', '--model', type=str, required=True,
149 |                       help='Model name')
150 |   
151 |   args = parser.parse_args()
152 |   model_dir = os.path.join("./logs", args.model)
153 | 
154 |   if not os.path.exists(model_dir):
155 |     os.makedirs(model_dir)
156 | 
157 |   config_path = args.config
158 |   config_save_path = os.path.join(model_dir, "config.json")
159 |   if init:
160 |     with open(config_path, "r") as f:
161 |       data = f.read()
162 |     with open(config_save_path, "w") as f:
163 |       f.write(data)
164 |   else:
165 |     with open(config_save_path, "r") as f:
166 |       data = f.read()
167 |   config = json.loads(data)
168 |   
169 |   hparams = HParams(**config)
170 |   hparams.model_dir = model_dir
171 |   return hparams
172 | 
173 | 
174 | def get_hparams_from_dir(model_dir):
175 |   config_save_path = os.path.join(model_dir, "config.json")
176 |   with open(config_save_path, "r") as f:
177 |     data = f.read()
178 |   config = json.loads(data)
179 | 
180 |   hparams =HParams(**config)
181 |   hparams.model_dir = model_dir
182 |   return hparams
183 | 
184 | 
185 | def get_hparams_from_file(config_path):
186 |   with open(config_path, "r") as f:
187 |     data = f.read()
188 |   config = json.loads(data)
189 | 
190 |   hparams =HParams(**config)
191 |   return hparams
192 | 
193 | 
194 | def check_git_hash(model_dir):
195 |   source_dir = os.path.dirname(os.path.realpath(__file__))
196 |   if not os.path.exists(os.path.join(source_dir, ".git")):
197 |     logger.warn("{} is not a git repository, therefore hash value comparison will be ignored.".format(
198 |       source_dir
199 |     ))
200 |     return
201 | 
202 |   cur_hash = subprocess.getoutput("git rev-parse HEAD")
203 | 
204 |   path = os.path.join(model_dir, "githash")
205 |   if os.path.exists(path):
206 |     saved_hash = open(path).read()
207 |     if saved_hash != cur_hash:
208 |       logger.warn("git hash values are different. {}(saved) != {}(current)".format(
209 |         saved_hash[:8], cur_hash[:8]))
210 |   else:
211 |     open(path, "w").write(cur_hash)
212 | 
213 | 
214 | def get_logger(model_dir, filename="train.log"):
215 |   global logger
216 |   logger = logging.getLogger(os.path.basename(model_dir))
217 |   logger.setLevel(logging.DEBUG)
218 |   
219 |   formatter = logging.Formatter("%(asctime)s\t%(name)s\t%(levelname)s\t%(message)s")
220 |   if not os.path.exists(model_dir):
221 |     os.makedirs(model_dir)
222 |   h = logging.FileHandler(os.path.join(model_dir, filename))
223 |   h.setLevel(logging.DEBUG)
224 |   h.setFormatter(formatter)
225 |   logger.addHandler(h)
226 |   return logger
227 | 
228 | 
229 | class HParams():
230 |   def __init__(self, **kwargs):
231 |     for k, v in kwargs.items():
232 |       if type(v) == dict:
233 |         v = HParams(**v)
234 |       self[k] = v
235 |     
236 |   def keys(self):
237 |     return self.__dict__.keys()
238 | 
239 |   def items(self):
240 |     return self.__dict__.items()
241 | 
242 |   def values(self):
243 |     return self.__dict__.values()
244 | 
245 |   def __len__(self):
246 |     return len(self.__dict__)
247 | 
248 |   def __getitem__(self, key):
249 |     return getattr(self, key)
250 | 
251 |   def __setitem__(self, key, value):
252 |     return setattr(self, key, value)
253 | 
254 |   def __contains__(self, key):
255 |     return key in self.__dict__
256 | 
257 |   def __repr__(self):
258 |     return self.__dict__.__repr__()
259 | 


--------------------------------------------------------------------------------