├── .gitignore
├── LICENSE
├── README.md
├── configs
├── svs
│ ├── data.yaml
│ ├── model.yaml
│ └── train.yaml
└── tts
│ ├── data.yaml
│ ├── model.yaml
│ └── train.yaml
├── dataset
├── __init__.py
├── dataset_svs.py
├── dataset_tts.py
├── espnet_texts
│ ├── __init__.py
│ ├── cleaners.py
│ ├── cmudict.py
│ ├── dict.py
│ ├── numbers.py
│ └── symbols.py
└── texts
│ ├── __init__.py
│ ├── cleaners.py
│ ├── cmudict.py
│ ├── numbers.py
│ ├── pinyin.py
│ └── symbols.py
├── lexicon
├── libritts-lexicon.txt
└── pinyin-lexicon.txt
├── loss
├── __init__.py
├── fastspeech2_loss.py
└── loss.py
├── models
├── __init__.py
├── discriminator.py
├── fastspeech2.py
└── xiaoice2.py
├── modules
├── __init__.py
├── conv
│ └── __init__.py
├── transformer
│ ├── Constants.py
│ ├── Layers.py
│ ├── Models.py
│ ├── Modules.py
│ ├── SubLayers.py
│ └── __init__.py
└── variance
│ ├── __init__.py
│ └── modules.py
├── pics
├── 2085003136_145600.png
├── after_2085003136_145600.png
├── before_2085003136_145600.png
├── before_mel_l2_loss.png
├── post_mel_l2_loss.png
└── xs1_before_2085003136_145600.png
├── preprocess
├── audio_preprocess.py
└── data_prep.py
├── pyutils
├── __init__.py
├── gen_duration_from_tg.py
├── logger.py
├── mask.py
├── optimizer.py
├── parse_options.sh
├── plot.py
├── save_and_load.py
└── scheduler.py
├── run.sh
├── train.py
├── train_gan.py
└── utils
/.gitignore:
--------------------------------------------------------------------------------
1 | # Byte-compiled / optimized / DLL files
2 | __pycache__/
3 | *.py[cod]
4 | *$py.class
5 |
6 | # C extensions
7 | *.so
8 |
9 | # Distribution / packaging
10 | .Python
11 | build/
12 | develop-eggs/
13 | dist/
14 | downloads/
15 | eggs/
16 | .eggs/
17 | lib/
18 | lib64/
19 | parts/
20 | sdist/
21 | var/
22 | wheels/
23 | share/python-wheels/
24 | *.egg-info/
25 | .installed.cfg
26 | *.egg
27 | MANIFEST
28 |
29 | # PyInstaller
30 | # Usually these files are written by a python script from a template
31 | # before PyInstaller builds the exe, so as to inject date/other infos into it.
32 | *.manifest
33 | *.spec
34 |
35 | # Installer logs
36 | pip-log.txt
37 | pip-delete-this-directory.txt
38 |
39 | # Unit test / coverage reports
40 | htmlcov/
41 | .tox/
42 | .nox/
43 | .coverage
44 | .coverage.*
45 | .cache
46 | nosetests.xml
47 | coverage.xml
48 | *.cover
49 | *.py,cover
50 | .hypothesis/
51 | .pytest_cache/
52 | cover/
53 |
54 | # Translations
55 | *.mo
56 | *.pot
57 |
58 | # Django stuff:
59 | *.log
60 | local_settings.py
61 | db.sqlite3
62 | db.sqlite3-journal
63 |
64 | # Flask stuff:
65 | instance/
66 | .webassets-cache
67 |
68 | # Scrapy stuff:
69 | .scrapy
70 |
71 | # Sphinx documentation
72 | docs/_build/
73 |
74 | # PyBuilder
75 | .pybuilder/
76 | target/
77 |
78 | # Jupyter Notebook
79 | .ipynb_checkpoints
80 |
81 | # IPython
82 | profile_default/
83 | ipython_config.py
84 |
85 | # pyenv
86 | # For a library or package, you might want to ignore these files since the code is
87 | # intended to run in multiple environments; otherwise, check them in:
88 | # .python-version
89 |
90 | # pipenv
91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies
93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not
94 | # install all needed dependencies.
95 | #Pipfile.lock
96 |
97 | # poetry
98 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
99 | # This is especially recommended for binary packages to ensure reproducibility, and is more
100 | # commonly ignored for libraries.
101 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
102 | #poetry.lock
103 |
104 | # pdm
105 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
106 | #pdm.lock
107 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
108 | # in version control.
109 | # https://pdm.fming.dev/#use-with-ide
110 | .pdm.toml
111 |
112 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
113 | __pypackages__/
114 |
115 | # Celery stuff
116 | celerybeat-schedule
117 | celerybeat.pid
118 |
119 | # SageMath parsed files
120 | *.sage.py
121 |
122 | # Environments
123 | .env
124 | .venv
125 | env/
126 | venv/
127 | ENV/
128 | env.bak/
129 | venv.bak/
130 |
131 | # Spyder project settings
132 | .spyderproject
133 | .spyproject
134 |
135 | # Rope project settings
136 | .ropeproject
137 |
138 | # mkdocs documentation
139 | /site
140 |
141 | # mypy
142 | .mypy_cache/
143 | .dmypy.json
144 | dmypy.json
145 |
146 | # Pyre type checker
147 | .pyre/
148 |
149 | # pytype static type analyzer
150 | .pytype/
151 |
152 | # Cython debug symbols
153 | cython_debug/
154 |
155 | # PyCharm
156 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can
157 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
158 | # and can be added to the global gitignore or merged into this file. For a more nuclear
159 | # option (not recommended) you can uncomment the following to ignore the entire idea folder.
160 | #.idea/
161 | #
162 | job.sh
163 | data
164 | data/*
165 | exp
166 | exp/*
167 | *.out
168 | wandb
169 | wandb/*
170 | .nfs*
171 | local_run.sh
172 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | BSD 3-Clause License
2 |
3 | Copyright (c) 2023, zengchang233
4 |
5 | Redistribution and use in source and binary forms, with or without
6 | modification, are permitted provided that the following conditions are met:
7 |
8 | 1. Redistributions of source code must retain the above copyright notice, this
9 | list of conditions and the following disclaimer.
10 |
11 | 2. Redistributions in binary form must reproduce the above copyright notice,
12 | this list of conditions and the following disclaimer in the documentation
13 | and/or other materials provided with the distribution.
14 |
15 | 3. Neither the name of the copyright holder nor the names of its
16 | contributors may be used to endorse or promote products derived from
17 | this software without specific prior written permission.
18 |
19 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
22 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
23 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
25 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
26 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
27 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # [XiaoiceSing2](https://www.isca-speech.org/archive/interspeech_2023/chunhui23_interspeech.html)
2 | The source code for the paper [XiaoiceSing2](https://www.isca-speech.org/archive/interspeech_2023/chunhui23_interspeech.html) (interspeech2023)
3 |
4 | [Demo page](https://wavelandspeech.github.io/xiaoice2/)
5 |
6 | ## Notice
7 |
8 | I am busy with job-hunting now. I will update other modules, including the [HiFi-WaveGAN](https://arxiv.org/abs/2210.12740) after my final decision.
9 |
10 | ## Implementation (developping)
11 |
12 | - [x] fastspeech2-based generator
13 | - [x] discriminator group, including segment discriminators and detail discriminators
14 | - [ ] ConvFFT block
15 |
16 | ## Dataset and preparation
17 |
18 | - [x] opencpop 
19 | - [ ] kiritan 
20 | - [ ] CSD 
21 | - [ ] m4singer 
22 | - [ ] NUS48E
23 |
24 | Kaldi style preparation
25 |
26 | - wav.scp
27 | - utt2spk
28 | - spk2utt
29 | - text
30 |
31 | ```
32 | ./run.sh --start-stage 1 --stop-stage 1 # extract melspectrogram, f0, energy, and statistical value
33 | ```
34 |
35 | ## Training
36 |
37 | ```
38 | ./run.sh --start-stage 2 --stop-stage 2
39 | ```
40 |
41 | ### Real and generated melspectrogram (145600 training steps)
42 |
43 | Real(left) XiaoiceSing(middle) XiaoiceSing2(right)
44 |
45 |
50 |
51 | ### L2 loss curve for melspectrogram
52 |
53 | L2 loss before post-processing(left) L2 loss after post-processing(right)
54 |
55 |
56 |
57 |
58 |
"
9 | EOS_WORD = ""
10 |
--------------------------------------------------------------------------------
/modules/transformer/Layers.py:
--------------------------------------------------------------------------------
1 | from collections import OrderedDict
2 |
3 | import torch
4 | import torch.nn as nn
5 | import numpy as np
6 | from torch.nn import functional as F
7 |
8 | try:
9 | from modules_v2.transformer.sublayer import MultiHeadAttention, PositionwiseFeedForward, MultiLayeredConv1d
10 | from modules_v2.transformer.embedding import PositionalEncoding
11 | from modules_v2.transformer.layer import EncoderLayer
12 | except (ImportError, ModuleNotFoundError):
13 | import sys
14 | import os
15 | filepath = os.path.dirname(os.path.abspath(__file__))
16 | sys.path.insert(0, filepath)
17 | from SubLayers import MultiHeadAttention, PositionwiseFeedForward
18 |
19 | class FFTBlock(torch.nn.Module):
20 | """FFT Block"""
21 |
22 | def __init__(self, d_model, n_head, d_k, d_v, d_inner, kernel_size, dropout=0.1):
23 | super(FFTBlock, self).__init__()
24 | self.slf_attn = MultiHeadAttention(n_head, d_model, d_k, d_v, dropout=dropout)
25 | self.pos_ffn = PositionwiseFeedForward(
26 | d_model, d_inner, kernel_size, dropout=dropout
27 | )
28 |
29 | def forward(self, enc_input, mask=None, slf_attn_mask=None):
30 | enc_output, enc_slf_attn = self.slf_attn(
31 | enc_input, enc_input, enc_input, mask=slf_attn_mask
32 | )
33 | enc_output = enc_output.masked_fill(mask.unsqueeze(-1), 0)
34 |
35 | enc_output = self.pos_ffn(enc_output)
36 | enc_output = enc_output.masked_fill(mask.unsqueeze(-1), 0)
37 |
38 | return enc_output, enc_slf_attn
39 |
40 |
41 | class ConvNorm(torch.nn.Module):
42 | def __init__(
43 | self,
44 | in_channels,
45 | out_channels,
46 | kernel_size=1,
47 | stride=1,
48 | padding=None,
49 | dilation=1,
50 | bias=True,
51 | w_init_gain="linear",
52 | ):
53 | super(ConvNorm, self).__init__()
54 |
55 | if padding is None:
56 | assert kernel_size % 2 == 1
57 | padding = int(dilation * (kernel_size - 1) / 2)
58 |
59 | self.conv = torch.nn.Conv1d(
60 | in_channels,
61 | out_channels,
62 | kernel_size=kernel_size,
63 | stride=stride,
64 | padding=padding,
65 | dilation=dilation,
66 | bias=bias,
67 | )
68 |
69 | def forward(self, signal):
70 | conv_signal = self.conv(signal)
71 |
72 | return conv_signal
73 |
74 |
75 | class PostNet(nn.Module):
76 | """
77 | PostNet: Five 1-d convolution with 512 channels and kernel size 5
78 | """
79 |
80 | def __init__(
81 | self,
82 | n_mels=80,
83 | postnet_embedding_dim=512,
84 | postnet_kernel_size=5,
85 | postnet_n_convolutions=5,
86 | ):
87 |
88 | super(PostNet, self).__init__()
89 | self.convolutions = nn.ModuleList()
90 |
91 | self.convolutions.append(
92 | nn.Sequential(
93 | ConvNorm(
94 | n_mels,
95 | postnet_embedding_dim,
96 | kernel_size=postnet_kernel_size,
97 | stride=1,
98 | padding=int((postnet_kernel_size - 1) / 2),
99 | dilation=1,
100 | w_init_gain="tanh",
101 | ),
102 | nn.BatchNorm1d(postnet_embedding_dim),
103 | )
104 | )
105 |
106 | for i in range(1, postnet_n_convolutions - 1):
107 | self.convolutions.append(
108 | nn.Sequential(
109 | ConvNorm(
110 | postnet_embedding_dim,
111 | postnet_embedding_dim,
112 | kernel_size=postnet_kernel_size,
113 | stride=1,
114 | padding=int((postnet_kernel_size - 1) / 2),
115 | dilation=1,
116 | w_init_gain="tanh",
117 | ),
118 | nn.BatchNorm1d(postnet_embedding_dim),
119 | )
120 | )
121 |
122 | self.convolutions.append(
123 | nn.Sequential(
124 | ConvNorm(
125 | postnet_embedding_dim,
126 | n_mels,
127 | kernel_size=postnet_kernel_size,
128 | stride=1,
129 | padding=int((postnet_kernel_size - 1) / 2),
130 | dilation=1,
131 | w_init_gain="linear",
132 | ),
133 | nn.BatchNorm1d(n_mels),
134 | )
135 | )
136 |
137 | def forward(self, x):
138 | x = x.contiguous().transpose(1, 2)
139 |
140 | for i in range(len(self.convolutions) - 1):
141 | x = F.dropout(torch.tanh(self.convolutions[i](x)), 0.5, self.training)
142 | x = F.dropout(self.convolutions[-1](x), 0.5, self.training)
143 |
144 | x = x.contiguous().transpose(1, 2)
145 | return x
146 |
147 | if __name__ == "__main__":
148 | import sys
149 | sys.path.insert(0, '/home/zengchang/code/acoustic_v2/modules_v2/transformer')
150 | fft_block = FFTBlock(512, 8, 64, 64, 2048, [3,3])
151 | x = torch.randn(2, 100, 512)
152 | mask = torch.ones(2, 100).bool()
153 | slf_attn_mask = torch.ones(2, 100, 100).bool()
154 | y, attn = fft_block(x, mask, slf_attn_mask)
155 | print(y.shape)
156 | print(attn.shape)
--------------------------------------------------------------------------------
/modules/transformer/Models.py:
--------------------------------------------------------------------------------
1 | import torch
2 | import torch.nn as nn
3 | import numpy as np
4 |
5 | try:
6 | import modules.transformer.Constants as Constants
7 | from modules.transformer.Layers import FFTBlock
8 | from dataset.texts.symbols import symbols
9 | except:
10 | import sys
11 | import os
12 | filepath = '/'.join(os.path.dirname(os.path.abspath(__file__)).split('/')[:-2])
13 | print(filepath)
14 | sys.path.insert(0, filepath)
15 | import modules.transformer.Constants as Constants
16 | from modules.transformer.Layers import FFTBlock
17 | from dataset.texts.symbols import symbols
18 |
19 |
20 | def get_sinusoid_encoding_table(n_position, d_hid, padding_idx=None):
21 | """ Sinusoid position encoding table """
22 |
23 | def cal_angle(position, hid_idx):
24 | return position / np.power(10000, 2 * (hid_idx // 2) / d_hid)
25 |
26 | def get_posi_angle_vec(position):
27 | return [cal_angle(position, hid_j) for hid_j in range(d_hid)]
28 |
29 | sinusoid_table = np.array(
30 | [get_posi_angle_vec(pos_i) for pos_i in range(n_position)]
31 | )
32 |
33 | sinusoid_table[:, 0::2] = np.sin(sinusoid_table[:, 0::2]) # dim 2i
34 | sinusoid_table[:, 1::2] = np.cos(sinusoid_table[:, 1::2]) # dim 2i+1
35 |
36 | if padding_idx is not None:
37 | # zero vector for padding dimension
38 | sinusoid_table[padding_idx] = 0.0
39 |
40 | return torch.FloatTensor(sinusoid_table)
41 |
42 |
43 | class Encoder(nn.Module):
44 | """ Encoder """
45 |
46 | def __init__(
47 | self, max_seq_len, n_src_vocab, d_word_vec,
48 | n_layers, n_head, d_model, d_inner, max_note_pitch,
49 | max_note_duration, kernel_size, dropout=0.1
50 | ):
51 | super(Encoder, self).__init__()
52 |
53 | n_position = max_seq_len + 1
54 | n_src_vocab = n_src_vocab
55 | d_word_vec = d_word_vec
56 | n_layers = n_layers
57 | n_head = n_head
58 | d_k = d_v = (d_word_vec // n_head)
59 | d_model = d_model
60 | d_inner = d_inner
61 | kernel_size = kernel_size
62 | dropout = dropout
63 |
64 | # self.max_seq_len = config["max_seq_len"]
65 | self.max_seq_len = max_seq_len
66 | self.d_model = d_model
67 |
68 | self.src_word_emb = nn.Embedding(
69 | n_src_vocab, d_word_vec, padding_idx = Constants.PAD
70 | )
71 | self.note_pitch_emb = nn.Embedding(
72 | max_note_pitch, d_word_vec, padding_idx = Constants.PAD
73 | )
74 | self.note_duration_emb = nn.Embedding(
75 | max_note_duration, d_word_vec, padding_idx = Constants.PAD
76 | )
77 | self.position_enc = nn.Parameter(
78 | get_sinusoid_encoding_table(n_position, d_word_vec).unsqueeze(0),
79 | requires_grad=False,
80 | )
81 |
82 | self.layer_stack = nn.ModuleList(
83 | [
84 | FFTBlock(
85 | d_model, n_head, d_k, d_v, d_inner, kernel_size, dropout=dropout
86 | )
87 | for _ in range(n_layers)
88 | ]
89 | )
90 |
91 | def forward(self, src_seq, note_pitchs, note_durations, mask, return_attns=False):
92 |
93 | enc_slf_attn_list = []
94 | batch_size, max_len = src_seq.shape[0], src_seq.shape[1]
95 |
96 | # -- Prepare masks
97 | slf_attn_mask = mask.unsqueeze(1).expand(-1, max_len, -1)
98 |
99 | # -- Forward
100 | if not self.training and src_seq.shape[1] > self.max_seq_len:
101 | enc_output = self.src_word_emb(src_seq) + get_sinusoid_encoding_table(
102 | src_seq.shape[1], self.d_model
103 | )[: src_seq.shape[1], :].unsqueeze(0).expand(batch_size, -1, -1).to(
104 | src_seq.device
105 | )
106 | else:
107 | # print("training!!!!!!!!!!")
108 | enc_output = self.src_word_emb(src_seq) \
109 | + self.note_pitch_emb(note_pitchs) \
110 | + self.note_duration_emb(note_durations) \
111 | + self.position_enc[:, :max_len, :].expand(batch_size, -1, -1)
112 |
113 | for enc_layer in self.layer_stack:
114 | enc_output, enc_slf_attn = enc_layer(
115 | enc_output, mask=mask, slf_attn_mask=slf_attn_mask
116 | )
117 | if return_attns:
118 | enc_slf_attn_list += [enc_slf_attn]
119 |
120 | return enc_output
121 |
122 |
123 | class Decoder(nn.Module):
124 | """ Decoder """
125 |
126 | def __init__(
127 | self, max_seq_len, d_word_vec,
128 | n_layers, n_head, d_model, d_inner,
129 | kernel_size, dropout=0.1
130 | ):
131 | super(Decoder, self).__init__()
132 |
133 | n_position = max_seq_len + 1
134 | d_word_vec = d_word_vec
135 | n_layers = n_layers
136 | n_head = n_head
137 | d_k = d_v = (d_word_vec // n_head)
138 | d_model = d_model
139 | d_inner = d_inner
140 | kernel_size = kernel_size
141 | dropout = dropout
142 |
143 | # self.max_seq_len = config["max_seq_len"]
144 | self.max_seq_len = max_seq_len
145 | self.d_model = d_model
146 |
147 | self.position_enc = nn.Parameter(
148 | get_sinusoid_encoding_table(n_position, d_word_vec).unsqueeze(0),
149 | requires_grad=False,
150 | )
151 |
152 | self.layer_stack = nn.ModuleList(
153 | [
154 | FFTBlock(
155 | d_model, n_head, d_k, d_v, d_inner, kernel_size, dropout=dropout
156 | )
157 | for _ in range(n_layers)
158 | ]
159 | )
160 |
161 | def forward(self, enc_seq, mask, return_attns=False):
162 |
163 | dec_slf_attn_list = []
164 | batch_size, max_len = enc_seq.shape[0], enc_seq.shape[1]
165 |
166 | # -- Forward
167 | if not self.training and enc_seq.shape[1] > self.max_seq_len:
168 | # -- Prepare masks
169 | slf_attn_mask = mask.unsqueeze(1).expand(-1, max_len, -1)
170 | dec_output = enc_seq + get_sinusoid_encoding_table(
171 | enc_seq.shape[1], self.d_model
172 | )[: enc_seq.shape[1], :].unsqueeze(0).expand(batch_size, -1, -1).to(
173 | enc_seq.device
174 | )
175 | else:
176 | max_len = min(max_len, self.max_seq_len)
177 |
178 | # -- Prepare masks
179 | slf_attn_mask = mask.unsqueeze(1).expand(-1, max_len, -1)
180 | dec_output = enc_seq[:, :max_len, :] + self.position_enc[
181 | :, :max_len, :
182 | ].expand(batch_size, -1, -1)
183 | mask = mask[:, :max_len]
184 | slf_attn_mask = slf_attn_mask[:, :, :max_len]
185 |
186 | for dec_layer in self.layer_stack:
187 | dec_output, dec_slf_attn = dec_layer(
188 | dec_output, mask=mask, slf_attn_mask=slf_attn_mask
189 | )
190 | if return_attns:
191 | dec_slf_attn_list += [dec_slf_attn]
192 |
193 | return dec_output, mask
194 |
195 | if __name__ == "__main__":
196 | encoder = Encoder({"max_seq_len": 100, "transformer": {"encoder_hidden": 256, "encoder_layer": 6, "encoder_head": 4, "conv_filter_size": 1024, "conv_kernel_size": [3,3], "encoder_dropout": 0.1}})
197 | decoder = Decoder({"max_seq_len": 100, "transformer": {"decoder_hidden": 256, "decoder_layer": 6, "decoder_head": 4, "conv_filter_size": 1024, "conv_kernel_size": [3,3], "decoder_dropout": 0.1}})
198 | src_seq = torch.randint(0, 100, (2, 100))
199 | mask = torch.ones((2, 100)).bool()
200 | enc_output = encoder(src_seq, mask)
201 | dec_output, mask = decoder(enc_output, mask)
202 | print(dec_output.shape, mask.shape)
203 |
--------------------------------------------------------------------------------
/modules/transformer/Modules.py:
--------------------------------------------------------------------------------
1 | import torch
2 | import torch.nn as nn
3 | import numpy as np
4 |
5 |
6 | class ScaledDotProductAttention(nn.Module):
7 | """ Scaled Dot-Product Attention """
8 |
9 | def __init__(self, temperature):
10 | super().__init__()
11 | self.temperature = temperature
12 | self.softmax = nn.Softmax(dim=2)
13 |
14 | def forward(self, q, k, v, mask=None):
15 |
16 | attn = torch.bmm(q, k.transpose(1, 2))
17 | attn = attn / self.temperature
18 |
19 | if mask is not None:
20 | attn = attn.masked_fill(mask, -np.inf)
21 |
22 | attn = self.softmax(attn)
23 | output = torch.bmm(attn, v)
24 |
25 | return output, attn
26 |
--------------------------------------------------------------------------------
/modules/transformer/SubLayers.py:
--------------------------------------------------------------------------------
1 | import torch.nn as nn
2 | import torch.nn.functional as F
3 | import numpy as np
4 |
5 | from Modules import ScaledDotProductAttention
6 |
7 |
8 | class MultiHeadAttention(nn.Module):
9 | """ Multi-Head Attention module """
10 |
11 | def __init__(self, n_head, d_model, d_k, d_v, dropout=0.1):
12 | super().__init__()
13 |
14 | self.n_head = n_head
15 | self.d_k = d_k
16 | self.d_v = d_v
17 |
18 | self.w_qs = nn.Linear(d_model, n_head * d_k)
19 | self.w_ks = nn.Linear(d_model, n_head * d_k)
20 | self.w_vs = nn.Linear(d_model, n_head * d_v)
21 |
22 | self.attention = ScaledDotProductAttention(temperature=np.power(d_k, 0.5))
23 | self.layer_norm = nn.LayerNorm(d_model)
24 |
25 | self.fc = nn.Linear(n_head * d_v, d_model)
26 |
27 | self.dropout = nn.Dropout(dropout)
28 |
29 | def forward(self, q, k, v, mask=None):
30 |
31 | d_k, d_v, n_head = self.d_k, self.d_v, self.n_head
32 |
33 | sz_b, len_q, _ = q.size()
34 | sz_b, len_k, _ = k.size()
35 | sz_b, len_v, _ = v.size()
36 |
37 | residual = q
38 |
39 | q = self.w_qs(q).view(sz_b, len_q, n_head, d_k)
40 | k = self.w_ks(k).view(sz_b, len_k, n_head, d_k)
41 | v = self.w_vs(v).view(sz_b, len_v, n_head, d_v)
42 | q = q.permute(2, 0, 1, 3).contiguous().view(-1, len_q, d_k) # (n*b) x lq x dk
43 | k = k.permute(2, 0, 1, 3).contiguous().view(-1, len_k, d_k) # (n*b) x lk x dk
44 | v = v.permute(2, 0, 1, 3).contiguous().view(-1, len_v, d_v) # (n*b) x lv x dv
45 |
46 | mask = mask.repeat(n_head, 1, 1) # (n*b) x .. x ..
47 | output, attn = self.attention(q, k, v, mask=mask)
48 |
49 | output = output.view(n_head, sz_b, len_q, d_v)
50 | output = (
51 | output.permute(1, 2, 0, 3).contiguous().view(sz_b, len_q, -1)
52 | ) # b x lq x (n*dv)
53 |
54 | output = self.dropout(self.fc(output))
55 | output = self.layer_norm(output + residual)
56 |
57 | return output, attn
58 |
59 |
60 | class PositionwiseFeedForward(nn.Module):
61 | """ A two-feed-forward-layer module """
62 |
63 | def __init__(self, d_in, d_hid, kernel_size, dropout=0.1):
64 | super().__init__()
65 |
66 | # Use Conv1D
67 | # position-wise
68 | self.w_1 = nn.Conv1d(
69 | d_in,
70 | d_hid,
71 | kernel_size=kernel_size[0],
72 | padding=(kernel_size[0] - 1) // 2,
73 | )
74 | # position-wise
75 | self.w_2 = nn.Conv1d(
76 | d_hid,
77 | d_in,
78 | kernel_size=kernel_size[1],
79 | padding=(kernel_size[1] - 1) // 2,
80 | )
81 |
82 | self.layer_norm = nn.LayerNorm(d_in)
83 | self.dropout = nn.Dropout(dropout)
84 |
85 | def forward(self, x):
86 | residual = x
87 | output = x.transpose(1, 2)
88 | output = self.w_2(F.relu(self.w_1(output)))
89 | output = output.transpose(1, 2)
90 | output = self.dropout(output)
91 | output = self.layer_norm(output + residual)
92 |
93 | return output
94 |
--------------------------------------------------------------------------------
/modules/transformer/__init__.py:
--------------------------------------------------------------------------------
1 | from .Constants import *
2 | from .Layers import *
3 | from .SubLayers import *
4 | from .Models import *
5 | from .Modules import *
--------------------------------------------------------------------------------
/modules/variance/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/zengchang233/xiaoicesing2/b2d0436ac73337123a7fe75a7be21ed385b48098/modules/variance/__init__.py
--------------------------------------------------------------------------------
/modules/variance/modules.py:
--------------------------------------------------------------------------------
1 | from collections import OrderedDict
2 |
3 | import torch
4 | import torch.nn as nn
5 | import numpy as np
6 |
7 | from pyutils import get_mask_from_lengths, pad
8 |
9 | def f02pitch(f0):
10 | #f0 =f0 + 0.01
11 | return np.log2(f0 / 27.5) * 12 + 21
12 |
13 | class VarianceAdaptor(nn.Module):
14 | """Variance Adaptor"""
15 |
16 | def __init__(self, data_config, model_config):
17 | super(VarianceAdaptor, self).__init__()
18 | self.duration_predictor = VariancePredictor(**model_config['variance_predictor'])
19 | self.length_regulator = LengthRegulator()
20 | self.pitch_predictor = VariancePredictor(**model_config['variance_predictor'])
21 | self.uv_predictor = VariancePredictor(**model_config['variance_predictor'])
22 | self.energy_predictor = VariancePredictor(**model_config['variance_predictor'])
23 |
24 | self.uv_threshold = model_config['uv_threshold']
25 |
26 | self.pitch_feature_level = data_config["pitch"]["feature"]
27 | self.energy_feature_level = data_config["energy"]["feature"]
28 | assert self.pitch_feature_level in ["phoneme_level", "frame_level"]
29 | assert self.energy_feature_level in ["phoneme_level", "frame_level"]
30 |
31 | pitch_quantization = model_config["variance_embedding"]["pitch_quantization"]
32 | energy_quantization = model_config["variance_embedding"]["energy_quantization"]
33 | n_bins = model_config["variance_embedding"]["n_bins"]
34 | assert pitch_quantization in ["linear", "log"]
35 | assert energy_quantization in ["linear", "log"]
36 |
37 | pitch_min_max = f02pitch(np.load(data_config['f0_min_max']))
38 | pitch_min, pitch_max = pitch_min_max[0][0], pitch_min_max[0][1]
39 | # print(np.load(data_config['energy_min_max']))
40 |
41 | energy_min_max = np.load(data_config['energy_min_max'])
42 | energy_min, energy_max = energy_min_max[0][0] + 1e-4, energy_min_max[0][1]
43 |
44 | if pitch_quantization == "log":
45 | self.pitch_bins = nn.Parameter(
46 | torch.exp(
47 | torch.linspace(np.log(pitch_min), np.log(pitch_max), n_bins - 1)
48 | ),
49 | requires_grad=False,
50 | )
51 | else:
52 | self.pitch_bins = nn.Parameter(
53 | torch.linspace(pitch_min, pitch_max, n_bins - 1),
54 | requires_grad=False,
55 | )
56 | if energy_quantization == "log":
57 | self.energy_bins = nn.Parameter(
58 | torch.exp(
59 | torch.linspace(np.log(energy_min + 1e-6), np.log(energy_max), n_bins - 1)
60 | ),
61 | requires_grad=False,
62 | )
63 | else:
64 | self.energy_bins = nn.Parameter(
65 | torch.linspace(energy_min, energy_max, n_bins - 1),
66 | requires_grad=False,
67 | )
68 |
69 | self.pitch_embedding = nn.Embedding(
70 | n_bins, model_config["transformer"]["encoder"]["d_word_vec"]
71 | )
72 | self.energy_embedding = nn.Embedding(
73 | n_bins, model_config["transformer"]["encoder"]["d_word_vec"]
74 | )
75 | self.uv_embedding = nn.Embedding(
76 | 2, model_config['transformer']['encoder']['d_word_vec']
77 | )
78 |
79 | def get_uv_embedding(self, x, target, mask, control=1.0):
80 | prediction = self.uv_predictor(x, mask)
81 | if target is not None:
82 | embedding = self.uv_embedding(target.to(torch.int64))
83 | else:
84 | prediction = prediction * control
85 | prediction = torch.sigmoid(prediction)
86 | for i in range(prediction.shape[0]):
87 | prediction[i] = prediction[i] >= self.uv_threshold # (B, max_frames, 1)
88 |
89 | embedding = self.uv_embedding(prediction.long())
90 |
91 | return prediction, embedding
92 |
93 | def get_pitch_embedding(self, x, target, mask, control):
94 | prediction = self.pitch_predictor(x, mask)
95 | if target is not None:
96 | embedding = self.pitch_embedding(torch.bucketize(target, self.pitch_bins))
97 | else:
98 | prediction = prediction * control
99 | embedding = self.pitch_embedding(
100 | torch.bucketize(prediction, self.pitch_bins)
101 | )
102 | return prediction, embedding
103 |
104 | def get_energy_embedding(self, x, target, mask, control):
105 | prediction = self.energy_predictor(x, mask)
106 | if target is not None:
107 | embedding = self.energy_embedding(torch.bucketize(target, self.energy_bins))
108 | else:
109 | prediction = prediction * control
110 | embedding = self.energy_embedding(
111 | torch.bucketize(prediction, self.energy_bins)
112 | )
113 | return prediction, embedding
114 |
115 | def forward(
116 | self,
117 | x,
118 | src_mask,
119 | mel_mask=None,
120 | max_len=None,
121 | pitch_target=None,
122 | energy_target=None,
123 | uv_target=None,
124 | duration_target=None,
125 | p_control=1.0,
126 | e_control=1.0,
127 | d_control=1.0,
128 | uv_control=1.0
129 | ):
130 |
131 | log_duration_prediction = self.duration_predictor(x, src_mask)
132 | if self.pitch_feature_level == "phoneme_level":
133 | pitch_prediction, pitch_embedding = self.get_pitch_embedding(
134 | x, pitch_target, src_mask, p_control
135 | )
136 | x = x + pitch_embedding
137 | if self.energy_feature_level == "phoneme_level":
138 | energy_prediction, energy_embedding = self.get_energy_embedding(
139 | x, energy_target, src_mask, p_control
140 | )
141 | x = x + energy_embedding
142 |
143 | if duration_target is not None:
144 | x, mel_len = self.length_regulator(x, duration_target, max_len)
145 | duration_rounded = duration_target
146 | else:
147 | duration_rounded = torch.clamp(
148 | (torch.round(torch.exp(log_duration_prediction) - 1) * d_control),
149 | min=0,
150 | )
151 | x, mel_len = self.length_regulator(x, duration_rounded, max_len)
152 | mel_mask = get_mask_from_lengths(mel_len)
153 |
154 | if self.pitch_feature_level == "frame_level":
155 | pitch_prediction, pitch_embedding = self.get_pitch_embedding(
156 | x, pitch_target, mel_mask, p_control
157 | )
158 | x = x + pitch_embedding
159 | if self.energy_feature_level == "frame_level":
160 | energy_prediction, energy_embedding = self.get_energy_embedding(
161 | x, energy_target, mel_mask, p_control
162 | )
163 | x = x + energy_embedding
164 |
165 | uv_prediction, uv_embedding = self.get_uv_embedding(
166 | x, uv_target, mel_mask, uv_control
167 | )
168 | x = x + uv_embedding
169 |
170 | return (
171 | x,
172 | pitch_prediction,
173 | energy_prediction,
174 | uv_prediction,
175 | log_duration_prediction,
176 | duration_rounded,
177 | mel_len,
178 | mel_mask,
179 | )
180 |
181 |
182 | class LengthRegulator(nn.Module):
183 | """Length Regulator"""
184 |
185 | def __init__(self):
186 | super(LengthRegulator, self).__init__()
187 |
188 | def LR(self, x, duration, max_len):
189 | device = x.device
190 | output = list()
191 | mel_len = list()
192 | for batch, expand_target in zip(x, duration):
193 | expanded = self.expand(batch, expand_target)
194 | output.append(expanded)
195 | mel_len.append(expanded.shape[0])
196 |
197 | if max_len is not None:
198 | output = pad(output, max_len)
199 | else:
200 | output = pad(output)
201 |
202 | return output, torch.LongTensor(mel_len).to(device)
203 |
204 | def expand(self, batch, predicted):
205 | out = list()
206 |
207 | for i, vec in enumerate(batch):
208 | expand_size = predicted[i].item()
209 | out.append(vec.expand(max(int(expand_size), 0), -1))
210 | out = torch.cat(out, 0)
211 |
212 | return out
213 |
214 | def forward(self, x, duration, max_len):
215 | output, mel_len = self.LR(x, duration, max_len)
216 | return output, mel_len
217 |
218 |
219 | class VariancePredictor(nn.Module):
220 | """Duration, Pitch and Energy Predictor"""
221 |
222 | def __init__(
223 | self, input_size, filter_size,
224 | kernel_size, dropout
225 | ):
226 | super(VariancePredictor, self).__init__()
227 |
228 | # self.input_size = model_config["transformer"]["encoder_hidden"]
229 | # self.filter_size = model_config["variance_predictor"]["filter_size"]
230 | # self.kernel = model_config["variance_predictor"]["kernel_size"]
231 | # self.conv_output_size = model_config["variance_predictor"]["filter_size"]
232 | # self.dropout = model_config["variance_predictor"]["dropout"]
233 | self.input_size = input_size
234 | self.filter_size = filter_size
235 | self.kernel = kernel_size
236 | self.conv_output_size = filter_size
237 | self.dropout = dropout
238 |
239 | self.conv_layer = nn.Sequential(
240 | OrderedDict(
241 | [
242 | (
243 | "conv1d_1",
244 | Conv(
245 | self.input_size,
246 | self.filter_size,
247 | kernel_size=self.kernel,
248 | padding=(self.kernel - 1) // 2,
249 | ),
250 | ),
251 | ("relu_1", nn.ReLU()),
252 | ("layer_norm_1", nn.LayerNorm(self.filter_size)),
253 | ("dropout_1", nn.Dropout(self.dropout)),
254 | (
255 | "conv1d_2",
256 | Conv(
257 | self.filter_size,
258 | self.filter_size,
259 | kernel_size=self.kernel,
260 | padding=1,
261 | ),
262 | ),
263 | ("relu_2", nn.ReLU()),
264 | ("layer_norm_2", nn.LayerNorm(self.filter_size)),
265 | ("dropout_2", nn.Dropout(self.dropout)),
266 | ]
267 | )
268 | )
269 |
270 | self.linear_layer = nn.Linear(self.conv_output_size, 1)
271 |
272 | def forward(self, encoder_output, mask):
273 | out = self.conv_layer(encoder_output)
274 | out = self.linear_layer(out)
275 | out = out.squeeze(-1)
276 |
277 | if mask is not None:
278 | out = out.masked_fill(mask, 0.0)
279 |
280 | return out
281 |
282 |
283 | class Conv(nn.Module):
284 | """
285 | Convolution Module
286 | """
287 |
288 | def __init__(
289 | self,
290 | in_channels,
291 | out_channels,
292 | kernel_size=1,
293 | stride=1,
294 | padding=0,
295 | dilation=1,
296 | bias=True,
297 | w_init="linear",
298 | ):
299 | """
300 | :param in_channels: dimension of input
301 | :param out_channels: dimension of output
302 | :param kernel_size: size of kernel
303 | :param stride: size of stride
304 | :param padding: size of padding
305 | :param dilation: dilation rate
306 | :param bias: boolean. if True, bias is included.
307 | :param w_init: str. weight inits with xavier initialization.
308 | """
309 | super(Conv, self).__init__()
310 |
311 | self.conv = nn.Conv1d(
312 | in_channels,
313 | out_channels,
314 | kernel_size=kernel_size,
315 | stride=stride,
316 | padding=padding,
317 | dilation=dilation,
318 | bias=bias,
319 | )
320 |
321 | def forward(self, x):
322 | x = x.contiguous().transpose(1, 2)
323 | x = self.conv(x)
324 | x = x.contiguous().transpose(1, 2)
325 |
326 | return x
327 |
328 | if __name__ == "__main__":
329 | import yaml
330 | with open('/home/zengchang/code/acoustic_v2/configs/data.yaml', 'r') as f:
331 | data_config = yaml.load(f, Loader = yaml.FullLoader)
332 | with open('/home/zengchang/code/acoustic_v2/configs/model.yaml', 'r') as f:
333 | model_config = yaml.load(f, Loader = yaml.FullLoader)
334 | model = VarianceAdaptor(data_config, model_config['generator'])
335 | print(model)
336 |
--------------------------------------------------------------------------------
/pics/2085003136_145600.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/zengchang233/xiaoicesing2/b2d0436ac73337123a7fe75a7be21ed385b48098/pics/2085003136_145600.png
--------------------------------------------------------------------------------
/pics/after_2085003136_145600.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/zengchang233/xiaoicesing2/b2d0436ac73337123a7fe75a7be21ed385b48098/pics/after_2085003136_145600.png
--------------------------------------------------------------------------------
/pics/before_2085003136_145600.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/zengchang233/xiaoicesing2/b2d0436ac73337123a7fe75a7be21ed385b48098/pics/before_2085003136_145600.png
--------------------------------------------------------------------------------
/pics/before_mel_l2_loss.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/zengchang233/xiaoicesing2/b2d0436ac73337123a7fe75a7be21ed385b48098/pics/before_mel_l2_loss.png
--------------------------------------------------------------------------------
/pics/post_mel_l2_loss.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/zengchang233/xiaoicesing2/b2d0436ac73337123a7fe75a7be21ed385b48098/pics/post_mel_l2_loss.png
--------------------------------------------------------------------------------
/pics/xs1_before_2085003136_145600.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/zengchang233/xiaoicesing2/b2d0436ac73337123a7fe75a7be21ed385b48098/pics/xs1_before_2085003136_145600.png
--------------------------------------------------------------------------------
/preprocess/audio_preprocess.py:
--------------------------------------------------------------------------------
1 | import argparse
2 | import os
3 | import sys
4 | import librosa
5 | import pyworld
6 | import parselmouth
7 | import soundfile as sf
8 | import numpy as np
9 | import yaml
10 | from tqdm import tqdm
11 | from sklearn.preprocessing import StandardScaler
12 | from ipdb import set_trace
13 | from pyutils import f02pitch, pitch2f0, pitchxuv
14 | cwd=os.path.dirname(os.path.realpath(__file__))
15 | sys.path.insert(0, cwd)
16 |
17 | def resample_wav(wav, src_sr, tgt_sr):
18 | return librosa.resample(wav, orig_sr=src_sr, target_sr=tgt_sr)
19 |
20 | def _resize_f0(x, target_len):
21 | source = np.array(x)
22 | source[source < 0.001] = np.nan
23 | target = np.interp(np.arange(0, len(source) * target_len, len(source)) / target_len, np.arange(0, len(source)), source)
24 | res = np.nan_to_num(target)
25 | return res
26 |
27 | def compute_f0_dio(wav, p_len=None, sampling_rate=48000, hop_length=240):
28 | if p_len is None:
29 | p_len = wav.shape[0]//hop_length
30 | f0, t = pyworld.dio(
31 | wav.astype(np.double),
32 | fs=sampling_rate,
33 | f0_ceil=800,
34 | frame_period=1000 * hop_length / sampling_rate
35 | )
36 | f0 = pyworld.stonemask(wav.astype(np.double), f0, t, sampling_rate)
37 | for index, pitch in enumerate(f0):
38 | f0[index] = round(pitch, 1)
39 | return _resize_f0(f0, p_len)
40 |
41 | def compute_f0_parselmouth(wav, p_len=None, sampling_rate=48000, hop_length=240):
42 | x = wav
43 | if p_len is None:
44 | p_len = x.shape[0]//hop_length
45 | else:
46 | assert abs(p_len-x.shape[0]//hop_length) < 4, "pad length error"
47 | time_step = hop_length / sampling_rate * 1000
48 | f0_min = 50
49 | f0_max = 1100
50 | f0 = parselmouth.Sound(x, sampling_rate).to_pitch_ac(
51 | time_step=time_step / 1000, voicing_threshold=0.6,
52 | pitch_floor=f0_min, pitch_ceiling=f0_max).selected_array['frequency']
53 |
54 | pad_size=(p_len - len(f0) + 1) // 2
55 | if(pad_size>0 or p_len - len(f0) - pad_size>0):
56 | f0 = np.pad(f0,[[pad_size,p_len - len(f0) - pad_size]], mode='constant')
57 | return f0
58 |
59 | def interpolate_f0(f0):
60 | data = np.reshape(f0, (f0.size, 1))
61 |
62 | vuv_vector = np.zeros((data.size, 1), dtype=np.float32)
63 | vuv_vector[data > 0.0] = 1.0
64 | vuv_vector[data <= 0.0] = 0.0
65 |
66 | ip_data = data
67 |
68 | frame_number = data.size
69 | last_value = 0.0
70 | for i in range(frame_number):
71 | if data[i] <= 0.0:
72 | j = i + 1
73 | for j in range(i + 1, frame_number):
74 | if data[j] > 0.0:
75 | break
76 | if j < frame_number - 1:
77 | if last_value > 0.0:
78 | step = (data[j] - data[i - 1]) / float(j - i)
79 | for k in range(i, j):
80 | ip_data[k] = data[i - 1] + step * (k - i + 1)
81 | else:
82 | for k in range(i, j):
83 | ip_data[k] = data[j]
84 | else:
85 | for k in range(i, frame_number):
86 | ip_data[k] = last_value
87 | else:
88 | ip_data[i] = data[i] # this may not be necessary
89 | last_value = data[i]
90 |
91 | return ip_data[:,0], vuv_vector[:,0]
92 |
93 | def read_scp(scp_file):
94 | filelists = []
95 | with open(scp_file, 'r') as f:
96 | for line in f:
97 | line = line.rstrip()
98 | filelists.append(line)
99 | return filelists
100 |
101 | def spec_normalize(feat):
102 | '''
103 | params:
104 | feat: T, F
105 | '''
106 | return (feat - feat.mean(axis = 0, keepdims = True)) / (feat.std(axis = 0, keepdims = True) + 2e-12)
107 |
108 | def pad_wav(wav, config):
109 | padded_wav = np.pad(wav, (int((config['n_fft']-config['hop_length'])/2), int((config['n_fft']-config['hop_length'])/2)), mode='reflect')
110 | return padded_wav
111 |
112 | def extract_spec_with_energy(wav, filepath, config, spec_scaler = None, energy_scaler = None):
113 | '''
114 | (T, F/C)
115 | '''
116 | wav = pad_wav(wav, config)
117 | stft = librosa.stft(
118 | wav,
119 | n_fft = config['n_fft'],
120 | hop_length = config['hop_length'],
121 | win_length = config['win_length'],
122 | window = 'hann',
123 | center = False,
124 | pad_mode = 'reflect'
125 | )
126 | # set_trace()
127 | spec = np.abs(stft).transpose(1, 0)
128 | energy = np.sqrt((spec**2).sum(axis = 1))
129 | energy = energy.reshape(-1, 1)
130 | if spec_scaler is not None:
131 | spec_scaler.partial_fit(spec)
132 | if energy_scaler is not None:
133 | energy_scaler.partial_fit(energy)
134 | suffix = filepath.split('.')[-1]
135 | spec_filepath = filepath.replace(f'.{suffix}', '.spec.npy')
136 | np.save(spec_filepath, spec)
137 | suffix = filepath.split('.')[-1]
138 | energy_filepath = filepath.replace(f'.{suffix}', '.en.npy')
139 | np.save(energy_filepath, energy)
140 | # return spec, energy
141 |
142 | def extract_mel(wav, filepath, config, mel_scaler):
143 | '''
144 | log mel + spec normalization
145 | (T, F/C)
146 | '''
147 | wav = pad_wav(wav, config)
148 | mel_spec = librosa.feature.melspectrogram(
149 | y = wav,
150 | sr = config['sampling_rate'],
151 | n_fft = config['n_fft'],
152 | hop_length = config['hop_length'],
153 | win_length = config['win_length'],
154 | window = 'hann',
155 | n_mels = config['n_mels'],
156 | fmin = config['fmin'],
157 | fmax = config['fmax'],
158 | center = False,
159 | pad_mode = 'reflect'
160 | )
161 | log_mel_spec = np.log(mel_spec + 1e-9).transpose(1, 0)
162 | # normalized_log_mel_spec = spec_normalize(log_mel_spec)
163 | mel_scaler.partial_fit(log_mel_spec)
164 | suffix = filepath.split('.')[-1]
165 | mel_filepath = filepath.replace(f'.{suffix}', '.mel.npy')
166 | np.save(mel_filepath, log_mel_spec)
167 |
168 | def extract_f0(filepath, config, f0_scaler = None):
169 | '''
170 | (T, 1)
171 | '''
172 | wav, sr = sf.read(filepath)
173 | wav = resample_wav(wav, sr, config['sampling_rate'])
174 | sr = config['sampling_rate']
175 | assert sr == config['sampling_rate'], "Sampling rate ({}) != {}, please fix it!".format(sr, config['sampling_rate'])
176 | # wav = pad_wav(wav, config) # don't padding for computing f0
177 | f0 = compute_f0_dio(
178 | wav,
179 | sampling_rate = config["sampling_rate"],
180 | hop_length = config["hop_length"]
181 | )
182 | f0, uv = interpolate_f0(f0)
183 | f0 = f0.reshape(-1, 1)
184 | if f0_scaler is not None:
185 | f0_scaler.partial_fit(f0)
186 | suffix = filepath.split('.')[-1]
187 | f0_filepath = filepath.replace(f'.{suffix}', '.f0.npy')
188 | uv_filepath = filepath.replace(f'.{suffix}', '.uv.npy')
189 | np.save(f0_filepath, f0)
190 | np.save(uv_filepath, uv)
191 |
192 | def process_one_utterance_spec(filepath, config, spec_scaler, mel_scaler, energy_scaler = None):
193 | wav, sr = sf.read(filepath)
194 | wav = resample_wav(wav, sr, config['sampling_rate'])
195 | sr = config['sampling_rate']
196 | assert sr == config['sampling_rate'], "Sampling rate ({}) != {}, please fix it!".format(sr, config['sampling_rate'])
197 | if args.spec:
198 | extract_spec_with_energy(wav, filepath, config, spec_scaler, energy_scaler)
199 | if args.mel:
200 | extract_mel(wav, filepath, config, mel_scaler)
201 |
202 | def normalize(filelists, mean, std, feature = 'f0'):
203 | '''
204 | normalize spec/mel_spec
205 | unnormalize f0/energy
206 | '''
207 | min_value = np.finfo(np.float64).max
208 | max_value = np.finfo(np.float64).min
209 | for filepath in filelists:
210 | suffix = filepath.split('.')[-1]
211 | filepath = filepath.replace(f'.{suffix}', f'.{feature}.npy')
212 | values = np.load(filepath)
213 | if feature in ['f0', 'en']:
214 | min_value = min(min_value, min(values))
215 | max_value = max(max_value, max(values))
216 | else:
217 | values = (np.load(filepath) - mean) / std
218 | np.save(filepath, values)
219 | return np.array([min_value, max_value]).reshape(1, -1)
220 |
221 | def parse_args():
222 | parser = argparse.ArgumentParser()
223 | parser.add_argument("--data-config", dest = "data_config", type = str, default = "", help = "data config path")
224 | parser.add_argument("--spec", action = "store_true", help = "extract stft spec feature")
225 | parser.add_argument("--mel", action = "store_true", help = "extract mel feature")
226 | parser.add_argument("--f0", action = "store_true", help = "extract f0 and uv")
227 | parser.add_argument("--energy", action = "store_true", help = "extract energy")
228 | parser.add_argument("--stat", action = "store_true", help = "Count the statistical numbers (mean and std) for energy and f0")
229 |
230 | args = parser.parse_args()
231 | return args
232 |
233 | def main(args):
234 | with open(args.data_config, 'r') as f:
235 | data_config = yaml.load(f, Loader = yaml.FullLoader)
236 | filelists = []
237 | with open(data_config['audio_manifest'], 'r') as f:
238 | for line in f:
239 | line = line.rstrip().split(' ')[-1]
240 | filelists.append(line)
241 | args.scp_file = data_config['audio_manifest']
242 |
243 | spec_scaler = StandardScaler()
244 | mel_scaler = StandardScaler()
245 | f0_scaler = None
246 | energy_scaler = None
247 | if args.stat:
248 | f0_scaler = StandardScaler()
249 | energy_scaler = StandardScaler()
250 |
251 | print("Extracting features...")
252 | for filepath in tqdm(filelists):
253 | if args.spec or args.mel:
254 | try:
255 | process_one_utterance_spec(filepath, data_config, spec_scaler, mel_scaler, energy_scaler)
256 | except:
257 | print(filepath)
258 | if args.f0:
259 | try:
260 | extract_f0(filepath, data_config, f0_scaler)
261 | except:
262 | print(filepath)
263 |
264 | if args.stat:
265 | if args.spec:
266 | spec_mean = spec_scaler.mean_.reshape(1, -1)
267 | spec_std = spec_scaler.scale_.reshape(1, -1)
268 | np.save(os.path.join(os.path.dirname(args.scp_file), 'spec_mean.npy'), spec_mean)
269 | np.save(os.path.join(os.path.dirname(args.scp_file), 'spec_std.npy'), spec_std)
270 | normalize(filelists, spec_mean, spec_std, feature = 'spec')
271 |
272 | if args.mel:
273 | mel_mean = mel_scaler.mean_.reshape(1, -1)
274 | mel_std = mel_scaler.scale_.reshape(1, -1)
275 | np.save(os.path.join(os.path.dirname(args.scp_file), 'mel_mean.npy'), mel_mean)
276 | np.save(os.path.join(os.path.dirname(args.scp_file), 'mel_std.npy'), mel_std)
277 | normalize(filelists, mel_mean, mel_std, feature = 'mel')
278 |
279 | if args.f0:
280 | print("Calculating f0 stats...")
281 | f0_mean = f0_scaler.mean_.reshape(1, -1)
282 | f0_std = f0_scaler.scale_.reshape(1, -1)
283 | np.save(os.path.join(os.path.dirname(args.scp_file), 'f0_mean.npy'), f0_mean)
284 | np.save(os.path.join(os.path.dirname(args.scp_file), 'f0_std.npy'), f0_std)
285 | f0_min_max = normalize(filelists, f0_mean, f0_std, feature = 'f0')
286 | np.save(os.path.join(os.path.dirname(args.scp_file), 'f0_min_max.npy'), f0_min_max)
287 |
288 | if args.energy:
289 | print("Calculating energy stats...")
290 | energy_mean = energy_scaler.mean_.reshape(1, -1)
291 | energy_std = energy_scaler.scale_.reshape(1, -1)
292 | np.save(os.path.join(os.path.dirname(args.scp_file), 'energy_mean.npy'), energy_mean)
293 | np.save(os.path.join(os.path.dirname(args.scp_file), 'energy_std.npy'), energy_std)
294 | energy_min_max = normalize(filelists, energy_mean, energy_std, feature = 'en')
295 | np.save(os.path.join(os.path.dirname(args.scp_file), 'energy_min_max.npy'), energy_min_max)
296 |
297 | if __name__ == "__main__":
298 | args = parse_args()
299 | print(args)
300 | main(args)
301 |
--------------------------------------------------------------------------------
/preprocess/data_prep.py:
--------------------------------------------------------------------------------
1 | import os
2 | import librosa
3 | import numpy as np
4 | from scipy.io import wavfile
5 | from tqdm import tqdm
6 |
7 | def prepare_aishell3(config):
8 | pass
9 |
10 |
11 | def prepare_align(config):
12 | in_dir = config["path"]["corpus_path"]
13 | out_dir = config["path"]["raw_path"]
14 | sampling_rate = config["preprocessing"]["audio"]["sampling_rate"]
15 | max_wav_value = config["preprocessing"]["audio"]["max_wav_value"]
16 | for dataset in ["train", "test"]:
17 | print("Processing {}ing set...".format(dataset))
18 | with open(os.path.join(in_dir, dataset, "content.txt"), encoding="utf-8") as f:
19 | for line in tqdm(f):
20 | wav_name, text = line.strip("\n").split("\t")
21 | speaker = wav_name[:7]
22 | text = text.split(" ")[1::2]
23 | wav_path = os.path.join(in_dir, dataset, "wav", speaker, wav_name)
24 | if os.path.exists(wav_path):
25 | os.makedirs(os.path.join(out_dir, speaker), exist_ok=True)
26 | wav, _ = librosa.load(wav_path, sampling_rate)
27 | wav = wav / max(abs(wav)) * max_wav_value
28 | wavfile.write(
29 | os.path.join(out_dir, speaker, wav_name),
30 | sampling_rate,
31 | wav.astype(np.int16),
32 | )
33 | with open(
34 | os.path.join(out_dir, speaker, "{}.lab".format(wav_name[:11])),
35 | "w",
36 | ) as f1:
37 | f1.write(" ".join(text))
38 |
--------------------------------------------------------------------------------
/pyutils/__init__.py:
--------------------------------------------------------------------------------
1 | from .save_and_load import *
2 | from .plot import *
3 | from .logger import *
4 | from .mask import *
5 | from .logger import *
6 | from .optimizer import *
7 | from . import scheduler
8 | import torch
9 | import numpy as np
10 |
11 | def f02pitch(f0):
12 | #f0 =f0 + 0.01
13 | return np.log2(f0 / 27.5) * 12 + 21
14 |
15 | def pitch2f0(pitch):
16 | f0 = np.exp2((pitch - 21 ) / 12) * 27.5
17 | for i in range(len(f0)):
18 | if f0[i] <= 10:
19 | f0[i] = 0
20 | return f0
21 |
22 | def pitchxuv(pitch, uv, to_f0 = False):
23 | result = pitch * uv
24 | if to_f0:
25 | result = pitch2f0(result)
26 | return result
27 |
28 | def initialize(model, init_type="pytorch"):
29 | """Initialize Transformer module
30 |
31 | :param torch.nn.Module model: core instance
32 | :param str init_type: initialization type
33 | """
34 | if init_type == "pytorch":
35 | return
36 |
37 | # weight init
38 | for p in model.parameters():
39 | if p.dim() > 1:
40 | if init_type == "xavier_uniform":
41 | torch.nn.init.xavier_uniform_(p.data)
42 | elif init_type == "xavier_normal":
43 | torch.nn.init.xavier_normal_(p.data)
44 | elif init_type == "kaiming_uniform":
45 | torch.nn.init.kaiming_uniform_(p.data, nonlinearity="relu")
46 | elif init_type == "kaiming_normal":
47 | torch.nn.init.kaiming_normal_(p.data, nonlinearity="relu")
48 | else:
49 | raise ValueError("Unknown initialization: " + init_type)
50 | # bias init
51 | for p in model.parameters():
52 | if p.dim() == 1:
53 | p.data.zero_()
54 |
55 | # reset some loss with default init
56 | for m in model.modules():
57 | if isinstance(m, (torch.nn.Embedding, torch.nn.LayerNorm)):
58 | m.reset_parameters()
59 |
60 | def get_mask_from_lengths(lengths, max_len=None):
61 | device = lengths.device
62 | batch_size = lengths.shape[0]
63 | if max_len is None:
64 | max_len = torch.max(lengths).item()
65 |
66 | ids = torch.arange(0, max_len).unsqueeze(0).expand(batch_size, -1).to(device)
67 | mask = ids >= lengths.unsqueeze(1).expand(-1, max_len)
68 |
69 | return mask
70 |
71 | def pad(input_ele, mel_max_length=None):
72 | if mel_max_length:
73 | max_len = mel_max_length
74 | else:
75 | max_len = max([input_ele[i].size(0) for i in range(len(input_ele))])
76 |
77 | out_list = list()
78 | for i, batch in enumerate(input_ele):
79 | if len(batch.shape) == 1:
80 | one_batch_padded = F.pad(
81 | batch, (0, max_len - batch.size(0)), "constant", 0.0
82 | )
83 | elif len(batch.shape) == 2:
84 | one_batch_padded = F.pad(
85 | batch, (0, 0, 0, max_len - batch.size(0)), "constant", 0.0
86 | )
87 | out_list.append(one_batch_padded)
88 | out_padded = torch.stack(out_list)
89 | return out_padded
90 |
--------------------------------------------------------------------------------
/pyutils/gen_duration_from_tg.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 | import argparse
15 | import os
16 | from pathlib import Path
17 |
18 | import librosa
19 | import numpy as np
20 | import yaml
21 | from praatio import textgrid
22 | from yacs.config import CfgNode
23 | from tqdm import tqdm
24 |
25 |
26 | def readtg(tg_path, sample_rate=24000, n_shift=300):
27 | alignment = textgrid.openTextgrid(tg_path, includeEmptyIntervals=True)
28 | phones = []
29 | ends = []
30 | for interval in alignment.tierDict["phones"].entryList:
31 | phone = interval.label
32 | phones.append(phone)
33 | ends.append(interval.end)
34 | frame_pos = librosa.time_to_frames(ends, sr=sample_rate, hop_length=n_shift)
35 | durations = np.diff(frame_pos, prepend=0)
36 | assert len(durations) == len(phones)
37 | # merge "" and sp in the end
38 | if phones[-1] == "" and len(phones) > 1 and phones[-2] == "sp":
39 | phones = phones[:-1]
40 | durations[-2] += durations[-1]
41 | durations = durations[:-1]
42 | # replace the last "sp" with "sil" in MFA1.x
43 | phones[-1] = "sil" if phones[-1] == "sp" else phones[-1]
44 | # replace the edge "" with "sil", replace the inner "" with "sp"
45 | new_phones = []
46 | for i, phn in enumerate(phones):
47 | if phn == "":
48 | if i in {0, len(phones) - 1}:
49 | new_phones.append("sil")
50 | else:
51 | new_phones.append("sp")
52 | else:
53 | new_phones.append(phn)
54 | phones = new_phones
55 | results = ""
56 | for (p, d) in zip(phones, durations):
57 | results += p + " " + str(d) + " "
58 | return results.strip()
59 |
60 |
61 | # assume that the directory structure of inputdir is inputdir/speaker/*.TextGrid
62 | # in MFA1.x, there are blank labels("") in the end, and maybe "sp" before it
63 | # in MFA2.x, there are blank labels("") in the begin and the end, while no "sp" and "sil" anymore
64 | # we replace it with "sil"
65 | def gen_duration_from_textgrid(inputdir, output, sample_rate=24000,
66 | n_shift=300):
67 | # key: utt_id, value: (speaker, phn_durs)
68 | durations_dict = {}
69 | list_dir = os.listdir(inputdir)
70 | speakers = [dir for dir in list_dir if os.path.isdir(inputdir / dir)]
71 | for speaker in speakers:
72 | subdir = inputdir / speaker
73 | for file in tqdm(os.listdir(subdir)):
74 | if file.endswith(".TextGrid"):
75 | tg_path = subdir / file
76 | name = file.split(".")[0]
77 | durations_dict[name] = (speaker, readtg(
78 | tg_path, sample_rate=sample_rate, n_shift=n_shift))
79 | with open(output, "w") as wf:
80 | for name in sorted(durations_dict.keys()):
81 | wf.write(name + "|" + durations_dict[name][0] + "|" +
82 | durations_dict[name][1] + "\n")
83 |
84 |
85 | def main():
86 | # parse config and args
87 | parser = argparse.ArgumentParser(
88 | description="Preprocess audio and then extract features.")
89 | parser.add_argument(
90 | "--inputdir",
91 | default=None,
92 | type=str,
93 | help="directory to alignment files.")
94 | parser.add_argument(
95 | "--output", type=str, required=True, help="output duration file.")
96 | parser.add_argument("--sample-rate", type=int, help="the sample of wavs.")
97 | parser.add_argument(
98 | "--n-shift",
99 | type=int,
100 | help="the n_shift of time_to_freames, also called hop_length.")
101 | parser.add_argument(
102 | "--config", type=str, help="config file with fs and n_shift.")
103 |
104 | args = parser.parse_args()
105 | with open(args.config) as f:
106 | config = CfgNode(yaml.safe_load(f))
107 |
108 | inputdir = Path(args.inputdir).expanduser()
109 | output = Path(args.output).expanduser()
110 | print(config)
111 | # import sys
112 | # sys.exit(0)
113 | gen_duration_from_textgrid(inputdir, output, config.sampling_rate, config.hop_length)
114 |
115 |
116 | if __name__ == "__main__":
117 | main()
--------------------------------------------------------------------------------
/pyutils/logger.py:
--------------------------------------------------------------------------------
1 | import logging
2 |
3 | def get_logger(logging_file):
4 | logger = logging.getLogger()
5 | logger.setLevel(logging.INFO)
6 |
7 | formatter = logging.Formatter(
8 | "%(asctime)s-%(filename)s[line:%(lineno)d]-%(levelname)s: %(message)s"
9 | )
10 |
11 | file_log_handler = logging.FileHandler(logging_file, mode = 'w')
12 | file_log_handler.setLevel(logging.INFO)
13 | file_log_handler.setFormatter(formatter)
14 |
15 | # stream_log_handler = logging.StreamHandler()
16 | # stream_log_handler.setLevel(logging.INFO)
17 | # stream_log_handler.setFormatter(formatter)
18 |
19 | logger.addHandler(file_log_handler)
20 | # logger.addHandler(stream_log_handler)
21 |
22 | return logger
--------------------------------------------------------------------------------
/pyutils/optimizer.py:
--------------------------------------------------------------------------------
1 | import torch
2 | import numpy as np
3 | from ipdb import set_trace
4 | from torch.optim import (
5 | SGD,
6 | Adam,
7 | AdamW,
8 | RMSprop,
9 | RAdam,
10 | NAdam,
11 | ASGD
12 | )
13 |
14 | class NoamOpt():
15 | "Optim wrapper that implements rate."
16 |
17 | def __init__(self, optimizer, model_size, warmup, factor = 1.0):
18 | '''
19 | model_size: d_model
20 | factor: factor
21 | warmup: warmup step
22 | optimizer: optimizer (Adam default)
23 | '''
24 | self.optimizer = optimizer
25 | self._step = 0
26 | self.warmup = warmup
27 | self.factor = factor
28 | self.model_size = model_size
29 | self._rate = 0
30 |
31 | @property
32 | def param_groups(self):
33 | return self.optimizer.param_groups
34 |
35 | def step(self):
36 | "Update parameters and rate"
37 | self._step += 1
38 | rate = self.rate()
39 | for p in self.optimizer.param_groups:
40 | p["lr"] = rate
41 | self._rate = rate
42 | self.optimizer.step()
43 |
44 | def rate(self, step=None):
45 | "Implement `lrate` above"
46 | if step is None:
47 | step = self._step
48 | return (
49 | self.factor
50 | * self.model_size ** (-0.5)
51 | * min(step ** (-0.5), step * self.warmup ** (-1.5))
52 | )
53 |
54 | def zero_grad(self):
55 | self.optimizer.zero_grad()
56 |
57 | def state_dict(self):
58 | return {
59 | "_step": self._step,
60 | "warmup": self.warmup,
61 | "factor": self.factor,
62 | "model_size": self.model_size,
63 | "_rate": self._rate,
64 | "optimizer": self.optimizer.state_dict(),
65 | }
66 |
67 | def load_state_dict(self, state_dict):
68 | for key, value in state_dict.items():
69 | if key == "optimizer":
70 | self.optimizer.load_state_dict(state_dict["optimizer"])
71 | else:
72 | setattr(self, key, value)
73 |
74 | class ScheduledOptimD():
75 | ''' A simple wrapper class for learning rate scheduling '''
76 |
77 | def __init__(self, optimizer, init_lr, n_warmup_steps, current_steps):
78 | self.optimizer = optimizer
79 | self.n_warmup_steps = n_warmup_steps
80 | self.n_current_steps = current_steps
81 | self.init_lr = init_lr
82 |
83 | def step_and_update_lr_frozen(self, learning_rate_frozen):
84 | for param_group in self.optimizer.param_groups:
85 | param_group['lr'] = learning_rate_frozen
86 | self.optimizer.step()
87 |
88 | def step_and_update_lr(self):
89 | self._update_learning_rate()
90 | self.optimizer.step()
91 |
92 | def get_learning_rate(self):
93 | learning_rate = 0.0
94 | for param_group in self.optimizer.param_groups:
95 | learning_rate = param_group['lr']
96 |
97 | return learning_rate
98 |
99 | def zero_grad(self):
100 | # print(self.init_lr)
101 | self.optimizer.zero_grad()
102 |
103 | def set_current_steps(self, step):
104 | self.n_current_steps = step
105 |
106 | def _get_lr_scale(self):
107 | # set_trace()
108 | return np.min([
109 | np.power(self.n_current_steps, -0.5),
110 | np.power(self.n_warmup_steps, -1.5) * self.n_current_steps])
111 |
112 | def _update_learning_rate(self):
113 | ''' Learning rate scheduling per step '''
114 |
115 | lr = self.init_lr * self._get_lr_scale()
116 |
117 | for param_group in self.optimizer.param_groups:
118 | param_group['lr'] = lr
119 |
120 | def state_dict(self):
121 | return {
122 | "_step": self.n_current_steps,
123 | "warmup": self.n_warmup_steps,
124 | "factor": self.init_lr,
125 | "_rate": self.get_learning_rate(),
126 | "optimizer": self.optimizer.state_dict(),
127 | }
128 |
129 | def load_state_dict(self, state_dict):
130 | for key, value in state_dict.items():
131 | if key == "optimizer":
132 | self.optimizer.load_state_dict(state_dict["optimizer"])
133 | else:
134 | setattr(self, key, value)
135 |
136 | def get_g_opt(model, optim, d_model, warmup, factor):
137 | base = torch.optim.Adam(model.parameters(), lr = 0, betas = (0.9, 0.98), eps = 1e-9)
138 | return NoamOpt(base, d_model, warmup, factor)
139 |
140 | def get_d_opt(model, optim, warmup, factor, current_step):
141 | base = torch.optim.Adam(model.parameters(), lr = 0, betas = (0.9, 0.98), eps = 1e-9)
142 | return ScheduledOptimD(base, factor, warmup, current_step)
143 |
--------------------------------------------------------------------------------
/pyutils/parse_options.sh:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env bash
2 |
3 | # Copyright 2012 Johns Hopkins University (Author: Daniel Povey);
4 | # Arnab Ghoshal, Karel Vesely
5 |
6 | # Licensed under the Apache License, Version 2.0 (the "License");
7 | # you may not use this file except in compliance with the License.
8 | # You may obtain a copy of the License at
9 | #
10 | # http://www.apache.org/licenses/LICENSE-2.0
11 | #
12 | # THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
13 | # KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
14 | # WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE,
15 | # MERCHANTABLITY OR NON-INFRINGEMENT.
16 | # See the Apache 2 License for the specific language governing permissions and
17 | # limitations under the License.
18 |
19 |
20 | # Parse command-line options.
21 | # To be sourced by another script (as in ". parse_options.sh").
22 | # Option format is: --option-name arg
23 | # and shell variable "option_name" gets set to value "arg."
24 | # The exception is --help, which takes no arguments, but prints the
25 | # $help_message variable (if defined).
26 |
27 |
28 | ###
29 | ### The --config file options have lower priority to command line
30 | ### options, so we need to import them first...
31 | ###
32 |
33 | # Now import all the configs specified by command-line, in left-to-right order
34 | for ((argpos=1; argpos<$#; argpos++)); do
35 | if [ "${!argpos}" == "--config" ]; then
36 | argpos_plus1=$((argpos+1))
37 | config=${!argpos_plus1}
38 | [ ! -r $config ] && echo "$0: missing config '$config'" && exit 1
39 | . $config # source the config file.
40 | fi
41 | done
42 |
43 |
44 | ###
45 | ### Now we process the command line options
46 | ###
47 | while true; do
48 | [ -z "${1:-}" ] && break; # break if there are no arguments
49 | case "$1" in
50 | # If the enclosing script is called with --help option, print the help
51 | # message and exit. Scripts should put help messages in $help_message
52 | --help|-h) if [ -z "$help_message" ]; then echo "No help found." 1>&2;
53 | else printf "$help_message\n" 1>&2 ; fi;
54 | exit 0 ;;
55 | --*=*) echo "$0: options to scripts must be of the form --name value, got '$1'"
56 | exit 1 ;;
57 | # If the first command-line argument begins with "--" (e.g. --foo-bar),
58 | # then work out the variable name as $name, which will equal "foo_bar".
59 | --*) name=`echo "$1" | sed s/^--// | sed s/-/_/g`;
60 | # Next we test whether the variable in question is undefned-- if so it's
61 | # an invalid option and we die. Note: $0 evaluates to the name of the
62 | # enclosing script.
63 | # The test [ -z ${foo_bar+xxx} ] will return true if the variable foo_bar
64 | # is undefined. We then have to wrap this test inside "eval" because
65 | # foo_bar is itself inside a variable ($name).
66 | eval '[ -z "${'$name'+xxx}" ]' && echo "$0: invalid option $1" 1>&2 && exit 1;
67 |
68 | oldval="`eval echo \\$$name`";
69 | # Work out whether we seem to be expecting a Boolean argument.
70 | if [ "$oldval" == "true" ] || [ "$oldval" == "false" ]; then
71 | was_bool=true;
72 | else
73 | was_bool=false;
74 | fi
75 |
76 | # Set the variable to the right value-- the escaped quotes make it work if
77 | # the option had spaces, like --cmd "queue.pl -sync y"
78 | eval $name=\"$2\";
79 |
80 | # Check that Boolean-valued arguments are really Boolean.
81 | if $was_bool && [[ "$2" != "true" && "$2" != "false" ]]; then
82 | echo "$0: expected \"true\" or \"false\": $1 $2" 1>&2
83 | exit 1;
84 | fi
85 | shift 2;
86 | ;;
87 | *) break;
88 | esac
89 | done
90 |
91 |
92 | # Check for an empty argument to the --cmd option, which can easily occur as a
93 | # result of scripting errors.
94 | [ ! -z "${cmd+xxx}" ] && [ -z "$cmd" ] && echo "$0: empty argument to --cmd option" 1>&2 && exit 1;
95 |
96 |
97 | true; # so this script returns exit code 0.
--------------------------------------------------------------------------------
/pyutils/plot.py:
--------------------------------------------------------------------------------
1 | import matplotlib.pyplot as plt
2 | import librosa
3 | import soundfile as sf
4 | import numpy as np
5 | import logging
6 | import argparse
7 | logging.getLogger('matplotlib.font_manager').disabled = True
8 |
9 | def specplot(spec,
10 | pic_path = 'exp/test/melspectrograms/spec.png',
11 | **kwargs):
12 | """Plot the log mel spectrogram of audio."""
13 | fig = plt.figure()
14 | plt.imshow(spec, origin = 'lower', cmap = plt.cm.magma, aspect='auto')
15 | plt.colorbar()
16 | fig.savefig(pic_path)
17 | plt.close()
18 |
19 | def specplot_from_audio(filename = None,
20 | audio = None,
21 | rate = None,
22 | rotate=False,
23 | n_ffts = 1024,
24 | pic_path = 'exp/test/spectrograms/spec.png',
25 | **kwargs):
26 | """Plot the log magnitude spectrogram of audio."""
27 | if filename is not None:
28 | audio, rate = sf.read(filename)
29 | hop_length = kwargs.get('hop_length', None)
30 | win_length = kwargs.get('win_length', None)
31 | stft = librosa.stft(
32 | audio,
33 | n_fft = n_ffts,
34 | hop_length = hop_length,
35 | win_length = win_length
36 | )
37 | mag, phase = librosa.magphase(stft)
38 | logmag = np.log10(mag)
39 | fig = plt.figure()
40 | plt.imshow(logmag, cmap = plt.cm.magma, origin = 'lower', aspect = 'auto')
41 | plt.colorbar()
42 | fig.savefig(pic_path)
43 | plt.close()
44 |
45 | def melspecplot(mel_spec,
46 | pic_path = 'exp/test/melspectrograms/melspec.png',
47 | **kwargs):
48 | """Plot the log mel spectrogram of audio."""
49 | fig = plt.figure()
50 | plt.imshow(mel_spec, origin = 'lower', cmap = plt.cm.magma, aspect='auto')
51 | plt.colorbar()
52 | fig.savefig(pic_path)
53 | plt.close()
54 |
55 | def melspecplot_from_audio(filename = None,
56 | audio = None,
57 | rate = None,
58 | rotate = False,
59 | n_ffts = 1024,
60 | pic_path = 'exp/test/melspectrograms/melspec.png',
61 | **kwargs):
62 | """Plot the log mel spectrogram of audio."""
63 | if filename is not None:
64 | audio, rate = sf.read(filename)
65 | hop_length = kwargs.get('hop_length', None)
66 | win_length = kwargs.get('win_length', None)
67 | n_mels = kwargs.get('n_mels', 23)
68 | mel_spec = librosa.feature.melspectrogram(
69 | y = audio,
70 | sr = rate,
71 | n_fft = n_ffts,
72 | hop_length = hop_length,
73 | win_length = win_length,
74 | n_mels = n_mels
75 | )
76 | mel_spec = librosa.power_to_db(mel_spec, ref=np.max)
77 | plt.imshow(mel_spec, cmap = plt.cm.magma, origin = 'lower', aspect = 'auto')
78 | plt.savefig(pic_path)
79 | plt.close()
80 |
81 | def get_args():
82 | parser = argparse.ArgumentParser()
83 | parser.add_argument('--filename', type=str, default=None)
84 | parser.add_argument('--output', type=str, default=None)
85 | parser.add_argument('--mean', type=str, default=None)
86 | parser.add_argument('--std', type=str, default=None)
87 | return parser.parse_args()
88 |
89 | if __name__ == "__main__":
90 | args = get_args()
91 | import numpy as np
92 | # specplot_from_audio(args.filename, pic_path = args.output + '.aspec.png')
93 | data = np.load(args.filename).T
94 | mean = np.load(args.mean).T # (n_fft + 1, T)
95 | std = np.load(args.std).T
96 | if 'spec' in args.filename:
97 | specplot(data, mean, std, args.output + '.spec.png')
98 | if 'mel' in args.filename:
99 | melspecplot(data, args.output + '.melspec.png')
--------------------------------------------------------------------------------
/pyutils/save_and_load.py:
--------------------------------------------------------------------------------
1 | import os
2 | import glob
3 | import re
4 | import sys
5 | import argparse
6 | import logging
7 | import json
8 | import subprocess
9 | import warnings
10 | import random
11 | import functools
12 |
13 | import librosa
14 | import numpy as np
15 | from scipy.io.wavfile import read
16 | import torch
17 | from torch.nn import functional as F
18 | # from modules.commons import sequence_mask
19 | logging.basicConfig(stream=sys.stdout, level=logging.INFO)
20 | logger = logging
21 |
22 | def latest_checkpoint_path(dir_path, regex="G_*.pth"):
23 | f_list = glob.glob(os.path.join(dir_path, regex))
24 | f_list.sort(key=lambda f: int("".join(filter(str.isdigit, f))))
25 | x = f_list[-1]
26 | print(x)
27 | return x
28 |
29 | def load_checkpoint(checkpoint_path, model, optimizer=None, scheduler=None, skip_optimizer=False):
30 | assert os.path.isfile(checkpoint_path)
31 | checkpoint_dict = torch.load(checkpoint_path, map_location='cpu')
32 | iteration = checkpoint_dict['iteration']
33 | learning_rate = checkpoint_dict['learning_rate']
34 | if optimizer is not None and not skip_optimizer and checkpoint_dict['optimizer'] is not None:
35 | optimizer.load_state_dict(checkpoint_dict['optimizer'])
36 | if scheduler is not None and not skip_optimizer and checkpoint_dict['scheduler'] is not None:
37 | scheduler.load_state_dict(checkpoint_dict['scheduler'])
38 | saved_state_dict = checkpoint_dict['model']
39 | if hasattr(model, 'module'):
40 | state_dict = model.module.state_dict()
41 | else:
42 | state_dict = model.state_dict()
43 | new_state_dict = {}
44 | for k, v in state_dict.items():
45 | try:
46 | # assert "dec" in k or "disc" in k
47 | # print("load", k)
48 | new_state_dict[k] = saved_state_dict[k]
49 | assert saved_state_dict[k].shape == v.shape, (saved_state_dict[k].shape, v.shape)
50 | except:
51 | print("error, %s is not in the checkpoint" % k)
52 | logger.info("%s is not in the checkpoint" % k)
53 | new_state_dict[k] = v
54 | if hasattr(model, 'module'):
55 | model.module.load_state_dict(new_state_dict)
56 | else:
57 | model.load_state_dict(new_state_dict)
58 | print("load")
59 | logger.info("Loaded checkpoint '{}' (iteration {})".format(
60 | checkpoint_path, iteration))
61 | return model, optimizer, scheduler, learning_rate, iteration
62 |
63 | def save_checkpoint(model, optimizer, scheduler, learning_rate, iteration, checkpoint_path):
64 | logger.info("Saving model and optimizer state at iteration {} to {}".format(
65 | iteration, checkpoint_path))
66 | if hasattr(model, 'module'):
67 | state_dict = model.module.state_dict()
68 | else:
69 | state_dict = model.state_dict()
70 | torch.save({'model': state_dict,
71 | 'iteration': iteration,
72 | 'optimizer': optimizer.state_dict(),
73 | 'scheduler': scheduler.state_dict(),
74 | 'learning_rate': learning_rate},
75 | checkpoint_path)
76 |
77 | def clean_checkpoints(path_to_models='logs/44k/', n_ckpts_to_keep=2, sort_by_time=True):
78 | """Freeing up space by deleting saved ckpts
79 |
80 | Arguments:
81 | path_to_models -- Path to the model directory
82 | n_ckpts_to_keep -- Number of ckpts to keep, excluding G_0.pth and D_0.pth
83 | sort_by_time -- True -> chronologically delete ckpts
84 | False -> lexicographically delete ckpts
85 | """
86 | ckpts_files = [f for f in os.listdir(path_to_models) if os.path.isfile(os.path.join(path_to_models, f))]
87 | name_key = (lambda _f: int(re.compile('._(\d+)\.pth').match(_f).group(1)))
88 | time_key = (lambda _f: os.path.getmtime(os.path.join(path_to_models, _f)))
89 | sort_key = time_key if sort_by_time else name_key
90 | x_sorted = lambda _x: sorted([f for f in ckpts_files if f.startswith(_x) and not f.endswith('_0.pth')], key=sort_key)
91 | to_del = [os.path.join(path_to_models, fn) for fn in
92 | (x_sorted('G')[:-n_ckpts_to_keep] + x_sorted('D')[:-n_ckpts_to_keep])]
93 | del_info = lambda fn: logger.info(f".. Free up space by deleting ckpt {fn}")
94 | del_routine = lambda x: [os.remove(x), del_info(x)]
95 | rs = [del_routine(fn) for fn in to_del]
96 |
97 | class HParams():
98 | def __init__(self, **kwargs):
99 | for k, v in kwargs.items():
100 | if type(v) == dict:
101 | v = HParams(**v)
102 | self[k] = v
103 |
104 | def keys(self):
105 | return self.__dict__.keys()
106 |
107 | def items(self):
108 | return self.__dict__.items()
109 |
110 | def values(self):
111 | return self.__dict__.values()
112 |
113 | def __len__(self):
114 | return len(self.__dict__)
115 |
116 | def __getitem__(self, key):
117 | return getattr(self, key)
118 |
119 | def __setitem__(self, key, value):
120 | return setattr(self, key, value)
121 |
122 | def __contains__(self, key):
123 | return key in self.__dict__
124 |
125 | def __repr__(self):
126 | return self.__dict__.__repr__()
127 |
128 |
--------------------------------------------------------------------------------
/pyutils/scheduler.py:
--------------------------------------------------------------------------------
1 | import math
2 | from torch.optim.lr_scheduler import _LRScheduler
3 | import torch
4 |
5 | class WarmupLR(_LRScheduler):
6 | """The WarmupLR scheduler
7 |
8 | This scheduler is almost same as NoamLR Scheduler except for following
9 | difference:
10 |
11 | NoamLR:
12 | lr = optimizer.lr * model_size ** -0.5
13 | * min(step ** -0.5, step * warmup_step ** -1.5)
14 | WarmupLR:
15 | lr = optimizer.lr * warmup_step ** 0.5
16 | * min(step ** -0.5, step * warmup_step ** -1.5)
17 |
18 | Note that the maximum lr equals to optimizer.lr in this scheduler.
19 |
20 | """
21 |
22 | def __init__(
23 | self,
24 | optimizer,
25 | warmup_steps = 25000,
26 | last_epoch = -1,
27 | ):
28 | self.warmup_steps = warmup_steps
29 |
30 | # __init__() must be invoked before setting field
31 | # because step() is also invoked in __init__()
32 | super().__init__(optimizer, last_epoch)
33 |
34 | def __repr__(self):
35 | return f"{self.__class__.__name__}(warmup_steps={self.warmup_steps})"
36 |
37 | def get_lr(self):
38 | step_num = self.last_epoch + 1
39 | if self.warmup_steps == 0:
40 | return [
41 | lr * step_num ** -0.5
42 | for lr in self.base_lrs
43 | ]
44 | else:
45 | return [
46 | lr
47 | * self.warmup_steps ** 0.5
48 | * min(step_num ** -0.5, step_num * self.warmup_steps ** -1.5)
49 | for lr in self.base_lrs
50 | ]
51 |
52 | def set_step(self, step: int):
53 | self.last_epoch = step
54 |
55 | class BaseClass:
56 | '''
57 | Base Class for learning rate scheduler
58 | '''
59 |
60 | def __init__(self,
61 | optimizer,
62 | num_epochs,
63 | epoch_iter,
64 | initial_lr,
65 | final_lr,
66 | warm_up_epoch=6,
67 | scale_ratio=1.0,
68 | warm_from_zero=False):
69 | '''
70 | warm_up_epoch: the first warm_up_epoch is the multiprocess warm-up stage
71 | scale_ratio: multiplied to the current lr in the multiprocess training
72 | process
73 | '''
74 | self.optimizer = optimizer
75 | self.max_iter = num_epochs * epoch_iter
76 | self.initial_lr = initial_lr
77 | self.final_lr = final_lr
78 | self.scale_ratio = scale_ratio
79 | self.current_iter = 0
80 | self.warm_up_iter = warm_up_epoch * epoch_iter
81 | self.warm_from_zero = warm_from_zero
82 |
83 | def get_multi_process_coeff(self):
84 | lr_coeff = 1.0 * self.scale_ratio
85 | if self.current_iter < self.warm_up_iter:
86 | if self.warm_from_zero:
87 | lr_coeff = self.scale_ratio * self.current_iter / self.warm_up_iter
88 | elif self.scale_ratio > 1:
89 | lr_coeff = (self.scale_ratio -
90 | 1) * self.current_iter / self.warm_up_iter + 1.0
91 |
92 | return lr_coeff
93 |
94 | def get_current_lr(self):
95 | '''
96 | This function should be implemented in the child class
97 | '''
98 | return 0.0
99 |
100 | def get_lr(self):
101 | return self.optimizer.param_groups[0]['lr']
102 |
103 | def set_lr(self):
104 | current_lr = self.get_current_lr()
105 | for param_group in self.optimizer.param_groups:
106 | param_group['lr'] = current_lr
107 |
108 | def step(self, current_iter=None):
109 | if current_iter is not None:
110 | self.current_iter = current_iter
111 |
112 | self.set_lr()
113 | self.current_iter += 1
114 |
115 | def step_return_lr(self, current_iter=None):
116 | if current_iter is not None:
117 | self.current_iter = current_iter
118 |
119 | current_lr = self.get_current_lr()
120 | self.current_iter += 1
121 |
122 | return current_lr
123 |
124 | class ExponentialDecrease(BaseClass):
125 |
126 | def __init__(self,
127 | optimizer,
128 | num_epochs,
129 | epoch_iter,
130 | initial_lr,
131 | final_lr,
132 | warm_up_epoch=6,
133 | scale_ratio=1.0,
134 | warm_from_zero=False):
135 | super().__init__(optimizer, num_epochs, epoch_iter, initial_lr,
136 | final_lr, warm_up_epoch, scale_ratio, warm_from_zero)
137 |
138 | def get_current_lr(self):
139 | lr_coeff = self.get_multi_process_coeff()
140 | current_lr = lr_coeff * self.initial_lr * math.exp(
141 | (self.current_iter / self.max_iter) *
142 | math.log(self.final_lr / self.initial_lr))
143 | return current_lr
144 |
145 | class TriAngular2(BaseClass):
146 | '''
147 | The implementation of https://arxiv.org/pdf/1506.01186.pdf
148 | '''
149 |
150 | def __init__(self,
151 | optimizer,
152 | num_epochs,
153 | epoch_iter,
154 | initial_lr,
155 | final_lr,
156 | warm_up_epoch=6,
157 | scale_ratio=1.0,
158 | cycle_step=2,
159 | reduce_lr_diff_ratio=0.5):
160 | super().__init__(optimizer, num_epochs, epoch_iter, initial_lr,
161 | final_lr, warm_up_epoch, scale_ratio)
162 |
163 | self.reduce_lr_diff_ratio = reduce_lr_diff_ratio
164 | self.cycle_iter = cycle_step * epoch_iter
165 | self.step_size = self.cycle_iter // 2
166 |
167 | self.max_lr = initial_lr
168 | self.min_lr = final_lr
169 | self.gap = self.max_lr - self.min_lr
170 |
171 | def get_current_lr(self):
172 | lr_coeff = self.get_multi_process_coeff()
173 | point = self.current_iter % self.cycle_iter
174 | cycle_index = self.current_iter // self.cycle_iter
175 |
176 | self.max_lr = self.min_lr + self.gap * self.reduce_lr_diff_ratio**cycle_index
177 |
178 | if point <= self.step_size:
179 | current_lr = self.min_lr + (self.max_lr -
180 | self.min_lr) * point / self.step_size
181 | else:
182 | current_lr = self.max_lr - (self.max_lr - self.min_lr) * (
183 | point - self.step_size) / self.step_size
184 |
185 | current_lr = lr_coeff * current_lr
186 |
187 | return current_lr
188 |
189 |
190 | def show_lr_curve(scheduler):
191 | import matplotlib.pyplot as plt
192 |
193 | lr_list = []
194 | for current_lr in range(0, scheduler.max_iter):
195 | lr_list.append(scheduler.step_return_lr(current_lr))
196 | data_index = list(range(1, len(lr_list) + 1))
197 |
198 | plt.plot(data_index, lr_list, '-o', markersize=1)
199 | plt.legend(loc='best')
200 | plt.xlabel("Iteration")
201 | plt.ylabel("LR")
202 |
203 | plt.show()
204 |
205 |
206 | if __name__ == '__main__':
207 | optimizer = None
208 | num_epochs = 6
209 | epoch_iter = 500
210 | initial_lr = 0.6
211 | final_lr = 0.1
212 | warm_up_epoch = 2
213 | scale_ratio = 4
214 | scheduler = ExponentialDecrease(optimizer, num_epochs, epoch_iter,
215 | initial_lr, final_lr, warm_up_epoch,
216 | scale_ratio)
217 | # scheduler = TriAngular2(optimizer,
218 | # num_epochs,
219 | # epoch_iter,
220 | # initial_lr,
221 | # final_lr,
222 | # warm_up_epoch,
223 | # scale_ratio,
224 | # cycle_step=2,
225 | # reduce_lr_diff_ratio=0.5)
226 |
227 | show_lr_curve(scheduler)
228 |
--------------------------------------------------------------------------------
/run.sh:
--------------------------------------------------------------------------------
1 | #! /bin/bash
2 |
3 | export PYTHONPATH=$(pwd):$PYTHONPATH
4 | start_stage=2
5 | stop_stage=2
6 | step=
7 | input=
8 | output=
9 |
10 | echo $PYTHONPATH
11 |
12 | echo "$0 $@" # Print the command line for logging
13 |
14 | . ./utils/parse_options.sh
15 |
16 | if [ ${start_stage} -le 0 ] && [ ${stop_stage} -ge 0 ]; then
17 | echo "stage 0: Data Preparation"
18 | python3 preprocess/data_prep.py
19 | fi
20 |
21 | if [ ${start_stage} -le 1 ] && [ ${stop_stage} -ge 1 ]; then
22 | echo "stage 1: Feature Extraction"
23 | python preprocess/audio_preprocess.py --data-config configs/data.yaml \
24 | --spec \
25 | --mel \
26 | --f0 \
27 | --energy \
28 | --stat
29 | fi
30 |
31 | if [ ${start_stage} -le 2 ] && [ ${stop_stage} -ge 2 ]; then
32 | echo "stage 2: Training"
33 | export CUDA_VISIBLE_DEVICES=0,1
34 | python train_gan.py --data-config configs/svs/data.yaml \
35 | --model-config configs/svs/model.yaml \
36 | --train-config configs/svs/train.yaml \
37 | --num-gpus 2 \
38 | --dist-url 'tcp://localhost:30305'
39 | fi
40 |
41 | if [ ${start_stage} -le 3 ] && [ ${stop_stage} -ge 3 ]; then
42 | echo "stage 3: Synthesizing"
43 | python synthesize.py --exp-name ${name} \
44 | --step ${step} \
45 | --input ${input} \
46 | --output ${output}
47 | fi
48 |
--------------------------------------------------------------------------------
/train.py:
--------------------------------------------------------------------------------
1 | import os
2 | import yaml
3 | import argparse
4 | import math
5 | import logging
6 |
7 | import torch
8 | import torch.nn as nn
9 | from torch.utils.data import DataLoader
10 | import torch.multiprocessing as mp
11 |
12 | from dataset import SVSDataset, SVSCollate
13 | from models import Xiaoice2 as Generator
14 | from loss import FastSpeech2Loss
15 | import pyutils
16 | from pyutils import (
17 | load_checkpoint,
18 | save_checkpoint,
19 | clean_checkpoints,
20 | latest_checkpoint_path,
21 | melspecplot,
22 | get_logger
23 | )
24 |
25 | import wandb
26 |
27 | logging.basicConfig(format = "%(asctime)s-%(filename)s[line:%(lineno)d]-%(levelname)s: %(message)s", level = logging.INFO)
28 |
29 | class Trainer():
30 | def __init__(self, rank, args, data_configs, model_configs, train_configs):
31 | self.rank = rank
32 | self.device = torch.device('cuda:{:d}'.format(rank))
33 | trainset = SVSDataset(data_configs)
34 | collate_fn = SVSCollate()
35 | sampler = torch.utils.data.DistributedSampler(trainset) if args.num_gpus > 1 else None
36 | self.trainloader = DataLoader(
37 | trainset,
38 | shuffle = False,
39 | sampler = sampler,
40 | collate_fn = collate_fn,
41 | batch_size = train_configs['batch_size'],
42 | pin_memory = True,
43 | num_workers = train_configs['num_workers'],
44 | prefetch_factor = 10
45 | )
46 |
47 | model_configs['generator']['transformer']['encoder']['n_src_vocab'] = trainset.get_phone_number() + 1
48 | model_configs['generator']['spk_num'] = trainset.get_spk_number()
49 |
50 | if args.num_gpus > 1:
51 | self.models = (
52 | nn.parallel.DistributedDataParallel(
53 | Generator(
54 | data_configs,
55 | model_configs['generator']
56 | ).to(self.device),
57 | device_ids = [rank]
58 | ),
59 | )
60 | else:
61 | self.models = (
62 | Generator(
63 | data_configs,
64 | model_configs['generator']
65 | ).to(self.device),
66 | )
67 | self.data_configs = data_configs
68 | self.model_configs = model_configs
69 | self.train_configs = train_configs
70 | self.args = args
71 |
72 | try:
73 | self.g_optimizer = getattr(
74 | torch.optim, train_configs['g_optimizer']
75 | )(self.models[0].parameters(), **train_configs['g_optimizer_args'])
76 |
77 | self.g_scheduler = getattr(
78 | pyutils.scheduler, train_configs['g_scheduler']
79 | )(self.g_optimizer, **train_configs['g_scheduler_args'])
80 | except:
81 | raise NotImplementedError("Unknown optimizer or scheduler")
82 |
83 | self.fs2loss = FastSpeech2Loss(data_configs)
84 |
85 | if self.rank == 0:
86 | self._make_exp_dir()
87 | self.logger = get_logger(os.path.join(self.args.exp_name, 'logs/train.log'))
88 |
89 | try:
90 | latest_ckpt_path = latest_checkpoint_path(
91 | os.path.join(self.args.exp_name, 'models'),
92 | 'G_*.pth'
93 | )
94 | _, _, _, _, epoch_str = load_checkpoint(
95 | latest_ckpt_path,
96 | self.models[0],
97 | self.g_optimizer,
98 | self.g_scheduler,
99 | False
100 | )
101 | self.start_epoch = max(epoch_str, 1)
102 | name = latest_ckpt_path
103 | self.total_step = int(name[name.rfind("_")+1:name.rfind(".")]) + 1
104 | except Exception:
105 | print("Load old checkpoint failed...")
106 | print("Start a new training...")
107 | self.start_epoch = 1
108 | self.total_step = 0
109 |
110 | self.epochs = self.train_configs['epochs']
111 |
112 | def _dump_args_and_config(self, filename, config):
113 | with open(os.path.join(self.args.exp_name, 'configs', filename) + '.yaml', 'w') as f:
114 | yaml.dump(config, f)
115 |
116 | def _make_exp_dir(self):
117 | os.makedirs(self.args.exp_name, exist_ok=True)
118 | os.makedirs(os.path.join(self.args.exp_name, 'configs'), exist_ok=True)
119 | os.makedirs(os.path.join(self.args.exp_name, 'models'), exist_ok=True)
120 | os.makedirs(os.path.join(self.args.exp_name, 'audios'), exist_ok=True)
121 | os.makedirs(os.path.join(self.args.exp_name, 'spectrograms'), exist_ok=True)
122 | os.makedirs(os.path.join(self.args.exp_name, 'melspectrograms'), exist_ok=True)
123 | os.makedirs(os.path.join(self.args.exp_name, 'eval_results'), exist_ok=True)
124 | os.makedirs(os.path.join(self.args.exp_name, 'logs'), exist_ok = True)
125 | with open(os.path.join(self.args.exp_name, 'model_arch.txt'), 'w') as f:
126 | for model in self.models:
127 | print(model, file = f)
128 | self._dump_args_and_config('args', vars(self.args))
129 | self._dump_args_and_config('data', self.data_configs)
130 | self._dump_args_and_config('model', self.model_configs)
131 | self._dump_args_and_config('train', self.train_configs)
132 |
133 | def train(self):
134 | for epoch in range(self.start_epoch, self.epochs + 1, 1):
135 | self.train_epoch(epoch)
136 |
137 | def train_epoch(self, epoch):
138 | self.total_loss = 0.0
139 | for batch_idx, data in enumerate(self.trainloader):
140 | output, postnet_output = self.train_batch(data, epoch, batch_idx)
141 | self.g_scheduler.step()
142 |
143 | if self.rank == 0 and self.total_step % self.train_configs['save_interval'] == 0:
144 | ckpt_path = os.path.join(self.args.exp_name, 'models', 'G_{}.pth'.format(self.total_step))
145 | save_checkpoint(
146 | self.models[0],
147 | self.g_optimizer,
148 | self.g_scheduler,
149 | self.g_scheduler.get_lr()[0],
150 | epoch,
151 | ckpt_path
152 | )
153 | length = data['mel_lens'][0]
154 | real_pic_path = os.path.join(self.args.exp_name, 'melspectrograms', '{}_{}.png'.format(data['uttids'][0], self.total_step))
155 | before_pic_path = os.path.join(self.args.exp_name, 'melspectrograms', 'before_{}_{}.png'.format(data['uttids'][0], self.total_step))
156 | after_pic_path = os.path.join(self.args.exp_name, 'melspectrograms', 'after_{}_{}.png'.format(data['uttids'][0], self.total_step))
157 | melspecplot(data['mels'][0][:length, :].transpose(1, 0).numpy(), real_pic_path) # (n_mels, T)
158 | melspecplot(output[0][:length, :].transpose(1, 0).detach().cpu().numpy(), before_pic_path)
159 | melspecplot(postnet_output[0][:length, :].transpose(1, 0).detach().cpu().numpy(), after_pic_path)
160 |
161 | if self.rank == 0:
162 | clean_checkpoints(os.path.join(self.args.exp_name, 'models'), n_ckpts_to_keep = self.train_configs['ckpt_clean'])
163 |
164 | def _move_to_device(self, data):
165 | new_data = {}
166 | for k, v in data.items():
167 | if type(v) is torch.Tensor:
168 | new_data[k] = v.to(self.device)
169 | return new_data
170 |
171 | def train_batch(self, data, epoch, step):
172 | for model in self.models:
173 | model.train()
174 | new_data = self._move_to_device(data)
175 |
176 | self.g_optimizer.zero_grad()
177 | loss, report_keys, output, postnet_output = self.models[0](**new_data)
178 | loss.backward()
179 | grad_norm = nn.utils.clip_grad_norm_(self.models[0].parameters(), self.train_configs['grad_clip'])
180 | if math.isnan(grad_norm):
181 | raise ZeroDivisionError('Grad norm is nan')
182 | self.g_optimizer.step()
183 | self.total_loss += loss.item()
184 |
185 | self.total_step += 1
186 | if self.rank == 0:
187 | self.print_msg(epoch, step, report_keys) #, accuracy.item())
188 | wandb_log_dict = {
189 | 'train/avg_g_loss': self.total_loss / (step + 1),
190 | 'train/g_lr': self.g_scheduler.get_lr()[0]
191 | }
192 | for k, v in report_keys.items():
193 | wandb_log_dict['train/' + k] = v
194 | wandb.log(wandb_log_dict)
195 | return output, postnet_output
196 |
197 | def print_msg(self, epoch, step, report_keys):
198 | if self.total_step % self.train_configs['log_interval'] == 0:
199 | temp = ''
200 | for k, v in report_keys.items():
201 | temp += '{}: {:.6f} '.format(k, v)
202 | message = ('[Epoch: {} Step: {} Total steps: {}] ' + temp).format(
203 | epoch, step + 1, self.total_step
204 | )
205 | self.logger.info(message)
206 |
207 | def parse_args():
208 | parser = argparse.ArgumentParser()
209 | parser.add_argument('--data-config', dest = 'data_config', type = str, default = './conf/data.yaml')
210 | parser.add_argument('--model-config', dest = 'model_config', type = str, default = './conf/model.yaml')
211 | parser.add_argument('--train-config', dest = 'train_config', type = str, default = './conf/train.yaml')
212 | parser.add_argument('--num-gpus', dest = 'num_gpus', type = int, default = 1)
213 | parser.add_argument('--exp-name', dest = 'exp_name', type = str, default = 'default')
214 | parser.add_argument('--dist-backend', dest = 'dist_backend', type = str, default = 'nccl')
215 | parser.add_argument('--dist-url', dest = 'dist_url', type = str, default = 'tcp://localhost:30302')
216 | return parser.parse_args()
217 |
218 | def main(rank, args, configs):
219 | if args.num_gpus > 1:
220 | torch.cuda.set_device(rank)
221 | torch.distributed.init_process_group(
222 | backend = args.dist_backend,
223 | init_method = args.dist_url,
224 | world_size = args.num_gpus,
225 | rank = rank
226 | )
227 |
228 | data_configs, model_configs, train_configs = configs
229 | args.exp_name = train_configs['wandb_args']['group'] + '-' + \
230 | train_configs['wandb_args']['job_type'] + '-' + \
231 | train_configs['wandb_args']['name']
232 | args.exp_name = os.path.join('exp', args.exp_name)
233 |
234 | # wandb initialization
235 | if train_configs['wandb']:
236 | wandb_configs = vars(args)
237 | for config in configs:
238 | wandb_configs.update(config)
239 | wandb.init(
240 | **train_configs['wandb_args'],
241 | config = wandb_configs
242 | )
243 |
244 | trainer = Trainer(rank, args, data_configs, model_configs, train_configs)
245 | trainer.train()
246 |
247 | if train_configs['wandb']:
248 | wandb.finish()
249 |
250 | if __name__ == "__main__":
251 | args = parse_args()
252 | args.exp_name = os.path.join('exp', args.exp_name)
253 | with open(args.data_config, 'r') as f:
254 | data_configs = yaml.load(f, Loader = yaml.FullLoader)
255 | with open(args.model_config, 'r') as f:
256 | model_configs = yaml.load(f, Loader = yaml.FullLoader)
257 | with open(args.train_config, 'r') as f:
258 | train_configs = yaml.load(f, Loader = yaml.FullLoader)
259 | configs = (data_configs, model_configs, train_configs)
260 |
261 | num_gpus = torch.cuda.device_count()
262 | if args.num_gpus > 1:
263 | mp.spawn(main, nprocs = num_gpus, args = (args, configs))
264 | else:
265 | main(0, args, configs)
266 |
--------------------------------------------------------------------------------
/train_gan.py:
--------------------------------------------------------------------------------
1 | import os
2 | import yaml
3 | import argparse
4 | import math
5 | import shutil
6 | import logging
7 |
8 | import torch
9 | import torch.nn as nn
10 | from torch.utils.data import DataLoader
11 | import torch.multiprocessing as mp
12 |
13 | from dataset import SVSDataset as Dataset
14 | from dataset import SVSCollate as Collate
15 | from models import Xiaoice2 as Generator
16 | from models import Discriminator as Discriminator
17 | from loss import FastSpeech2Loss, FeatLoss, LSGANGLoss, LSGANDLoss
18 | import pyutils
19 | from pyutils import (
20 | load_checkpoint,
21 | save_checkpoint,
22 | clean_checkpoints,
23 | latest_checkpoint_path,
24 | melspecplot,
25 | get_logger
26 | )
27 |
28 | import wandb
29 |
30 | logging.basicConfig(format = "%(asctime)s-%(filename)s[line:%(lineno)d]-%(levelname)s: %(message)s", level = logging.INFO)
31 |
32 | python_script = os.path.realpath(__file__)
33 |
34 | class Trainer():
35 | def __init__(self, rank, args, data_configs, model_configs, train_configs):
36 | self.rank = rank
37 | self.device = torch.device('cuda:{:d}'.format(rank))
38 | trainset = Dataset(data_configs)
39 | collate_fn = Collate()
40 | sampler = torch.utils.data.DistributedSampler(trainset) if args.num_gpus > 1 else None
41 | self.trainloader = DataLoader(
42 | trainset,
43 | shuffle = False,
44 | sampler = sampler,
45 | collate_fn = collate_fn,
46 | batch_size = train_configs['batch_size'],
47 | pin_memory = True,
48 | num_workers = train_configs['num_workers'],
49 | prefetch_factor = 10
50 | )
51 |
52 | model_configs['generator']['transformer']['encoder']['n_src_vocab'] = trainset.get_phone_number() + 1
53 | model_configs['generator']['spk_num'] = trainset.get_spk_number()
54 |
55 | if args.num_gpus > 1:
56 | self.models = (
57 | nn.parallel.DistributedDataParallel(
58 | Generator(
59 | data_configs,
60 | model_configs['generator']
61 | ).to(self.device),
62 | device_ids = [rank]
63 | ),
64 | nn.parallel.DistributedDataParallel(
65 | Discriminator().to(self.device),
66 | device_ids = [rank]
67 | )
68 | )
69 | else:
70 | self.models = (
71 | Generator(
72 | data_configs,
73 | model_configs['generator']
74 | ).to(self.device),
75 | Discriminator().to(self.device)
76 | )
77 | self.data_configs = data_configs
78 | self.model_configs = model_configs
79 | self.train_configs = train_configs
80 | self.args = args
81 |
82 | try:
83 | self.g_optimizer = getattr(
84 | torch.optim, train_configs['g_optimizer']
85 | )(self.models[0].parameters(), **train_configs['g_optimizer_args'])
86 |
87 | self.g_scheduler = getattr(
88 | pyutils.scheduler, train_configs['g_scheduler']
89 | )(self.g_optimizer, **train_configs['g_scheduler_args'])
90 |
91 | self.d_optimizer = getattr(
92 | torch.optim, train_configs['d_optimizer']
93 | )(self.models[1].parameters(), **train_configs['d_optimizer_args'])
94 |
95 | self.d_scheduler = getattr(
96 | pyutils.scheduler, train_configs['d_scheduler']
97 | )(self.d_optimizer, **train_configs['d_scheduler_args'])
98 | except:
99 | raise NotImplementedError("Unknown optimizer or scheduler")
100 |
101 | self.fs2loss = FastSpeech2Loss(data_configs)
102 | self.feat_loss = FeatLoss(train_configs['feat_loss_weight'])
103 | self.adv_g_loss = LSGANGLoss(train_configs['adv_g_loss_weight'])
104 | self.adv_d_loss = LSGANDLoss()
105 |
106 | if self.rank == 0:
107 | self._make_exp_dir()
108 | self.logger = get_logger(os.path.join(self.args.exp_name, 'logs/train.log'))
109 |
110 | try:
111 | latest_gckpt_path = latest_checkpoint_path(
112 | os.path.join(self.args.exp_name, 'models'),
113 | 'G_*.pth'
114 | )
115 | latest_dckpt_path = latest_checkpoint_path(
116 | os.path.join(self.args.exp_name, 'models'),
117 | 'D_*.pth'
118 | )
119 | _, _, _, _, epoch_str = load_checkpoint(
120 | latest_gckpt_path,
121 | self.models[0],
122 | self.g_optimizer,
123 | self.g_scheduler,
124 | False
125 | )
126 | _, _, _, _, epoch_str = load_checkpoint(
127 | latest_dckpt_path,
128 | self.models[1],
129 | self.d_optimizer,
130 | self.d_scheduler,
131 | False
132 | )
133 | self.start_epoch = max(epoch_str, 1)
134 | name = latest_gckpt_path
135 | self.total_step = int(name[name.rfind("_")+1:name.rfind(".")])+1
136 | except Exception:
137 | print("Load old checkpoint failed...")
138 | print("Start a new training...")
139 | self.start_epoch = 1
140 | self.total_step = 0
141 |
142 | self.epochs = self.train_configs['epochs']
143 | self.start_disc_steps = self.train_configs['start_disc_steps']
144 |
145 | def _dump_args_and_config(self, filename, config):
146 | with open(os.path.join(self.args.exp_name, 'conf', filename) + '.yaml', 'w') as f:
147 | yaml.dump(config, f)
148 |
149 | def _make_exp_dir(self):
150 | os.makedirs(self.args.exp_name, exist_ok=True)
151 | os.makedirs(os.path.join(self.args.exp_name, 'conf'), exist_ok=True)
152 | os.makedirs(os.path.join(self.args.exp_name, 'models'), exist_ok=True)
153 | os.makedirs(os.path.join(self.args.exp_name, 'audios'), exist_ok=True)
154 | os.makedirs(os.path.join(self.args.exp_name, 'spectrograms'), exist_ok=True)
155 | os.makedirs(os.path.join(self.args.exp_name, 'melspectrograms'), exist_ok=True)
156 | os.makedirs(os.path.join(self.args.exp_name, 'eval_results'), exist_ok=True)
157 | os.makedirs(os.path.join(self.args.exp_name, 'logs'), exist_ok = True)
158 | with open(os.path.join(self.args.exp_name, 'model_arch.txt'), 'w') as f:
159 | for model in self.models:
160 | print(model, file = f)
161 | self._dump_args_and_config('args', vars(self.args))
162 | self._dump_args_and_config('data', self.data_configs)
163 | self._dump_args_and_config('model', self.model_configs)
164 | self._dump_args_and_config('train', self.train_configs)
165 | basename = os.path.basename(python_script)
166 | shutil.copyfile(python_script, os.path.join(self.args.exp_name, basename))
167 |
168 | def train(self):
169 | for epoch in range(self.start_epoch, self.epochs + 1, 1):
170 | self.train_epoch(epoch)
171 |
172 | def train_epoch(self, epoch):
173 | self.total_g_loss = 0.0
174 | self.total_d_loss = 0.0
175 | for batch_idx, data in enumerate(self.trainloader):
176 | output, postnet_output = self.train_batch(data, epoch, batch_idx)
177 | self.g_scheduler.step()
178 | self.d_scheduler.step()
179 |
180 | if self.rank == 0 and self.total_step % self.train_configs['save_interval'] == 0:
181 | gckpt_path = os.path.join(self.args.exp_name, 'models', 'G_{}.pth'.format(self.total_step))
182 | save_checkpoint(
183 | self.models[0],
184 | self.g_optimizer,
185 | self.g_scheduler,
186 | self.g_scheduler.get_lr()[0],
187 | epoch,
188 | gckpt_path
189 | )
190 | dckpt_path = os.path.join(self.args.exp_name, 'models', 'D_{}.pth'.format(self.total_step))
191 | save_checkpoint(
192 | self.models[1],
193 | self.d_optimizer,
194 | self.d_scheduler,
195 | self.d_scheduler.get_lr()[0],
196 | epoch,
197 | dckpt_path
198 | )
199 | length = data['mel_lens'][0]
200 | real_pic_path = os.path.join(self.args.exp_name, 'melspectrograms', '{}_{}.png'.format(data['uttids'][0], self.total_step))
201 | before_pic_path = os.path.join(self.args.exp_name, 'melspectrograms', 'before_{}_{}.png'.format(data['uttids'][0], self.total_step))
202 | after_pic_path = os.path.join(self.args.exp_name, 'melspectrograms', 'after_{}_{}.png'.format(data['uttids'][0], self.total_step))
203 | melspecplot(data['mels'][0][:length, :].transpose(1, 0).numpy(), real_pic_path) # (n_mels, T)
204 | melspecplot(output[0][:length, :].transpose(1, 0).detach().cpu().numpy(), before_pic_path)
205 | melspecplot(postnet_output[0][:length, :].transpose(1, 0).detach().cpu().numpy(), after_pic_path)
206 |
207 | if self.rank == 0:
208 | clean_checkpoints(
209 | os.path.join(self.args.exp_name, 'models'),
210 | n_ckpts_to_keep = self.train_configs['ckpt_clean']
211 | )
212 |
213 | def _move_to_device(self, data):
214 | new_data = {}
215 | for k, v in data.items():
216 | if type(v) is torch.Tensor:
217 | new_data[k] = v.to(self.device)
218 | return new_data
219 |
220 | def train_batch(self, data, epoch, step):
221 | for model in self.models:
222 | model.train()
223 | new_data = self._move_to_device(data)
224 |
225 | # loss, report_keys, output, postnet_output = self.models[0](**new_data)
226 | self.g_optimizer.zero_grad()
227 | g_loss, report_keys, output, postnet_output = self.models[0](**new_data)
228 | if self.total_step >= self.train_configs['start_disc_steps']:
229 | d_fake, random_N = self.models[1](output, new_data['mel_lens'], new_data['mels'])
230 | feat_loss, feat_loss_report_keys = self.feat_loss(d_fake)
231 | adv_g_loss, adv_gloss_report_keys = self.adv_g_loss(d_fake)
232 | g_loss += feat_loss
233 | g_loss += adv_g_loss
234 | report_keys.update(feat_loss_report_keys)
235 | report_keys.update(adv_gloss_report_keys)
236 | g_loss.backward()
237 |
238 | grad_norm = nn.utils.clip_grad_norm_(self.models[0].parameters(), self.train_configs['grad_clip'])
239 | if math.isnan(grad_norm):
240 | raise ZeroDivisionError('Grad norm is nan')
241 | self.g_optimizer.step()
242 | self.total_g_loss += g_loss.item()
243 |
244 | if self.total_step >= self.train_configs['start_disc_steps']:
245 | self.d_optimizer.zero_grad()
246 | d_fake, _ = self.models[1](
247 | output.detach(),
248 | new_data['mel_lens'],
249 | new_data['mels'],
250 | random_N
251 | )
252 | adv_d_loss, adv_dloss_report_keys = self.adv_d_loss(d_fake)
253 | adv_d_loss.backward()
254 | report_keys.update(adv_dloss_report_keys)
255 | grad_norm = nn.utils.clip_grad_norm_(self.models[1].parameters(), self.train_configs['grad_clip'])
256 | if math.isnan(grad_norm):
257 | raise ZeroDivisionError('Grad norm is nan')
258 | self.d_optimizer.step()
259 | self.total_d_loss += adv_d_loss.item()
260 |
261 | self.total_step += 1
262 | if self.rank == 0:
263 | self.print_msg(epoch, step, report_keys) #, accuracy.item())
264 | wandb_log_dict = {
265 | 'train/avg_g_loss': self.total_g_loss / (step + 1),
266 | 'train/avg_d_loss': self.total_d_loss / (step + 1),
267 | 'train/g_lr': self.g_scheduler.get_lr()[0],
268 | 'train/d_lr': self.d_scheduler.get_lr()[0]
269 | }
270 | for k, v in report_keys.items():
271 | wandb_log_dict['train/' + k] = v
272 | if self.train_configs['wandb']:
273 | wandb.log(wandb_log_dict)
274 | return output, postnet_output
275 |
276 | def print_msg(self, epoch, step, report_keys):
277 | if self.total_step % self.train_configs['log_interval'] == 0:
278 | temp = ''
279 | for k, v in report_keys.items():
280 | temp += '{}: {:.6f} '.format(k, v)
281 | message = ('[Epoch: {} Step: {} Total steps: {}] ' + temp).format(
282 | epoch, step + 1, self.total_step
283 | )
284 | self.logger.info(message)
285 |
286 | def parse_args():
287 | parser = argparse.ArgumentParser()
288 | parser.add_argument('--data-config', dest = 'data_config', type = str, default = './conf/data.yaml')
289 | parser.add_argument('--model-config', dest = 'model_config', type = str, default = './conf/model.yaml')
290 | parser.add_argument('--train-config', dest = 'train_config', type = str, default = './conf/train.yaml')
291 | parser.add_argument('--num-gpus', dest = 'num_gpus', type = int, default = 1)
292 | # parser.add_argument('--exp-name', dest = 'exp_name', type = str, default = 'default')
293 | parser.add_argument('--dist-backend', dest = 'dist_backend', type = str, default = 'nccl')
294 | parser.add_argument('--dist-url', dest = 'dist_url', type = str, default = 'tcp://localhost:30302')
295 | return parser.parse_args()
296 |
297 | def main(rank, args, configs):
298 | if args.num_gpus > 1:
299 | torch.cuda.set_device(rank)
300 | torch.distributed.init_process_group(
301 | backend = args.dist_backend,
302 | init_method = args.dist_url,
303 | world_size = args.num_gpus,
304 | rank = rank
305 | )
306 |
307 | data_configs, model_configs, train_configs = configs
308 | args.exp_name = train_configs['wandb_args']['group'] + '-' + \
309 | train_configs['wandb_args']['job_type'] + '-' + \
310 | train_configs['wandb_args']['name']
311 | args.exp_name = os.path.join('exp', args.exp_name)
312 |
313 | # wandb initialization
314 | if train_configs['wandb']:
315 | wandb_configs = vars(args)
316 | for config in configs:
317 | wandb_configs.update(config)
318 | wandb.init(
319 | **train_configs['wandb_args'],
320 | config = wandb_configs
321 | )
322 |
323 | trainer = Trainer(rank, args, data_configs, model_configs, train_configs)
324 | trainer.train()
325 |
326 | if train_configs['wandb']:
327 | wandb.finish()
328 |
329 | if __name__ == "__main__":
330 | args = parse_args()
331 | with open(args.data_config, 'r') as f:
332 | data_configs = yaml.load(f, Loader = yaml.FullLoader)
333 | with open(args.model_config, 'r') as f:
334 | model_configs = yaml.load(f, Loader = yaml.FullLoader)
335 | with open(args.train_config, 'r') as f:
336 | train_configs = yaml.load(f, Loader = yaml.FullLoader)
337 | configs = (data_configs, model_configs, train_configs)
338 |
339 | num_gpus = torch.cuda.device_count()
340 | if args.num_gpus > 1:
341 | mp.spawn(main, nprocs = num_gpus, args = (args, configs))
342 | else:
343 | main(0, args, configs)
344 |
--------------------------------------------------------------------------------
/utils:
--------------------------------------------------------------------------------
1 | /home/smg/zengchang/apps/kaldi/egs/wsj/s5/utils
--------------------------------------------------------------------------------
47 | 
48 | 
49 |