├── .gitignore
├── LICENSE
├── NeuralSeq
├── LICENSE
├── README.md
├── configs
│ ├── config_base.yaml
│ ├── singing
│ │ ├── base.yaml
│ │ └── fs2.yaml
│ └── tts
│ │ ├── base.yaml
│ │ ├── base_zh.yaml
│ │ ├── emotion
│ │ ├── base_text2mel.yaml
│ │ └── pre_align.py
│ │ ├── fs2.yaml
│ │ ├── hifigan.yaml
│ │ ├── libritts
│ │ ├── base_text2mel.yaml
│ │ ├── fs2.yaml
│ │ ├── pre_align.py
│ │ └── pwg.yaml
│ │ ├── lj
│ │ ├── base_mel2wav.yaml
│ │ ├── base_text2mel.yaml
│ │ ├── fs2.yaml
│ │ ├── hifigan.yaml
│ │ └── pwg.yaml
│ │ └── pwg.yaml
├── data_gen
│ └── tts
│ │ ├── base_binarizer.py
│ │ ├── base_binarizer_emotion.py
│ │ ├── base_preprocess.py
│ │ ├── binarizer_zh.py
│ │ ├── data_gen_utils.py
│ │ ├── emotion
│ │ ├── audio.py
│ │ ├── inference.py
│ │ ├── model.py
│ │ ├── params_data.py
│ │ ├── params_model.py
│ │ └── test_emotion.py
│ │ ├── txt_processors
│ │ ├── __init__.py
│ │ ├── base_text_processor.py
│ │ ├── en.py
│ │ ├── zh.py
│ │ └── zh_g2pM.py
│ │ └── wav_processors
│ │ ├── __init__.py
│ │ ├── base_processor.py
│ │ └── common_processors.py
├── egs
│ ├── datasets
│ │ └── audio
│ │ │ ├── emotion
│ │ │ ├── base_text2mel.yaml
│ │ │ └── pre_align.py
│ │ │ ├── libritts
│ │ │ ├── base_text2mel.yaml
│ │ │ ├── fs2.yaml
│ │ │ ├── pre_align.py
│ │ │ └── pwg.yaml
│ │ │ ├── lj
│ │ │ ├── base_mel2wav.yaml
│ │ │ ├── preprocess.py
│ │ │ └── pwg.yaml
│ │ │ └── vctk
│ │ │ ├── base_mel2wav.yaml
│ │ │ ├── fs2.yaml
│ │ │ ├── pre_align.py
│ │ │ └── pwg.yaml
│ └── egs_bases
│ │ ├── config_base.yaml
│ │ ├── svs
│ │ ├── base.yaml
│ │ ├── lj_ds_beta6.yaml
│ │ ├── midi
│ │ │ ├── cascade
│ │ │ │ └── opencs
│ │ │ │ │ ├── aux_rel.yaml
│ │ │ │ │ ├── ds60_rel.yaml
│ │ │ │ │ └── opencpop_statis.yaml
│ │ │ ├── e2e
│ │ │ │ ├── opencpop
│ │ │ │ │ ├── ds1000-10dil.yaml
│ │ │ │ │ ├── ds1000.yaml
│ │ │ │ │ └── ds100_adj_rel.yaml
│ │ │ │ └── popcs
│ │ │ │ │ └── ds100_adj_rel.yaml
│ │ │ └── pe.yaml
│ │ ├── popcs_ds_beta6.yaml
│ │ ├── popcs_ds_beta6_offline.yaml
│ │ └── popcs_fs2.yaml
│ │ └── tts
│ │ ├── base.yaml
│ │ ├── base_zh.yaml
│ │ ├── fs2.yaml
│ │ ├── fs2_adv.yaml
│ │ ├── ps.yaml
│ │ ├── ps_flow.yaml
│ │ ├── ps_flow_small.yaml
│ │ └── vocoder
│ │ ├── base.yaml
│ │ ├── hifigan.yaml
│ │ └── pwg.yaml
├── gitattributes
├── inference
│ ├── svs
│ │ ├── base_svs_infer.py
│ │ ├── ds_cascade.py
│ │ ├── ds_e2e.py
│ │ └── opencpop
│ │ │ ├── cpop_pinyin2ph.txt
│ │ │ └── map.py
│ └── tts
│ │ ├── GenerSpeech.py
│ │ ├── PortaSpeech.py
│ │ └── base_tts_infer.py
├── modules
│ ├── GenerSpeech
│ │ ├── config
│ │ │ └── generspeech.yaml
│ │ ├── model
│ │ │ ├── generspeech.py
│ │ │ ├── glow_modules.py
│ │ │ ├── mixstyle.py
│ │ │ ├── prosody_util.py
│ │ │ └── wavenet.py
│ │ └── task
│ │ │ ├── dataset.py
│ │ │ └── generspeech.py
│ ├── __init__.py
│ ├── commons
│ │ ├── align_ops.py
│ │ ├── common_layers.py
│ │ ├── conv.py
│ │ ├── espnet_positional_embedding.py
│ │ ├── normalizing_flow
│ │ │ ├── glow_modules.py
│ │ │ ├── res_flow.py
│ │ │ └── utils.py
│ │ ├── rel_transformer.py
│ │ ├── ssim.py
│ │ ├── transformer.py
│ │ └── wavenet.py
│ ├── diff
│ │ ├── candidate_decoder.py
│ │ ├── diffusion.py
│ │ ├── net.py
│ │ └── shallow_diffusion_tts.py
│ ├── diffsinger_midi
│ │ └── fs2.py
│ ├── fastspeech
│ │ ├── fs2.py
│ │ ├── pe.py
│ │ └── tts_modules.py
│ ├── hifigan
│ │ ├── hifigan.py
│ │ └── mel_utils.py
│ ├── parallel_wavegan
│ │ ├── __init__.py
│ │ ├── layers
│ │ │ ├── __init__.py
│ │ │ ├── causal_conv.py
│ │ │ ├── pqmf.py
│ │ │ ├── residual_block.py
│ │ │ ├── residual_stack.py
│ │ │ ├── tf_layers.py
│ │ │ └── upsample.py
│ │ ├── losses
│ │ │ ├── __init__.py
│ │ │ └── stft_loss.py
│ │ ├── models
│ │ │ ├── __init__.py
│ │ │ ├── melgan.py
│ │ │ ├── parallel_wavegan.py
│ │ │ └── source.py
│ │ ├── optimizers
│ │ │ ├── __init__.py
│ │ │ └── radam.py
│ │ ├── stft_loss.py
│ │ └── utils
│ │ │ ├── __init__.py
│ │ │ └── utils.py
│ └── syntaspeech
│ │ ├── multi_window_disc.py
│ │ ├── syntactic_graph_buider.py
│ │ ├── syntactic_graph_encoder.py
│ │ └── syntaspeech.py
├── tasks
│ ├── base_task.py
│ ├── run.py
│ ├── svs
│ │ ├── __init__.py
│ │ ├── diffsinger_task.py
│ │ ├── diffspeech_task.py
│ │ └── task.py
│ ├── tts
│ │ ├── dataset_utils.py
│ │ ├── fs2.py
│ │ ├── fs2_adv.py
│ │ ├── fs2_utils.py
│ │ ├── pe.py
│ │ ├── ps.py
│ │ ├── ps_adv.py
│ │ ├── ps_flow.py
│ │ ├── synta.py
│ │ ├── tts.py
│ │ ├── tts_base.py
│ │ └── tts_utils.py
│ └── vocoder
│ │ ├── dataset_utils.py
│ │ └── vocoder_base.py
├── utils
│ ├── __init__.py
│ ├── audio.py
│ ├── ckpt_utils.py
│ ├── cwt.py
│ ├── dtw.py
│ ├── hparams.py
│ ├── indexed_datasets.py
│ ├── multiprocess_utils.py
│ ├── os_utils.py
│ ├── pitch_utils.py
│ ├── pl_utils.py
│ ├── plot.py
│ ├── text_encoder.py
│ ├── text_norm.py
│ ├── training_utils.py
│ └── tts_utils.py
└── vocoders
│ ├── __init__.py
│ ├── base_vocoder.py
│ ├── hifigan.py
│ ├── pwg.py
│ └── vocoder_utils.py
├── README.md
├── assets
├── 2bf90e35.wav
├── 5d67d1b9.wav
├── 7cb0d24f.wav
├── 7ef0ec0b.wav
├── README.md
├── Track 4.wav
├── a-group-of-sheep-are-baaing.wav
├── a2i.png
├── asr.png
├── b973e878.wav
├── detection.png
├── drums-and-music-playing-with-a-man-speaking.wav
├── fd5cf55e.wav
├── i2a-1.png
├── i2a-2.png
├── inpaint-1.png
├── inpaint-2.png
├── m2b.png
├── mix1.wav
├── sound_extraction.png
├── style_transfer_tts.png
├── t2a.png
├── t2i.png
├── t2s.png
├── tsd.png
└── tts.png
├── audio-chatgpt.py
├── audio_detection
├── __init__.py
├── audio_infer
│ ├── __init__.py
│ ├── metadata
│ │ ├── black_list
│ │ │ ├── groundtruth_weak_label_evaluation_set.csv
│ │ │ └── groundtruth_weak_label_testing_set.csv
│ │ └── class_labels_indices.csv
│ ├── pytorch
│ │ ├── evaluate.py
│ │ ├── finetune_template.py
│ │ ├── inference.py
│ │ ├── losses.py
│ │ ├── main.py
│ │ ├── models.py
│ │ └── pytorch_utils.py
│ ├── results
│ │ └── YDlWd7Wmdi1E.png
│ └── utils
│ │ ├── config.py
│ │ ├── crash.py
│ │ ├── create_black_list.py
│ │ ├── create_indexes.py
│ │ ├── data_generator.py
│ │ ├── dataset.py
│ │ ├── plot_for_paper.py
│ │ ├── plot_statistics.py
│ │ └── utilities.py
└── target_sound_detection
│ └── src
│ ├── models.py
│ └── utils.py
├── audio_to_text
├── __init__.py
├── captioning
│ ├── __init__.py
│ ├── models
│ │ ├── __init__.py
│ │ ├── base_model.py
│ │ ├── decoder.py
│ │ ├── encoder.py
│ │ ├── transformer_model.py
│ │ └── utils.py
│ └── utils
│ │ ├── README.md
│ │ ├── __init__.py
│ │ ├── bert
│ │ ├── create_sent_embedding.py
│ │ └── create_word_embedding.py
│ │ ├── build_vocab.py
│ │ ├── build_vocab_ltp.py
│ │ ├── build_vocab_spacy.py
│ │ ├── eval_round_robin.py
│ │ ├── fasttext
│ │ └── create_word_embedding.py
│ │ ├── lr_scheduler.py
│ │ ├── model_eval_diff.py
│ │ ├── predict_nn.py
│ │ ├── remove_optimizer.py
│ │ ├── report_results.py
│ │ ├── tokenize_caption.py
│ │ ├── train_util.py
│ │ └── word2vec
│ │ └── create_word_embedding.py
└── inference_waveform.py
├── download.sh
├── mono2binaural
└── src
│ ├── models.py
│ ├── utils.py
│ └── warping.py
├── requirements.txt
├── run.md
├── sound_extraction
├── model
│ ├── LASSNet.py
│ ├── film.py
│ ├── modules.py
│ ├── resunet_film.py
│ └── text_encoder.py
└── utils
│ ├── create_mixtures.py
│ ├── stft.py
│ └── wav_io.py
└── text_to_audio
└── Make_An_Audio
├── configs
├── img_to_audio
│ └── img2audio_args.yaml
├── inpaint
│ └── txt2audio_args.yaml
└── text_to_audio
│ ├── clap_args.yaml
│ ├── hifigan_args.yaml
│ └── txt2audio_args.yaml
├── ldm
├── data
│ └── extract_mel_spectrogram.py
├── lr_scheduler.py
├── models
│ ├── autoencoder.py
│ ├── autoencoder_multi.py
│ └── diffusion
│ │ ├── __init__.py
│ │ ├── classifier.py
│ │ ├── ddim.py
│ │ ├── ddpm.py
│ │ ├── ddpm_audio.py
│ │ ├── ddpm_audio_inpaint.py
│ │ └── plms.py
├── modules
│ ├── attention.py
│ ├── diffusionmodules
│ │ ├── __init__.py
│ │ ├── custom_openaimodel.py
│ │ ├── model.py
│ │ ├── openaimodel.py
│ │ └── util.py
│ ├── discriminator
│ │ ├── model.py
│ │ └── multi_window_disc.py
│ ├── distributions
│ │ ├── __init__.py
│ │ └── distributions.py
│ ├── ema.py
│ ├── encoders
│ │ ├── CLAP
│ │ │ ├── CLAPWrapper.py
│ │ │ ├── __init__.py
│ │ │ ├── audio.py
│ │ │ ├── clap.py
│ │ │ ├── config.yml
│ │ │ └── utils.py
│ │ ├── __init__.py
│ │ ├── modules.py
│ │ └── open_clap
│ │ │ ├── __init__.py
│ │ │ ├── bert.py
│ │ │ ├── bpe_simple_vocab_16e6.txt.gz
│ │ │ ├── factory.py
│ │ │ ├── feature_fusion.py
│ │ │ ├── htsat.py
│ │ │ ├── linear_probe.py
│ │ │ ├── loss.py
│ │ │ ├── model.py
│ │ │ ├── model_configs
│ │ │ ├── HTSAT-base.json
│ │ │ ├── HTSAT-large.json
│ │ │ ├── HTSAT-tiny-win-1536.json
│ │ │ ├── HTSAT-tiny.json
│ │ │ ├── PANN-10.json
│ │ │ ├── PANN-14-fmax-18k.json
│ │ │ ├── PANN-14-fmax-8k-20s.json
│ │ │ ├── PANN-14-tiny-transformer.json
│ │ │ ├── PANN-14-win-1536.json
│ │ │ ├── PANN-14.json
│ │ │ ├── PANN-6.json
│ │ │ ├── RN101-quickgelu.json
│ │ │ ├── RN101.json
│ │ │ ├── RN50-quickgelu.json
│ │ │ ├── RN50.json
│ │ │ ├── RN50x16.json
│ │ │ ├── RN50x4.json
│ │ │ ├── ViT-B-16.json
│ │ │ ├── ViT-B-32-quickgelu.json
│ │ │ ├── ViT-B-32.json
│ │ │ └── ViT-L-14.json
│ │ │ ├── openai.py
│ │ │ ├── pann_model.py
│ │ │ ├── pretrained.py
│ │ │ ├── timm_model.py
│ │ │ ├── tokenizer.py
│ │ │ ├── transform.py
│ │ │ ├── utils.py
│ │ │ └── version.py
│ ├── image_degradation
│ │ ├── __init__.py
│ │ ├── bsrgan.py
│ │ ├── bsrgan_light.py
│ │ ├── utils
│ │ │ └── test.png
│ │ └── utils_image.py
│ ├── losses_audio
│ │ ├── __init__.py
│ │ ├── contperceptual.py
│ │ ├── contperceptual_dis.py
│ │ ├── lpaps.py
│ │ ├── vggishish
│ │ │ ├── config
│ │ │ │ ├── melception.yaml
│ │ │ │ └── vggish.yaml
│ │ │ ├── data
│ │ │ │ ├── train_means_stds_melspec_10s_22050hz.txt
│ │ │ │ ├── vggsound.csv
│ │ │ │ ├── vggsound_test.txt
│ │ │ │ ├── vggsound_train.txt
│ │ │ │ └── vggsound_valid.txt
│ │ │ ├── dataset.py
│ │ │ ├── logger.py
│ │ │ ├── loss.py
│ │ │ ├── metrics.py
│ │ │ ├── model.py
│ │ │ ├── predict.py
│ │ │ ├── train_melception.py
│ │ │ ├── train_vggishish.py
│ │ │ └── transforms.py
│ │ └── vqperceptual.py
│ └── x_transformer.py
└── util.py
├── useful_ckpts
└── CLAP
│ └── config.yml
├── vocoder
├── bigvgan
│ ├── __init__.py
│ ├── activations.py
│ ├── alias_free_torch
│ │ ├── __init__.py
│ │ ├── act.py
│ │ ├── filter.py
│ │ └── resample.py
│ └── models.py
├── hifigan
│ └── modules.py
└── logs
│ └── hifi_0127
│ └── args.yml
└── wav_evaluation
└── models
├── CLAPWrapper.py
├── __init__.py
├── audio.py
├── clap.py
└── utils.py
/.gitignore:
--------------------------------------------------------------------------------
1 | # JetBrains PyCharm IDE
2 | .idea/
3 | .github/
4 | .circleci/
5 |
6 | # Byte-compiled / optimized / DLL files
7 | *__pycache__/
8 | __pycache__/
9 | *.py[cod]
10 | *$py.class
11 |
12 | # C extensions
13 | *.so
14 |
15 | # macOS dir files
16 | .DS_Store
17 |
18 | # Distribution / packaging
19 | .Python
20 | env/
21 | build/
22 | develop-eggs/
23 | dist/
24 | downloads/
25 | eggs/
26 | .eggs/
27 | lib/
28 | lib64/
29 | parts/
30 | sdist/
31 | var/
32 | wheels/
33 | *.egg-info/
34 | .installed.cfg
35 | *.egg
36 |
37 | # Checkpoints
38 | checkpoints
39 |
40 | # PyInstaller
41 | # Usually these files are written by a python script from a template
42 | # before PyInstaller builds the exe, so as to inject date/other infos into it.
43 | *.manifest
44 | *.spec
45 |
46 | # Installer logs
47 | pip-log.txt
48 | pip-delete-this-directory.txt
49 |
50 | # Unit test / coverage reports
51 | htmlcov/
52 | .tox/
53 | .coverage
54 | .coverage.*
55 | .cache
56 | nosetests.xml
57 | coverage.xml
58 | *.cover
59 | .hypothesis/
60 |
61 | # Translations
62 | *.mo
63 | *.pot
64 |
65 | # Django stuff:
66 | *.log
67 | local_settings.py
68 |
69 | # Flask stuff:
70 | instance/
71 | .webassets-cache
72 |
73 | # Scrapy stuff:
74 | .scrapy
75 |
76 | # Sphinx documentation
77 | docs/_build/
78 |
79 | # PyBuilder
80 | target/
81 |
82 | # Jupyter Notebook
83 | .ipynb_checkpoints
84 |
85 | # pyenv
86 | .python-version
87 |
88 | # celery beat schedule file
89 | celerybeat-schedule
90 |
91 | # SageMath parsed files
92 | *.sage.py
93 |
94 | # dotenv
95 | .env
96 |
97 | # virtualenv
98 | .venv
99 | venv/
100 | ENV/
101 |
102 | # Spyder project settings
103 | .spyderproject
104 | .spyproject
105 |
106 | # Rope project settings
107 | .ropeproject
108 |
109 | # mkdocs documentation
110 | /site
111 |
112 | # mypy
113 | .mypy_cache/
114 |
115 | # Generated files
116 | /fairseq/temporal_convolution_tbc
117 | /fairseq/modules/*_layer/*_forward.cu
118 | /fairseq/modules/*_layer/*_backward.cu
119 | /fairseq/version.py
120 |
121 | # data
122 | data-bin/
123 |
124 | # reranking
125 | /examples/reranking/rerank_data
126 |
127 | # Cython-generated C++ source files
128 | /fairseq/data/data_utils_fast.cpp
129 | /fairseq/data/token_block_utils_fast.cpp
130 |
131 | # VSCODE
132 | .vscode/ftp-sync.json
133 | .vscode/settings.json
134 |
135 | # Experimental Folder
136 | experimental/*
137 |
138 | # Weights and Biases logs
139 | wandb/
140 |
141 | # Hydra artifacts
142 | nohup.out
143 | multirun
144 | outputs
145 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AIGC-Audio/AudioGPT/a674543c537bb3a60b8f521df76b97a627c1c379/LICENSE
--------------------------------------------------------------------------------
/NeuralSeq/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2021 Jinglin Liu
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
--------------------------------------------------------------------------------
/NeuralSeq/README.md:
--------------------------------------------------------------------------------
1 | In this directory, we support FastSpeech, GenerSpeech, SyntaSpeech, DiffSinger
--------------------------------------------------------------------------------
/NeuralSeq/configs/config_base.yaml:
--------------------------------------------------------------------------------
1 | # task
2 | binary_data_dir: ''
3 | work_dir: '' # experiment directory.
4 | infer: false # infer
5 | seed: 1234
6 | debug: false
7 | save_codes:
8 | - configs
9 | - modules
10 | - tasks
11 | - utils
12 | - usr
13 |
14 | #############
15 | # dataset
16 | #############
17 | ds_workers: 1
18 | test_num: 100
19 | valid_num: 100
20 | endless_ds: false
21 | sort_by_len: true
22 |
23 | #########
24 | # train and eval
25 | #########
26 | load_ckpt: ''
27 | save_ckpt: true
28 | save_best: false
29 | num_ckpt_keep: 3
30 | clip_grad_norm: 0
31 | accumulate_grad_batches: 1
32 | log_interval: 100
33 | num_sanity_val_steps: 5 # steps of validation at the beginning
34 | check_val_every_n_epoch: 10
35 | val_check_interval: 2000
36 | max_epochs: 1000
37 | max_updates: 160000
38 | max_tokens: 31250
39 | max_sentences: 100000
40 | max_eval_tokens: -1
41 | max_eval_sentences: -1
42 | test_input_dir: ''
43 |
--------------------------------------------------------------------------------
/NeuralSeq/configs/singing/base.yaml:
--------------------------------------------------------------------------------
1 | base_config:
2 | - configs/tts/base.yaml
3 | - configs/tts/base_zh.yaml
4 |
5 |
6 | datasets: []
7 | test_prefixes: []
8 | test_num: 0
9 | valid_num: 0
10 |
11 | pre_align_cls: data_gen.singing.pre_align.SingingPreAlign
12 | binarizer_cls: data_gen.singing.binarize.SingingBinarizer
13 | pre_align_args:
14 | use_tone: false # for ZH
15 | forced_align: mfa
16 | use_sox: true
17 | hop_size: 128 # Hop size.
18 | fft_size: 512 # FFT size.
19 | win_size: 512 # FFT size.
20 | max_frames: 8000
21 | fmin: 50 # Minimum freq in mel basis calculation.
22 | fmax: 11025 # Maximum frequency in mel basis calculation.
23 | pitch_type: frame
24 |
25 | hidden_size: 256
26 | mel_loss: "ssim:0.5|l1:0.5"
27 | lambda_f0: 0.0
28 | lambda_uv: 0.0
29 | lambda_energy: 0.0
30 | lambda_ph_dur: 0.0
31 | lambda_sent_dur: 0.0
32 | lambda_word_dur: 0.0
33 | predictor_grad: 0.0
34 | use_spk_embed: true
35 | use_spk_id: false
36 |
37 | max_tokens: 20000
38 | max_updates: 400000
39 | num_spk: 100
40 | save_f0: true
41 | use_gt_dur: true
42 | use_gt_f0: true
43 |
--------------------------------------------------------------------------------
/NeuralSeq/configs/singing/fs2.yaml:
--------------------------------------------------------------------------------
1 | base_config:
2 | - configs/tts/fs2.yaml
3 | - configs/singing/base.yaml
4 |
--------------------------------------------------------------------------------
/NeuralSeq/configs/tts/base.yaml:
--------------------------------------------------------------------------------
1 | # task
2 | base_config: configs/config_base.yaml
3 | task_cls: ''
4 | #############
5 | # dataset
6 | #############
7 | raw_data_dir: ''
8 | processed_data_dir: ''
9 | binary_data_dir: ''
10 | dict_dir: ''
11 | pre_align_cls: ''
12 | binarizer_cls: data_gen.tts.base_binarizer.BaseBinarizer
13 | pre_align_args:
14 | use_tone: true # for ZH
15 | forced_align: mfa
16 | use_sox: false
17 | txt_processor: en
18 | allow_no_txt: false
19 | denoise: false
20 | binarization_args:
21 | shuffle: false
22 | with_txt: true
23 | with_wav: false
24 | with_align: true
25 | with_spk_embed: true
26 | with_f0: true
27 | with_f0cwt: true
28 |
29 | loud_norm: false
30 | endless_ds: true
31 | reset_phone_dict: true
32 |
33 | test_num: 100
34 | valid_num: 100
35 | max_frames: 1550
36 | max_input_tokens: 1550
37 | audio_num_mel_bins: 80
38 | audio_sample_rate: 22050
39 | hop_size: 256 # For 22050Hz, 275 ~= 12.5 ms (0.0125 * sample_rate)
40 | win_size: 1024 # For 22050Hz, 1100 ~= 50 ms (If None, win_size: fft_size) (0.05 * sample_rate)
41 | fmin: 80 # Set this to 55 if your speaker is male! if female, 95 should help taking off noise. (To test depending on dataset. Pitch info: male~[65, 260], female~[100, 525])
42 | fmax: 7600 # To be increased/reduced depending on data.
43 | fft_size: 1024 # Extra window size is filled with 0 paddings to match this parameter
44 | min_level_db: -100
45 | num_spk: 1
46 | mel_vmin: -6
47 | mel_vmax: 1.5
48 | ds_workers: 4
49 |
50 | #########
51 | # model
52 | #########
53 | dropout: 0.1
54 | enc_layers: 4
55 | dec_layers: 4
56 | hidden_size: 384
57 | num_heads: 2
58 | prenet_dropout: 0.5
59 | prenet_hidden_size: 256
60 | stop_token_weight: 5.0
61 | enc_ffn_kernel_size: 9
62 | dec_ffn_kernel_size: 9
63 | ffn_act: gelu
64 | ffn_padding: 'SAME'
65 |
66 |
67 | ###########
68 | # optimization
69 | ###########
70 | lr: 2.0
71 | warmup_updates: 8000
72 | optimizer_adam_beta1: 0.9
73 | optimizer_adam_beta2: 0.98
74 | weight_decay: 0
75 | clip_grad_norm: 1
76 |
77 |
78 | ###########
79 | # train and eval
80 | ###########
81 | max_tokens: 30000
82 | max_sentences: 100000
83 | max_eval_sentences: 1
84 | max_eval_tokens: 60000
85 | train_set_name: 'train'
86 | valid_set_name: 'valid'
87 | test_set_name: 'test'
88 | vocoder: pwg
89 | vocoder_ckpt: ''
90 | profile_infer: false
91 | out_wav_norm: false
92 | save_gt: false
93 | save_f0: false
94 | gen_dir_name: ''
95 | use_denoise: false
96 |
--------------------------------------------------------------------------------
/NeuralSeq/configs/tts/base_zh.yaml:
--------------------------------------------------------------------------------
1 | pre_align_args:
2 | txt_processor: zh_g2pM
3 | binarizer_cls: data_gen.tts.binarizer_zh.ZhBinarizer
--------------------------------------------------------------------------------
/NeuralSeq/configs/tts/emotion/base_text2mel.yaml:
--------------------------------------------------------------------------------
1 | raw_data_dir: 'data/raw/ESD'
2 | processed_data_dir: 'data/processed/emotion'
3 | binary_data_dir: 'data/binary/emotion'
4 | pre_align_cls: configs.tts.emotion.pre_align.EmoPreAlign
5 | audio_sample_rate: 16000
6 | binarization_args:
7 | shuffle: true
8 | binarizer_cls: data_gen.tts.base_binarizer_emotion.EmotionBinarizer
9 | use_spk_id: true
10 | test_num: 200
11 | num_spk: 10
12 | pitch_type: frame
13 | min_frames: 128
14 | num_test_samples: 30
15 | mel_loss: "ssim:0.5|l1:0.5"
16 | vocoder_ckpt: ''
17 | use_emotion: true
--------------------------------------------------------------------------------
/NeuralSeq/configs/tts/emotion/pre_align.py:
--------------------------------------------------------------------------------
1 | import os
2 |
3 | from data_gen.tts.base_preprocess import BasePreprocessor
4 | import glob
5 | import re
6 |
7 | class EmoPreAlign(BasePreprocessor):
8 |
9 | def meta_data(self):
10 | spks = ['0012', '0011', '0013', '0014', '0015', '0016', '0017', '0018', '0019', '0020']
11 | pattern = re.compile('[\t\n ]+')
12 | for spk in spks:
13 | for line in open(f"{self.raw_data_dir}/{spk}/{spk}.txt", 'r'): # 打开文件
14 | line = re.sub(pattern, ' ', line)
15 | if line == ' ': continue
16 | split_ = line.split(' ')
17 | txt = ' '.join(split_[1: -2])
18 | item_name = split_[0]
19 | emotion = split_[-2]
20 | wav_fn = f'{self.raw_data_dir}/{spk}/{emotion}/{item_name}.wav'
21 | yield item_name, wav_fn, txt, spk, emotion
22 |
23 |
24 | if __name__ == "__main__":
25 | EmoPreAlign().process()
26 |
--------------------------------------------------------------------------------
/NeuralSeq/configs/tts/fs2.yaml:
--------------------------------------------------------------------------------
1 | base_config: configs/tts/base.yaml
2 | task_cls: tasks.tts.fs2.FastSpeech2Task
3 |
4 | # model
5 | hidden_size: 256
6 | dropout: 0.1
7 | encoder_type: fft # fft|tacotron|tacotron2|conformer
8 | encoder_K: 8 # for tacotron encoder
9 | decoder_type: fft # fft|rnn|conv|conformer
10 | use_pos_embed: true
11 |
12 | # duration
13 | predictor_hidden: -1
14 | predictor_kernel: 5
15 | predictor_layers: 2
16 | dur_predictor_kernel: 3
17 | dur_predictor_layers: 2
18 | predictor_dropout: 0.5
19 |
20 | # pitch and energy
21 | use_pitch_embed: true
22 | pitch_type: ph # frame|ph|cwt
23 | use_uv: true
24 | cwt_hidden_size: 128
25 | cwt_layers: 2
26 | cwt_loss: l1
27 | cwt_add_f0_loss: false
28 | cwt_std_scale: 0.8
29 |
30 | pitch_ar: false
31 | #pitch_embed_type: 0q
32 | pitch_loss: 'l1' # l1|l2|ssim
33 | pitch_norm: log
34 | use_energy_embed: false
35 |
36 | # reference encoder and speaker embedding
37 | use_spk_id: false
38 | use_split_spk_id: false
39 | use_spk_embed: false
40 | use_var_enc: false
41 | lambda_commit: 0.25
42 | ref_norm_layer: bn
43 | pitch_enc_hidden_stride_kernel:
44 | - 0,2,5 # conv_hidden_size, conv_stride, conv_kernel_size. conv_hidden_size=0: use hidden_size
45 | - 0,2,5
46 | - 0,2,5
47 | dur_enc_hidden_stride_kernel:
48 | - 0,2,3 # conv_hidden_size, conv_stride, conv_kernel_size. conv_hidden_size=0: use hidden_size
49 | - 0,2,3
50 | - 0,1,3
51 |
52 |
53 | # mel
54 | mel_loss: l1:0.5|ssim:0.5 # l1|l2|gdl|ssim or l1:0.5|ssim:0.5
55 |
56 | # loss lambda
57 | lambda_f0: 1.0
58 | lambda_uv: 1.0
59 | lambda_energy: 0.1
60 | lambda_ph_dur: 1.0
61 | lambda_sent_dur: 1.0
62 | lambda_word_dur: 1.0
63 | predictor_grad: 0.1
64 |
65 | # train and eval
66 | pretrain_fs_ckpt: ''
67 | warmup_updates: 2000
68 | max_tokens: 32000
69 | max_sentences: 100000
70 | max_eval_sentences: 1
71 | max_updates: 120000
72 | num_valid_plots: 5
73 | num_test_samples: 0
74 | test_ids: []
75 | use_gt_dur: false
76 | use_gt_f0: false
77 |
78 | # exp
79 | dur_loss: mse # huber|mol
80 | norm_type: gn
--------------------------------------------------------------------------------
/NeuralSeq/configs/tts/hifigan.yaml:
--------------------------------------------------------------------------------
1 | base_config: configs/tts/pwg.yaml
2 | task_cls: tasks.vocoder.hifigan.HifiGanTask
3 | resblock: "1"
4 | adam_b1: 0.8
5 | adam_b2: 0.99
6 | upsample_rates: [ 8,8,2,2 ]
7 | upsample_kernel_sizes: [ 16,16,4,4 ]
8 | upsample_initial_channel: 128
9 | resblock_kernel_sizes: [ 3,7,11 ]
10 | resblock_dilation_sizes: [ [ 1,3,5 ], [ 1,3,5 ], [ 1,3,5 ] ]
11 |
12 | lambda_mel: 45.0
13 |
14 | max_samples: 8192
15 | max_sentences: 16
16 |
17 | generator_params:
18 | lr: 0.0002 # Generator's learning rate.
19 | aux_context_window: 0 # Context window size for auxiliary feature.
20 | discriminator_optimizer_params:
21 | lr: 0.0002 # Discriminator's learning rate.
--------------------------------------------------------------------------------
/NeuralSeq/configs/tts/libritts/base_text2mel.yaml:
--------------------------------------------------------------------------------
1 | raw_data_dir: 'data/raw/LibriTTS'
2 | processed_data_dir: 'data/processed/libritts'
3 | binary_data_dir: 'data/binary/libritts'
4 | pre_align_cls: configs.tts.libritts.pre_align.LibrittsPreAlign
5 | binarization_args:
6 | shuffle: true
7 | use_spk_id: true
8 | test_num: 200
9 | num_spk: 2320
10 | pitch_type: frame
11 | min_frames: 128
12 | num_test_samples: 30
13 | mel_loss: "ssim:0.5|l1:0.5"
14 | vocoder_ckpt: ''
--------------------------------------------------------------------------------
/NeuralSeq/configs/tts/libritts/fs2.yaml:
--------------------------------------------------------------------------------
1 | base_config:
2 | - configs/tts/fs2.yaml
3 | - ./base_text2mel.yaml
4 |
--------------------------------------------------------------------------------
/NeuralSeq/configs/tts/libritts/pre_align.py:
--------------------------------------------------------------------------------
1 | import os
2 |
3 | from data_gen.tts.base_preprocess import BasePreprocessor
4 | import glob
5 |
6 |
7 | class LibrittsPreAlign(BasePreprocessor):
8 | def meta_data(self):
9 | wav_fns = sorted(glob.glob(f'{self.raw_data_dir}/*/*/*.wav'))
10 | for wav_fn in wav_fns:
11 | item_name = os.path.basename(wav_fn)[:-4]
12 | txt_fn = f'{wav_fn[:-4]}.normalized.txt'
13 | with open(txt_fn, 'r') as f:
14 | txt = f.readlines()
15 | f.close()
16 | spk = item_name.split("_")[0]
17 | # Example:
18 | #
19 | # 'item_name': '103_1241_000000_000001'
20 | # 'wav_fn': 'LibriTTS/train-clean-100/103/1241/103_1241_000000_000001.wav'
21 | # 'txt': 'matthew Cuthbert is surprised'
22 | # 'spk_name': '103'
23 | yield {'item_name': item_name, 'wav_fn': wav_fn, 'txt': txt[0], 'spk_name': spk}
24 |
25 |
26 | if __name__ == "__main__":
27 | LibrittsPreAlign().process()
28 |
--------------------------------------------------------------------------------
/NeuralSeq/configs/tts/libritts/pwg.yaml:
--------------------------------------------------------------------------------
1 | base_config: egs/egs_bases/tts/vocoder/pwg.yaml
2 | raw_data_dir: 'data/raw/LibriTTS'
3 | processed_data_dir: 'data/processed/libritts'
4 | binary_data_dir: 'data/binary/libritts_wav'
5 | generator_params:
6 | kernel_size: 5
7 | num_spk: 400
8 | max_samples: 20480
9 |
--------------------------------------------------------------------------------
/NeuralSeq/configs/tts/lj/base_mel2wav.yaml:
--------------------------------------------------------------------------------
1 | raw_data_dir: 'data/raw/LJSpeech-1.1'
2 | processed_data_dir: 'data/processed/ljspeech'
3 | binary_data_dir: 'data/binary/ljspeech_wav'
4 |
--------------------------------------------------------------------------------
/NeuralSeq/configs/tts/lj/base_text2mel.yaml:
--------------------------------------------------------------------------------
1 | raw_data_dir: 'data/raw/LJSpeech-1.1'
2 | processed_data_dir: 'data/processed/ljspeech'
3 | binary_data_dir: 'data/binary/ljspeech'
4 | pre_align_cls: data_gen.tts.lj.pre_align.LJPreAlign
5 |
6 | pitch_type: cwt
7 | mel_loss: l1
8 | num_test_samples: 20
9 | test_ids: [ 68, 70, 74, 87, 110, 172, 190, 215, 231, 294,
10 | 316, 324, 402, 422, 485, 500, 505, 508, 509, 519 ]
11 | use_energy_embed: false
12 | test_num: 523
13 | valid_num: 348
--------------------------------------------------------------------------------
/NeuralSeq/configs/tts/lj/fs2.yaml:
--------------------------------------------------------------------------------
1 | base_config:
2 | - configs/tts/fs2.yaml
3 | - configs/tts/lj/base_text2mel.yaml
--------------------------------------------------------------------------------
/NeuralSeq/configs/tts/lj/hifigan.yaml:
--------------------------------------------------------------------------------
1 | base_config:
2 | - configs/tts/hifigan.yaml
3 | - configs/tts/lj/base_mel2wav.yaml
--------------------------------------------------------------------------------
/NeuralSeq/configs/tts/lj/pwg.yaml:
--------------------------------------------------------------------------------
1 | base_config:
2 | - configs/tts/pwg.yaml
3 | - configs/tts/lj/base_mel2wav.yaml
--------------------------------------------------------------------------------
/NeuralSeq/data_gen/tts/binarizer_zh.py:
--------------------------------------------------------------------------------
1 | import os
2 |
3 | os.environ["OMP_NUM_THREADS"] = "1"
4 |
5 | from data_gen.tts.txt_processors.zh_g2pM import ALL_SHENMU
6 | from data_gen.tts.base_binarizer import BaseBinarizer, BinarizationError
7 | from data_gen.tts.data_gen_utils import get_mel2ph
8 | from utils.hparams import set_hparams, hparams
9 | import numpy as np
10 |
11 |
12 | class ZhBinarizer(BaseBinarizer):
13 | @staticmethod
14 | def get_align(tg_fn, ph, mel, phone_encoded, res):
15 | if tg_fn is not None and os.path.exists(tg_fn):
16 | _, dur = get_mel2ph(tg_fn, ph, mel, hparams)
17 | else:
18 | raise BinarizationError(f"Align not found")
19 | ph_list = ph.split(" ")
20 | assert len(dur) == len(ph_list)
21 | mel2ph = []
22 | # 分隔符的时长分配给韵母
23 | dur_cumsum = np.pad(np.cumsum(dur), [1, 0], mode='constant', constant_values=0)
24 | for i in range(len(dur)):
25 | p = ph_list[i]
26 | if p[0] != '<' and not p[0].isalpha():
27 | uv_ = res['f0'][dur_cumsum[i]:dur_cumsum[i + 1]] == 0
28 | j = 0
29 | while j < len(uv_) and not uv_[j]:
30 | j += 1
31 | dur[i - 1] += j
32 | dur[i] -= j
33 | if dur[i] < 100:
34 | dur[i - 1] += dur[i]
35 | dur[i] = 0
36 | # 声母和韵母等长
37 | for i in range(len(dur)):
38 | p = ph_list[i]
39 | if p in ALL_SHENMU:
40 | p_next = ph_list[i + 1]
41 | if not (dur[i] > 0 and p_next[0].isalpha() and p_next not in ALL_SHENMU):
42 | print(f"assert dur[i] > 0 and p_next[0].isalpha() and p_next not in ALL_SHENMU, "
43 | f"dur[i]: {dur[i]}, p: {p}, p_next: {p_next}.")
44 | continue
45 | total = dur[i + 1] + dur[i]
46 | dur[i] = total // 2
47 | dur[i + 1] = total - dur[i]
48 | for i in range(len(dur)):
49 | mel2ph += [i + 1] * dur[i]
50 | mel2ph = np.array(mel2ph)
51 | if mel2ph.max() - 1 >= len(phone_encoded):
52 | raise BinarizationError(f"| Align does not match: {(mel2ph.max() - 1, len(phone_encoded))}")
53 | res['mel2ph'] = mel2ph
54 | res['dur'] = dur
55 |
56 |
57 | if __name__ == "__main__":
58 | set_hparams()
59 | ZhBinarizer().process()
60 |
--------------------------------------------------------------------------------
/NeuralSeq/data_gen/tts/emotion/model.py:
--------------------------------------------------------------------------------
1 |
2 | from data_gen.tts.emotion.params_model import *
3 | from data_gen.tts.emotion.params_data import *
4 | from torch.nn.utils import clip_grad_norm_
5 | from scipy.optimize import brentq
6 | from torch import nn
7 | import numpy as np
8 | import torch
9 |
10 |
11 | class EmotionEncoder(nn.Module):
12 | def __init__(self, device, loss_device):
13 | super().__init__()
14 | self.loss_device = loss_device
15 |
16 | # Network defition
17 | self.lstm = nn.LSTM(input_size=mel_n_channels,
18 | hidden_size=model_hidden_size,
19 | num_layers=model_num_layers,
20 | batch_first=True).to(device)
21 | self.linear = nn.Linear(in_features=model_hidden_size,
22 | out_features=model_embedding_size).to(device)
23 | self.relu = torch.nn.ReLU().to(device)
24 |
25 |
26 | # Cosine similarity scaling (with fixed initial parameter values)
27 | self.similarity_weight = nn.Parameter(torch.tensor([10.])).to(loss_device)
28 | self.similarity_bias = nn.Parameter(torch.tensor([-5.])).to(loss_device)
29 |
30 | # Loss
31 | self.loss_fn = nn.CrossEntropyLoss().to(loss_device)
32 |
33 | def do_gradient_ops(self):
34 | # Gradient scale
35 | self.similarity_weight.grad *= 0.01
36 | self.similarity_bias.grad *= 0.01
37 |
38 | # Gradient clipping
39 | clip_grad_norm_(self.parameters(), 3, norm_type=2)
40 |
41 | def forward(self, utterances, hidden_init=None):
42 | """
43 | Computes the embeddings of a batch of utterance spectrograms.
44 |
45 | :param utterances: batch of mel-scale filterbanks of same duration as a tensor of shape
46 | (batch_size, n_frames, n_channels)
47 | :param hidden_init: initial hidden state of the LSTM as a tensor of shape (num_layers,
48 | batch_size, hidden_size). Will default to a tensor of zeros if None.
49 | :return: the embeddings as a tensor of shape (batch_size, embedding_size)
50 | """
51 | # Pass the input through the LSTM layers and retrieve all outputs, the final hidden state
52 | # and the final cell state.
53 | out, (hidden, cell) = self.lstm(utterances, hidden_init)
54 |
55 | # We take only the hidden state of the last layer
56 | embeds_raw = self.relu(self.linear(hidden[-1]))
57 |
58 | # L2-normalize it
59 | embeds = embeds_raw / torch.norm(embeds_raw, dim=1, keepdim=True)
60 |
61 | return embeds
62 |
63 | def inference(self, utterances, hidden_init=None):
64 | """
65 | Computes the embeddings of a batch of utterance spectrograms.
66 |
67 | :param utterances: batch of mel-scale filterbanks of same duration as a tensor of shape
68 | (batch_size, n_frames, n_channels)
69 | :param hidden_init: initial hidden state of the LSTM as a tensor of shape (num_layers,
70 | batch_size, hidden_size). Will default to a tensor of zeros if None.
71 | :return: the embeddings as a tensor of shape (batch_size, embedding_size)
72 | """
73 | # Pass the input through the LSTM layers and retrieve all outputs, the final hidden state
74 | # and the final cell state.
75 |
76 | out, (hidden, cell) = self.lstm(utterances, hidden_init)
77 |
78 | return hidden[-1]
--------------------------------------------------------------------------------
/NeuralSeq/data_gen/tts/emotion/params_data.py:
--------------------------------------------------------------------------------
1 |
2 | ## Mel-filterbank
3 | mel_window_length = 25 # In milliseconds
4 | mel_window_step = 10 # In milliseconds
5 | mel_n_channels = 40
6 |
7 |
8 | ## Audio
9 | sampling_rate = 16000
10 | # Number of spectrogram frames in a partial utterance
11 | partials_n_frames = 160 # 1600 ms
12 | # Number of spectrogram frames at inference
13 | inference_n_frames = 80 # 800 ms
14 |
15 |
16 | ## Voice Activation Detection
17 | # Window size of the VAD. Must be either 10, 20 or 30 milliseconds.
18 | # This sets the granularity of the VAD. Should not need to be changed.
19 | vad_window_length = 30 # In milliseconds
20 | # Number of frames to average together when performing the moving average smoothing.
21 | # The larger this value, the larger the VAD variations must be to not get smoothed out.
22 | vad_moving_average_width = 8
23 | # Maximum number of consecutive silent frames a segment can have.
24 | vad_max_silence_length = 6
25 |
26 |
27 | ## Audio volume normalization
28 | audio_norm_target_dBFS = -30
29 |
30 |
--------------------------------------------------------------------------------
/NeuralSeq/data_gen/tts/emotion/params_model.py:
--------------------------------------------------------------------------------
1 |
2 | ## Model parameters
3 | model_hidden_size = 256
4 | model_embedding_size = 256
5 | model_num_layers = 3
6 |
7 |
8 | ## Training parameters
9 | learning_rate_init = 1e-4
10 | speakers_per_batch = 6
11 | utterances_per_speaker = 20
12 |
--------------------------------------------------------------------------------
/NeuralSeq/data_gen/tts/txt_processors/__init__.py:
--------------------------------------------------------------------------------
1 | from . import en
--------------------------------------------------------------------------------
/NeuralSeq/data_gen/tts/txt_processors/base_text_processor.py:
--------------------------------------------------------------------------------
1 | from data_gen.tts.data_gen_utils import is_sil_phoneme
2 |
3 | REGISTERED_TEXT_PROCESSORS = {}
4 |
5 | def register_txt_processors(name):
6 | def _f(cls):
7 | REGISTERED_TEXT_PROCESSORS[name] = cls
8 | return cls
9 |
10 | return _f
11 |
12 |
13 | def get_txt_processor_cls(name):
14 | return REGISTERED_TEXT_PROCESSORS.get(name, None)
15 |
16 |
17 | class BaseTxtProcessor:
18 | @staticmethod
19 | def sp_phonemes():
20 | return ['|']
21 |
22 | @classmethod
23 | def process(cls, txt, preprocess_args):
24 | raise NotImplementedError
25 |
26 | @classmethod
27 | def postprocess(cls, txt_struct, preprocess_args):
28 | # remove sil phoneme in head and tail
29 | while len(txt_struct) > 0 and is_sil_phoneme(txt_struct[0][0]):
30 | txt_struct = txt_struct[1:]
31 | while len(txt_struct) > 0 and is_sil_phoneme(txt_struct[-1][0]):
32 | txt_struct = txt_struct[:-1]
33 | if preprocess_args['with_phsep']:
34 | txt_struct = cls.add_bdr(txt_struct)
35 | if preprocess_args['add_eos_bos']:
36 | txt_struct = [["<BOS>", ["<BOS>"]]] + txt_struct + [["<EOS>", ["<EOS>"]]]
37 | return txt_struct
38 |
39 | @classmethod
40 | def add_bdr(cls, txt_struct):
41 | txt_struct_ = []
42 | for i, ts in enumerate(txt_struct):
43 | txt_struct_.append(ts)
44 | if i != len(txt_struct) - 1 and \
45 | not is_sil_phoneme(txt_struct[i][0]) and not is_sil_phoneme(txt_struct[i + 1][0]):
46 | txt_struct_.append(['|', ['|']])
47 | return txt_struct_
--------------------------------------------------------------------------------
/NeuralSeq/data_gen/tts/txt_processors/en.py:
--------------------------------------------------------------------------------
1 | import re
2 | import unicodedata
3 |
4 | from g2p_en import G2p
5 | from g2p_en.expand import normalize_numbers
6 | from nltk import pos_tag
7 | from nltk.tokenize import TweetTokenizer
8 |
9 | from data_gen.tts.txt_processors.base_text_processor import BaseTxtProcessor, register_txt_processors
10 | from data_gen.tts.data_gen_utils import is_sil_phoneme, PUNCS
11 |
12 | class EnG2p(G2p):
13 | word_tokenize = TweetTokenizer().tokenize
14 |
15 | def __call__(self, text):
16 | # preprocessing
17 | words = EnG2p.word_tokenize(text)
18 | tokens = pos_tag(words) # tuples of (word, tag)
19 |
20 | # steps
21 | prons = []
22 | for word, pos in tokens:
23 | if re.search("[a-z]", word) is None:
24 | pron = [word]
25 |
26 | elif word in self.homograph2features: # Check homograph
27 | pron1, pron2, pos1 = self.homograph2features[word]
28 | if pos.startswith(pos1):
29 | pron = pron1
30 | else:
31 | pron = pron2
32 | elif word in self.cmu: # lookup CMU dict
33 | pron = self.cmu[word][0]
34 | else: # predict for oov
35 | pron = self.predict(word)
36 |
37 | prons.extend(pron)
38 | prons.extend([" "])
39 |
40 | return prons[:-1]
41 |
42 |
43 | @register_txt_processors('en')
44 | class TxtProcessor(BaseTxtProcessor):
45 | g2p = EnG2p()
46 |
47 | @staticmethod
48 | def preprocess_text(text):
49 | text = normalize_numbers(text)
50 | text = ''.join(char for char in unicodedata.normalize('NFD', text)
51 | if unicodedata.category(char) != 'Mn') # Strip accents
52 | text = text.lower()
53 | text = re.sub("[\'\"()]+", "", text)
54 | text = re.sub("[-]+", " ", text)
55 | text = re.sub(f"[^ a-z{PUNCS}]", "", text)
56 | text = re.sub(f" ?([{PUNCS}]) ?", r"\1", text) # !! -> !
57 | text = re.sub(f"([{PUNCS}])+", r"\1", text) # !! -> !
58 | text = text.replace("i.e.", "that is")
59 | text = text.replace("i.e.", "that is")
60 | text = text.replace("etc.", "etc")
61 | text = re.sub(f"([{PUNCS}])", r" \1 ", text)
62 | text = re.sub(rf"\s+", r" ", text)
63 | return text
64 |
65 | @classmethod
66 | def process(cls, txt, preprocess_args):
67 | txt = cls.preprocess_text(txt).strip()
68 | phs = cls.g2p(txt)
69 | txt_struct = [[w, []] for w in txt.split(" ")]
70 | i_word = 0
71 | for p in phs:
72 | if p == ' ':
73 | i_word += 1
74 | else:
75 | txt_struct[i_word][1].append(p)
76 | txt_struct = cls.postprocess(txt_struct, preprocess_args)
77 | return txt_struct, txt
--------------------------------------------------------------------------------
/NeuralSeq/data_gen/tts/txt_processors/zh.py:
--------------------------------------------------------------------------------
1 | import re
2 | import jieba
3 | from pypinyin import pinyin, Style
4 | from data_gen.tts.data_gen_utils import PUNCS
5 | from data_gen.tts.txt_processors.base_text_processor import BaseTxtProcessor
6 | from utils.text_norm import NSWNormalizer
7 |
8 |
9 | class TxtProcessor(BaseTxtProcessor):
10 | table = {ord(f): ord(t) for f, t in zip(
11 | u'：，。！？【】（）％＃＠＆１２３４５６７８９０',
12 | u':,.!?[]()%#@&1234567890')}
13 |
14 | @staticmethod
15 | def preprocess_text(text):
16 | text = text.translate(TxtProcessor.table)
17 | text = NSWNormalizer(text).normalize(remove_punc=False)
18 | text = re.sub("[\'\"()]+", "", text)
19 | text = re.sub("[-]+", " ", text)
20 | text = re.sub(f"[^ A-Za-z\u4e00-\u9fff{PUNCS}]", "", text)
21 | text = re.sub(f"([{PUNCS}])+", r"\1", text) # !! -> !
22 | text = re.sub(f"([{PUNCS}])", r" \1 ", text)
23 | text = re.sub(rf"\s+", r"", text)
24 | text = re.sub(rf"[A-Za-z]+", r"quot;, text)
25 | return text
26 |
27 | @classmethod
28 | def process(cls, txt, pre_align_args):
29 | txt = cls.preprocess_text(txt)
30 | shengmu = pinyin(txt, style=Style.INITIALS) # https://blog.csdn.net/zhoulei124/article/details/89055403
31 | yunmu_finals = pinyin(txt, style=Style.FINALS)
32 | yunmu_tone3 = pinyin(txt, style=Style.FINALS_TONE3)
33 | yunmu = [[t[0] + '5'] if t[0] == f[0] else t for f, t in zip(yunmu_finals, yunmu_tone3)] \
34 | if pre_align_args['use_tone'] else yunmu_finals
35 |
36 | assert len(shengmu) == len(yunmu)
37 | phs = ["|"]
38 | for a, b, c in zip(shengmu, yunmu, yunmu_finals):
39 | if a[0] == c[0]:
40 | phs += [a[0], "|"]
41 | else:
42 | phs += [a[0], b[0], "|"]
43 | return phs, txt
44 |
--------------------------------------------------------------------------------
/NeuralSeq/data_gen/tts/txt_processors/zh_g2pM.py:
--------------------------------------------------------------------------------
1 | import re
2 | import jieba
3 | from pypinyin import pinyin, Style
4 | from data_gen.tts.data_gen_utils import PUNCS
5 | from data_gen.tts.txt_processors import zh
6 | from g2pM import G2pM
7 |
8 | ALL_SHENMU = ['zh', 'ch', 'sh', 'b', 'p', 'm', 'f', 'd', 't', 'n', 'l', 'g', 'k', 'h', 'j',
9 | 'q', 'x', 'r', 'z', 'c', 's', 'y', 'w']
10 | ALL_YUNMU = ['a', 'ai', 'an', 'ang', 'ao', 'e', 'ei', 'en', 'eng', 'er', 'i', 'ia', 'ian',
11 | 'iang', 'iao', 'ie', 'in', 'ing', 'iong', 'iu', 'ng', 'o', 'ong', 'ou',
12 | 'u', 'ua', 'uai', 'uan', 'uang', 'ui', 'un', 'uo', 'v', 'van', 've', 'vn']
13 |
14 |
15 | class TxtProcessor(zh.TxtProcessor):
16 | model = G2pM()
17 |
18 | @staticmethod
19 | def sp_phonemes():
20 | return ['|', '#']
21 |
22 | @classmethod
23 | def process(cls, txt, pre_align_args):
24 | txt = cls.preprocess_text(txt)
25 | ph_list = cls.model(txt, tone=pre_align_args['use_tone'], char_split=True)
26 | seg_list = '#'.join(jieba.cut(txt))
27 | assert len(ph_list) == len([s for s in seg_list if s != '#']), (ph_list, seg_list)
28 |
29 | # 加入词边界'#'
30 | ph_list_ = []
31 | seg_idx = 0
32 | for p in ph_list:
33 | p = p.replace("u:", "v")
34 | if seg_list[seg_idx] == '#':
35 | ph_list_.append('#')
36 | seg_idx += 1
37 | else:
38 | ph_list_.append("|")
39 | seg_idx += 1
40 | if re.findall('[\u4e00-\u9fff]', p):
41 | if pre_align_args['use_tone']:
42 | p = pinyin(p, style=Style.TONE3, strict=True)[0][0]
43 | if p[-1] not in ['1', '2', '3', '4', '5']:
44 | p = p + '5'
45 | else:
46 | p = pinyin(p, style=Style.NORMAL, strict=True)[0][0]
47 |
48 | finished = False
49 | if len([c.isalpha() for c in p]) > 1:
50 | for shenmu in ALL_SHENMU:
51 | if p.startswith(shenmu) and not p.lstrip(shenmu).isnumeric():
52 | ph_list_ += [shenmu, p.lstrip(shenmu)]
53 | finished = True
54 | break
55 | if not finished:
56 | ph_list_.append(p)
57 |
58 | ph_list = ph_list_
59 |
60 | # 去除静音符号周围的词边界标记 [..., '#', ',', '#', ...]
61 | sil_phonemes = list(PUNCS) + TxtProcessor.sp_phonemes()
62 | ph_list_ = []
63 | for i in range(0, len(ph_list), 1):
64 | if ph_list[i] != '#' or (ph_list[i - 1] not in sil_phonemes and ph_list[i + 1] not in sil_phonemes):
65 | ph_list_.append(ph_list[i])
66 | ph_list = ph_list_
67 | return ph_list, txt
68 |
69 |
70 | if __name__ == '__main__':
71 | phs, txt = TxtProcessor.process('他来到了，网易杭研大厦', {'use_tone': True})
72 | print(phs)
73 |
--------------------------------------------------------------------------------
/NeuralSeq/data_gen/tts/wav_processors/__init__.py:
--------------------------------------------------------------------------------
1 | from . import base_processor
2 | from . import common_processors
3 |
--------------------------------------------------------------------------------
/NeuralSeq/data_gen/tts/wav_processors/base_processor.py:
--------------------------------------------------------------------------------
1 | REGISTERED_WAV_PROCESSORS = {}
2 |
3 |
4 | def register_wav_processors(name):
5 | def _f(cls):
6 | REGISTERED_WAV_PROCESSORS[name] = cls
7 | return cls
8 |
9 | return _f
10 |
11 |
12 | def get_wav_processor_cls(name):
13 | return REGISTERED_WAV_PROCESSORS.get(name, None)
14 |
15 |
16 | class BaseWavProcessor:
17 | @property
18 | def name(self):
19 | raise NotImplementedError
20 |
21 | def output_fn(self, input_fn):
22 | return f'{input_fn[:-4]}_{self.name}.wav'
23 |
24 | def process(self, input_fn, sr, tmp_dir, processed_dir, item_name, preprocess_args):
25 | raise NotImplementedError
26 |
--------------------------------------------------------------------------------
/NeuralSeq/data_gen/tts/wav_processors/common_processors.py:
--------------------------------------------------------------------------------
1 | import os
2 | import subprocess
3 | import librosa
4 | import numpy as np
5 | from data_gen.tts.wav_processors.base_processor import BaseWavProcessor, register_wav_processors
6 | from data_gen.tts.data_gen_utils import trim_long_silences
7 | from utils.audio import save_wav, rnnoise
8 | from utils.hparams import hparams
9 |
10 |
11 | @register_wav_processors(name='sox_to_wav')
12 | class ConvertToWavProcessor(BaseWavProcessor):
13 | @property
14 | def name(self):
15 | return 'ToWav'
16 |
17 | def process(self, input_fn, sr, tmp_dir, processed_dir, item_name, preprocess_args):
18 | if input_fn[-4:] == '.wav':
19 | return input_fn, sr
20 | else:
21 | output_fn = self.output_fn(input_fn)
22 | subprocess.check_call(f'sox -v 0.95 "{input_fn}" -t wav "{output_fn}"', shell=True)
23 | return output_fn, sr
24 |
25 |
26 | @register_wav_processors(name='sox_resample')
27 | class ResampleProcessor(BaseWavProcessor):
28 | @property
29 | def name(self):
30 | return 'Resample'
31 |
32 | def process(self, input_fn, sr, tmp_dir, processed_dir, item_name, preprocess_args):
33 | output_fn = self.output_fn(input_fn)
34 | sr_file = librosa.core.get_samplerate(input_fn)
35 | if sr != sr_file:
36 | subprocess.check_call(f'sox -v 0.95 "{input_fn}" -r{sr} "{output_fn}"', shell=True)
37 | y, _ = librosa.core.load(input_fn, sr=sr)
38 | y, _ = librosa.effects.trim(y)
39 | save_wav(y, output_fn, sr)
40 | return output_fn, sr
41 | else:
42 | return input_fn, sr
43 |
44 |
45 | @register_wav_processors(name='trim_sil')
46 | class TrimSILProcessor(BaseWavProcessor):
47 | @property
48 | def name(self):
49 | return 'TrimSIL'
50 |
51 | def process(self, input_fn, sr, tmp_dir, processed_dir, item_name, preprocess_args):
52 | output_fn = self.output_fn(input_fn)
53 | y, _ = librosa.core.load(input_fn, sr=sr)
54 | y, _ = librosa.effects.trim(y)
55 | save_wav(y, output_fn, sr)
56 | return output_fn
57 |
58 |
59 | @register_wav_processors(name='trim_all_sil')
60 | class TrimAllSILProcessor(BaseWavProcessor):
61 | @property
62 | def name(self):
63 | return 'TrimSIL'
64 |
65 | def process(self, input_fn, sr, tmp_dir, processed_dir, item_name, preprocess_args):
66 | output_fn = self.output_fn(input_fn)
67 | y, audio_mask, _ = trim_long_silences(
68 | input_fn, vad_max_silence_length=preprocess_args.get('vad_max_silence_length', 12))
69 | save_wav(y, output_fn, sr)
70 | if preprocess_args['save_sil_mask']:
71 | os.makedirs(f'{processed_dir}/sil_mask', exist_ok=True)
72 | np.save(f'{processed_dir}/sil_mask/{item_name}.npy', audio_mask)
73 | return output_fn, sr
74 |
75 |
76 | @register_wav_processors(name='denoise')
77 | class DenoiseProcessor(BaseWavProcessor):
78 | @property
79 | def name(self):
80 | return 'Denoise'
81 |
82 | def process(self, input_fn, sr, tmp_dir, processed_dir, item_name, preprocess_args):
83 | output_fn = self.output_fn(input_fn)
84 | rnnoise(input_fn, output_fn, out_sample_rate=sr)
85 | return output_fn, sr
86 |
--------------------------------------------------------------------------------
/NeuralSeq/egs/datasets/audio/emotion/base_text2mel.yaml:
--------------------------------------------------------------------------------
1 | raw_data_dir: 'data/raw/ESD'
2 | processed_data_dir: 'data/processed/emotion'
3 | binary_data_dir: 'data/binary/emotion'
4 | pre_align_cls: egs.datasets.audio.emotion.pre_align.EmoPreAlign
5 | audio_sample_rate: 16000
6 | binarization_args:
7 | shuffle: true
8 | binarizer_cls: data_gen.tts.base_binarizer_emotion.EmotionBinarizer
9 | use_spk_id: true
10 | test_num: 200
11 | num_spk: 10
12 | pitch_type: frame
13 | min_frames: 128
14 | num_test_samples: 30
15 | mel_loss: "ssim:0.5|l1:0.5"
16 | vocoder_ckpt: ''
17 | use_emotion: true
--------------------------------------------------------------------------------
/NeuralSeq/egs/datasets/audio/emotion/pre_align.py:
--------------------------------------------------------------------------------
1 | import os
2 |
3 | from data_gen.tts.base_preprocess import BasePreprocessor
4 | import glob
5 | import re
6 |
7 | class EmoPreAlign(BasePreprocessor):
8 |
9 | def meta_data(self):
10 | spks = ['0012', '0011', '0013', '0014', '0015', '0016', '0017', '0018', '0019', '0020']
11 | pattern = re.compile('[\t\n ]+')
12 | for spk in spks:
13 | for line in open(f"{self.raw_data_dir}/{spk}/{spk}.txt", 'r'): # 打开文件
14 | line = re.sub(pattern, ' ', line)
15 | if line == ' ': continue
16 | split_ = line.split(' ')
17 | txt = ' '.join(split_[1: -2])
18 | item_name = split_[0]
19 | emotion = split_[-2]
20 | wav_fn = f'{self.raw_data_dir}/{spk}/{emotion}/{item_name}.wav'
21 | yield item_name, wav_fn, txt, spk, emotion
22 |
23 |
24 | if __name__ == "__main__":
25 | EmoPreAlign().process()
26 |
--------------------------------------------------------------------------------
/NeuralSeq/egs/datasets/audio/libritts/base_text2mel.yaml:
--------------------------------------------------------------------------------
1 | raw_data_dir: 'data/raw/LibriTTS'
2 | processed_data_dir: 'data/processed/libritts'
3 | binary_data_dir: 'data/binary/libritts'
4 | pre_align_cls: egs.datasets.audio.libritts.pre_align.LibrittsPreAlign
5 | binarization_args:
6 | shuffle: true
7 | use_spk_id: true
8 | test_num: 200
9 | num_spk: 2320
10 | pitch_type: frame
11 | min_frames: 128
12 | num_test_samples: 30
13 | mel_loss: "ssim:0.5|l1:0.5"
14 | vocoder_ckpt: ''
--------------------------------------------------------------------------------
/NeuralSeq/egs/datasets/audio/libritts/fs2.yaml:
--------------------------------------------------------------------------------
1 | base_config:
2 | - egs/egs_bases/tts/fs2.yaml
3 | - ./base_text2mel.yaml
4 |
--------------------------------------------------------------------------------
/NeuralSeq/egs/datasets/audio/libritts/pre_align.py:
--------------------------------------------------------------------------------
1 | import os
2 |
3 | from data_gen.tts.base_preprocess import BasePreprocessor
4 | import glob
5 |
6 |
7 | class LibrittsPreAlign(BasePreprocessor):
8 | def meta_data(self):
9 | wav_fns = sorted(glob.glob(f'{self.raw_data_dir}/*/*/*.wav'))
10 | for wav_fn in wav_fns:
11 | item_name = os.path.basename(wav_fn)[:-4]
12 | txt_fn = f'{wav_fn[:-4]}.normalized.txt'
13 | with open(txt_fn, 'r') as f:
14 | txt = f.readlines()
15 | f.close()
16 | spk = item_name.split("_")[0]
17 | # Example:
18 | #
19 | # 'item_name': '103_1241_000000_000001'
20 | # 'wav_fn': 'LibriTTS/train-clean-100/103/1241/103_1241_000000_000001.wav'
21 | # 'txt': 'matthew Cuthbert is surprised'
22 | # 'spk_name': '103'
23 | yield {'item_name': item_name, 'wav_fn': wav_fn, 'txt': txt[0], 'spk_name': spk}
24 |
25 |
26 | if __name__ == "__main__":
27 | LibrittsPreAlign().process()
28 |
--------------------------------------------------------------------------------
/NeuralSeq/egs/datasets/audio/libritts/pwg.yaml:
--------------------------------------------------------------------------------
1 | base_config: egs/egs_bases/tts/vocoder/pwg.yaml
2 | raw_data_dir: 'data/raw/LibriTTS'
3 | processed_data_dir: 'data/processed/libritts'
4 | binary_data_dir: 'data/binary/libritts_wav'
5 | generator_params:
6 | kernel_size: 5
7 | num_spk: 400
8 | max_samples: 20480
9 |
--------------------------------------------------------------------------------
/NeuralSeq/egs/datasets/audio/lj/base_mel2wav.yaml:
--------------------------------------------------------------------------------
1 | raw_data_dir: 'data/raw/LJSpeech-1.1'
2 | processed_data_dir: 'data/processed/ljspeech'
3 | binary_data_dir: 'data/binary/ljspeech_wav'
4 | binarization_args:
5 | with_spk_embed: false
--------------------------------------------------------------------------------
/NeuralSeq/egs/datasets/audio/lj/preprocess.py:
--------------------------------------------------------------------------------
1 | from data_gen.tts.base_preprocess import BasePreprocessor
2 |
3 |
4 | class LJPreprocess(BasePreprocessor):
5 | def meta_data(self):
6 | for l in open(f'{self.raw_data_dir}/metadata.csv').readlines():
7 | item_name, _, txt = l.strip().split("|")
8 | wav_fn = f"{self.raw_data_dir}/wavs/{item_name}.wav"
9 | yield {'item_name': item_name, 'wav_fn': wav_fn, 'txt': txt}
10 |
--------------------------------------------------------------------------------
/NeuralSeq/egs/datasets/audio/lj/pwg.yaml:
--------------------------------------------------------------------------------
1 | base_config:
2 | - egs/egs_bases/tts/vocoder/pwg.yaml
3 | - ./base_mel2wav.yaml
--------------------------------------------------------------------------------
/NeuralSeq/egs/datasets/audio/vctk/base_mel2wav.yaml:
--------------------------------------------------------------------------------
1 | raw_data_dir: 'data/raw/VCTK-Corpus'
2 | processed_data_dir: 'data/processed/vctk'
3 | binary_data_dir: 'data/binary/vctk_wav'
4 |
--------------------------------------------------------------------------------
/NeuralSeq/egs/datasets/audio/vctk/fs2.yaml:
--------------------------------------------------------------------------------
1 | base_config:
2 | - egs/egs_bases/tts/fs2.yaml
3 | raw_data_dir: 'data/raw/VCTK-Corpus'
4 | processed_data_dir: 'data/processed/vctk'
5 | binary_data_dir: 'data/binary/vctk'
6 | pre_align_cls: egs.datasets.audio.vctk.pre_align.VCTKPreAlign
7 | use_spk_id: true
8 | test_num: 200
9 | num_spk: 400
10 | binarization_args:
11 | shuffle: true
12 | trim_eos_bos: true
--------------------------------------------------------------------------------
/NeuralSeq/egs/datasets/audio/vctk/pre_align.py:
--------------------------------------------------------------------------------
1 | import os
2 |
3 | from data_gen.tts.base_pre_align import BasePreAlign
4 | import glob
5 |
6 |
7 | class VCTKPreAlign(BasePreAlign):
8 | def meta_data(self):
9 | wav_fns = glob.glob(f'{self.raw_data_dir}/wav48/*/*.wav')
10 | for wav_fn in wav_fns:
11 | item_name = os.path.basename(wav_fn)[:-4]
12 | spk = item_name.split("_")[0]
13 | txt_fn = wav_fn.split("/")
14 | txt_fn[-1] = f'{item_name}.txt'
15 | txt_fn[-3] = f'txt'
16 | txt_fn = "/".join(txt_fn)
17 | if os.path.exists(txt_fn) and os.path.exists(wav_fn):
18 | yield item_name, wav_fn, (self.load_txt, txt_fn), spk
19 |
20 |
21 | if __name__ == "__main__":
22 | VCTKPreAlign().process()
23 |
--------------------------------------------------------------------------------
/NeuralSeq/egs/datasets/audio/vctk/pwg.yaml:
--------------------------------------------------------------------------------
1 | base_config:
2 | - egs/egs_bases/tts/vocoder/pwg.yaml
3 | - ./base_mel2wav.yaml
4 |
5 | num_spk: 400
6 | max_samples: 20480
7 |
--------------------------------------------------------------------------------
/NeuralSeq/egs/egs_bases/config_base.yaml:
--------------------------------------------------------------------------------
1 | # task
2 | binary_data_dir: ''
3 | work_dir: '' # experiment directory.
4 | infer: false # inference
5 | amp: false
6 | seed: 1234
7 | debug: false
8 | save_codes: []
9 | # - configs
10 | # - modules
11 | # - tasks
12 | # - utils
13 | # - usr
14 |
15 | #############
16 | # dataset
17 | #############
18 | ds_workers: 1
19 | test_num: 100
20 | endless_ds: false
21 | sort_by_len: true
22 |
23 | #########
24 | # train and eval
25 | #########
26 | print_nan_grads: false
27 | load_ckpt: ''
28 | save_best: true
29 | num_ckpt_keep: 3
30 | clip_grad_norm: 0
31 | accumulate_grad_batches: 1
32 | tb_log_interval: 100
33 | num_sanity_val_steps: 5 # steps of validation at the beginning
34 | check_val_every_n_epoch: 10
35 | val_check_interval: 2000
36 | valid_monitor_key: 'val_loss'
37 | valid_monitor_mode: 'min'
38 | max_epochs: 1000
39 | max_updates: 1000000
40 | max_tokens: 31250
41 | max_sentences: 100000
42 | max_valid_tokens: -1
43 | max_valid_sentences: -1
44 | test_input_dir: ''
45 | resume_from_checkpoint: 0
46 | rename_tmux: true
--------------------------------------------------------------------------------
/NeuralSeq/egs/egs_bases/svs/base.yaml:
--------------------------------------------------------------------------------
1 | task_cls: tasks.svs.task.DiffFsTask
2 | pitch_type: frame
3 | timesteps: 100
4 | dilation_cycle_length: 1
5 | residual_layers: 20
6 | residual_channels: 256
7 | lr: 0.001
8 | decay_steps: 50000
9 | keep_bins: 80
10 | spec_min: [ ]
11 | spec_max: [ ]
12 |
13 | content_cond_steps: [ ] # [ 0, 10000 ]
14 | spk_cond_steps: [ ] # [ 0, 10000 ]
15 | # train and eval
16 | fs2_ckpt: ''
17 | max_updates: 400000
18 | # max_updates: 200000
19 | use_gt_dur: true
20 | use_gt_f0: true
21 | gen_tgt_spk_id: -1
22 | max_sentences: 48
23 | num_sanity_val_steps: 1
24 | num_valid_plots: 1
25 |
--------------------------------------------------------------------------------
/NeuralSeq/egs/egs_bases/svs/lj_ds_beta6.yaml:
--------------------------------------------------------------------------------
1 | base_config:
2 | - configs/tts/lj/fs2.yaml
3 | - ./base.yaml
4 | # spec_min and spec_max are calculated on the training set.
5 | spec_min: [ -4.7574, -4.6783, -4.6431, -4.5832, -4.5390, -4.6771, -4.8089, -4.7672,
6 | -4.5784, -4.7755, -4.7150, -4.8919, -4.8271, -4.7389, -4.6047, -4.7759,
7 | -4.6799, -4.8201, -4.7823, -4.8262, -4.7857, -4.7545, -4.9358, -4.9733,
8 | -5.1134, -5.1395, -4.9016, -4.8434, -5.0189, -4.8460, -5.0529, -4.9510,
9 | -5.0217, -5.0049, -5.1831, -5.1445, -5.1015, -5.0281, -4.9887, -4.9916,
10 | -4.9785, -4.9071, -4.9488, -5.0342, -4.9332, -5.0650, -4.8924, -5.0875,
11 | -5.0483, -5.0848, -5.1809, -5.0677, -5.0015, -5.0792, -5.0636, -5.2413,
12 | -5.1421, -5.1710, -5.3256, -5.0511, -5.1186, -5.0057, -5.0446, -5.1173,
13 | -5.0325, -5.1085, -5.0053, -5.0755, -5.1176, -5.1004, -5.2153, -5.2757,
14 | -5.3025, -5.2867, -5.2918, -5.3328, -5.2731, -5.2985, -5.2400, -5.2211 ]
15 | spec_max: [ -0.5982, -0.0778, 0.1205, 0.2747, 0.4657, 0.5123, 0.5684, 0.7093,
16 | 0.6461, 0.6420, 0.7316, 0.7715, 0.7681, 0.8349, 0.7815, 0.7591,
17 | 0.7910, 0.7433, 0.7352, 0.6869, 0.6854, 0.6623, 0.5353, 0.6492,
18 | 0.6909, 0.6106, 0.5761, 0.5936, 0.5638, 0.4054, 0.4545, 0.3589,
19 | 0.3037, 0.3380, 0.1599, 0.2433, 0.2741, 0.2130, 0.1569, 0.1911,
20 | 0.2324, 0.1586, 0.1221, 0.0341, -0.0558, 0.0553, -0.1153, -0.0933,
21 | -0.1171, -0.0050, -0.1519, -0.1629, -0.0522, -0.0739, -0.2069, -0.2405,
22 | -0.1244, -0.2116, -0.1361, -0.1575, -0.1442, 0.0513, -0.1567, -0.2000,
23 | 0.0086, -0.0698, 0.1385, 0.0941, 0.1864, 0.1225, 0.2176, 0.2566,
24 | 0.1670, 0.1007, 0.1444, 0.0888, 0.1998, 0.2414, 0.2932, 0.3047 ]
25 |
26 | task_cls: tasks.svs.diffspeech_task.DiffSpeechTask
27 | vocoder: vocoders.hifigan.HifiGAN
28 | vocoder_ckpt: checkpoints/0414_hifi_lj_1
29 | num_valid_plots: 10
30 | use_gt_dur: false
31 | use_gt_f0: false
32 | pitch_type: cwt
33 | pitch_extractor: 'parselmouth'
34 | max_updates: 160000
35 | lr: 0.001
36 | timesteps: 100
37 | K_step: 71
38 | diff_loss_type: l1
39 | diff_decoder_type: 'wavenet'
40 | schedule_type: 'linear'
41 | max_beta: 0.06
42 | fs2_ckpt: checkpoints/fs2_lj_1/model_ckpt_steps_150000.ckpt
43 | save_gt: true
--------------------------------------------------------------------------------
/NeuralSeq/egs/egs_bases/svs/midi/cascade/opencs/aux_rel.yaml:
--------------------------------------------------------------------------------
1 | base_config:
2 | - configs/singing/fs2.yaml
3 | - egs/egs_bases/svs/midi/cascade/opencs/opencpop_statis.yaml
4 |
5 | audio_sample_rate: 24000
6 | hop_size: 128 # Hop size.
7 | fft_size: 512 # FFT size.
8 | win_size: 512 # FFT size.
9 | fmin: 30
10 | fmax: 12000
11 | min_level_db: -120
12 |
13 | binarization_args:
14 | with_wav: true
15 | with_spk_embed: false
16 | with_align: true
17 | raw_data_dir: 'data/raw/opencpop/segments'
18 | processed_data_dir: 'xxx'
19 | binarizer_cls: data_gen.singing.binarize.OpencpopBinarizer
20 |
21 |
22 | binary_data_dir: 'data/binary/opencpop-midi-dp'
23 | use_midi: true # for midi exp
24 | use_gt_f0: false # for midi exp
25 | use_gt_dur: false # for further midi exp
26 | lambda_f0: 1.0
27 | lambda_uv: 1.0
28 | #lambda_energy: 0.1
29 | lambda_ph_dur: 1.0
30 | lambda_sent_dur: 1.0
31 | lambda_word_dur: 1.0
32 | predictor_grad: 0.1
33 | pe_enable: false
34 | pe_ckpt: ''
35 |
36 | num_spk: 1
37 | test_prefixes: [
38 | '2044',
39 | '2086',
40 | '2092',
41 | '2093',
42 | '2100',
43 | ]
44 |
45 | task_cls: tasks.svs.diffsinger_task.AuxDecoderMIDITask
46 | #vocoder: tasks.svs.singingvocoder.highgan.HighGAN
47 | #vocoder_ckpt: checkpoints/h_2_model/checkpoint-530000steps.pkl
48 | vocoder: vocoders.hifigan.HifiGAN
49 | vocoder_ckpt: checkpoints/0109_hifigan_bigpopcs_hop128
50 |
51 | use_nsf: true
52 |
53 | # config for experiments
54 | max_frames: 5000
55 | max_tokens: 40000
56 | predictor_layers: 5
57 | rel_pos: true
58 | dur_predictor_layers: 5 # *
59 |
60 | use_spk_embed: false
61 | num_valid_plots: 10
62 | max_updates: 160000
63 | save_gt: true
--------------------------------------------------------------------------------
/NeuralSeq/egs/egs_bases/svs/midi/cascade/opencs/ds60_rel.yaml:
--------------------------------------------------------------------------------
1 | base_config:
2 | - egs/egs_bases/svs/popcs_ds_beta6.yaml
3 | - egs/egs_bases/svs/midi/cascade/opencs/opencpop_statis.yaml
4 |
5 | binarizer_cls: data_gen.singing.binarize.OpencpopBinarizer
6 | binary_data_dir: 'data/binary/opencpop-midi-dp'
7 |
8 | #switch_midi2f0_step: 174000
9 | use_midi: true # for midi exp
10 | use_gt_f0: false # for midi exp
11 | use_gt_dur: false # for further midi exp
12 | lambda_f0: 1.0
13 | lambda_uv: 1.0
14 | #lambda_energy: 0.1
15 | lambda_ph_dur: 1.0
16 | lambda_sent_dur: 1.0
17 | lambda_word_dur: 1.0
18 | predictor_grad: 0.1
19 | pe_enable: false
20 | pe_ckpt: ''
21 |
22 | fs2_ckpt: 'checkpoints/0302_opencpop_fs_midi/model_ckpt_steps_160000.ckpt' #
23 | #num_valid_plots: 0
24 | task_cls: tasks.svs.diffsinger_task.DiffSingerMIDITask
25 |
26 | K_step: 60
27 | max_tokens: 36000
28 | predictor_layers: 5
29 | dilation_cycle_length: 4 # *
30 | rel_pos: true
31 | dur_predictor_layers: 5 # *
32 | max_updates: 160000
33 | gaussian_start: false
34 | mask_uv_prob: 0.15
--------------------------------------------------------------------------------
/NeuralSeq/egs/egs_bases/svs/midi/cascade/opencs/opencpop_statis.yaml:
--------------------------------------------------------------------------------
1 | spec_min: [-6., -6., -6., -6., -6., -6., -6., -6., -6., -6., -6., -6.,
2 | -6., -6., -6., -6., -6., -6., -6., -6., -6., -6., -6., -6.,
3 | -6., -6., -6., -6., -6., -6., -6., -6., -6., -6., -6., -6.,
4 | -6., -6., -6., -6., -6., -6., -6., -6., -6., -6., -6., -6.,
5 | -6., -6., -6., -6., -6., -6., -6., -6., -6., -6., -6., -6.,
6 | -6., -6., -6., -6., -6., -6., -6., -6., -6., -6., -6., -6.,
7 | -6., -6., -6., -6., -6., -6., -6., -6.]
8 | spec_max: [-7.9453e-01, -8.1116e-01, -6.1631e-01, -3.0679e-01, -1.3863e-01,
9 | -5.0652e-02, -1.1563e-01, -1.0679e-01, -9.1068e-02, -6.2174e-02,
10 | -7.5302e-02, -7.2217e-02, -6.3815e-02, -7.3299e-02, 7.3610e-03,
11 | -7.2508e-02, -5.0234e-02, -1.6534e-01, -2.6928e-01, -2.0782e-01,
12 | -2.0823e-01, -1.1702e-01, -7.0128e-02, -6.5868e-02, -1.2675e-02,
13 | 1.5121e-03, -8.9902e-02, -2.1392e-01, -2.3789e-01, -2.8922e-01,
14 | -3.0405e-01, -2.3029e-01, -2.2088e-01, -2.1542e-01, -2.9367e-01,
15 | -3.0137e-01, -3.8281e-01, -4.3590e-01, -2.8681e-01, -4.6855e-01,
16 | -5.7485e-01, -4.7022e-01, -5.4266e-01, -4.4848e-01, -6.4120e-01,
17 | -6.8700e-01, -6.4860e-01, -7.6436e-01, -4.9971e-01, -7.1068e-01,
18 | -6.9724e-01, -6.1487e-01, -5.5843e-01, -6.9773e-01, -5.7502e-01,
19 | -7.0919e-01, -8.2431e-01, -8.4213e-01, -9.0431e-01, -8.2840e-01,
20 | -7.7945e-01, -8.2758e-01, -8.7699e-01, -1.0532e+00, -1.0766e+00,
21 | -1.1198e+00, -1.0185e+00, -9.8983e-01, -1.0001e+00, -1.0756e+00,
22 | -1.0024e+00, -1.0304e+00, -1.0579e+00, -1.0188e+00, -1.0500e+00,
23 | -1.0842e+00, -1.0923e+00, -1.1223e+00, -1.2381e+00, -1.6467e+00]
24 |
25 | mel_vmin: -6. #-6.
26 | mel_vmax: 1.5
27 | wav2spec_eps: 1e-6
28 |
29 | raw_data_dir: 'data/raw/opencpop/segments'
30 | processed_data_dir: 'xxx'
31 | binary_data_dir: 'data/binary/opencpop-midi-dp'
32 | datasets: [
33 | 'opencpop',
34 | ]
35 | test_prefixes: [
36 | '2044',
37 | '2086',
38 | '2092',
39 | '2093',
40 | '2100',
41 | ]
42 |
--------------------------------------------------------------------------------
/NeuralSeq/egs/egs_bases/svs/midi/e2e/opencpop/ds1000-10dil.yaml:
--------------------------------------------------------------------------------
1 | base_config:
2 | - egs/egs_bases/svs/popcs_ds_beta6.yaml
3 | - egs/egs_bases/svs/midi/cascade/opencs/opencpop_statis.yaml
4 |
5 | binarizer_cls: data_gen.singing.binarize.OpencpopBinarizer
6 | binary_data_dir: 'data/binary/opencpop-midi-dp'
7 |
8 | #switch_midi2f0_step: 174000
9 | use_midi: true # for midi exp
10 | use_gt_dur: false # for further midi exp
11 | lambda_ph_dur: 1.0
12 | lambda_sent_dur: 1.0
13 | lambda_word_dur: 1.0
14 | predictor_grad: 0.1
15 | dur_predictor_layers: 5 # *
16 |
17 |
18 | fs2_ckpt: '' #
19 | #num_valid_plots: 0
20 | task_cls: tasks.svs.diffsinger_task.DiffSingerMIDITask
21 |
22 | timesteps: 1000
23 | K_step: 1000
24 | max_beta: 0.02
25 | max_tokens: 36000
26 | max_updates: 320000
27 | gaussian_start: True
28 |
29 | use_pitch_embed: false
30 | use_gt_f0: false # for midi exp
31 |
32 | lambda_f0: 0.
33 | lambda_uv: 0.
34 | dilation_cycle_length: 10 # *
35 | rel_pos: true
36 | predictor_layers: 5
37 | pe_enable: true
38 | pe_ckpt: 'checkpoints/0102_xiaoma_pe'
39 |
--------------------------------------------------------------------------------
/NeuralSeq/egs/egs_bases/svs/midi/e2e/opencpop/ds1000.yaml:
--------------------------------------------------------------------------------
1 | base_config:
2 | - egs/egs_bases/svs/popcs_ds_beta6.yaml
3 | - egs/egs_bases/svs/midi/cascade/opencs/opencpop_statis.yaml
4 |
5 | binarizer_cls: data_gen.singing.binarize.OpencpopBinarizer
6 | binary_data_dir: 'data/binary/opencpop-midi-dp'
7 |
8 | #switch_midi2f0_step: 174000
9 | use_midi: true # for midi exp
10 | use_gt_dur: false # for further midi exp
11 | lambda_ph_dur: 1.0
12 | lambda_sent_dur: 1.0
13 | lambda_word_dur: 1.0
14 | predictor_grad: 0.1
15 | dur_predictor_layers: 5 # *
16 |
17 |
18 | fs2_ckpt: '' #
19 | #num_valid_plots: 0
20 | task_cls: tasks.svs.diffsinger_task.DiffSingerMIDITask
21 |
22 | # for diffusion schedule
23 | timesteps: 1000
24 | K_step: 1000
25 | max_beta: 0.02
26 | max_tokens: 36000
27 | max_updates: 320000
28 | gaussian_start: True
29 | pndm_speedup: 10
30 |
31 | use_pitch_embed: false
32 | use_gt_f0: false # for midi exp
33 |
34 | lambda_f0: 0.
35 | lambda_uv: 0.
36 | dilation_cycle_length: 4 # *
37 | rel_pos: true
38 | predictor_layers: 5
39 | pe_enable: true
40 | pe_ckpt: 'checkpoints/0102_xiaoma_pe'
41 |
42 |
43 |
--------------------------------------------------------------------------------
/NeuralSeq/egs/egs_bases/svs/midi/e2e/opencpop/ds100_adj_rel.yaml:
--------------------------------------------------------------------------------
1 | base_config:
2 | - egs/egs_bases/svs/popcs_ds_beta6.yaml
3 | - egs/egs_bases/svs/midi/cascade/opencs/opencpop_statis.yaml
4 |
5 | binarizer_cls: data_gen.singing.binarize.OpencpopBinarizer
6 | binary_data_dir: 'data/binary/opencpop-midi-dp'
7 |
8 | #switch_midi2f0_step: 174000
9 | use_midi: true # for midi exp
10 | use_gt_dur: false # for further midi exp
11 | lambda_ph_dur: 1.0
12 | lambda_sent_dur: 1.0
13 | lambda_word_dur: 1.0
14 | predictor_grad: 0.1
15 | dur_predictor_layers: 5 # *
16 |
17 |
18 | fs2_ckpt: '' #
19 | #num_valid_plots: 0
20 | task_cls: tasks.svs.diffsinger_task.DiffSingerMIDITask
21 |
22 | K_step: 100
23 | max_tokens: 36000
24 | max_updates: 160000
25 | gaussian_start: True
26 |
27 | use_pitch_embed: false
28 | use_gt_f0: false # for midi exp
29 |
30 | lambda_f0: 0.
31 | lambda_uv: 0.
32 | dilation_cycle_length: 4 # *
33 | rel_pos: true
34 | predictor_layers: 5
35 | pe_enable: true
36 | pe_ckpt: 'checkpoints/0102_xiaoma_pe'
37 |
--------------------------------------------------------------------------------
/NeuralSeq/egs/egs_bases/svs/midi/e2e/popcs/ds100_adj_rel.yaml:
--------------------------------------------------------------------------------
1 | base_config:
2 | - egs/egs_bases/svs/popcs_ds_beta6.yaml
3 | - egs/egs_bases/svs/midi/cascade/popcs/popcs_statis.yaml
4 |
5 | binarizer_cls: data_gen.singing.binarize.MidiSingingBinarizer
6 | binary_data_dir: 'data/binary/popcs-midi-dp'
7 |
8 | #switch_midi2f0_step: 174000
9 | use_midi: true # for midi exp
10 | use_gt_dur: false # for further midi exp
11 | lambda_ph_dur: 1.0
12 | lambda_sent_dur: 1.0
13 | lambda_word_dur: 1.0
14 | predictor_grad: 0.1
15 | dur_predictor_layers: 5 # *
16 |
17 |
18 | fs2_ckpt: '' #
19 | #num_valid_plots: 0
20 | task_cls: tasks.svs.diffsinger_task.DiffSingerMIDITask
21 |
22 | K_step: 100
23 | max_tokens: 40000
24 | max_updates: 160000
25 | gaussian_start: True
26 |
27 | use_pitch_embed: false
28 | use_gt_f0: false # for midi exp
29 |
30 | lambda_f0: 0.
31 | lambda_uv: 0.
32 | dilation_cycle_length: 4 # *
33 | rel_pos: true
34 | predictor_layers: 5
35 | pe_enable: true
36 | pe_ckpt: 'checkpoints/0102_xiaoma_pe'
37 |
--------------------------------------------------------------------------------
/NeuralSeq/egs/egs_bases/svs/midi/pe.yaml:
--------------------------------------------------------------------------------
1 | base_config:
2 | - configs/tts/lj/fs2.yaml
3 |
4 | max_frames: 8000
5 | audio_sample_rate: 24000
6 | hop_size: 128 # Hop size.
7 | fft_size: 512 # FFT size.
8 | win_size: 512 # FFT size.
9 | fmin: 30
10 | fmax: 12000
11 | min_level_db: -120
12 |
13 | binary_data_dir: 'xxx'
14 |
15 | pitch_type: frame
16 | task_cls: tasks.tts.pe.PitchExtractionTask
17 | pitch_extractor_conv_layers: 2
18 |
19 |
20 | # config for experiments
21 | max_tokens: 20000
22 | use_spk_embed: false
23 | num_valid_plots: 10
24 | max_updates: 60000
--------------------------------------------------------------------------------
/NeuralSeq/egs/egs_bases/svs/popcs_ds_beta6.yaml:
--------------------------------------------------------------------------------
1 | base_config:
2 | - configs/tts/fs2.yaml
3 | - configs/singing/base.yaml
4 | - ./base.yaml
5 |
6 | audio_sample_rate: 24000
7 | hop_size: 128 # Hop size.
8 | fft_size: 512 # FFT size.
9 | win_size: 512 # FFT size.
10 | fmin: 30
11 | fmax: 12000
12 | min_level_db: -120
13 |
14 | binarization_args:
15 | with_wav: true
16 | with_spk_embed: false
17 | with_align: true
18 | raw_data_dir: 'data/raw/popcs'
19 | processed_data_dir: 'data/processed/popcs'
20 | binary_data_dir: 'data/binary/popcs-pmf0'
21 | num_spk: 1
22 | datasets: [
23 | 'popcs',
24 | ]
25 | test_prefixes: [
26 | 'popcs-说散就散',
27 | 'popcs-隐形的翅膀',
28 | ]
29 |
30 | spec_min: [-6.8276, -7.0270, -6.8142, -7.1429, -7.6669, -7.6000, -7.1148, -6.9640,
31 | -6.8414, -6.6596, -6.6880, -6.7439, -6.7986, -7.4940, -7.7845, -7.6586,
32 | -6.9288, -6.7639, -6.9118, -6.8246, -6.7183, -7.1769, -6.9794, -7.4513,
33 | -7.3422, -7.5623, -6.9610, -6.8158, -6.9595, -6.8403, -6.5688, -6.6356,
34 | -7.0209, -6.5002, -6.7819, -6.5232, -6.6927, -6.5701, -6.5531, -6.7069,
35 | -6.6462, -6.4523, -6.5954, -6.4264, -6.4487, -6.7070, -6.4025, -6.3042,
36 | -6.4008, -6.3857, -6.3903, -6.3094, -6.2491, -6.3518, -6.3566, -6.4168,
37 | -6.2481, -6.3624, -6.2858, -6.2575, -6.3638, -6.4520, -6.1835, -6.2754,
38 | -6.1253, -6.1645, -6.0638, -6.1262, -6.0710, -6.1039, -6.4428, -6.1363,
39 | -6.1054, -6.1252, -6.1797, -6.0235, -6.0758, -5.9453, -6.0213, -6.0446]
40 | spec_max: [ 0.2645, 0.0583, -0.2344, -0.0184, 0.1227, 0.1533, 0.1103, 0.1212,
41 | 0.2421, 0.1809, 0.2134, 0.3161, 0.3301, 0.3289, 0.2667, 0.2421,
42 | 0.2581, 0.2600, 0.1394, 0.1907, 0.1082, 0.1474, 0.1680, 0.2550,
43 | 0.1057, 0.0826, 0.0423, 0.1203, -0.0701, -0.0056, 0.0477, -0.0639,
44 | -0.0272, -0.0728, -0.1648, -0.0855, -0.2652, -0.1998, -0.1547, -0.2167,
45 | -0.4181, -0.5463, -0.4161, -0.4733, -0.6518, -0.5387, -0.4290, -0.4191,
46 | -0.4151, -0.3042, -0.3810, -0.4160, -0.4496, -0.2847, -0.4676, -0.4658,
47 | -0.4931, -0.4885, -0.5547, -0.5481, -0.6948, -0.7968, -0.8455, -0.8392,
48 | -0.8770, -0.9520, -0.8749, -0.7297, -0.8374, -0.8667, -0.7157, -0.9035,
49 | -0.9219, -0.8801, -0.9298, -0.9009, -0.9604, -1.0537, -1.0781, -1.3766]
50 |
51 | task_cls: tasks.svs.diffsinger_task.DiffSingerTask
52 | #vocoder: tasks.svs.singingvocoder.highgan.HighGAN
53 | #vocoder_ckpt: checkpoints/h_2_model/checkpoint-530000steps.pkl
54 | vocoder: vocoders.hifigan.HifiGAN
55 | vocoder_ckpt: checkpoints/0109_hifigan_bigpopcs_hop128
56 |
57 | pitch_extractor: 'parselmouth'
58 | # config for experiments
59 | use_spk_embed: false
60 | num_valid_plots: 10
61 | max_updates: 160000
62 | lr: 0.001
63 | timesteps: 100
64 | K_step: 51
65 | diff_loss_type: l1
66 | diff_decoder_type: 'wavenet'
67 | schedule_type: 'linear'
68 | max_beta: 0.06
69 | fs2_ckpt: ''
70 | use_nsf: true
--------------------------------------------------------------------------------
/NeuralSeq/egs/egs_bases/svs/popcs_ds_beta6_offline.yaml:
--------------------------------------------------------------------------------
1 | base_config:
2 | - ./popcs_ds_beta6.yaml
3 |
4 | fs2_ckpt: checkpoints/popcs_fs2_pmf0_1230/model_ckpt_steps_160000.ckpt # to be infer
5 | num_valid_plots: 0
6 | task_cls: tasks.svs.diffsinger_task.DiffSingerOfflineTask
7 |
8 | # tmp:
9 | #pe_enable: true
10 | #pe_ckpt: ''
11 | vocoder: vocoders.hifigan.HifiGAN
12 | vocoder_ckpt: checkpoints/0109_hifigan_bigpopcs_hop128
--------------------------------------------------------------------------------
/NeuralSeq/egs/egs_bases/svs/popcs_fs2.yaml:
--------------------------------------------------------------------------------
1 | base_config:
2 | - configs/singing/fs2.yaml
3 |
4 | audio_sample_rate: 24000
5 | hop_size: 128 # Hop size.
6 | fft_size: 512 # FFT size.
7 | win_size: 512 # FFT size.
8 | fmin: 30
9 | fmax: 12000
10 | min_level_db: -120
11 |
12 | binarization_args:
13 | with_wav: true
14 | with_spk_embed: false
15 | with_align: true
16 | raw_data_dir: 'data/raw/popcs'
17 | processed_data_dir: 'data/processed/popcs'
18 | binary_data_dir: 'data/binary/popcs-pmf0'
19 | num_spk: 1
20 | datasets: [
21 | 'popcs',
22 | ]
23 | test_prefixes: [
24 | 'popcs-说散就散',
25 | 'popcs-隐形的翅膀',
26 | ]
27 |
28 | task_cls: tasks.tts.fs2.FastSpeech2Task
29 | #vocoder: tasks.svs.singingvocoder.highgan.HighGAN
30 | #vocoder_ckpt: checkpoints/h_2_model/checkpoint-530000steps.pkl
31 | vocoder: vocoders.hifigan.HifiGAN
32 | vocoder_ckpt: checkpoints/0109_hifigan_bigpopcs_hop128
33 | use_nsf: true
34 |
35 | # config for experiments
36 | max_tokens: 18000
37 | use_spk_embed: false
38 | num_valid_plots: 10
39 | max_updates: 160000
40 | save_gt: true
41 |
42 | # tmp:
43 | #pe_enable: true
44 | #pe_ckpt: ''
--------------------------------------------------------------------------------
/NeuralSeq/egs/egs_bases/tts/base.yaml:
--------------------------------------------------------------------------------
1 | # task
2 | base_config: ../config_base.yaml
3 | task_cls: ''
4 | #############
5 | # dataset
6 | #############
7 | raw_data_dir: ''
8 | processed_data_dir: ''
9 | binary_data_dir: ''
10 | dict_dir: ''
11 | pre_align_cls: ''
12 | binarizer_cls: data_gen.tts.base_binarizer.BaseBinarizer
13 | pre_align_args:
14 | txt_processor: en
15 | use_tone: true # for ZH
16 | sox_resample: false
17 | sox_to_wav: false
18 | allow_no_txt: false
19 | trim_sil: false
20 | denoise: false
21 | binarization_args:
22 | shuffle: false
23 | with_txt: true
24 | with_wav: false
25 | with_align: true
26 | with_spk_embed: false
27 | with_spk_id: true
28 | with_f0: true
29 | with_f0cwt: false
30 | with_linear: false
31 | with_word: true
32 | trim_sil: false
33 | trim_eos_bos: false
34 | reset_phone_dict: true
35 | reset_word_dict: true
36 | word_size: 30000
37 | pitch_extractor: parselmouth
38 |
39 | loud_norm: false
40 | endless_ds: true
41 |
42 | test_num: 100
43 | min_frames: 0
44 | max_frames: 1548
45 | frames_multiple: 1
46 | max_input_tokens: 1550
47 | audio_num_mel_bins: 80
48 | audio_sample_rate: 22050
49 | hop_size: 256 # For 22050Hz, 275 ~= 12.5 ms (0.0125 * sample_rate)
50 | win_size: 1024 # For 22050Hz, 1100 ~= 50 ms (If None, win_size: fft_size) (0.05 * sample_rate)
51 | fmin: 80 # Set this to 55 if your speaker is male! if female, 95 should help taking off noise. (To test depending on dataset. Pitch info: male~[65, 260], female~[100, 525])
52 | fmax: 7600 # To be increased/reduced depending on data.
53 | fft_size: 1024 # Extra window size is filled with 0 paddings to match this parameter
54 | min_level_db: -100
55 | ref_level_db: 20
56 | griffin_lim_iters: 60
57 | num_spk: 1
58 | mel_vmin: -6
59 | mel_vmax: 1.5
60 | ds_workers: 1
61 |
62 | #########
63 | # model
64 | #########
65 | dropout: 0.1
66 | enc_layers: 4
67 | dec_layers: 4
68 | hidden_size: 256
69 | num_heads: 2
70 | enc_ffn_kernel_size: 9
71 | dec_ffn_kernel_size: 9
72 | ffn_act: gelu
73 | ffn_padding: 'SAME'
74 | use_spk_id: false
75 | use_split_spk_id: false
76 | use_spk_embed: false
77 |
78 |
79 | ###########
80 | # optimization
81 | ###########
82 | lr: 2.0
83 | scheduler: rsqrt # rsqrt|none
84 | warmup_updates: 8000
85 | optimizer_adam_beta1: 0.9
86 | optimizer_adam_beta2: 0.98
87 | weight_decay: 0
88 | clip_grad_norm: 1
89 | clip_grad_value: 0
90 |
91 |
92 | ###########
93 | # train and eval
94 | ###########
95 | max_tokens: 30000
96 | max_sentences: 100000
97 | max_valid_sentences: 1
98 | max_valid_tokens: 60000
99 | valid_infer_interval: 10000
100 | train_set_name: 'train'
101 | train_sets: ''
102 | valid_set_name: 'valid'
103 | test_set_name: 'test'
104 | num_test_samples: 0
105 | num_valid_plots: 10
106 | test_ids: [ ]
107 | vocoder_denoise_c: 0.0
108 | profile_infer: false
109 | out_wav_norm: false
110 | save_gt: true
111 | save_f0: false
112 | gen_dir_name: ''
--------------------------------------------------------------------------------
/NeuralSeq/egs/egs_bases/tts/base_zh.yaml:
--------------------------------------------------------------------------------
1 | base_config: ./base.yaml
2 | preprocess_args:
3 | txt_processor: zh
4 | use_tone: true
5 |
6 | word_size: 3000
--------------------------------------------------------------------------------
/NeuralSeq/egs/egs_bases/tts/fs2.yaml:
--------------------------------------------------------------------------------
1 | base_config: ./base.yaml
2 | task_cls: tasks.tts.fs2.FastSpeech2Task
3 |
4 | # model
5 | hidden_size: 256
6 | dropout: 0.1
7 | encoder_type: fft # rel_fft|fft|tacotron|tacotron2|conformer
8 | decoder_type: fft # fft|rnn|conv|conformer|wn
9 |
10 | # rnn enc/dec
11 | encoder_K: 8
12 | decoder_rnn_dim: 0 # for rnn decoder, 0 -> hidden_size * 2
13 |
14 | # fft enc/dec
15 | use_pos_embed: true
16 | dec_num_heads: 2
17 | dec_layers: 4
18 | ffn_hidden_size: 1024
19 | enc_ffn_kernel_size: 9
20 | dec_ffn_kernel_size: 9
21 |
22 | # conv enc/dec
23 | enc_dec_norm: ln
24 | conv_use_pos: false
25 | layers_in_block: 2
26 | enc_dilations: [ 1, 1, 1, 1 ]
27 | enc_kernel_size: 5
28 | dec_dilations: [ 1, 1, 1, 1 ] # for conv decoder
29 | dec_kernel_size: 5
30 | dur_loss: mse # huber|mol
31 |
32 | # duration
33 | predictor_hidden: -1
34 | predictor_kernel: 5
35 | predictor_layers: 2
36 | dur_predictor_kernel: 3
37 | dur_predictor_layers: 2
38 | predictor_dropout: 0.5
39 |
40 | # pitch and energy
41 | pitch_norm: standard # standard|log
42 | use_pitch_embed: true
43 | pitch_type: frame # frame|ph|cwt
44 | use_uv: true
45 | cwt_hidden_size: 128
46 | cwt_layers: 2
47 | cwt_loss: l1
48 | cwt_add_f0_loss: false
49 | cwt_std_scale: 0.8
50 |
51 | pitch_ar: false
52 | pitch_embed_type: 0
53 | pitch_loss: 'l1' # l1|l2|ssim
54 | pitch_ssim_win: 11
55 | use_energy_embed: false
56 |
57 | # reference encoder and speaker embedding
58 | use_ref_enc: false
59 | use_var_enc: false
60 | lambda_commit: 0.25
61 | var_enc_vq_codes: 64
62 | ref_norm_layer: bn
63 | dec_inp_add_noise: false
64 | sil_add_noise: false
65 | ref_hidden_stride_kernel:
66 | - 0,3,5 # conv_hidden_size, conv_stride, conv_kernel_size. conv_hidden_size=0: use hidden_size
67 | - 0,3,5
68 | - 0,2,5
69 | - 0,2,5
70 | - 0,2,5
71 | pitch_enc_hidden_stride_kernel:
72 | - 0,2,5 # conv_hidden_size, conv_stride, conv_kernel_size. conv_hidden_size=0: use hidden_size
73 | - 0,2,5
74 | - 0,2,5
75 | dur_enc_hidden_stride_kernel:
76 | - 0,2,3 # conv_hidden_size, conv_stride, conv_kernel_size. conv_hidden_size=0: use hidden_size
77 | - 0,2,3
78 | - 0,1,3
79 |
80 | # mel
81 | mel_loss: l1:0.5|ssim:0.5 # l1|l2|gdl|ssim or l1:0.5|ssim:0.5
82 |
83 | # loss lambda
84 | lambda_f0: 1.0
85 | lambda_uv: 1.0
86 | lambda_energy: 0.1
87 | lambda_ph_dur: 0.1
88 | lambda_sent_dur: 1.0
89 | lambda_word_dur: 1.0
90 | predictor_grad: 0.1
91 |
92 | # train and eval
93 | pretrain_fs_ckpt: ''
94 | warmup_updates: 2000
95 | max_tokens: 32000
96 | max_sentences: 100000
97 | max_valid_sentences: 1
98 | max_updates: 120000
99 | use_gt_dur: false
100 | use_gt_f0: false
101 | ds_workers: 2
102 | lr: 1.0
103 |
--------------------------------------------------------------------------------
/NeuralSeq/egs/egs_bases/tts/fs2_adv.yaml:
--------------------------------------------------------------------------------
1 | base_config: ./fs2.yaml
2 | task_cls: tasks.tts.fs2_adv.FastSpeech2AdvTask
3 |
4 | disc_win_num: 3
5 | disc_interval: 1
6 | disc_reduction: stack # stack|sum|none
7 | disc_start_steps: 0
8 | rerun_gen: false
9 |
10 | disc_norm: in
11 | mel_disc_hidden_size: 128
12 |
13 | # mel decoder
14 | mel_gan: true
15 | lambda_mel_adv: 0.1
16 | mel_hidden_size: 256
17 |
18 | # others
19 | dropout: 0.05
20 | pitch_embed_type: 0
21 | enc_ffn_kernel_size: 9
22 | dec_ffn_kernel_size: 9
23 | use_cond_disc: false
24 |
25 | optimizer_adam_beta1: 0.5
26 | optimizer_adam_beta2: 0.999
27 | generator_grad_norm: 5.0 # Generator's gradient norm.
28 | disc_hidden_size: 128
29 | disc_lr: 0.0001 # Discriminator's learning rate.
30 | discriminator_optimizer_params:
31 | eps: 1.0e-6 # Discriminator's epsilon.
32 | weight_decay: 0.0 # Discriminator's weight decay coefficient.
33 | discriminator_scheduler_params:
34 | step_size: 60000 # Discriminator's scheduler step size.
35 | gamma: 0.5 # D5iscriminator's scheduler gamma.
36 | # At each step size, lr will be multiplied by this parameter.
37 | discriminator_grad_norm: 1 # Discriminator's gradient norm.
38 |
39 | max_updates: 400000
40 | max_tokens: 30000
41 | max_sentences: 80
42 | val_check_interval: 2000
43 |
44 | gen_dir_name: ''
45 | num_ckpt_keep: 2
46 | save_best: false
47 |
48 |
--------------------------------------------------------------------------------
/NeuralSeq/egs/egs_bases/tts/ps.yaml:
--------------------------------------------------------------------------------
1 | base_config: ./fs2.yaml
2 |
3 | ###########################
4 | # models
5 | ###########################
6 | # encoders
7 | hidden_size: 192
8 | ffn_hidden_size: 768
9 | enc_ffn_kernel_size: 5
10 | enc_layers: 4
11 | dur_level: word
12 | encoder_type: rel_fft
13 | use_word_encoder: true
14 |
15 | # mix ling encoder
16 | word_enc_layers: 4
17 | word_encoder_type: rel_fft
18 | use_pitch_embed: false
19 | enc_prenet: true
20 | enc_pre_ln: true
21 | text_encoder_postnet: true
22 | dropout: 0.0
23 | add_word_pos: true
24 |
25 | # dur predictor
26 | dur_predictor_layers: 3
27 | dur_predictor_kernel: 5
28 | predictor_dropout: 0.2
29 |
30 | ## fvae
31 | use_fvae: true
32 | latent_size: 16
33 | fvae_encoder_type: conv
34 | fvae_decoder_type: conv
35 | fvae_enc_dec_hidden: 192
36 | fvae_kernel_size: 5
37 | fvae_enc_n_layers: 8
38 | fvae_dec_n_layers: 4
39 | fvae_strides: 4
40 | fvae_noise_scale: 1.0
41 |
42 | # prior flow
43 | use_prior_flow: true
44 | prior_flow_hidden: 64
45 | prior_flow_kernel_size: 3
46 | prior_flow_n_blocks: 4
47 |
48 | ###########################
49 | # training and inference
50 | ###########################
51 | lambda_kl: 1.0
52 | kl_min: 0.0
53 | lambda_sent_dur: 0.0
54 | kl_start_steps: 10000
55 | posterior_start_steps: 0
56 | frames_multiple: 4
57 | num_valid_plots: 10
58 | lr: 0.0002
59 | warmup_updates: 8000
60 | max_tokens: 40000
61 | valid_infer_interval: 10000
62 | max_sentences: 80
63 | max_updates: 480000
--------------------------------------------------------------------------------
/NeuralSeq/egs/egs_bases/tts/ps_flow.yaml:
--------------------------------------------------------------------------------
1 | base_config: ./ps2.yaml
2 | task_cls: tasks.tts.ps_flow.PortaSpeechFlowTask
3 |
4 | use_post_flow: true
5 | detach_postflow_input: true
6 | post_flow_lr: 0.001
7 | post_glow_hidden: 192
8 | post_glow_kernel_size: 3
9 | post_glow_n_blocks: 12
10 | post_glow_n_block_layers: 3
11 | post_share_cond_layers: false
12 | share_wn_layers: 4
13 | use_cond_proj: false
14 | use_latent_cond: false
15 | use_txt_cond: true
16 | sigmoid_scale: false
17 | post_glow_training_start: 160000
18 | noise_scale: 0.8
19 | infer_post_glow: true
20 | two_stage: true
--------------------------------------------------------------------------------
/NeuralSeq/egs/egs_bases/tts/ps_flow_small.yaml:
--------------------------------------------------------------------------------
1 | base_config: ./ps_flow.yaml
2 |
3 | ###########################
4 | # models
5 | ###########################
6 | # encoders
7 | hidden_size: 128
8 | ffn_hidden_size: 512
9 | enc_ffn_kernel_size: 3
10 | enc_layers: 3
11 | word_enc_layers: 3
12 |
13 | # dur predictor
14 | dur_predictor_layers: 3
15 | dur_predictor_kernel: 5
16 | predictor_dropout: 0.2
17 |
18 | ## fvae
19 | use_fvae: true
20 | latent_size: 16
21 | fvae_encoder_type: wn
22 | fvae_decoder_type: wn
23 | fvae_enc_dec_hidden: 128
24 | fvae_kernel_size: 3
25 | fvae_enc_n_layers: 8
26 | fvae_dec_n_layers: 3
27 | fvae_strides: 4
28 | fvae_noise_scale: 1.0
29 |
30 |
31 | # prior flow
32 | use_prior_flow: true
33 | prior_flow_hidden: 32
34 | prior_flow_kernel_size: 3
35 | prior_flow_n_blocks: 3
36 | # post flow
37 | post_glow_hidden: 128
38 | post_glow_kernel_size: 3
39 | post_glow_n_blocks: 8
40 | post_glow_n_block_layers: 3
41 | share_wn_layers: 4
42 | noise_scale: 0.6
--------------------------------------------------------------------------------
/NeuralSeq/egs/egs_bases/tts/vocoder/base.yaml:
--------------------------------------------------------------------------------
1 | base_config: ../base.yaml
2 | binarization_args:
3 | with_wav: true
4 | with_spk_embed: false
5 | with_align: false
6 | with_word: false
7 | with_txt: false
8 |
9 | ###########
10 | # train and eval
11 | ###########
12 | max_samples: 25600
13 | max_sentences: 5
14 | max_valid_sentences: 1
15 | max_updates: 1000000
16 | val_check_interval: 2000
17 |
18 | ###########################################################
19 | # FEATURE EXTRACTION SETTING #
20 | ###########################################################
21 | fft_size: 1024 # FFT size.
22 | hop_size: 256 # Hop size.
23 | win_length: null # Window length.
24 | # If set to null, it will be the same as fft_size.
25 | window: "hann" # Window function.
26 | num_mels: 80 # Number of mel basis.
27 | fmin: 80 # Minimum freq in mel basis calculation.
28 | fmax: 7600 # Maximum frequency in mel basis calculation.
29 | aux_context_window: 0 # Context window size for auxiliary feature.
30 | use_pitch_embed: false
31 |
32 | generator_grad_norm: 10 # Generator's gradient norm.
33 | discriminator_grad_norm: 1 # Discriminator's gradient norm.
34 | disc_start_steps: 40000 # Number of steps to start to train discriminator.
35 |
--------------------------------------------------------------------------------
/NeuralSeq/egs/egs_bases/tts/vocoder/hifigan.yaml:
--------------------------------------------------------------------------------
1 | base_config: ./base.yaml
2 | task_cls: tasks.vocoder.hifigan.HifiGanTask
3 | resblock: "1"
4 | adam_b1: 0.8
5 | adam_b2: 0.99
6 | upsample_rates: [ 8,8,2,2 ]
7 | upsample_kernel_sizes: [ 16,16,4,4 ]
8 | upsample_initial_channel: 512
9 | resblock_kernel_sizes: [ 3,7,11 ]
10 | resblock_dilation_sizes: [ [ 1,3,5 ], [ 1,3,5 ], [ 1,3,5 ] ]
11 |
12 | use_pitch_embed: false
13 | use_fm_loss: false
14 | use_ms_stft: false
15 |
16 | lambda_mel: 5.0
17 | lambda_mel_adv: 1.0
18 | lambda_cdisc: 4.0
19 | lambda_adv: 1.0
20 |
21 | lr: 0.0002 # Generator's learning rate.
22 | generator_scheduler_params:
23 | step_size: 600
24 | gamma: 0.999
25 | discriminator_scheduler_params:
26 | step_size: 600
27 | gamma: 0.999
28 | max_updates: 3000000
--------------------------------------------------------------------------------
/NeuralSeq/gitattributes:
--------------------------------------------------------------------------------
1 | *.7z filter=lfs diff=lfs merge=lfs -text
2 | *.arrow filter=lfs diff=lfs merge=lfs -text
3 | *.bin filter=lfs diff=lfs merge=lfs -text
4 | *.bz2 filter=lfs diff=lfs merge=lfs -text
5 | *.ftz filter=lfs diff=lfs merge=lfs -text
6 | *.gz filter=lfs diff=lfs merge=lfs -text
7 | *.h5 filter=lfs diff=lfs merge=lfs -text
8 | *.joblib filter=lfs diff=lfs merge=lfs -text
9 | *.lfs.* filter=lfs diff=lfs merge=lfs -text
10 | *.model filter=lfs diff=lfs merge=lfs -text
11 | *.msgpack filter=lfs diff=lfs merge=lfs -text
12 | *.npy filter=lfs diff=lfs merge=lfs -text
13 | *.npz filter=lfs diff=lfs merge=lfs -text
14 | *.onnx filter=lfs diff=lfs merge=lfs -text
15 | *.ot filter=lfs diff=lfs merge=lfs -text
16 | *.parquet filter=lfs diff=lfs merge=lfs -text
17 | *.pickle filter=lfs diff=lfs merge=lfs -text
18 | *.pkl filter=lfs diff=lfs merge=lfs -text
19 | *.pb filter=lfs diff=lfs merge=lfs -text
20 | *.pt filter=lfs diff=lfs merge=lfs -text
21 | *.pth filter=lfs diff=lfs merge=lfs -text
22 | *.rar filter=lfs diff=lfs merge=lfs -text
23 | saved_model/**/* filter=lfs diff=lfs merge=lfs -text
24 | *.tar.* filter=lfs diff=lfs merge=lfs -text
25 | *.tflite filter=lfs diff=lfs merge=lfs -text
26 | *.tgz filter=lfs diff=lfs merge=lfs -text
27 | *.wasm filter=lfs diff=lfs merge=lfs -text
28 | *.xz filter=lfs diff=lfs merge=lfs -text
29 | *.zip filter=lfs diff=lfs merge=lfs -text
30 | *.zstandard filter=lfs diff=lfs merge=lfs -text
31 | *tfevents* filter=lfs diff=lfs merge=lfs -text
32 | model_ckpt_steps* filter=lfs diff=lfs merge=lfs -text
33 | checkpoints/0831_opencpop_ds1000 filter=lfs diff=lfs merge=lfs -text
34 |
--------------------------------------------------------------------------------
/NeuralSeq/inference/svs/ds_cascade.py:
--------------------------------------------------------------------------------
1 | import torch
2 | from inference.svs.base_svs_infer import BaseSVSInfer
3 | from utils import load_ckpt
4 | from utils.hparams import hparams
5 | from modulesmodules.diff.shallow_diffusion_tts import GaussianDiffusion
6 | from tasks.svs.diffsinger_task import DIFF_DECODERS
7 |
8 | class DiffSingerCascadeInfer(BaseSVSInfer):
9 | def build_model(self):
10 | model = GaussianDiffusion(
11 | phone_encoder=self.ph_encoder,
12 | out_dims=hparams['audio_num_mel_bins'], denoise_fn=DIFF_DECODERS[hparams['diff_decoder_type']](hparams),
13 | timesteps=hparams['timesteps'],
14 | K_step=hparams['K_step'],
15 | loss_type=hparams['diff_loss_type'],
16 | spec_min=hparams['spec_min'], spec_max=hparams['spec_max'],
17 | )
18 | model.eval()
19 | load_ckpt(model, hparams['work_dir'], 'model')
20 | return model
21 |
22 | def forward_model(self, inp):
23 | sample = self.input_to_batch(inp)
24 | txt_tokens = sample['txt_tokens'] # [B, T_t]
25 | spk_id = sample.get('spk_ids')
26 | with torch.no_grad():
27 | output = self.model(txt_tokens, spk_id=spk_id, ref_mels=None, infer=True,
28 | pitch_midi=sample['pitch_midi'], midi_dur=sample['midi_dur'],
29 | is_slur=sample['is_slur'])
30 | mel_out = output['mel_out'] # [B, T,80]
31 | f0_pred = output['f0_denorm']
32 | wav_out = self.run_vocoder(mel_out, f0=f0_pred)
33 | wav_out = wav_out.cpu().numpy()
34 | return wav_out[0]
35 |
36 |
37 | if __name__ == '__main__':
38 | inp = {
39 | 'text': '小酒窝长睫毛AP是你最美的记号',
40 | 'notes': 'C#4/Db4 | F#4/Gb4 | G#4/Ab4 | A#4/Bb4 F#4/Gb4 | F#4/Gb4 C#4/Db4 | C#4/Db4 | rest | C#4/Db4 | A#4/Bb4 | G#4/Ab4 | A#4/Bb4 | G#4/Ab4 | F4 | C#4/Db4',
41 | 'notes_duration': '0.407140 | 0.376190 | 0.242180 | 0.509550 0.183420 | 0.315400 0.235020 | 0.361660 | 0.223070 | 0.377270 | 0.340550 | 0.299620 | 0.344510 | 0.283770 | 0.323390 | 0.360340',
42 | 'input_type': 'word'
43 | } # user input: Chinese characters
44 | c = {
45 | 'text': '小酒窝长睫毛AP是你最美的记号',
46 | 'ph_seq': 'x iao j iu w o ch ang ang j ie ie m ao AP sh i n i z ui m ei d e j i h ao',
47 | 'note_seq': 'C#4/Db4 C#4/Db4 F#4/Gb4 F#4/Gb4 G#4/Ab4 G#4/Ab4 A#4/Bb4 A#4/Bb4 F#4/Gb4 F#4/Gb4 F#4/Gb4 C#4/Db4 C#4/Db4 C#4/Db4 rest C#4/Db4 C#4/Db4 A#4/Bb4 A#4/Bb4 G#4/Ab4 G#4/Ab4 A#4/Bb4 A#4/Bb4 G#4/Ab4 G#4/Ab4 F4 F4 C#4/Db4 C#4/Db4',
48 | 'note_dur_seq': '0.407140 0.407140 0.376190 0.376190 0.242180 0.242180 0.509550 0.509550 0.183420 0.315400 0.315400 0.235020 0.361660 0.361660 0.223070 0.377270 0.377270 0.340550 0.340550 0.299620 0.299620 0.344510 0.344510 0.283770 0.283770 0.323390 0.323390 0.360340 0.360340',
49 | 'is_slur_seq': '0 0 0 0 0 0 0 0 1 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0',
50 | 'input_type': 'phoneme'
51 | } # input like Opencpop dataset.
52 | DiffSingerCascadeInfer.example_run(inp)
53 |
54 | # # CUDA_VISIBLE_DEVICES=1 python inference/svs/ds_cascade.py --config egs/egs_bases/svs/midi/cascade/opencs/ds60_rel.yaml --exp_name 0303_opencpop_ds58_midi
--------------------------------------------------------------------------------
/NeuralSeq/inference/svs/opencpop/map.py:
--------------------------------------------------------------------------------
1 | def cpop_pinyin2ph_func():
2 | # In the README file of opencpop dataset, they defined a "pinyin to phoneme mapping table"
3 | pinyin2phs = {'AP': 'AP', 'SP': 'SP'}
4 | with open('NeuralSeq/inference/svs/opencpop/cpop_pinyin2ph.txt') as rf:
5 | for line in rf.readlines():
6 | elements = [x.strip() for x in line.split('|') if x.strip() != '']
7 | pinyin2phs[elements[0]] = elements[1]
8 | return pinyin2phs
--------------------------------------------------------------------------------
/NeuralSeq/modules/GenerSpeech/config/generspeech.yaml:
--------------------------------------------------------------------------------
1 | base_config:
2 | - egs/egs_bases/tts/fs2.yaml
3 | - egs/datasets/audio/emotion/base_text2mel.yaml
4 |
5 | task_cls: modules.GenerSpeech.task.generspeech.GenerSpeechTask
6 |
7 | # emotion encoder
8 | emotion_encoder_path: checkpoints/Emotion_encoder.pt # set the emotion encoder path
9 |
10 | # vocoder
11 | vocoder: hifigan
12 | vocoder_ckpt: checkpoints/trainset_hifigan
13 |
14 | # dataset
15 | raw_data_dir: 'data/raw/training_set'
16 | processed_data_dir: 'data/processed/training_set'
17 | binary_data_dir: 'data/binary/training_set'
18 | test_input_dir: ''
19 |
20 | # process
21 | binarizer_cls: data_gen.tts.base_binarizer_emotion.EmotionBinarizer
22 | audio_sample_rate: 16000
23 | hop_size: 256 # For 22050Hz, 275 ~= 12.5 ms (0.0125 * sample_rate)
24 | win_size: 1024 # For 22050Hz, 1100 ~= 50 ms (If None, win_size: fft_size) (0.05 * sample_rate)
25 | fmin: 80 # Set this to 55 if your speaker is male! if female, 95 should help taking off noise. (To test depending on dataset. Pitch info: male~[65, 260], female~[100, 525])
26 | fmax: 7600 # To be increased/reduced depending on data.
27 | fft_size: 1024 # Extra window size is filled with 0 paddings to match this parameter
28 | min_level_db: -100
29 | ref_level_db: 20
30 |
31 | binarization_args:
32 | reset_phone_dict: true
33 | reset_word_dict: true
34 | shuffle: true
35 | trim_eos_bos: false
36 | trim_sil: false
37 | with_align: true
38 | with_f0: true
39 | with_f0cwt: false
40 | with_linear: false
41 | with_spk_embed: true
42 | with_spk_id: true
43 | with_txt: true
44 | with_wav: true
45 | with_word: true
46 |
47 | preprocess_cls: egs.datasets.audio.libritts.pre_align.LibrittsPreAlign
48 | preprocess_args:
49 | nsample_per_mfa_group: 1000
50 | # text process
51 | txt_processor: en
52 | use_mfa: true
53 | with_phsep: true
54 | reset_phone_dict: true
55 | reset_word_dict: true
56 | add_eos_bos: true
57 | # mfa
58 | mfa_group_shuffle: false
59 | mfa_offset: 0.02
60 | # wav processors
61 | wav_processors: []
62 | save_sil_mask: true
63 | vad_max_silence_length: 12
64 |
65 | # data
66 | word_dict_size: 10000
67 | num_spk: 500
68 | use_spk_embed: true
69 | use_spk_id: false
70 | use_word: true
71 | use_emotion: true
72 | use_gt_dur: false
73 | ref_audio: ''
74 | text: ''
75 |
76 | # training
77 | num_sanity_val_steps: -1
78 | max_updates: 300000
79 | max_sentences: 100000
80 | num_test_samples: 72
81 |
82 | ## glow
83 | post_glow_hidden: 128
84 | post_glow_kernel_size: 3
85 | post_glow_n_blocks: 8
86 | post_glow_n_block_layers: 3
87 | share_wn_layers: 4
88 | sigmoid_scale: false
89 | post_share_cond_layers: false
90 | use_txt_cond: true
91 | use_latent_cond: true
92 | noise_scale: 0.8
93 |
94 | # prosody extractor
95 | lambda_commit: 0.25
96 | vq_start: 20500
97 | vae_dropout: 0.0
98 | nVQ: 128
99 | forcing: 20000
100 | crop: false
101 | predictor_grad: 1.0
--------------------------------------------------------------------------------
/NeuralSeq/modules/GenerSpeech/model/mixstyle.py:
--------------------------------------------------------------------------------
1 | from modules.commons.common_layers import *
2 | import random
3 |
4 |
5 | class MixStyle(nn.Module):
6 | """MixStyle.
7 | Reference:
8 | Zhou et al. Domain Generalization with MixStyle. ICLR 2021.
9 | """
10 |
11 | def __init__(self, p=0.5, alpha=0.1, eps=1e-6, hidden_size=256):
12 | """
13 | Args:
14 | p (float): probability of using MixStyle.
15 | alpha (float): parameter of the Beta distribution.
16 | eps (float): scaling parameter to avoid numerical issues.
17 | mix (str): how to mix.
18 | """
19 | super().__init__()
20 | self.p = p
21 | self.beta = torch.distributions.Beta(alpha, alpha)
22 | self.eps = eps
23 | self.alpha = alpha
24 | self._activated = True
25 | self.hidden_size = hidden_size
26 | self.affine_layer = LinearNorm(
27 | hidden_size,
28 | 2 * hidden_size, # For both b (bias) g (gain)
29 | )
30 |
31 | def __repr__(self):
32 | return f'MixStyle(p={self.p}, alpha={self.alpha}, eps={self.eps})'
33 |
34 | def set_activation_status(self, status=True):
35 | self._activated = status
36 |
37 | def forward(self, x, spk_embed):
38 | if not self.training or not self._activated:
39 | return x
40 |
41 | if random.random() > self.p:
42 | return x
43 |
44 | B = x.size(0)
45 |
46 | mu, sig = torch.mean(x, dim=-1, keepdim=True), torch.std(x, dim=-1, keepdim=True)
47 | x_normed = (x - mu) / (sig + 1e-6) # [B, T, H_m]
48 |
49 | lmda = self.beta.sample((B, 1, 1))
50 | lmda = lmda.to(x.device)
51 |
52 | # Get Bias and Gain
53 | mu1, sig1 = torch.split(self.affine_layer(spk_embed), self.hidden_size, dim=-1) # [B, 1, 2 * H_m] --> 2 * [B, 1, H_m]
54 |
55 | # MixStyle
56 | perm = torch.randperm(B)
57 | mu2, sig2 = mu1[perm], sig1[perm]
58 |
59 | mu_mix = mu1*lmda + mu2 * (1-lmda)
60 | sig_mix = sig1*lmda + sig2 * (1-lmda)
61 |
62 | # Perform Scailing and Shifting
63 | return sig_mix * x_normed + mu_mix # [B, T, H_m]
64 |
--------------------------------------------------------------------------------
/NeuralSeq/modules/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AIGC-Audio/AudioGPT/a674543c537bb3a60b8f521df76b97a627c1c379/NeuralSeq/modules/__init__.py
--------------------------------------------------------------------------------
/NeuralSeq/modules/commons/align_ops.py:
--------------------------------------------------------------------------------
1 | import torch
2 | import torch.nn.functional as F
3 |
4 |
5 | def build_word_mask(x2word, y2word):
6 | return (x2word[:, :, None] == y2word[:, None, :]).long()
7 |
8 |
9 | def mel2ph_to_mel2word(mel2ph, ph2word):
10 | mel2word = (ph2word - 1).gather(1, (mel2ph - 1).clamp(min=0)) + 1
11 | mel2word = mel2word * (mel2ph > 0).long()
12 | return mel2word
13 |
14 |
15 | def clip_mel2token_to_multiple(mel2token, frames_multiple):
16 | max_frames = mel2token.shape[1] // frames_multiple * frames_multiple
17 | mel2token = mel2token[:, :max_frames]
18 | return mel2token
19 |
20 |
21 | def expand_states(h, mel2token):
22 | h = F.pad(h, [0, 0, 1, 0])
23 | mel2token_ = mel2token[..., None].repeat([1, 1, h.shape[-1]])
24 | h = torch.gather(h, 1, mel2token_) # [B, T, H]
25 | return h
26 |
--------------------------------------------------------------------------------
/NeuralSeq/modules/commons/normalizing_flow/res_flow.py:
--------------------------------------------------------------------------------
1 | import torch
2 | from torch import nn
3 | from modules.commons.conv import ConditionalConvBlocks
4 | from modules.commons.wavenet import WN
5 |
6 |
7 | class FlipLayer(nn.Module):
8 | def forward(self, x, nonpadding, cond=None, reverse=False):
9 | x = torch.flip(x, [1])
10 | return x
11 |
12 |
13 | class CouplingLayer(nn.Module):
14 | def __init__(self, c_in, hidden_size, kernel_size, n_layers, p_dropout=0, c_in_g=0, nn_type='wn'):
15 | super().__init__()
16 | self.channels = c_in
17 | self.hidden_size = hidden_size
18 | self.kernel_size = kernel_size
19 | self.n_layers = n_layers
20 | self.c_half = c_in // 2
21 |
22 | self.pre = nn.Conv1d(self.c_half, hidden_size, 1)
23 | if nn_type == 'wn':
24 | self.enc = WN(hidden_size, kernel_size, 1, n_layers, p_dropout=p_dropout,
25 | c_cond=c_in_g)
26 | elif nn_type == 'conv':
27 | self.enc = ConditionalConvBlocks(
28 | hidden_size, c_in_g, hidden_size, None, kernel_size,
29 | layers_in_block=1, is_BTC=False, num_layers=n_layers)
30 | self.post = nn.Conv1d(hidden_size, self.c_half, 1)
31 |
32 | def forward(self, x, nonpadding, cond=None, reverse=False):
33 | x0, x1 = x[:, :self.c_half], x[:, self.c_half:]
34 | x_ = self.pre(x0) * nonpadding
35 | x_ = self.enc(x_, nonpadding=nonpadding, cond=cond)
36 | m = self.post(x_)
37 | x1 = m + x1 if not reverse else x1 - m
38 | x = torch.cat([x0, x1], 1)
39 | return x * nonpadding
40 |
41 |
42 | class ResFlow(nn.Module):
43 | def __init__(self,
44 | c_in,
45 | hidden_size,
46 | kernel_size,
47 | n_flow_layers,
48 | n_flow_steps=4,
49 | c_cond=0,
50 | nn_type='wn'):
51 | super().__init__()
52 | self.flows = nn.ModuleList()
53 | for i in range(n_flow_steps):
54 | self.flows.append(
55 | CouplingLayer(c_in, hidden_size, kernel_size, n_flow_layers, c_in_g=c_cond, nn_type=nn_type))
56 | self.flows.append(FlipLayer())
57 |
58 | def forward(self, x, nonpadding, cond=None, reverse=False):
59 | for flow in (self.flows if not reverse else reversed(self.flows)):
60 | x = flow(x, nonpadding, cond=cond, reverse=reverse)
61 | return x
62 |
--------------------------------------------------------------------------------
/NeuralSeq/modules/commons/normalizing_flow/utils.py:
--------------------------------------------------------------------------------
1 | import torch
2 |
3 |
4 | def squeeze(x, x_mask=None, n_sqz=2):
5 | b, c, t = x.size()
6 |
7 | t = (t // n_sqz) * n_sqz
8 | x = x[:, :, :t]
9 | x_sqz = x.view(b, c, t // n_sqz, n_sqz)
10 | x_sqz = x_sqz.permute(0, 3, 1, 2).contiguous().view(b, c * n_sqz, t // n_sqz)
11 |
12 | if x_mask is not None:
13 | x_mask = x_mask[:, :, n_sqz - 1::n_sqz]
14 | else:
15 | x_mask = torch.ones(b, 1, t // n_sqz).to(device=x.device, dtype=x.dtype)
16 | return x_sqz * x_mask, x_mask
17 |
18 |
19 | def unsqueeze(x, x_mask=None, n_sqz=2):
20 | b, c, t = x.size()
21 |
22 | x_unsqz = x.view(b, n_sqz, c // n_sqz, t)
23 | x_unsqz = x_unsqz.permute(0, 2, 3, 1).contiguous().view(b, c // n_sqz, t * n_sqz)
24 |
25 | if x_mask is not None:
26 | x_mask = x_mask.unsqueeze(-1).repeat(1, 1, 1, n_sqz).view(b, 1, t * n_sqz)
27 | else:
28 | x_mask = torch.ones(b, 1, t * n_sqz).to(device=x.device, dtype=x.dtype)
29 | return x_unsqz * x_mask, x_mask
30 |
--------------------------------------------------------------------------------
/NeuralSeq/modules/hifigan/mel_utils.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | import torch
3 | import torch.utils.data
4 | from librosa.filters import mel as librosa_mel_fn
5 | from scipy.io.wavfile import read
6 |
7 | MAX_WAV_VALUE = 32768.0
8 |
9 |
10 | def load_wav(full_path):
11 | sampling_rate, data = read(full_path)
12 | return data, sampling_rate
13 |
14 |
15 | def dynamic_range_compression(x, C=1, clip_val=1e-5):
16 | return np.log(np.clip(x, a_min=clip_val, a_max=None) * C)
17 |
18 |
19 | def dynamic_range_decompression(x, C=1):
20 | return np.exp(x) / C
21 |
22 |
23 | def dynamic_range_compression_torch(x, C=1, clip_val=1e-5):
24 | return torch.log(torch.clamp(x, min=clip_val) * C)
25 |
26 |
27 | def dynamic_range_decompression_torch(x, C=1):
28 | return torch.exp(x) / C
29 |
30 |
31 | def spectral_normalize_torch(magnitudes):
32 | output = dynamic_range_compression_torch(magnitudes)
33 | return output
34 |
35 |
36 | def spectral_de_normalize_torch(magnitudes):
37 | output = dynamic_range_decompression_torch(magnitudes)
38 | return output
39 |
40 |
41 | mel_basis = {}
42 | hann_window = {}
43 |
44 |
45 | def mel_spectrogram(y, hparams, center=False, complex=False):
46 | # hop_size: 512 # For 22050Hz, 275 ~= 12.5 ms (0.0125 * sample_rate)
47 | # win_size: 2048 # For 22050Hz, 1100 ~= 50 ms (If None, win_size: fft_size) (0.05 * sample_rate)
48 | # fmin: 55 # Set this to 55 if your speaker is male! if female, 95 should help taking off noise. (To test depending on dataset. Pitch info: male~[65, 260], female~[100, 525])
49 | # fmax: 10000 # To be increased/reduced depending on data.
50 | # fft_size: 2048 # Extra window size is filled with 0 paddings to match this parameter
51 | # n_fft, num_mels, sampling_rate, hop_size, win_size, fmin, fmax,
52 | n_fft = hparams['fft_size']
53 | num_mels = hparams['audio_num_mel_bins']
54 | sampling_rate = hparams['audio_sample_rate']
55 | hop_size = hparams['hop_size']
56 | win_size = hparams['win_size']
57 | fmin = hparams['fmin']
58 | fmax = hparams['fmax']
59 | y = y.clamp(min=-1., max=1.)
60 | global mel_basis, hann_window
61 | if fmax not in mel_basis:
62 | mel = librosa_mel_fn(sampling_rate, n_fft, num_mels, fmin, fmax)
63 | mel_basis[str(fmax) + '_' + str(y.device)] = torch.from_numpy(mel).float().to(y.device)
64 | hann_window[str(y.device)] = torch.hann_window(win_size).to(y.device)
65 |
66 | y = torch.nn.functional.pad(y.unsqueeze(1), (int((n_fft - hop_size) / 2), int((n_fft - hop_size) / 2)),
67 | mode='reflect')
68 | y = y.squeeze(1)
69 |
70 | spec = torch.stft(y, n_fft, hop_length=hop_size, win_length=win_size, window=hann_window[str(y.device)],
71 | center=center, pad_mode='reflect', normalized=False, onesided=True)
72 |
73 | if not complex:
74 | spec = torch.sqrt(spec.pow(2).sum(-1) + (1e-9))
75 | spec = torch.matmul(mel_basis[str(fmax) + '_' + str(y.device)], spec)
76 | spec = spectral_normalize_torch(spec)
77 | else:
78 | B, C, T, _ = spec.shape
79 | spec = spec.transpose(1, 2) # [B, T, n_fft, 2]
80 | return spec
81 |
--------------------------------------------------------------------------------
/NeuralSeq/modules/parallel_wavegan/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AIGC-Audio/AudioGPT/a674543c537bb3a60b8f521df76b97a627c1c379/NeuralSeq/modules/parallel_wavegan/__init__.py
--------------------------------------------------------------------------------
/NeuralSeq/modules/parallel_wavegan/layers/__init__.py:
--------------------------------------------------------------------------------
1 | from .causal_conv import * # NOQA
2 | from .pqmf import * # NOQA
3 | from .residual_block import * # NOQA
4 | from modules.parallel_wavegan.layers.residual_stack import * # NOQA
5 | from .upsample import * # NOQA
6 |
--------------------------------------------------------------------------------
/NeuralSeq/modules/parallel_wavegan/layers/causal_conv.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | # Copyright 2020 Tomoki Hayashi
4 | # MIT License (https://opensource.org/licenses/MIT)
5 |
6 | """Causal convolusion layer modules."""
7 |
8 |
9 | import torch
10 |
11 |
12 | class CausalConv1d(torch.nn.Module):
13 | """CausalConv1d module with customized initialization."""
14 |
15 | def __init__(self, in_channels, out_channels, kernel_size,
16 | dilation=1, bias=True, pad="ConstantPad1d", pad_params={"value": 0.0}):
17 | """Initialize CausalConv1d module."""
18 | super(CausalConv1d, self).__init__()
19 | self.pad = getattr(torch.nn, pad)((kernel_size - 1) * dilation, **pad_params)
20 | self.conv = torch.nn.Conv1d(in_channels, out_channels, kernel_size,
21 | dilation=dilation, bias=bias)
22 |
23 | def forward(self, x):
24 | """Calculate forward propagation.
25 |
26 | Args:
27 | x (Tensor): Input tensor (B, in_channels, T).
28 |
29 | Returns:
30 | Tensor: Output tensor (B, out_channels, T).
31 |
32 | """
33 | return self.conv(self.pad(x))[:, :, :x.size(2)]
34 |
35 |
36 | class CausalConvTranspose1d(torch.nn.Module):
37 | """CausalConvTranspose1d module with customized initialization."""
38 |
39 | def __init__(self, in_channels, out_channels, kernel_size, stride, bias=True):
40 | """Initialize CausalConvTranspose1d module."""
41 | super(CausalConvTranspose1d, self).__init__()
42 | self.deconv = torch.nn.ConvTranspose1d(
43 | in_channels, out_channels, kernel_size, stride, bias=bias)
44 | self.stride = stride
45 |
46 | def forward(self, x):
47 | """Calculate forward propagation.
48 |
49 | Args:
50 | x (Tensor): Input tensor (B, in_channels, T_in).
51 |
52 | Returns:
53 | Tensor: Output tensor (B, out_channels, T_out).
54 |
55 | """
56 | return self.deconv(x)[:, :, :-self.stride]
57 |
--------------------------------------------------------------------------------
/NeuralSeq/modules/parallel_wavegan/layers/residual_stack.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | # Copyright 2020 Tomoki Hayashi
4 | # MIT License (https://opensource.org/licenses/MIT)
5 |
6 | """Residual stack module in MelGAN."""
7 |
8 | import torch
9 |
10 | from . import CausalConv1d
11 |
12 |
13 | class ResidualStack(torch.nn.Module):
14 | """Residual stack module introduced in MelGAN."""
15 |
16 | def __init__(self,
17 | kernel_size=3,
18 | channels=32,
19 | dilation=1,
20 | bias=True,
21 | nonlinear_activation="LeakyReLU",
22 | nonlinear_activation_params={"negative_slope": 0.2},
23 | pad="ReflectionPad1d",
24 | pad_params={},
25 | use_causal_conv=False,
26 | ):
27 | """Initialize ResidualStack module.
28 |
29 | Args:
30 | kernel_size (int): Kernel size of dilation convolution layer.
31 | channels (int): Number of channels of convolution layers.
32 | dilation (int): Dilation factor.
33 | bias (bool): Whether to add bias parameter in convolution layers.
34 | nonlinear_activation (str): Activation function module name.
35 | nonlinear_activation_params (dict): Hyperparameters for activation function.
36 | pad (str): Padding function module name before dilated convolution layer.
37 | pad_params (dict): Hyperparameters for padding function.
38 | use_causal_conv (bool): Whether to use causal convolution.
39 |
40 | """
41 | super(ResidualStack, self).__init__()
42 |
43 | # defile residual stack part
44 | if not use_causal_conv:
45 | assert (kernel_size - 1) % 2 == 0, "Not support even number kernel size."
46 | self.stack = torch.nn.Sequential(
47 | getattr(torch.nn, nonlinear_activation)(**nonlinear_activation_params),
48 | getattr(torch.nn, pad)((kernel_size - 1) // 2 * dilation, **pad_params),
49 | torch.nn.Conv1d(channels, channels, kernel_size, dilation=dilation, bias=bias),
50 | getattr(torch.nn, nonlinear_activation)(**nonlinear_activation_params),
51 | torch.nn.Conv1d(channels, channels, 1, bias=bias),
52 | )
53 | else:
54 | self.stack = torch.nn.Sequential(
55 | getattr(torch.nn, nonlinear_activation)(**nonlinear_activation_params),
56 | CausalConv1d(channels, channels, kernel_size, dilation=dilation,
57 | bias=bias, pad=pad, pad_params=pad_params),
58 | getattr(torch.nn, nonlinear_activation)(**nonlinear_activation_params),
59 | torch.nn.Conv1d(channels, channels, 1, bias=bias),
60 | )
61 |
62 | # defile extra layer for skip connection
63 | self.skip_layer = torch.nn.Conv1d(channels, channels, 1, bias=bias)
64 |
65 | def forward(self, c):
66 | """Calculate forward propagation.
67 |
68 | Args:
69 | c (Tensor): Input tensor (B, channels, T).
70 |
71 | Returns:
72 | Tensor: Output tensor (B, chennels, T).
73 |
74 | """
75 | return self.stack(c) + self.skip_layer(c)
76 |
--------------------------------------------------------------------------------
/NeuralSeq/modules/parallel_wavegan/losses/__init__.py:
--------------------------------------------------------------------------------
1 | from .stft_loss import * # NOQA
2 |
--------------------------------------------------------------------------------
/NeuralSeq/modules/parallel_wavegan/models/__init__.py:
--------------------------------------------------------------------------------
1 | from .melgan import * # NOQA
2 | from .parallel_wavegan import * # NOQA
3 |
--------------------------------------------------------------------------------
/NeuralSeq/modules/parallel_wavegan/optimizers/__init__.py:
--------------------------------------------------------------------------------
1 | from torch.optim import * # NOQA
2 | from .radam import * # NOQA
3 |
--------------------------------------------------------------------------------
/NeuralSeq/modules/parallel_wavegan/utils/__init__.py:
--------------------------------------------------------------------------------
1 | from .utils import * # NOQA
2 |
--------------------------------------------------------------------------------
/NeuralSeq/tasks/run.py:
--------------------------------------------------------------------------------
1 | import importlib
2 | from utils.hparams import set_hparams, hparams
3 |
4 |
5 | def run_task():
6 | assert hparams['task_cls'] != ''
7 | pkg = ".".join(hparams["task_cls"].split(".")[:-1])
8 | cls_name = hparams["task_cls"].split(".")[-1]
9 | task_cls = getattr(importlib.import_module(pkg), cls_name)
10 | task_cls.start()
11 |
12 |
13 | if __name__ == '__main__':
14 | set_hparams()
15 | run_task()
16 |
--------------------------------------------------------------------------------
/NeuralSeq/tasks/svs/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AIGC-Audio/AudioGPT/a674543c537bb3a60b8f521df76b97a627c1c379/NeuralSeq/tasks/svs/__init__.py
--------------------------------------------------------------------------------
/NeuralSeq/tasks/tts/synta.py:
--------------------------------------------------------------------------------
1 | import os
2 | import torch
3 | import torch.nn.functional as F
4 | from torch import nn
5 |
6 | from modules.tts.syntaspeech.syntaspeech import SyntaSpeech
7 | from tasks.tts.ps_adv import PortaSpeechAdvTask
8 | from utils.hparams import hparams
9 |
10 |
11 | class SyntaSpeechTask(PortaSpeechAdvTask):
12 | def build_tts_model(self):
13 | ph_dict_size = len(self.token_encoder)
14 | word_dict_size = len(self.word_encoder)
15 | self.model = SyntaSpeech(ph_dict_size, word_dict_size, hparams)
16 |
17 | self.gen_params = [p for p in self.model.parameters() if p.requires_grad]
18 | self.dp_params = [p for k, p in self.model.named_parameters() if (('dur_predictor' in k) and p.requires_grad)]
19 | self.gen_params_except_dp = [p for k, p in self.model.named_parameters() if (('dur_predictor' not in k) and p.requires_grad)]
20 | self.bert_params = [p for k, p in self.model.named_parameters() if (('bert' in k) and p.requires_grad)]
21 | self.gen_params_except_bert_and_dp = [p for k, p in self.model.named_parameters() if ('dur_predictor' not in k) and ('bert' not in k) and p.requires_grad ]
22 |
23 | self.use_bert = True if len(self.bert_params) > 0 else False
24 |
25 |
--------------------------------------------------------------------------------
/NeuralSeq/tasks/tts/tts_utils.py:
--------------------------------------------------------------------------------
1 | import importlib
2 |
3 | from data_gen.tts.base_binarizer import BaseBinarizer
4 | from data_gen.tts.base_preprocess import BasePreprocessor
5 | from data_gen.tts.txt_processors.base_text_processor import get_txt_processor_cls
6 | from utils.hparams import hparams
7 |
8 |
9 | def parse_dataset_configs():
10 | max_tokens = hparams['max_tokens']
11 | max_sentences = hparams['max_sentences']
12 | max_valid_tokens = hparams['max_valid_tokens']
13 | if max_valid_tokens == -1:
14 | hparams['max_valid_tokens'] = max_valid_tokens = max_tokens
15 | max_valid_sentences = hparams['max_valid_sentences']
16 | if max_valid_sentences == -1:
17 | hparams['max_valid_sentences'] = max_valid_sentences = max_sentences
18 | return max_tokens, max_sentences, max_valid_tokens, max_valid_sentences
19 |
20 |
21 | def parse_mel_losses():
22 | mel_losses = hparams['mel_losses'].split("|")
23 | loss_and_lambda = {}
24 | for i, l in enumerate(mel_losses):
25 | if l == '':
26 | continue
27 | if ':' in l:
28 | l, lbd = l.split(":")
29 | lbd = float(lbd)
30 | else:
31 | lbd = 1.0
32 | loss_and_lambda[l] = lbd
33 | print("| Mel losses:", loss_and_lambda)
34 | return loss_and_lambda
35 |
36 |
37 | def load_data_preprocessor():
38 | preprocess_cls = hparams["preprocess_cls"]
39 | pkg = ".".join(preprocess_cls.split(".")[:-1])
40 | cls_name = preprocess_cls.split(".")[-1]
41 | preprocessor: BasePreprocessor = getattr(importlib.import_module(pkg), cls_name)()
42 | preprocess_args = {}
43 | preprocess_args.update(hparams['preprocess_args'])
44 | return preprocessor, preprocess_args
45 |
46 |
47 | def load_data_binarizer():
48 | binarizer_cls = hparams['binarizer_cls']
49 | pkg = ".".join(binarizer_cls.split(".")[:-1])
50 | cls_name = binarizer_cls.split(".")[-1]
51 | binarizer: BaseBinarizer = getattr(importlib.import_module(pkg), cls_name)()
52 | binarization_args = {}
53 | binarization_args.update(hparams['binarization_args'])
54 | return binarizer, binarization_args
--------------------------------------------------------------------------------
/NeuralSeq/tasks/vocoder/vocoder_base.py:
--------------------------------------------------------------------------------
1 | import os
2 |
3 | import torch
4 | import torch.distributed as dist
5 | from torch.utils.data import DistributedSampler
6 |
7 | from tasks.base_task import BaseTask
8 | from tasks.base_task import data_loader
9 | from tasks.vocoder.dataset_utils import VocoderDataset, EndlessDistributedSampler
10 | from utils.hparams import hparams
11 |
12 |
13 | class VocoderBaseTask(BaseTask):
14 | def __init__(self):
15 | super(VocoderBaseTask, self).__init__()
16 | self.max_sentences = hparams['max_sentences']
17 | self.max_valid_sentences = hparams['max_valid_sentences']
18 | if self.max_valid_sentences == -1:
19 | hparams['max_valid_sentences'] = self.max_valid_sentences = self.max_sentences
20 | self.dataset_cls = VocoderDataset
21 |
22 | @data_loader
23 | def train_dataloader(self):
24 | train_dataset = self.dataset_cls('train', shuffle=True)
25 | return self.build_dataloader(train_dataset, True, self.max_sentences, hparams['endless_ds'])
26 |
27 | @data_loader
28 | def val_dataloader(self):
29 | valid_dataset = self.dataset_cls('valid', shuffle=False)
30 | return self.build_dataloader(valid_dataset, False, self.max_valid_sentences)
31 |
32 | @data_loader
33 | def test_dataloader(self):
34 | test_dataset = self.dataset_cls('test', shuffle=False)
35 | return self.build_dataloader(test_dataset, False, self.max_valid_sentences)
36 |
37 | def build_dataloader(self, dataset, shuffle, max_sentences, endless=False):
38 | world_size = 1
39 | rank = 0
40 | if dist.is_initialized():
41 | world_size = dist.get_world_size()
42 | rank = dist.get_rank()
43 | sampler_cls = DistributedSampler if not endless else EndlessDistributedSampler
44 | train_sampler = sampler_cls(
45 | dataset=dataset,
46 | num_replicas=world_size,
47 | rank=rank,
48 | shuffle=shuffle,
49 | )
50 | return torch.utils.data.DataLoader(
51 | dataset=dataset,
52 | shuffle=False,
53 | collate_fn=dataset.collater,
54 | batch_size=max_sentences,
55 | num_workers=dataset.num_workers,
56 | sampler=train_sampler,
57 | pin_memory=True,
58 | )
59 |
60 | def test_start(self):
61 | self.gen_dir = os.path.join(hparams['work_dir'],
62 | f'generated_{self.trainer.global_step}_{hparams["gen_dir_name"]}')
63 | os.makedirs(self.gen_dir, exist_ok=True)
64 |
65 | def test_end(self, outputs):
66 | return {}
67 |
--------------------------------------------------------------------------------
/NeuralSeq/utils/ckpt_utils.py:
--------------------------------------------------------------------------------
1 | import glob
2 | import logging
3 | import os
4 | import re
5 | import torch
6 |
7 |
8 | def get_last_checkpoint(work_dir, steps=None):
9 | checkpoint = None
10 | last_ckpt_path = None
11 | ckpt_paths = get_all_ckpts(work_dir, steps)
12 | if len(ckpt_paths) > 0:
13 | last_ckpt_path = ckpt_paths[0]
14 | checkpoint = torch.load(last_ckpt_path, map_location='cpu')
15 | logging.info(f'load module from checkpoint: {last_ckpt_path}')
16 | return checkpoint, last_ckpt_path
17 |
18 |
19 | def get_all_ckpts(work_dir, steps=None):
20 | if steps is None:
21 | ckpt_path_pattern = f'{work_dir}/model_ckpt_steps_*.ckpt'
22 | else:
23 | ckpt_path_pattern = f'{work_dir}/model_ckpt_steps_{steps}.ckpt'
24 | return sorted(glob.glob(ckpt_path_pattern),
25 | key=lambda x: -int(re.findall('.*steps\_(\d+)\.ckpt', x)[0]))
26 |
27 |
28 | def load_ckpt(cur_model, ckpt_base_dir, model_name='model', force=True, strict=True):
29 | if os.path.isfile(ckpt_base_dir):
30 | base_dir = os.path.dirname(ckpt_base_dir)
31 | ckpt_path = ckpt_base_dir
32 | checkpoint = torch.load(ckpt_base_dir, map_location='cpu')
33 | else:
34 | base_dir = ckpt_base_dir
35 | checkpoint, ckpt_path = get_last_checkpoint(ckpt_base_dir)
36 | if checkpoint is not None:
37 | state_dict = checkpoint["state_dict"]
38 | if len([k for k in state_dict.keys() if '.' in k]) > 0:
39 | state_dict = {k[len(model_name) + 1:]: v for k, v in state_dict.items()
40 | if k.startswith(f'{model_name}.')}
41 | else:
42 | if '.' not in model_name:
43 | state_dict = state_dict[model_name]
44 | else:
45 | base_model_name = model_name.split('.')[0]
46 | rest_model_name = model_name[len(base_model_name) + 1:]
47 | state_dict = {
48 | k[len(rest_model_name) + 1:]: v for k, v in state_dict[base_model_name].items()
49 | if k.startswith(f'{rest_model_name}.')}
50 | if not strict:
51 | cur_model_state_dict = cur_model.state_dict()
52 | unmatched_keys = []
53 | for key, param in state_dict.items():
54 | if key in cur_model_state_dict:
55 | new_param = cur_model_state_dict[key]
56 | if new_param.shape != param.shape:
57 | unmatched_keys.append(key)
58 | print("| Unmatched keys: ", key, new_param.shape, param.shape)
59 | for key in unmatched_keys:
60 | del state_dict[key]
61 | cur_model.load_state_dict(state_dict, strict=strict)
62 | print(f"| load '{model_name}' from '{ckpt_path}'.")
63 | else:
64 | e_msg = f"| ckpt not found in {base_dir}."
65 | if force:
66 | assert False, e_msg
67 | else:
68 | print(e_msg)
69 |
--------------------------------------------------------------------------------
/NeuralSeq/utils/indexed_datasets.py:
--------------------------------------------------------------------------------
1 | import pickle
2 | from copy import deepcopy
3 |
4 | import numpy as np
5 |
6 |
7 | class IndexedDataset:
8 | def __init__(self, path, num_cache=1):
9 | super().__init__()
10 | self.path = path
11 | self.data_file = None
12 | self.data_offsets = np.load(f"{path}.idx", allow_pickle=True).item()['offsets']
13 | self.data_file = open(f"{path}.data", 'rb', buffering=-1)
14 | self.cache = []
15 | self.num_cache = num_cache
16 |
17 | def check_index(self, i):
18 | if i < 0 or i >= len(self.data_offsets) - 1:
19 | raise IndexError('index out of range')
20 |
21 | def __del__(self):
22 | if self.data_file:
23 | self.data_file.close()
24 |
25 | def __getitem__(self, i):
26 | self.check_index(i)
27 | if self.num_cache > 0:
28 | for c in self.cache:
29 | if c[0] == i:
30 | return c[1]
31 | self.data_file.seek(self.data_offsets[i])
32 | b = self.data_file.read(self.data_offsets[i + 1] - self.data_offsets[i])
33 | item = pickle.loads(b)
34 | if self.num_cache > 0:
35 | self.cache = [(i, deepcopy(item))] + self.cache[:-1]
36 | return item
37 |
38 | def __len__(self):
39 | return len(self.data_offsets) - 1
40 |
41 | class IndexedDatasetBuilder:
42 | def __init__(self, path):
43 | self.path = path
44 | self.out_file = open(f"{path}.data", 'wb')
45 | self.byte_offsets = [0]
46 |
47 | def add_item(self, item):
48 | s = pickle.dumps(item)
49 | bytes = self.out_file.write(s)
50 | self.byte_offsets.append(self.byte_offsets[-1] + bytes)
51 |
52 | def finalize(self):
53 | self.out_file.close()
54 | np.save(open(f"{self.path}.idx", 'wb'), {'offsets': self.byte_offsets})
55 |
56 |
57 | if __name__ == "__main__":
58 | import random
59 | from tqdm import tqdm
60 | ds_path = '/tmp/indexed_ds_example'
61 | size = 100
62 | items = [{"a": np.random.normal(size=[10000, 10]),
63 | "b": np.random.normal(size=[10000, 10])} for i in range(size)]
64 | builder = IndexedDatasetBuilder(ds_path)
65 | for i in tqdm(range(size)):
66 | builder.add_item(items[i])
67 | builder.finalize()
68 | ds = IndexedDataset(ds_path)
69 | for i in tqdm(range(10000)):
70 | idx = random.randint(0, size - 1)
71 | assert (ds[idx]['a'] == items[idx]['a']).all()
72 |
--------------------------------------------------------------------------------
/NeuralSeq/utils/multiprocess_utils.py:
--------------------------------------------------------------------------------
1 | import os
2 | import traceback
3 | from multiprocessing import Queue, Process
4 |
5 |
6 | def chunked_worker(worker_id, map_func, args, results_queue=None, init_ctx_func=None):
7 | ctx = init_ctx_func(worker_id) if init_ctx_func is not None else None
8 | for job_idx, arg in args:
9 | try:
10 | if ctx is not None:
11 | res = map_func(*arg, ctx=ctx)
12 | else:
13 | res = map_func(*arg)
14 | results_queue.put((job_idx, res))
15 | except:
16 | traceback.print_exc()
17 | results_queue.put((job_idx, None))
18 |
19 | def chunked_multiprocess_run(map_func, args, num_workers=None, ordered=True, init_ctx_func=None, q_max_size=1000):
20 | args = zip(range(len(args)), args)
21 | args = list(args)
22 | n_jobs = len(args)
23 | if num_workers is None:
24 | num_workers = int(os.getenv('N_PROC', os.cpu_count()))
25 | results_queues = []
26 | if ordered:
27 | for i in range(num_workers):
28 | results_queues.append(Queue(maxsize=q_max_size // num_workers))
29 | else:
30 | results_queue = Queue(maxsize=q_max_size)
31 | for i in range(num_workers):
32 | results_queues.append(results_queue)
33 | workers = []
34 | for i in range(num_workers):
35 | args_worker = args[i::num_workers]
36 | p = Process(target=chunked_worker, args=(
37 | i, map_func, args_worker, results_queues[i], init_ctx_func), daemon=True)
38 | workers.append(p)
39 | p.start()
40 | for n_finished in range(n_jobs):
41 | results_queue = results_queues[n_finished % num_workers]
42 | job_idx, res = results_queue.get()
43 | assert job_idx == n_finished or not ordered, (job_idx, n_finished)
44 | yield res
45 | for w in workers:
46 | w.join()
47 | w.close()
48 |
49 | def multiprocess_run_tqdm(map_func, args, num_workers=None, ordered=True, init_ctx_func=None,
50 | multithread=False, desc=None):
51 | for i, res in tqdm(enumerate(
52 | multiprocess_run(map_func, args, num_workers, ordered, init_ctx_func, multithread)),
53 | total=len(args), desc=desc):
54 | yield i, res
--------------------------------------------------------------------------------
/NeuralSeq/utils/os_utils.py:
--------------------------------------------------------------------------------
1 | import os
2 | import subprocess
3 |
4 |
5 | def link_file(from_file, to_file):
6 | subprocess.check_call(
7 | f'ln -s "`realpath --relative-to="{os.path.dirname(to_file)}" "{from_file}"`" "{to_file}"', shell=True)
8 |
9 |
10 | def move_file(from_file, to_file):
11 | subprocess.check_call(f'mv "{from_file}" "{to_file}"', shell=True)
12 |
13 |
14 | def copy_file(from_file, to_file):
15 | subprocess.check_call(f'cp -r "{from_file}" "{to_file}"', shell=True)
16 |
17 |
18 | def remove_file(*fns):
19 | for f in fns:
20 | subprocess.check_call(f'rm -rf "{f}"', shell=True)
--------------------------------------------------------------------------------
/NeuralSeq/utils/pitch_utils.py:
--------------------------------------------------------------------------------
1 | #########
2 | # world
3 | ##########
4 | import librosa
5 | import numpy as np
6 | import torch
7 |
8 | gamma = 0
9 | mcepInput = 3 # 0 for dB, 3 for magnitude
10 | alpha = 0.45
11 | en_floor = 10 ** (-80 / 20)
12 | FFT_SIZE = 2048
13 |
14 |
15 | f0_bin = 256
16 | f0_max = 1100.0
17 | f0_min = 50.0
18 | f0_mel_min = 1127 * np.log(1 + f0_min / 700)
19 | f0_mel_max = 1127 * np.log(1 + f0_max / 700)
20 |
21 |
22 | def f0_to_coarse(f0):
23 | is_torch = isinstance(f0, torch.Tensor)
24 | f0_mel = 1127 * (1 + f0 / 700).log() if is_torch else 1127 * np.log(1 + f0 / 700)
25 | f0_mel[f0_mel > 0] = (f0_mel[f0_mel > 0] - f0_mel_min) * (f0_bin - 2) / (f0_mel_max - f0_mel_min) + 1
26 |
27 | f0_mel[f0_mel <= 1] = 1
28 | f0_mel[f0_mel > f0_bin - 1] = f0_bin - 1
29 | f0_coarse = (f0_mel + 0.5).long() if is_torch else np.rint(f0_mel).astype(np.int)
30 | assert f0_coarse.max() <= 255 and f0_coarse.min() >= 1, (f0_coarse.max(), f0_coarse.min())
31 | return f0_coarse
32 |
33 |
34 | def norm_f0(f0, uv, hparams):
35 | is_torch = isinstance(f0, torch.Tensor)
36 | if hparams['pitch_norm'] == 'standard':
37 | f0 = (f0 - hparams['f0_mean']) / hparams['f0_std']
38 | if hparams['pitch_norm'] == 'log':
39 | f0 = torch.log2(f0) if is_torch else np.log2(f0)
40 | if uv is not None and hparams['use_uv']:
41 | f0[uv > 0] = 0
42 | return f0
43 |
44 |
45 | def norm_interp_f0(f0, hparams):
46 | is_torch = isinstance(f0, torch.Tensor)
47 | if is_torch:
48 | device = f0.device
49 | f0 = f0.data.cpu().numpy()
50 | uv = f0 == 0
51 | f0 = norm_f0(f0, uv, hparams)
52 | if sum(uv) == len(f0):
53 | f0[uv] = 0
54 | elif sum(uv) > 0:
55 | f0[uv] = np.interp(np.where(uv)[0], np.where(~uv)[0], f0[~uv])
56 | uv = torch.FloatTensor(uv)
57 | f0 = torch.FloatTensor(f0)
58 | if is_torch:
59 | f0 = f0.to(device)
60 | return f0, uv
61 |
62 |
63 | def denorm_f0(f0, uv, hparams, pitch_padding=None, min=None, max=None):
64 | if hparams['pitch_norm'] == 'standard':
65 | f0 = f0 * hparams['f0_std'] + hparams['f0_mean']
66 | if hparams['pitch_norm'] == 'log':
67 | f0 = 2 ** f0
68 | if min is not None:
69 | f0 = f0.clamp(min=min)
70 | if max is not None:
71 | f0 = f0.clamp(max=max)
72 | if uv is not None and hparams['use_uv']:
73 | f0[uv > 0] = 0
74 | if pitch_padding is not None:
75 | f0[pitch_padding] = 0
76 | return f0
77 |
--------------------------------------------------------------------------------
/NeuralSeq/utils/plot.py:
--------------------------------------------------------------------------------
1 | import matplotlib.pyplot as plt
2 | import numpy as np
3 | import torch
4 |
5 | LINE_COLORS = ['w', 'r', 'y', 'cyan', 'm', 'b', 'lime']
6 |
7 |
8 | def spec_to_figure(spec, vmin=None, vmax=None):
9 | if isinstance(spec, torch.Tensor):
10 | spec = spec.cpu().numpy()
11 | fig = plt.figure(figsize=(12, 6))
12 | plt.pcolor(spec.T, vmin=vmin, vmax=vmax)
13 | return fig
14 |
15 |
16 | def spec_f0_to_figure(spec, f0s, figsize=None):
17 | max_y = spec.shape[1]
18 | if isinstance(spec, torch.Tensor):
19 | spec = spec.detach().cpu().numpy()
20 | f0s = {k: f0.detach().cpu().numpy() for k, f0 in f0s.items()}
21 | f0s = {k: f0 / 10 for k, f0 in f0s.items()}
22 | fig = plt.figure(figsize=(12, 6) if figsize is None else figsize)
23 | plt.pcolor(spec.T)
24 | for i, (k, f0) in enumerate(f0s.items()):
25 | plt.plot(f0.clip(0, max_y), label=k, c=LINE_COLORS[i], linewidth=1, alpha=0.8)
26 | plt.legend()
27 | return fig
28 |
29 |
30 | def dur_to_figure(dur_gt, dur_pred, txt):
31 | dur_gt = dur_gt.long().cpu().numpy()
32 | dur_pred = dur_pred.long().cpu().numpy()
33 | dur_gt = np.cumsum(dur_gt)
34 | dur_pred = np.cumsum(dur_pred)
35 | fig = plt.figure(figsize=(12, 6))
36 | for i in range(len(dur_gt)):
37 | shift = (i % 8) + 1
38 | plt.text(dur_gt[i], shift, txt[i])
39 | plt.text(dur_pred[i], 10 + shift, txt[i])
40 | plt.vlines(dur_gt[i], 0, 10, colors='b') # blue is gt
41 | plt.vlines(dur_pred[i], 10, 20, colors='r') # red is pred
42 | return fig
43 |
44 |
45 | def f0_to_figure(f0_gt, f0_cwt=None, f0_pred=None):
46 | fig = plt.figure()
47 | f0_gt = f0_gt.cpu().numpy()
48 | plt.plot(f0_gt, color='r', label='gt')
49 | if f0_cwt is not None:
50 | f0_cwt = f0_cwt.cpu().numpy()
51 | plt.plot(f0_cwt, color='b', label='cwt')
52 | if f0_pred is not None:
53 | f0_pred = f0_pred.cpu().numpy()
54 | plt.plot(f0_pred, color='green', label='pred')
55 | plt.legend()
56 | return fig
57 |
--------------------------------------------------------------------------------
/NeuralSeq/utils/training_utils.py:
--------------------------------------------------------------------------------
1 | from utils.hparams import hparams
2 |
3 |
4 | class RSQRTSchedule(object):
5 | def __init__(self, optimizer):
6 | super().__init__()
7 | self.optimizer = optimizer
8 | self.constant_lr = hparams['lr']
9 | self.warmup_updates = hparams['warmup_updates']
10 | self.hidden_size = hparams['hidden_size']
11 | self.lr = hparams['lr']
12 | for param_group in optimizer.param_groups:
13 | param_group['lr'] = self.lr
14 | self.step(0)
15 |
16 | def step(self, num_updates):
17 | constant_lr = self.constant_lr
18 | warmup = min(num_updates / self.warmup_updates, 1.0)
19 | rsqrt_decay = max(self.warmup_updates, num_updates) ** -0.5
20 | rsqrt_hidden = self.hidden_size ** -0.5
21 | self.lr = max(constant_lr * warmup * rsqrt_decay * rsqrt_hidden, 1e-7)
22 | for param_group in self.optimizer.param_groups:
23 | param_group['lr'] = self.lr
24 | return self.lr
25 |
26 | def get_lr(self):
27 | return self.optimizer.param_groups[0]['lr']
28 |
--------------------------------------------------------------------------------
/NeuralSeq/vocoders/__init__.py:
--------------------------------------------------------------------------------
1 | from vocoders import hifigan
2 |
--------------------------------------------------------------------------------
/NeuralSeq/vocoders/base_vocoder.py:
--------------------------------------------------------------------------------
1 | import importlib
2 | VOCODERS = {}
3 |
4 |
5 | def register_vocoder(cls):
6 | VOCODERS[cls.__name__.lower()] = cls
7 | VOCODERS[cls.__name__] = cls
8 | return cls
9 |
10 |
11 | def get_vocoder_cls(hparams):
12 | if hparams['vocoder'] in VOCODERS:
13 | return VOCODERS[hparams['vocoder']]
14 | else:
15 | vocoder_cls = hparams['vocoder']
16 | pkg = ".".join(vocoder_cls.split(".")[:-1])
17 | cls_name = vocoder_cls.split(".")[-1]
18 | vocoder_cls = getattr(importlib.import_module(pkg), cls_name)
19 | return vocoder_cls
20 |
21 |
22 | class BaseVocoder:
23 | def spec2wav(self, mel):
24 | """
25 |
26 | :param mel: [T, 80]
27 | :return: wav: [T']
28 | """
29 |
30 | raise NotImplementedError
31 |
32 | @staticmethod
33 | def wav2spec(wav_fn):
34 | """
35 |
36 | :param wav_fn: str
37 | :return: wav, mel: [T, 80]
38 | """
39 | raise NotImplementedError
40 |
--------------------------------------------------------------------------------
/NeuralSeq/vocoders/hifigan.py:
--------------------------------------------------------------------------------
1 | import glob
2 | import json
3 | import os
4 | import re
5 |
6 | import librosa
7 | import torch
8 |
9 | import utils
10 | from modules.hifigan.hifigan import HifiGanGenerator
11 | from utils.hparams import hparams, set_hparams
12 | from vocoders.base_vocoder import register_vocoder
13 | from vocoders.pwg import PWG
14 | from vocoders.vocoder_utils import denoise
15 |
16 |
17 | def load_model(config_path, checkpoint_path):
18 | device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
19 | ckpt_dict = torch.load(checkpoint_path, map_location="cpu")
20 | if '.yaml' in config_path:
21 | config = set_hparams(config_path, global_hparams=False)
22 | state = ckpt_dict["state_dict"]["model_gen"]
23 | elif '.json' in config_path:
24 | config = json.load(open(config_path, 'r'))
25 | state = ckpt_dict["generator"]
26 |
27 | model = HifiGanGenerator(config)
28 | model.load_state_dict(state, strict=True)
29 | model.remove_weight_norm()
30 | model = model.eval().to(device)
31 | print(f"| Loaded model parameters from {checkpoint_path}.")
32 | print(f"| HifiGAN device: {device}.")
33 | return model, config, device
34 |
35 |
36 | total_time = 0
37 |
38 |
39 | @register_vocoder
40 | class HifiGAN(PWG):
41 | def __init__(self):
42 | base_dir = hparams['vocoder_ckpt']
43 | config_path = f'{base_dir}/config.yaml'
44 | if os.path.exists(config_path):
45 | ckpt = sorted(glob.glob(f'{base_dir}/model_ckpt_steps_*.ckpt'), key=
46 | lambda x: int(re.findall(f'{base_dir}/model_ckpt_steps_(\d+).ckpt', x)[0]))[-1]
47 | print('| load HifiGAN: ', ckpt)
48 | self.model, self.config, self.device = load_model(config_path=config_path, checkpoint_path=ckpt)
49 | else:
50 | config_path = f'{base_dir}/config.json'
51 | ckpt = f'{base_dir}/generator_v1'
52 | if os.path.exists(config_path):
53 | self.model, self.config, self.device = load_model(config_path=config_path, checkpoint_path=ckpt)
54 |
55 | def spec2wav(self, mel, **kwargs):
56 | device = self.device
57 | with torch.no_grad():
58 | c = torch.FloatTensor(mel).unsqueeze(0).transpose(2, 1).to(device)
59 | with utils.Timer('hifigan', print_time=hparams['profile_infer']):
60 | f0 = kwargs.get('f0')
61 | if f0 is not None and hparams.get('use_nsf'):
62 | f0 = torch.FloatTensor(f0[None, :]).to(device)
63 | y = self.model(c, f0).view(-1)
64 | else:
65 | y = self.model(c).view(-1)
66 | wav_out = y.cpu().numpy()
67 | if hparams.get('vocoder_denoise_c', 0.0) > 0:
68 | wav_out = denoise(wav_out, v=hparams['vocoder_denoise_c'])
69 | return wav_out
70 |
71 | # @staticmethod
72 | # def wav2spec(wav_fn, **kwargs):
73 | # wav, _ = librosa.core.load(wav_fn, sr=hparams['audio_sample_rate'])
74 | # wav_torch = torch.FloatTensor(wav)[None, :]
75 | # mel = mel_spectrogram(wav_torch, hparams).numpy()[0]
76 | # return wav, mel.T
77 |
--------------------------------------------------------------------------------
/NeuralSeq/vocoders/vocoder_utils.py:
--------------------------------------------------------------------------------
1 | import librosa
2 |
3 | from utils.hparams import hparams
4 | import numpy as np
5 |
6 |
7 | def denoise(wav, v=0.1):
8 | spec = librosa.stft(y=wav, n_fft=hparams['fft_size'], hop_length=hparams['hop_size'],
9 | win_length=hparams['win_size'], pad_mode='constant')
10 | spec_m = np.abs(spec)
11 | spec_m = np.clip(spec_m - v, a_min=0, a_max=None)
12 | spec_a = np.angle(spec)
13 |
14 | return librosa.istft(spec_m * np.exp(1j * spec_a), hop_length=hparams['hop_size'],
15 | win_length=hparams['win_size'])
16 |
--------------------------------------------------------------------------------
/assets/2bf90e35.wav:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AIGC-Audio/AudioGPT/a674543c537bb3a60b8f521df76b97a627c1c379/assets/2bf90e35.wav
--------------------------------------------------------------------------------
/assets/5d67d1b9.wav:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AIGC-Audio/AudioGPT/a674543c537bb3a60b8f521df76b97a627c1c379/assets/5d67d1b9.wav
--------------------------------------------------------------------------------
/assets/7cb0d24f.wav:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AIGC-Audio/AudioGPT/a674543c537bb3a60b8f521df76b97a627c1c379/assets/7cb0d24f.wav
--------------------------------------------------------------------------------
/assets/7ef0ec0b.wav:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AIGC-Audio/AudioGPT/a674543c537bb3a60b8f521df76b97a627c1c379/assets/7ef0ec0b.wav
--------------------------------------------------------------------------------
/assets/Track 4.wav:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AIGC-Audio/AudioGPT/a674543c537bb3a60b8f521df76b97a627c1c379/assets/Track 4.wav
--------------------------------------------------------------------------------
/assets/a-group-of-sheep-are-baaing.wav:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AIGC-Audio/AudioGPT/a674543c537bb3a60b8f521df76b97a627c1c379/assets/a-group-of-sheep-are-baaing.wav
--------------------------------------------------------------------------------
/assets/a2i.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AIGC-Audio/AudioGPT/a674543c537bb3a60b8f521df76b97a627c1c379/assets/a2i.png
--------------------------------------------------------------------------------
/assets/asr.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AIGC-Audio/AudioGPT/a674543c537bb3a60b8f521df76b97a627c1c379/assets/asr.png
--------------------------------------------------------------------------------
/assets/b973e878.wav:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AIGC-Audio/AudioGPT/a674543c537bb3a60b8f521df76b97a627c1c379/assets/b973e878.wav
--------------------------------------------------------------------------------
/assets/detection.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AIGC-Audio/AudioGPT/a674543c537bb3a60b8f521df76b97a627c1c379/assets/detection.png
--------------------------------------------------------------------------------
/assets/drums-and-music-playing-with-a-man-speaking.wav:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AIGC-Audio/AudioGPT/a674543c537bb3a60b8f521df76b97a627c1c379/assets/drums-and-music-playing-with-a-man-speaking.wav
--------------------------------------------------------------------------------
/assets/fd5cf55e.wav:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AIGC-Audio/AudioGPT/a674543c537bb3a60b8f521df76b97a627c1c379/assets/fd5cf55e.wav
--------------------------------------------------------------------------------
/assets/i2a-1.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AIGC-Audio/AudioGPT/a674543c537bb3a60b8f521df76b97a627c1c379/assets/i2a-1.png
--------------------------------------------------------------------------------
/assets/i2a-2.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AIGC-Audio/AudioGPT/a674543c537bb3a60b8f521df76b97a627c1c379/assets/i2a-2.png
--------------------------------------------------------------------------------
/assets/inpaint-1.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AIGC-Audio/AudioGPT/a674543c537bb3a60b8f521df76b97a627c1c379/assets/inpaint-1.png
--------------------------------------------------------------------------------
/assets/inpaint-2.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AIGC-Audio/AudioGPT/a674543c537bb3a60b8f521df76b97a627c1c379/assets/inpaint-2.png
--------------------------------------------------------------------------------
/assets/m2b.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AIGC-Audio/AudioGPT/a674543c537bb3a60b8f521df76b97a627c1c379/assets/m2b.png
--------------------------------------------------------------------------------
/assets/mix1.wav:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AIGC-Audio/AudioGPT/a674543c537bb3a60b8f521df76b97a627c1c379/assets/mix1.wav
--------------------------------------------------------------------------------
/assets/sound_extraction.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AIGC-Audio/AudioGPT/a674543c537bb3a60b8f521df76b97a627c1c379/assets/sound_extraction.png
--------------------------------------------------------------------------------
/assets/style_transfer_tts.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AIGC-Audio/AudioGPT/a674543c537bb3a60b8f521df76b97a627c1c379/assets/style_transfer_tts.png
--------------------------------------------------------------------------------
/assets/t2a.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AIGC-Audio/AudioGPT/a674543c537bb3a60b8f521df76b97a627c1c379/assets/t2a.png
--------------------------------------------------------------------------------
/assets/t2i.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AIGC-Audio/AudioGPT/a674543c537bb3a60b8f521df76b97a627c1c379/assets/t2i.png
--------------------------------------------------------------------------------
/assets/t2s.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AIGC-Audio/AudioGPT/a674543c537bb3a60b8f521df76b97a627c1c379/assets/t2s.png
--------------------------------------------------------------------------------
/assets/tsd.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AIGC-Audio/AudioGPT/a674543c537bb3a60b8f521df76b97a627c1c379/assets/tsd.png
--------------------------------------------------------------------------------
/assets/tts.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AIGC-Audio/AudioGPT/a674543c537bb3a60b8f521df76b97a627c1c379/assets/tts.png
--------------------------------------------------------------------------------
/audio_detection/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AIGC-Audio/AudioGPT/a674543c537bb3a60b8f521df76b97a627c1c379/audio_detection/__init__.py
--------------------------------------------------------------------------------
/audio_detection/audio_infer/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AIGC-Audio/AudioGPT/a674543c537bb3a60b8f521df76b97a627c1c379/audio_detection/audio_infer/__init__.py
--------------------------------------------------------------------------------
/audio_detection/audio_infer/pytorch/evaluate.py:
--------------------------------------------------------------------------------
1 | from sklearn import metrics
2 |
3 | from pytorch_utils import forward
4 |
5 |
6 | class Evaluator(object):
7 | def __init__(self, model):
8 | """Evaluator.
9 |
10 | Args:
11 | model: object
12 | """
13 | self.model = model
14 |
15 | def evaluate(self, data_loader):
16 | """Forward evaluation data and calculate statistics.
17 |
18 | Args:
19 | data_loader: object
20 |
21 | Returns:
22 | statistics: dict,
23 | {'average_precision': (classes_num,), 'auc': (classes_num,)}
24 | """
25 |
26 | # Forward
27 | output_dict = forward(
28 | model=self.model,
29 | generator=data_loader,
30 | return_target=True)
31 |
32 | clipwise_output = output_dict['clipwise_output'] # (audios_num, classes_num)
33 | target = output_dict['target'] # (audios_num, classes_num)
34 |
35 | average_precision = metrics.average_precision_score(
36 | target, clipwise_output, average=None)
37 |
38 | auc = metrics.roc_auc_score(target, clipwise_output, average=None)
39 |
40 | statistics = {'average_precision': average_precision, 'auc': auc}
41 |
42 | return statistics
--------------------------------------------------------------------------------
/audio_detection/audio_infer/pytorch/losses.py:
--------------------------------------------------------------------------------
1 | import torch
2 | import torch.nn.functional as F
3 |
4 |
5 | def clip_bce(output_dict, target_dict):
6 | """Binary crossentropy loss.
7 | """
8 | return F.binary_cross_entropy(
9 | output_dict['clipwise_output'], target_dict['target'])
10 |
11 |
12 | def get_loss_func(loss_type):
13 | if loss_type == 'clip_bce':
14 | return clip_bce
--------------------------------------------------------------------------------
/audio_detection/audio_infer/results/YDlWd7Wmdi1E.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AIGC-Audio/AudioGPT/a674543c537bb3a60b8f521df76b97a627c1c379/audio_detection/audio_infer/results/YDlWd7Wmdi1E.png
--------------------------------------------------------------------------------
/audio_detection/audio_infer/utils/crash.py:
--------------------------------------------------------------------------------
1 | import sys
2 |
3 | class ExceptionHook:
4 | instance = None
5 | def __call__(self, *args, **kwargs):
6 | if self.instance is None:
7 | from IPython.core import ultratb
8 | self.instance = ultratb.FormattedTB(mode='Plain',
9 | color_scheme='Linux', call_pdb=1)
10 | return self.instance(*args, **kwargs)
11 |
12 | sys.excepthook = ExceptionHook()
13 |
--------------------------------------------------------------------------------
/audio_detection/audio_infer/utils/create_black_list.py:
--------------------------------------------------------------------------------
1 | import argparse
2 | import csv
3 | import os
4 |
5 | from utilities import create_folder
6 |
7 |
8 | def dcase2017task4(args):
9 | """Create black list. Black list is a list of audio ids that will be
10 | skipped in training.
11 | """
12 |
13 | # Augments & parameters
14 | workspace = args.workspace
15 |
16 | # Black list from DCASE 2017 Task 4
17 | test_weak_csv = 'metadata/black_list/groundtruth_weak_label_testing_set.csv'
18 | evaluation_weak_csv = 'metadata/black_list/groundtruth_weak_label_evaluation_set.csv'
19 |
20 | black_list_csv = os.path.join(workspace, 'black_list', 'dcase2017task4.csv')
21 | create_folder(os.path.dirname(black_list_csv))
22 |
23 | def get_id_sets(csv_path):
24 | with open(csv_path, 'r') as fr:
25 | reader = csv.reader(fr, delimiter='\t')
26 | lines = list(reader)
27 |
28 | ids_set = []
29 |
30 | for line in lines:
31 | """line: ['-5QrBL6MzLg_60.000_70.000.wav', '60.000', '70.000', 'Train horn']"""
32 | ids_set.append(line[0][0 : 11])
33 |
34 | ids_set = list(set(ids_set))
35 | return ids_set
36 |
37 | test_ids_set = get_id_sets(test_weak_csv)
38 | evaluation_ids_set = get_id_sets(evaluation_weak_csv)
39 |
40 | full_ids_set = test_ids_set + evaluation_ids_set
41 |
42 | # Write black list
43 | fw = open(black_list_csv, 'w')
44 |
45 | for id in full_ids_set:
46 | fw.write('{}\n'.format(id))
47 |
48 | print('Write black list to {}'.format(black_list_csv))
49 |
50 |
51 | if __name__ == '__main__':
52 | parser = argparse.ArgumentParser(description='')
53 | subparsers = parser.add_subparsers(dest='mode')
54 |
55 | parser_dcase2017task4 = subparsers.add_parser('dcase2017task4')
56 | parser_dcase2017task4.add_argument('--workspace', type=str, required=True)
57 |
58 | args = parser.parse_args()
59 |
60 | if args.mode == 'dcase2017task4':
61 | dcase2017task4(args)
62 |
63 | else:
64 | raise Exception('Error argument!')
--------------------------------------------------------------------------------
/audio_to_text/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AIGC-Audio/AudioGPT/a674543c537bb3a60b8f521df76b97a627c1c379/audio_to_text/__init__.py
--------------------------------------------------------------------------------
/audio_to_text/captioning/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AIGC-Audio/AudioGPT/a674543c537bb3a60b8f521df76b97a627c1c379/audio_to_text/captioning/__init__.py
--------------------------------------------------------------------------------
/audio_to_text/captioning/models/__init__.py:
--------------------------------------------------------------------------------
1 | from .base_model import *
2 | from .transformer_model import *
3 |
4 |
--------------------------------------------------------------------------------
/audio_to_text/captioning/utils/README.md:
--------------------------------------------------------------------------------
1 | # Utils
2 |
3 | Scripts in this directory are used as utility functions.
4 |
5 | ## BERT Pretrained Embeddings
6 |
7 | You can load pretrained word embeddings in Google [BERT](https://github.com/google-research/bert#pre-trained-models) instead of training word embeddings from scratch. The scripts in `utils/bert` need a BERT server in the background. We use BERT server from [bert-as-service](https://github.com/hanxiao/bert-as-service).
8 |
9 | To use bert-as-service, you need to first install the repository. It is recommended that you create a new environment with Tensorflow 1.3 to run BERT server since it is incompatible with Tensorflow 2.x.
10 |
11 | After successful installation of [bert-as-service](https://github.com/hanxiao/bert-as-service), downloading and running the BERT server needs to execute:
12 |
13 | ```bash
14 | bash scripts/prepare_bert_server.sh <path-to-server> <num-workers> zh
15 | ```
16 |
17 | By default, server based on BERT base Chinese model is running in the background. You can change to other models by changing corresponding model name and path in `scripts/prepare_bert_server.sh`.
18 |
19 | To extract BERT word embeddings, you need to execute `utils/bert/create_word_embedding.py`.
20 |
--------------------------------------------------------------------------------
/audio_to_text/captioning/utils/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AIGC-Audio/AudioGPT/a674543c537bb3a60b8f521df76b97a627c1c379/audio_to_text/captioning/utils/__init__.py
--------------------------------------------------------------------------------
/audio_to_text/captioning/utils/bert/create_word_embedding.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 |
3 | import sys
4 | import os
5 |
6 | from bert_serving.client import BertClient
7 | import numpy as np
8 | from tqdm import tqdm
9 | import fire
10 | import torch
11 |
12 | sys.path.append(os.getcwd())
13 | from utils.build_vocab import Vocabulary
14 |
15 | def main(vocab_file: str, output: str, server_hostname: str):
16 | client = BertClient(ip=server_hostname)
17 | vocabulary = torch.load(vocab_file)
18 | vocab_size = len(vocabulary)
19 |
20 | fake_embedding = client.encode(["test"]).reshape(-1)
21 | embed_size = fake_embedding.shape[0]
22 |
23 | print("Encoding words into embeddings with size: ", embed_size)
24 |
25 | embeddings = np.empty((vocab_size, embed_size))
26 | for i in tqdm(range(len(embeddings)), ascii=True):
27 | embeddings[i] = client.encode([vocabulary.idx2word[i]])
28 | np.save(output, embeddings)
29 |
30 |
31 | if __name__ == '__main__':
32 | fire.Fire(main)
33 |
34 |
35 |
--------------------------------------------------------------------------------
/audio_to_text/captioning/utils/fasttext/create_word_embedding.py:
--------------------------------------------------------------------------------
1 | # coding=utf-8
2 | #!/usr/bin/env python3
3 |
4 | import numpy as np
5 | import pandas as pd
6 | import torch
7 | from gensim.models import FastText
8 | from tqdm import tqdm
9 | import fire
10 |
11 | import sys
12 | import os
13 | sys.path.append(os.getcwd())
14 | from utils.build_vocab import Vocabulary
15 |
16 | def create_embedding(caption_file: str,
17 | vocab_file: str,
18 | embed_size: int,
19 | output: str,
20 | **fasttext_kwargs):
21 | caption_df = pd.read_json(caption_file)
22 | caption_df["tokens"] = caption_df["tokens"].apply(lambda x: ["<start>"] + [token for token in x] + ["<end>"])
23 |
24 | sentences = list(caption_df["tokens"].values)
25 | vocabulary = torch.load(vocab_file, map_location="cpu")
26 |
27 | epochs = fasttext_kwargs.get("epochs", 10)
28 | model = FastText(size=embed_size, min_count=1, **fasttext_kwargs)
29 | model.build_vocab(sentences=sentences)
30 | model.train(sentences=sentences, total_examples=len(sentences), epochs=epochs)
31 |
32 | word_embeddings = np.zeros((len(vocabulary), embed_size))
33 |
34 | with tqdm(total=len(vocabulary), ascii=True) as pbar:
35 | for word, idx in vocabulary.word2idx.items():
36 | if word == "<pad>" or word == "<unk>":
37 | continue
38 | word_embeddings[idx] = model.wv[word]
39 | pbar.update()
40 |
41 | np.save(output, word_embeddings)
42 |
43 | print("Finish writing fasttext embeddings to " + output)
44 |
45 |
46 | if __name__ == "__main__":
47 | fire.Fire(create_embedding)
48 |
49 |
50 |
51 |
--------------------------------------------------------------------------------
/audio_to_text/captioning/utils/predict_nn.py:
--------------------------------------------------------------------------------
1 | import json
2 | import random
3 | import argparse
4 | import numpy as np
5 | from tqdm import tqdm
6 | from h5py import File
7 | import sklearn.metrics
8 |
9 | random.seed(1)
10 |
11 | parser = argparse.ArgumentParser()
12 | parser.add_argument("train_feature", type=str)
13 | parser.add_argument("train_corpus", type=str)
14 | parser.add_argument("pred_feature", type=str)
15 | parser.add_argument("output_json", type=str)
16 |
17 | args = parser.parse_args()
18 | train_embs = []
19 | train_idx_to_audioid = []
20 | with File(args.train_feature, "r") as store:
21 | for audio_id, embedding in tqdm(store.items(), ascii=True):
22 | train_embs.append(embedding[()])
23 | train_idx_to_audioid.append(audio_id)
24 |
25 | train_annotation = json.load(open(args.train_corpus, "r"))["audios"]
26 | train_audioid_to_tokens = {}
27 | for item in train_annotation:
28 | audio_id = item["audio_id"]
29 | train_audioid_to_tokens[audio_id] = [cap_item["tokens"] for cap_item in item["captions"]]
30 | train_embs = np.stack(train_embs)
31 |
32 |
33 | pred_data = []
34 | pred_embs = []
35 | pred_idx_to_audioids = []
36 | with File(args.pred_feature, "r") as store:
37 | for audio_id, embedding in tqdm(store.items(), ascii=True):
38 | pred_embs.append(embedding[()])
39 | pred_idx_to_audioids.append(audio_id)
40 | pred_embs = np.stack(pred_embs)
41 |
42 | similarity = sklearn.metrics.pairwise.cosine_similarity(pred_embs, train_embs)
43 | for idx, audio_id in enumerate(pred_idx_to_audioids):
44 | train_idx = similarity[idx].argmax()
45 | pred_data.append({
46 | "filename": audio_id,
47 | "tokens": random.choice(train_audioid_to_tokens[train_idx_to_audioid[train_idx]])
48 | })
49 | json.dump({"predictions": pred_data}, open(args.output_json, "w"), ensure_ascii=False, indent=4)
50 |
--------------------------------------------------------------------------------
/audio_to_text/captioning/utils/remove_optimizer.py:
--------------------------------------------------------------------------------
1 | import argparse
2 | import torch
3 |
4 |
5 | def main(checkpoint):
6 | state_dict = torch.load(checkpoint, map_location="cpu")
7 | if "optimizer" in state_dict:
8 | del state_dict["optimizer"]
9 | if "lr_scheduler" in state_dict:
10 | del state_dict["lr_scheduler"]
11 | torch.save(state_dict, checkpoint)
12 |
13 |
14 | if __name__ == "__main__":
15 | parser = argparse.ArgumentParser()
16 | parser.add_argument("checkpoint", type=str)
17 | args = parser.parse_args()
18 | main(args.checkpoint)
19 |
--------------------------------------------------------------------------------
/audio_to_text/captioning/utils/report_results.py:
--------------------------------------------------------------------------------
1 | from pathlib import Path
2 | import argparse
3 | import numpy as np
4 |
5 | parser = argparse.ArgumentParser()
6 | parser.add_argument("--input", help="input filename", type=str, nargs="+")
7 | parser.add_argument("--output", help="output result file", default=None)
8 |
9 | args = parser.parse_args()
10 |
11 |
12 | scores = {}
13 | for path in args.input:
14 | with open(path, "r") as reader:
15 | for line in reader.readlines():
16 | metric, score = line.strip().split(": ")
17 | score = float(score)
18 | if metric not in scores:
19 | scores[metric] = []
20 | scores[metric].append(score)
21 |
22 | if len(scores) == 0:
23 | print("No experiment directory found, wrong path?")
24 | exit(1)
25 |
26 | with open(args.output, "w") as writer:
27 | print("Average results: ", file=writer)
28 | for metric, score in scores.items():
29 | score = np.array(score)
30 | mean = np.mean(score)
31 | std = np.std(score)
32 | print(f"{metric}: {mean:.3f} (±{std:.3f})", file=writer)
33 | print("", file=writer)
34 | print("Best results: ", file=writer)
35 | for metric, score in scores.items():
36 | score = np.max(score)
37 | print(f"{metric}: {score:.3f}", file=writer)
38 |
--------------------------------------------------------------------------------
/audio_to_text/captioning/utils/word2vec/create_word_embedding.py:
--------------------------------------------------------------------------------
1 | # coding=utf-8
2 | #!/usr/bin/env python3
3 |
4 | import numpy as np
5 | import pandas as pd
6 | import torch
7 | import gensim
8 | from gensim.models import Word2Vec
9 | from tqdm import tqdm
10 | import fire
11 |
12 | import sys
13 | import os
14 | sys.path.append(os.getcwd())
15 | from utils.build_vocab import Vocabulary
16 |
17 | def create_embedding(vocab_file: str,
18 | embed_size: int,
19 | output: str,
20 | caption_file: str = None,
21 | pretrained_weights_path: str = None,
22 | **word2vec_kwargs):
23 | vocabulary = torch.load(vocab_file, map_location="cpu")
24 |
25 | if pretrained_weights_path:
26 | model = gensim.models.KeyedVectors.load_word2vec_format(
27 | fname=pretrained_weights_path,
28 | binary=True,
29 | )
30 | if model.vector_size != embed_size:
31 | assert embed_size < model.vector_size, f"only reduce dimension, cannot add dimesion {model.vector_size} to {embed_size}"
32 | from sklearn.decomposition import PCA
33 | pca = PCA(n_components=embed_size)
34 | model.vectors = pca.fit_transform(model.vectors)
35 | else:
36 | caption_df = pd.read_json(caption_file)
37 | caption_df["tokens"] = caption_df["tokens"].apply(lambda x: ["<start>"] + [token for token in x] + ["<end>"])
38 | sentences = list(caption_df["tokens"].values)
39 | epochs = word2vec_kwargs.get("epochs", 10)
40 | if "epochs" in word2vec_kwargs:
41 | del word2vec_kwargs["epochs"]
42 | model = Word2Vec(size=embed_size, min_count=1, **word2vec_kwargs)
43 | model.build_vocab(sentences=sentences)
44 | model.train(sentences=sentences, total_examples=len(sentences), epochs=epochs)
45 |
46 | word_embeddings = np.random.randn(len(vocabulary), embed_size)
47 |
48 | if isinstance(model, gensim.models.word2vec.Word2Vec):
49 | model = model.wv
50 | with tqdm(total=len(vocabulary), ascii=True) as pbar:
51 | for word, idx in vocabulary.word2idx.items():
52 | try:
53 | word_embeddings[idx] = model.get_vector(word)
54 | except KeyError:
55 | print(f"word {word} not found in word2vec model, it is random initialized!")
56 | pbar.update()
57 |
58 | np.save(output, word_embeddings)
59 |
60 | print("Finish writing word2vec embeddings to " + output)
61 |
62 |
63 | if __name__ == "__main__":
64 | fire.Fire(create_embedding)
65 |
66 |
67 |
68 |
--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | --extra-index-url https://download.pytorch.org/whl/cu113
2 | accelerate
3 | addict==2.4.0
4 | aiofiles
5 | albumentations==1.3.0
6 | appdirs==1.4.4
7 | basicsr==1.4.2
8 | beautifulsoup4==4.10.0
9 | Cython==0.29.24
10 | diffusers
11 | einops==0.3.0
12 | espnet
13 | espnet_model_zoo
14 | ffmpeg-python
15 | g2p-en==2.1.0
16 | google==3.0.0
17 | gradio
18 | h5py
19 | imageio==2.9.0
20 | imageio-ffmpeg==0.4.2
21 | invisible-watermark>=0.1.5
22 | jieba
23 | kornia==0.6
24 | langchain==0.0.101
25 | librosa
26 | loguru
27 | miditoolkit==0.1.7
28 | mmcv==1.5.0
29 | mmdet==2.23.0
30 | mmengine==0.7.2
31 | moviepy==1.0.3
32 | numpy==1.23.1
33 | omegaconf==2.1.1
34 | open_clip_torch==2.0.2
35 | openai
36 | openai-whisper
37 | opencv-contrib-python==4.3.0.36
38 | praat-parselmouth==0.3.3
39 | prettytable==3.6.0
40 | proglog==0.1.9
41 | pycwt==0.3.0a22
42 | pyloudnorm==0.1.0
43 | pypinyin==0.43.0
44 | pytorch-lightning==1.5.0
45 | pytorch-ssim==0.1
46 | pyworld==0.3.0
47 | resampy==0.2.2
48 | Resemblyzer==0.1.1.dev0
49 | safetensors==0.2.7
50 | sklearn==0.0
51 | soundfile
52 | soupsieve==2.3
53 | streamlit==1.12.1
54 | streamlit-drawable-canvas==0.8.0
55 | tensorboardX==2.4
56 | test-tube>=0.7.5
57 | TextGrid==1.5
58 | timm==0.6.12
59 | torch==1.12.1
60 | torchaudio==0.12.1
61 | torch-fidelity==0.3.0
62 | torchlibrosa
63 | torchmetrics==0.6.0
64 | torchvision==0.13.1
65 | transformers==4.26.1
66 | typing-extensions==4.0.0
67 | uuid==1.30
68 | webdataset==0.2.5
69 | webrtcvad==2.0.10
70 | yapf==0.32.0
71 | git+https://github.com/openai/CLIP.git
--------------------------------------------------------------------------------
/run.md:
--------------------------------------------------------------------------------
1 | # Run AudioGPT
2 | ```
3 | # create a new environment
4 | conda create -n audiogpt python=3.8
5 |
6 | # prepare the basic environments
7 | pip install -r requirements.txt
8 |
9 | # download the foundation models you need
10 | bash download.sh
11 |
12 | # prepare your private openAI private key
13 | export OPENAI_API_KEY={Your_Private_Openai_Key}
14 |
15 | # Start AudioGPT !
16 | python audio-chatgpt.py
17 | ```
18 |
19 |
20 |
--------------------------------------------------------------------------------
/sound_extraction/model/LASSNet.py:
--------------------------------------------------------------------------------
1 | import torch
2 | import torch.nn as nn
3 | import torch.nn.functional as F
4 | from .text_encoder import Text_Encoder
5 | from .resunet_film import UNetRes_FiLM
6 |
7 | class LASSNet(nn.Module):
8 | def __init__(self, device='cuda'):
9 | super(LASSNet, self).__init__()
10 | self.text_embedder = Text_Encoder(device)
11 | self.UNet = UNetRes_FiLM(channels=1, cond_embedding_dim=256)
12 |
13 | def forward(self, x, caption):
14 | # x: (Batch, 1, T, 128))
15 | input_ids, attns_mask = self.text_embedder.tokenize(caption)
16 |
17 | cond_vec = self.text_embedder(input_ids, attns_mask)[0]
18 | dec_cond_vec = cond_vec
19 |
20 | mask = self.UNet(x, cond_vec, dec_cond_vec)
21 | mask = torch.sigmoid(mask)
22 | return mask
23 |
24 | def get_tokenizer(self):
25 | return self.text_embedder.tokenizer
26 |
--------------------------------------------------------------------------------
/sound_extraction/model/film.py:
--------------------------------------------------------------------------------
1 | import torch
2 | import torch.nn as nn
3 |
4 | class Film(nn.Module):
5 | def __init__(self, channels, cond_embedding_dim):
6 | super(Film, self).__init__()
7 | self.linear = nn.Sequential(
8 | nn.Linear(cond_embedding_dim, channels * 2),
9 | nn.ReLU(inplace=True),
10 | nn.Linear(channels * 2, channels),
11 | nn.ReLU(inplace=True)
12 | )
13 |
14 | def forward(self, data, cond_vec):
15 | """
16 | :param data: [batchsize, channels, samples] or [batchsize, channels, T, F] or [batchsize, channels, F, T]
17 | :param cond_vec: [batchsize, cond_embedding_dim]
18 | :return:
19 | """
20 | bias = self.linear(cond_vec) # [batchsize, channels]
21 | if len(list(data.size())) == 3:
22 | data = data + bias[..., None]
23 | elif len(list(data.size())) == 4:
24 | data = data + bias[..., None, None]
25 | else:
26 | print("Warning: The size of input tensor,", data.size(), "is not correct. Film is not working.")
27 | return data
--------------------------------------------------------------------------------
/sound_extraction/model/text_encoder.py:
--------------------------------------------------------------------------------
1 | import torch
2 | import torch.nn as nn
3 | from transformers import *
4 | import warnings
5 | warnings.filterwarnings('ignore')
6 | # pretrained model name: (model class, model tokenizer, output dimension, token style)
7 | MODELS = {
8 | 'prajjwal1/bert-mini': (BertModel, BertTokenizer),
9 | }
10 |
11 | class Text_Encoder(nn.Module):
12 | def __init__(self, device):
13 | super(Text_Encoder, self).__init__()
14 | self.base_model = 'prajjwal1/bert-mini'
15 | self.dropout = 0.1
16 |
17 | self.tokenizer = MODELS[self.base_model][1].from_pretrained(self.base_model)
18 |
19 | self.bert_layer = MODELS[self.base_model][0].from_pretrained(self.base_model,
20 | add_pooling_layer=False,
21 | hidden_dropout_prob=self.dropout,
22 | attention_probs_dropout_prob=self.dropout,
23 | output_hidden_states=True)
24 |
25 | self.linear_layer = nn.Sequential(nn.Linear(256, 256), nn.ReLU(inplace=True))
26 |
27 | self.device = device
28 |
29 | def tokenize(self, caption):
30 | # device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
31 | tokenized = self.tokenizer(caption, add_special_tokens=False, padding=True, return_tensors='pt')
32 | input_ids = tokenized['input_ids']
33 | attns_mask = tokenized['attention_mask']
34 |
35 | input_ids = input_ids.to(self.device)
36 | attns_mask = attns_mask.to(self.device)
37 | return input_ids, attns_mask
38 |
39 | def forward(self, input_ids, attns_mask):
40 | # input_ids, attns_mask = self.tokenize(caption)
41 | output = self.bert_layer(input_ids=input_ids, attention_mask=attns_mask)[0]
42 | cls_embed = output[:, 0, :]
43 | text_embed = self.linear_layer(cls_embed)
44 |
45 | return text_embed, output # text_embed: (batch, hidden_size)
--------------------------------------------------------------------------------
/sound_extraction/utils/wav_io.py:
--------------------------------------------------------------------------------
1 | import librosa
2 | import librosa.filters
3 | import math
4 | import numpy as np
5 | import scipy.io.wavfile
6 |
7 | def load_wav(path):
8 | max_length = 32000 * 10
9 | wav = librosa.core.load(path, sr=32000)[0]
10 | if len(wav) > max_length:
11 | audio = wav[0:max_length]
12 |
13 | # pad audio to max length, 10s for AudioCaps
14 | if len(wav) < max_length:
15 | # audio = torch.nn.functional.pad(audio, (0, self.max_length - audio.size(1)), 'constant')
16 | wav = np.pad(wav, (0, max_length - len(wav)), 'constant')
17 | wav = wav[...,None]
18 | return wav
19 |
20 |
21 | def save_wav(wav, path):
22 | wav *= 32767 / max(0.01, np.max(np.abs(wav)))
23 | scipy.io.wavfile.write(path, 32000, wav.astype(np.int16))
--------------------------------------------------------------------------------
/text_to_audio/Make_An_Audio/configs/img_to_audio/img2audio_args.yaml:
--------------------------------------------------------------------------------
1 | model:
2 | base_learning_rate: 1.0e-05
3 | target: ldm.models.diffusion.ddpm_audio.LatentDiffusion_audio
4 | params:
5 | linear_start: 0.00085
6 | linear_end: 0.0120
7 | num_timesteps_cond: 1
8 | log_every_t: 200
9 | timesteps: 1000
10 | first_stage_key: image
11 | cond_stage_key: caption
12 | image_size: 32 # unused
13 | mel_dim: 10 # 80 // 2^3
14 | mel_length: 78 # 624 // 2^3
15 | channels: 4
16 | cond_stage_trainable: false
17 | conditioning_key: crossattn
18 | monitor: val/loss_simple_ema
19 | scale_by_std: True
20 | use_ema: False
21 |
22 | scheduler_config: # 10000 warmup steps
23 | target: ldm.lr_scheduler.LambdaLinearScheduler
24 | params:
25 | warm_up_steps: [10000]
26 | cycle_lengths: [10000000000000]
27 | f_start: [1.e-6]
28 | f_max: [1.]
29 | f_min: [ 1.]
30 |
31 | unet_config:
32 | target: ldm.modules.diffusionmodules.custom_openaimodel.UNetModel
33 | params:
34 | image_size: 32 # ununsed
35 | in_channels: 4
36 | out_channels: 4
37 | model_channels: 256
38 | attention_resolutions:
39 | - 1
40 | - 2
41 | num_res_blocks: 2
42 | channel_mult: # num_down = len(ch_mult)-1
43 | - 1
44 | - 2
45 | num_head_channels: 32
46 | use_spatial_transformer: true
47 | transformer_depth: 1
48 | context_dim: 1024
49 | use_context_project: false
50 |
51 |
52 | first_stage_config:
53 | target: ldm.models.autoencoder.AutoencoderKL
54 | params:
55 | embed_dim: 4
56 | monitor: val/rec_loss
57 | ddconfig:
58 | double_z: true
59 | z_channels: 4
60 | resolution: 848
61 | in_channels: 1
62 | out_ch: 1
63 | ch: 128
64 | ch_mult: [ 1, 2, 2, 4 ] # num_down = len(ch_mult)-1
65 | num_res_blocks: 2
66 | attn_resolutions: [106, 212]
67 | dropout: 0.0
68 | lossconfig:
69 | target: torch.nn.Identity
70 |
71 | cond_stage_config:
72 | target: ldm.modules.encoders.modules.FrozenGlobalNormOpenCLIPEmbedder
73 | params:
74 | freeze: True
75 | delvisual: False
76 |
77 |
78 |
--------------------------------------------------------------------------------
/text_to_audio/Make_An_Audio/configs/inpaint/txt2audio_args.yaml:
--------------------------------------------------------------------------------
1 | model:
2 | base_learning_rate: 1.0e-05
3 | target: ldm.models.diffusion.ddpm_audio.LatentDiffusion_audio
4 | params:
5 | linear_start: 0.0015
6 | linear_end: 0.0205
7 | log_every_t: 100
8 | timesteps: 1000
9 | loss_type: l1
10 | first_stage_key: image
11 | cond_stage_key: masked_image
12 | image_size: 32 # unused
13 | mel_dim: 10 # 80 // 2^3
14 | mel_length: 106 # 848 // 2^3
15 | channels: 4
16 | concat_mode: true
17 | monitor: val/loss
18 | use_ema: False
19 |
20 | scheduler_config:
21 | target: ldm.lr_scheduler.LambdaWarmUpCosineScheduler
22 | params:
23 | verbosity_interval: 0
24 | warm_up_steps: 1000
25 | max_decay_steps: 50000
26 | lr_start: 0.001
27 | lr_max: 0.1
28 | lr_min: 0.0001
29 |
30 | unet_config:
31 | target: ldm.modules.diffusionmodules.openaimodel.UNetModel
32 | params:
33 | image_size: 32 # ununsed
34 | in_channels: 9 # 4 + 1 + 4
35 | out_channels: 4
36 | model_channels: 320
37 | attention_resolutions:
38 | - 1
39 | - 2
40 | num_res_blocks: 2
41 | channel_mult: # num_down = len(ch_mult)-1
42 | - 1
43 | - 2
44 | num_heads: 8
45 | resblock_updown: true
46 |
47 | first_stage_config:
48 | target: ldm.models.autoencoder.AutoencoderKL
49 | params:
50 | embed_dim: 4
51 | monitor: val/rec_loss
52 | ckpt_path: # /apdcephfs/share_1316500/nlphuang/results/Text_to_audio/ae15/2022-12-15T22-24-00_mixdata_kl_4_tile/epoch=000009-v2.ckpt
53 | ddconfig:
54 | double_z: true
55 | z_channels: 4
56 | resolution: 848
57 | in_channels: 1
58 | out_ch: 1
59 | ch: 128
60 | ch_mult: [ 1, 2, 2, 4 ] # num_down = len(ch_mult)-1
61 | num_res_blocks: 2
62 | attn_resolutions: [106, 212]
63 | dropout: 0.0
64 | lossconfig:
65 | target: torch.nn.Identity
66 |
67 | cond_stage_config: __is_first_stage__
68 |
69 |
--------------------------------------------------------------------------------
/text_to_audio/Make_An_Audio/configs/text_to_audio/clap_args.yaml:
--------------------------------------------------------------------------------
1 | # TEXT ENCODER CONFIG
2 | text_model: 'bert-base-uncased'
3 | text_len: 100
4 | transformer_embed_dim: 768
5 | freeze_text_encoder_weights: True
6 |
7 | # AUDIO ENCODER CONFIG
8 | audioenc_name: 'Cnn14'
9 | out_emb: 2048
10 | sampling_rate: 44100
11 | duration: 9
12 | fmin: 50
13 | fmax: 14000
14 | n_fft: 1028
15 | hop_size: 320
16 | mel_bins: 64
17 | window_size: 1024
18 |
19 | # PROJECTION SPACE CONFIG
20 | d_proj: 1024
21 | temperature: 0.003
22 |
23 | # TRAINING AND EVALUATION CONFIG
24 | num_classes: 527
25 | batch_size: 1024
26 | demo: False
27 |
--------------------------------------------------------------------------------
/text_to_audio/Make_An_Audio/configs/text_to_audio/hifigan_args.yaml:
--------------------------------------------------------------------------------
1 | adam_b1: 0.8
2 | adam_b2: 0.99
3 | batch_size: 24
4 | dist_config:
5 | dist_backend: nccl
6 | dist_url: tcp://localhost:54321
7 | world_size: 1
8 | fmax: 8000
9 | fmax_for_loss: null
10 | fmin: 0
11 | hop_size: 256
12 | learning_rate: 0.0002
13 | lr_decay: 0.999
14 | n_fft: 1024
15 | num_gpus: 0
16 | num_mels: 80
17 | num_workers: 4
18 | resblock: '1'
19 | resblock_dilation_sizes:
20 | - - 1
21 | - 3
22 | - 5
23 | - - 1
24 | - 3
25 | - 5
26 | - - 1
27 | - 3
28 | - 5
29 | resblock_kernel_sizes:
30 | - 3
31 | - 7
32 | - 11
33 | sampling_rate: 16000
34 | seed: 1234
35 | segment_size: 8192
36 | upsample_initial_channel: 512
37 | upsample_kernel_sizes:
38 | - 16
39 | - 16
40 | - 4
41 | - 4
42 | upsample_rates:
43 | - 8
44 | - 8
45 | - 2
46 | - 2
47 | win_size: 1024
48 |
--------------------------------------------------------------------------------
/text_to_audio/Make_An_Audio/configs/text_to_audio/txt2audio_args.yaml:
--------------------------------------------------------------------------------
1 | model:
2 | base_learning_rate: 1.0e-05
3 | target: ldm.models.diffusion.ddpm_audio.LatentDiffusion_audio
4 | params:
5 | linear_start: 0.00085
6 | linear_end: 0.0120
7 | num_timesteps_cond: 1
8 | log_every_t: 200
9 | timesteps: 1000
10 | first_stage_key: image
11 | cond_stage_key: caption
12 | image_size: 32 # unused
13 | mel_dim: 10 # 80 // 2^3
14 | mel_length: 78 # 624 // 2^3
15 | channels: 4
16 | cond_stage_trainable: false
17 | conditioning_key: crossattn
18 | monitor: val/loss_simple_ema
19 | scale_by_std: True
20 | use_ema: False
21 |
22 | scheduler_config: # 10000 warmup steps
23 | target: ldm.lr_scheduler.LambdaLinearScheduler
24 | params:
25 | warm_up_steps: [10000]
26 | cycle_lengths: [10000000000000]
27 | f_start: [1.e-6]
28 | f_max: [1.]
29 | f_min: [ 1.]
30 |
31 | unet_config:
32 | target: ldm.modules.diffusionmodules.openaimodel.UNetModel
33 | params:
34 | image_size: 32 # ununsed
35 | in_channels: 4
36 | out_channels: 4
37 | model_channels: 320
38 | attention_resolutions:
39 | - 1
40 | - 2
41 | num_res_blocks: 2
42 | channel_mult: # num_down = len(ch_mult)-1
43 | - 1
44 | - 2
45 | num_heads: 8
46 | use_spatial_transformer: true
47 | transformer_depth: 1
48 | context_dim: 1024
49 | use_checkpoint: true
50 | legacy: False
51 |
52 | first_stage_config:
53 | target: ldm.models.autoencoder.AutoencoderKL
54 | params:
55 | embed_dim: 4
56 | monitor: val/rec_loss
57 | ckpt_path:
58 | ddconfig:
59 | double_z: true
60 | z_channels: 4
61 | resolution: 848
62 | in_channels: 1
63 | out_ch: 1
64 | ch: 128
65 | ch_mult: [ 1, 2, 2, 4 ] # num_down = len(ch_mult)-1
66 | num_res_blocks: 2
67 | attn_resolutions: [106, 212]
68 | dropout: 0.0
69 | lossconfig:
70 | target: torch.nn.Identity
71 |
72 | cond_stage_config:
73 | target: ldm.modules.encoders.modules.FrozenCLAPEmbedder
74 | params:
75 | weights_path: useful_ckpts/CLAP/CLAP_weights_2022.pth
76 |
77 | ckpt_path: useful_ckpts/ta40multi_epoch=000085.ckpt
78 |
79 |
--------------------------------------------------------------------------------
/text_to_audio/Make_An_Audio/ldm/models/diffusion/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AIGC-Audio/AudioGPT/a674543c537bb3a60b8f521df76b97a627c1c379/text_to_audio/Make_An_Audio/ldm/models/diffusion/__init__.py
--------------------------------------------------------------------------------
/text_to_audio/Make_An_Audio/ldm/modules/diffusionmodules/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AIGC-Audio/AudioGPT/a674543c537bb3a60b8f521df76b97a627c1c379/text_to_audio/Make_An_Audio/ldm/modules/diffusionmodules/__init__.py
--------------------------------------------------------------------------------
/text_to_audio/Make_An_Audio/ldm/modules/distributions/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AIGC-Audio/AudioGPT/a674543c537bb3a60b8f521df76b97a627c1c379/text_to_audio/Make_An_Audio/ldm/modules/distributions/__init__.py
--------------------------------------------------------------------------------
/text_to_audio/Make_An_Audio/ldm/modules/distributions/distributions.py:
--------------------------------------------------------------------------------
1 | import torch
2 | import numpy as np
3 |
4 |
5 | class AbstractDistribution:
6 | def sample(self):
7 | raise NotImplementedError()
8 |
9 | def mode(self):
10 | raise NotImplementedError()
11 |
12 |
13 | class DiracDistribution(AbstractDistribution):
14 | def __init__(self, value):
15 | self.value = value
16 |
17 | def sample(self):
18 | return self.value
19 |
20 | def mode(self):
21 | return self.value
22 |
23 |
24 | class DiagonalGaussianDistribution(object):
25 | def __init__(self, parameters, deterministic=False):
26 | self.parameters = parameters
27 | self.mean, self.logvar = torch.chunk(parameters, 2, dim=1)
28 | self.logvar = torch.clamp(self.logvar, -30.0, 20.0)
29 | self.deterministic = deterministic
30 | self.std = torch.exp(0.5 * self.logvar)
31 | self.var = torch.exp(self.logvar)
32 | if self.deterministic:
33 | self.var = self.std = torch.zeros_like(self.mean).to(device=self.parameters.device)
34 |
35 | def sample(self):
36 | x = self.mean + self.std * torch.randn(self.mean.shape).to(device=self.parameters.device)
37 | return x
38 |
39 | def kl(self, other=None):
40 | if self.deterministic:
41 | return torch.Tensor([0.])
42 | else:
43 | if other is None:
44 | return 0.5 * torch.sum(torch.pow(self.mean, 2)
45 | + self.var - 1.0 - self.logvar,
46 | dim=[1, 2, 3])
47 | else:
48 | return 0.5 * torch.sum(
49 | torch.pow(self.mean - other.mean, 2) / other.var
50 | + self.var / other.var - 1.0 - self.logvar + other.logvar,
51 | dim=[1, 2, 3])
52 |
53 | def nll(self, sample, dims=[1,2,3]):
54 | if self.deterministic:
55 | return torch.Tensor([0.])
56 | logtwopi = np.log(2.0 * np.pi)
57 | return 0.5 * torch.sum(
58 | logtwopi + self.logvar + torch.pow(sample - self.mean, 2) / self.var,
59 | dim=dims)
60 |
61 | def mode(self):
62 | return self.mean
63 |
64 |
65 | def normal_kl(mean1, logvar1, mean2, logvar2):
66 | """
67 | source: https://github.com/openai/guided-diffusion/blob/27c20a8fab9cb472df5d6bdd6c8d11c8f430b924/guided_diffusion/losses.py#L12
68 | Compute the KL divergence between two gaussians.
69 | Shapes are automatically broadcasted, so batches can be compared to
70 | scalars, among other use cases.
71 | """
72 | tensor = None
73 | for obj in (mean1, logvar1, mean2, logvar2):
74 | if isinstance(obj, torch.Tensor):
75 | tensor = obj
76 | break
77 | assert tensor is not None, "at least one argument must be a Tensor"
78 |
79 | # Force variances to be Tensors. Broadcasting helps convert scalars to
80 | # Tensors, but it does not work for torch.exp().
81 | logvar1, logvar2 = [
82 | x if isinstance(x, torch.Tensor) else torch.tensor(x).to(tensor)
83 | for x in (logvar1, logvar2)
84 | ]
85 |
86 | return 0.5 * (
87 | -1.0
88 | + logvar2
89 | - logvar1
90 | + torch.exp(logvar1 - logvar2)
91 | + ((mean1 - mean2) ** 2) * torch.exp(-logvar2)
92 | )
93 |
--------------------------------------------------------------------------------
/text_to_audio/Make_An_Audio/ldm/modules/ema.py:
--------------------------------------------------------------------------------
1 | import torch
2 | from torch import nn
3 |
4 |
5 | class LitEma(nn.Module):
6 | def __init__(self, model, decay=0.9999, use_num_upates=True):
7 | super().__init__()
8 | if decay < 0.0 or decay > 1.0:
9 | raise ValueError('Decay must be between 0 and 1')
10 |
11 | self.m_name2s_name = {}
12 | self.register_buffer('decay', torch.tensor(decay, dtype=torch.float32))
13 | self.register_buffer('num_updates', torch.tensor(0,dtype=torch.int) if use_num_upates
14 | else torch.tensor(-1,dtype=torch.int))
15 |
16 | for name, p in model.named_parameters():
17 | if p.requires_grad:
18 | #remove as '.'-character is not allowed in buffers
19 | s_name = name.replace('.','')
20 | self.m_name2s_name.update({name:s_name})
21 | self.register_buffer(s_name,p.clone().detach().data)
22 |
23 | self.collected_params = []
24 |
25 | def forward(self,model):
26 | decay = self.decay
27 |
28 | if self.num_updates >= 0:
29 | self.num_updates += 1
30 | decay = min(self.decay,(1 + self.num_updates) / (10 + self.num_updates))
31 |
32 | one_minus_decay = 1.0 - decay
33 |
34 | with torch.no_grad():
35 | m_param = dict(model.named_parameters())
36 | shadow_params = dict(self.named_buffers())
37 |
38 | for key in m_param:
39 | if m_param[key].requires_grad:
40 | sname = self.m_name2s_name[key]
41 | shadow_params[sname] = shadow_params[sname].type_as(m_param[key])
42 | shadow_params[sname].sub_(one_minus_decay * (shadow_params[sname] - m_param[key]))
43 | else:
44 | assert not key in self.m_name2s_name
45 |
46 | def copy_to(self, model):
47 | m_param = dict(model.named_parameters())
48 | shadow_params = dict(self.named_buffers())
49 | for key in m_param:
50 | if m_param[key].requires_grad:
51 | m_param[key].data.copy_(shadow_params[self.m_name2s_name[key]].data)
52 | else:
53 | assert not key in self.m_name2s_name
54 |
55 | def store(self, parameters):
56 | """
57 | Save the current parameters for restoring later.
58 | Args:
59 | parameters: Iterable of `torch.nn.Parameter`; the parameters to be
60 | temporarily stored.
61 | """
62 | self.collected_params = [param.clone() for param in parameters]
63 |
64 | def restore(self, parameters):
65 | """
66 | Restore the parameters stored with the `store` method.
67 | Useful to validate the model with EMA parameters without affecting the
68 | original optimization process. Store the parameters before the
69 | `copy_to` method. After validation (or model saving), use this to
70 | restore the former parameters.
71 | Args:
72 | parameters: Iterable of `torch.nn.Parameter`; the parameters to be
73 | updated with the stored parameters.
74 | """
75 | for c_param, param in zip(self.collected_params, parameters):
76 | param.data.copy_(c_param.data)
77 |
--------------------------------------------------------------------------------
/text_to_audio/Make_An_Audio/ldm/modules/encoders/CLAP/__init__.py:
--------------------------------------------------------------------------------
1 | from . import clap
2 | from . import audio
3 | from . import utils
--------------------------------------------------------------------------------
/text_to_audio/Make_An_Audio/ldm/modules/encoders/CLAP/config.yml:
--------------------------------------------------------------------------------
1 | # TEXT ENCODER CONFIG
2 | text_model: 'bert-base-uncased'
3 | text_len: 100
4 | transformer_embed_dim: 768
5 | freeze_text_encoder_weights: True
6 |
7 | # AUDIO ENCODER CONFIG
8 | audioenc_name: 'Cnn14'
9 | out_emb: 2048
10 | sampling_rate: 44100
11 | duration: 5
12 | fmin: 50
13 | fmax: 14000
14 | n_fft: 1028
15 | hop_size: 320
16 | mel_bins: 64
17 | window_size: 1024
18 |
19 | # PROJECTION SPACE CONFIG
20 | d_proj: 1024
21 | temperature: 0.003
22 |
23 | # TRAINING AND EVALUATION CONFIG
24 | num_classes: 527
25 | batch_size: 1024
26 | demo: False
27 |
--------------------------------------------------------------------------------
/text_to_audio/Make_An_Audio/ldm/modules/encoders/CLAP/utils.py:
--------------------------------------------------------------------------------
1 | import argparse
2 | import yaml
3 | import sys
4 |
5 | def read_config_as_args(config_path,args=None,is_config_str=False):
6 | return_dict = {}
7 |
8 | if config_path is not None:
9 | if is_config_str:
10 | yml_config = yaml.load(config_path, Loader=yaml.FullLoader)
11 | else:
12 | with open(config_path, "r") as f:
13 | yml_config = yaml.load(f, Loader=yaml.FullLoader)
14 |
15 | if args != None:
16 | for k, v in yml_config.items():
17 | if k in args.__dict__:
18 | args.__dict__[k] = v
19 | else:
20 | sys.stderr.write("Ignored unknown parameter {} in yaml.\n".format(k))
21 | else:
22 | for k, v in yml_config.items():
23 | return_dict[k] = v
24 |
25 | args = args if args != None else return_dict
26 | return argparse.Namespace(**args)
27 |
--------------------------------------------------------------------------------
/text_to_audio/Make_An_Audio/ldm/modules/encoders/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AIGC-Audio/AudioGPT/a674543c537bb3a60b8f521df76b97a627c1c379/text_to_audio/Make_An_Audio/ldm/modules/encoders/__init__.py
--------------------------------------------------------------------------------
/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/__init__.py:
--------------------------------------------------------------------------------
1 | from .factory import list_models, create_model, create_model_and_transforms, add_model_config
2 | from .loss import ClipLoss, gather_features, LPLoss, lp_gather_features, LPMetrics
3 | from .model import CLAP, CLAPTextCfg, CLAPVisionCfg, CLAPAudioCfp, convert_weights_to_fp16, trace_model
4 | from .openai import load_openai_model, list_openai_models
5 | from .pretrained import list_pretrained, list_pretrained_tag_models, list_pretrained_model_tags,\
6 | get_pretrained_url, download_pretrained
7 | from .tokenizer import SimpleTokenizer, tokenize
8 | from .transform import image_transform
9 |
--------------------------------------------------------------------------------
/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/bert.py:
--------------------------------------------------------------------------------
1 | from transformers import BertTokenizer, BertModel
2 | tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
3 | model = BertModel.from_pretrained("bert-base-uncased")
4 | text = "Replace me by any text you'd like."
5 |
6 | def bert_embeddings(text):
7 | # text = "Replace me by any text you'd like."
8 | encoded_input = tokenizer(text, return_tensors='pt')
9 | output = model(**encoded_input)
10 | return output
11 |
12 | from transformers import RobertaTokenizer, RobertaModel
13 |
14 | tokenizer = RobertaTokenizer.from_pretrained('roberta-base')
15 | model = RobertaModel.from_pretrained('roberta-base')
16 | text = "Replace me by any text you'd like."
17 | def Roberta_embeddings(text):
18 | # text = "Replace me by any text you'd like."
19 | encoded_input = tokenizer(text, return_tensors='pt')
20 | output = model(**encoded_input)
21 | return output
22 |
23 | from transformers import BartTokenizer, BartModel
24 |
25 | tokenizer = BartTokenizer.from_pretrained('facebook/bart-base')
26 | model = BartModel.from_pretrained('facebook/bart-base')
27 | text = "Replace me by any text you'd like."
28 | def bart_embeddings(text):
29 | # text = "Replace me by any text you'd like."
30 | encoded_input = tokenizer(text, return_tensors='pt')
31 | output = model(**encoded_input)
32 | return output
--------------------------------------------------------------------------------
/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/bpe_simple_vocab_16e6.txt.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AIGC-Audio/AudioGPT/a674543c537bb3a60b8f521df76b97a627c1c379/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/bpe_simple_vocab_16e6.txt.gz
--------------------------------------------------------------------------------
/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/linear_probe.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | import torch.nn.functional as F
3 | from torch import nn
4 | from .model import MLPLayers
5 |
6 |
7 | class LinearProbe(nn.Module):
8 | def __init__(self, model, mlp, freeze, in_ch, out_ch, act=None):
9 | """
10 | Args:
11 | model: nn.Module
12 | mlp: bool, if True, then use the MLP layer as the linear probe module
13 | freeze: bool, if Ture, then freeze all the CLAP model's layers when training the linear probe
14 | in_ch: int, the output channel from CLAP model
15 | out_ch: int, the output channel from linear probe (class_num)
16 | act: torch.nn.functional, the activation function before the loss function
17 | """
18 | super().__init__()
19 | in_ch = 512
20 | self.clap_model = model
21 | self.clap_model.text_branch = None # to save memory
22 | self.freeze = freeze
23 | if mlp:
24 | self.lp_layer = MLPLayers(units=[in_ch, in_ch * 2, out_ch])
25 | else:
26 | self.lp_layer = nn.Linear(in_ch, out_ch)
27 |
28 | if self.freeze:
29 | for param in self.clap_model.parameters():
30 | param.requires_grad = False
31 |
32 | if act == 'None':
33 | self.act = None
34 | elif act == 'relu':
35 | self.act = nn.ReLU()
36 | elif act == 'elu':
37 | self.act = nn.ELU()
38 | elif act == 'prelu':
39 | self.act = nn.PReLU(num_parameters=in_ch)
40 | elif act == 'softmax':
41 | self.act = nn.Softmax(dim=-1)
42 | elif act == 'sigmoid':
43 | self.act = nn.Sigmoid()
44 |
45 | def forward(self, x, mix_lambda=None, device=None):
46 | """
47 | Args:
48 | x: waveform, torch.tensor [batch, t_samples] / batch of mel_spec and longer list
49 | mix_lambda: torch.tensor [batch], the mixup lambda
50 | Returns:
51 | class_prob: torch.tensor [batch, class_num]
52 |
53 | """
54 | # batchnorm cancel grandient
55 | if self.freeze:
56 | self.clap_model.eval()
57 |
58 | x = self.clap_model.audio_projection(
59 | self.clap_model.audio_branch(x, mixup_lambda=mix_lambda, device=device)["embedding"])
60 | out = self.lp_layer(x)
61 | if self.act is not None:
62 | out = self.act(out)
63 | return out
64 |
--------------------------------------------------------------------------------
/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model_configs/HTSAT-base.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 1024,
3 | "audio_cfg": {
4 | "audio_length": 1024,
5 | "clip_samples": 480000,
6 | "mel_bins": 64,
7 | "sample_rate": 48000,
8 | "window_size": 1024,
9 | "hop_size": 480,
10 | "fmin": 50,
11 | "fmax": 14000,
12 | "class_num": 527,
13 | "model_type": "HTSAT",
14 | "model_name": "base"
15 | },
16 | "text_cfg": {
17 | "context_length": 77,
18 | "vocab_size": 49408,
19 | "width": 512,
20 | "heads": 8,
21 | "layers": 12
22 | }
23 | }
--------------------------------------------------------------------------------
/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model_configs/HTSAT-large.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 2048,
3 | "audio_cfg": {
4 | "audio_length": 1024,
5 | "clip_samples": 480000,
6 | "mel_bins": 64,
7 | "sample_rate": 48000,
8 | "window_size": 1024,
9 | "hop_size": 480,
10 | "fmin": 50,
11 | "fmax": 14000,
12 | "class_num": 527,
13 | "model_type": "HTSAT",
14 | "model_name": "large"
15 | },
16 | "text_cfg": {
17 | "context_length": 77,
18 | "vocab_size": 49408,
19 | "width": 512,
20 | "heads": 8,
21 | "layers": 12
22 | }
23 | }
--------------------------------------------------------------------------------
/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model_configs/HTSAT-tiny-win-1536.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 768,
3 | "audio_cfg": {
4 | "audio_length": 1024,
5 | "clip_samples": 480000,
6 | "mel_bins": 64,
7 | "sample_rate": 48000,
8 | "window_size": 1536,
9 | "hop_size": 480,
10 | "fmin": 50,
11 | "fmax": 14000,
12 | "class_num": 527,
13 | "model_type": "HTSAT",
14 | "model_name": "tiny"
15 | },
16 | "text_cfg": {
17 | "context_length": 77,
18 | "vocab_size": 49408,
19 | "width": 512,
20 | "heads": 8,
21 | "layers": 12
22 | }
23 | }
--------------------------------------------------------------------------------
/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model_configs/HTSAT-tiny.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 768,
3 | "audio_cfg": {
4 | "audio_length": 1024,
5 | "clip_samples": 480000,
6 | "mel_bins": 64,
7 | "sample_rate": 48000,
8 | "window_size": 1024,
9 | "hop_size": 480,
10 | "fmin": 50,
11 | "fmax": 14000,
12 | "class_num": 527,
13 | "model_type": "HTSAT",
14 | "model_name": "tiny"
15 | },
16 | "text_cfg": {
17 | "context_length": 77,
18 | "vocab_size": 49408,
19 | "width": 512,
20 | "heads": 8,
21 | "layers": 12
22 | }
23 | }
--------------------------------------------------------------------------------
/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model_configs/PANN-10.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 1024,
3 | "audio_cfg": {
4 | "audio_length": 1024,
5 | "clip_samples": 480000,
6 | "mel_bins": 64,
7 | "sample_rate": 48000,
8 | "window_size": 1024,
9 | "hop_size": 480,
10 | "fmin": 50,
11 | "fmax": 14000,
12 | "class_num": 527,
13 | "model_type": "PANN",
14 | "model_name": "Cnn10"
15 | },
16 | "text_cfg": {
17 | "context_length": 77,
18 | "vocab_size": 49408,
19 | "width": 512,
20 | "heads": 8,
21 | "layers": 12
22 | }
23 | }
--------------------------------------------------------------------------------
/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model_configs/PANN-14-fmax-18k.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 2048,
3 | "audio_cfg": {
4 | "audio_length": 1024,
5 | "clip_samples": 480000,
6 | "mel_bins": 64,
7 | "sample_rate": 48000,
8 | "window_size": 1024,
9 | "hop_size": 480,
10 | "fmin": 50,
11 | "fmax": 18000,
12 | "class_num": 527,
13 | "model_type": "PANN",
14 | "model_name": "Cnn14"
15 | },
16 | "text_cfg": {
17 | "context_length": 77,
18 | "vocab_size": 49408,
19 | "width": 512,
20 | "heads": 8,
21 | "layers": 12
22 | }
23 | }
--------------------------------------------------------------------------------
/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model_configs/PANN-14-fmax-8k-20s.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 2048,
3 | "audio_cfg": {
4 | "audio_length": 1024,
5 | "clip_samples": 960000,
6 | "mel_bins": 64,
7 | "sample_rate": 48000,
8 | "window_size": 1024,
9 | "hop_size": 360,
10 | "fmin": 50,
11 | "fmax": 8000,
12 | "class_num": 527,
13 | "model_type": "PANN",
14 | "model_name": "Cnn14"
15 | },
16 | "text_cfg": {
17 | "context_length": 77,
18 | "vocab_size": 49408,
19 | "width": 512,
20 | "heads": 8,
21 | "layers": 12
22 | }
23 | }
--------------------------------------------------------------------------------
/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model_configs/PANN-14-tiny-transformer.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 2048,
3 | "audio_cfg": {
4 | "audio_length": 1024,
5 | "clip_samples": 480000,
6 | "mel_bins": 64,
7 | "sample_rate": 48000,
8 | "window_size": 1024,
9 | "hop_size": 480,
10 | "fmin": 50,
11 | "fmax": 14000,
12 | "class_num": 527,
13 | "model_type": "PANN",
14 | "model_name": "Cnn14"
15 | },
16 | "text_cfg": {
17 | "context_length": 77,
18 | "vocab_size": 49408,
19 | "width": 512,
20 | "heads": 8,
21 | "layers": 4
22 | }
23 | }
--------------------------------------------------------------------------------
/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model_configs/PANN-14-win-1536.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 2048,
3 | "audio_cfg": {
4 | "audio_length": 1024,
5 | "clip_samples": 480000,
6 | "mel_bins": 64,
7 | "sample_rate": 48000,
8 | "window_size": 1536,
9 | "hop_size": 480,
10 | "fmin": 50,
11 | "fmax": 14000,
12 | "class_num": 527,
13 | "model_type": "PANN",
14 | "model_name": "Cnn14"
15 | },
16 | "text_cfg": {
17 | "context_length": 77,
18 | "vocab_size": 49408,
19 | "width": 512,
20 | "heads": 8,
21 | "layers": 12
22 | }
23 | }
--------------------------------------------------------------------------------
/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model_configs/PANN-14.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 2048,
3 | "audio_cfg": {
4 | "audio_length": 1024,
5 | "clip_samples": 480000,
6 | "mel_bins": 64,
7 | "sample_rate": 48000,
8 | "window_size": 1024,
9 | "hop_size": 480,
10 | "fmin": 50,
11 | "fmax": 14000,
12 | "class_num": 527,
13 | "model_type": "PANN",
14 | "model_name": "Cnn14"
15 | },
16 | "text_cfg": {
17 | "context_length": 77,
18 | "vocab_size": 49408,
19 | "width": 512,
20 | "heads": 8,
21 | "layers": 12
22 | }
23 | }
--------------------------------------------------------------------------------
/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model_configs/PANN-6.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 512,
3 | "audio_cfg": {
4 | "audio_length": 1024,
5 | "clip_samples": 480000,
6 | "mel_bins": 64,
7 | "sample_rate": 48000,
8 | "window_size": 1024,
9 | "hop_size": 480,
10 | "fmin": 50,
11 | "fmax": 14000,
12 | "class_num": 527,
13 | "model_type": "PANN",
14 | "model_name": "Cnn6"
15 | },
16 | "text_cfg": {
17 | "context_length": 77,
18 | "vocab_size": 49408,
19 | "width": 512,
20 | "heads": 8,
21 | "layers": 12
22 | }
23 | }
--------------------------------------------------------------------------------
/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model_configs/RN101-quickgelu.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 512,
3 | "quick_gelu": true,
4 | "vision_cfg": {
5 | "image_size": 224,
6 | "layers": [
7 | 3,
8 | 4,
9 | 23,
10 | 3
11 | ],
12 | "width": 64,
13 | "patch_size": null
14 | },
15 | "text_cfg": {
16 | "context_length": 77,
17 | "vocab_size": 49408,
18 | "width": 512,
19 | "heads": 8,
20 | "layers": 12
21 | }
22 | }
--------------------------------------------------------------------------------
/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model_configs/RN101.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 512,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "layers": [
6 | 3,
7 | 4,
8 | 23,
9 | 3
10 | ],
11 | "width": 64,
12 | "patch_size": null
13 | },
14 | "text_cfg": {
15 | "context_length": 77,
16 | "vocab_size": 49408,
17 | "width": 512,
18 | "heads": 8,
19 | "layers": 12
20 | }
21 | }
--------------------------------------------------------------------------------
/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model_configs/RN50-quickgelu.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 1024,
3 | "quick_gelu": true,
4 | "vision_cfg": {
5 | "image_size": 224,
6 | "layers": [
7 | 3,
8 | 4,
9 | 6,
10 | 3
11 | ],
12 | "width": 64,
13 | "patch_size": null
14 | },
15 | "text_cfg": {
16 | "context_length": 77,
17 | "vocab_size": 49408,
18 | "width": 512,
19 | "heads": 8,
20 | "layers": 12
21 | }
22 | }
23 |
--------------------------------------------------------------------------------
/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model_configs/RN50.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 1024,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "layers": [
6 | 3,
7 | 4,
8 | 6,
9 | 3
10 | ],
11 | "width": 64,
12 | "patch_size": null
13 | },
14 | "text_cfg": {
15 | "context_length": 77,
16 | "vocab_size": 49408,
17 | "width": 512,
18 | "heads": 8,
19 | "layers": 12
20 | }
21 | }
--------------------------------------------------------------------------------
/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model_configs/RN50x16.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 768,
3 | "vision_cfg": {
4 | "image_size": 384,
5 | "layers": [
6 | 6,
7 | 8,
8 | 18,
9 | 8
10 | ],
11 | "width": 96,
12 | "patch_size": null
13 | },
14 | "text_cfg": {
15 | "context_length": 77,
16 | "vocab_size": 49408,
17 | "width": 768,
18 | "heads": 12,
19 | "layers": 12
20 | }
21 | }
--------------------------------------------------------------------------------
/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model_configs/RN50x4.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 640,
3 | "vision_cfg": {
4 | "image_size": 288,
5 | "layers": [
6 | 4,
7 | 6,
8 | 10,
9 | 6
10 | ],
11 | "width": 80,
12 | "patch_size": null
13 | },
14 | "text_cfg": {
15 | "context_length": 77,
16 | "vocab_size": 49408,
17 | "width": 640,
18 | "heads": 10,
19 | "layers": 12
20 | }
21 | }
--------------------------------------------------------------------------------
/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model_configs/ViT-B-16.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 512,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "layers": 12,
6 | "width": 768,
7 | "patch_size": 16
8 | },
9 | "text_cfg": {
10 | "context_length": 77,
11 | "vocab_size": 49408,
12 | "width": 512,
13 | "heads": 8,
14 | "layers": 12
15 | }
16 | }
--------------------------------------------------------------------------------
/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model_configs/ViT-B-32-quickgelu.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 512,
3 | "quick_gelu": true,
4 | "vision_cfg": {
5 | "image_size": 224,
6 | "layers": 12,
7 | "width": 768,
8 | "patch_size": 32
9 | },
10 | "text_cfg": {
11 | "context_length": 77,
12 | "vocab_size": 49408,
13 | "width": 512,
14 | "heads": 8,
15 | "layers": 12
16 | }
17 | }
--------------------------------------------------------------------------------
/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model_configs/ViT-B-32.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 512,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "layers": 12,
6 | "width": 768,
7 | "patch_size": 32
8 | },
9 | "text_cfg": {
10 | "context_length": 77,
11 | "vocab_size": 49408,
12 | "width": 512,
13 | "heads": 8,
14 | "layers": 12
15 | }
16 | }
--------------------------------------------------------------------------------
/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/model_configs/ViT-L-14.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 768,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "layers": 24,
6 | "width": 1024,
7 | "patch_size": 14
8 | },
9 | "text_cfg": {
10 | "context_length": 77,
11 | "vocab_size": 49408,
12 | "width": 768,
13 | "heads": 12,
14 | "layers": 12
15 | }
16 | }
--------------------------------------------------------------------------------
/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/transform.py:
--------------------------------------------------------------------------------
1 | from torchvision.transforms import Normalize, Compose, RandomResizedCrop, InterpolationMode, ToTensor, Resize, \
2 | CenterCrop
3 |
4 |
5 | def _convert_to_rgb(image):
6 | return image.convert('RGB')
7 |
8 |
9 | def image_transform(
10 | image_size: int,
11 | is_train: bool,
12 | mean=(0.48145466, 0.4578275, 0.40821073),
13 | std=(0.26862954, 0.26130258, 0.27577711)
14 | ):
15 | normalize = Normalize(mean=mean, std=std)
16 | if is_train:
17 | return Compose([
18 | RandomResizedCrop(image_size, scale=(0.9, 1.0), interpolation=InterpolationMode.BICUBIC),
19 | _convert_to_rgb,
20 | ToTensor(),
21 | normalize,
22 | ])
23 | else:
24 | return Compose([
25 | Resize(image_size, interpolation=InterpolationMode.BICUBIC),
26 | CenterCrop(image_size),
27 | _convert_to_rgb,
28 | ToTensor(),
29 | normalize,
30 | ])
31 |
--------------------------------------------------------------------------------
/text_to_audio/Make_An_Audio/ldm/modules/encoders/open_clap/version.py:
--------------------------------------------------------------------------------
1 | __version__ = '0.2.1'
2 |
--------------------------------------------------------------------------------
/text_to_audio/Make_An_Audio/ldm/modules/image_degradation/__init__.py:
--------------------------------------------------------------------------------
1 | from ldm.modules.image_degradation.bsrgan import degradation_bsrgan_variant as degradation_fn_bsr
2 | from ldm.modules.image_degradation.bsrgan_light import degradation_bsrgan_variant as degradation_fn_bsr_light
3 |
--------------------------------------------------------------------------------
/text_to_audio/Make_An_Audio/ldm/modules/image_degradation/utils/test.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AIGC-Audio/AudioGPT/a674543c537bb3a60b8f521df76b97a627c1c379/text_to_audio/Make_An_Audio/ldm/modules/image_degradation/utils/test.png
--------------------------------------------------------------------------------
/text_to_audio/Make_An_Audio/ldm/modules/losses_audio/__init__.py:
--------------------------------------------------------------------------------
1 | from ldm.modules.losses_audio.vqperceptual import DummyLoss
2 |
3 | # relative imports pain
4 | import os
5 | import sys
6 | path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'vggishish')
7 | sys.path.append(path)
8 |
--------------------------------------------------------------------------------
/text_to_audio/Make_An_Audio/ldm/modules/losses_audio/vggishish/config/melception.yaml:
--------------------------------------------------------------------------------
1 | seed: 1337
2 | log_code_state: True
3 | # patterns to ignore when backing up the code folder
4 | patterns_to_ignore: ['logs', '.git', '__pycache__', 'data', 'checkpoints', '*.pt']
5 |
6 | # data:
7 | mels_path: '/home/nvme/data/vggsound/features/melspec_10s_22050hz/'
8 | spec_shape: [80, 860]
9 | cropped_size: [80, 848]
10 | random_crop: False
11 |
12 | # train:
13 | device: 'cuda:0'
14 | batch_size: 8
15 | num_workers: 0
16 | optimizer: adam
17 | betas: [0.9, 0.999]
18 | momentum: 0.9
19 | learning_rate: 3e-4
20 | weight_decay: 0
21 | num_epochs: 100
22 | patience: 3
23 | logdir: './logs'
24 | cls_weights_in_loss: False
25 |
--------------------------------------------------------------------------------
/text_to_audio/Make_An_Audio/ldm/modules/losses_audio/vggishish/config/vggish.yaml:
--------------------------------------------------------------------------------
1 | seed: 1337
2 | log_code_state: True
3 | # patterns to ignore when backing up the code folder
4 | patterns_to_ignore: ['logs', '.git', '__pycache__']
5 |
6 | # data:
7 | mels_path: '/home/nvme/data/vggsound/features/melspec_10s_22050hz/'
8 | spec_shape: [80, 860]
9 | cropped_size: [80, 848]
10 | random_crop: False
11 |
12 | # model:
13 | # original vgg family except for MP is missing at the end
14 | # 'vggish': [64, 'MP', 128, 'MP', 256, 256, 'MP', 512, 512]
15 | # 'vgg11': [64, 'MP', 128, 'MP', 256, 256, 'MP', 512, 512, 'MP', 512, 512],
16 | # 'vgg13': [64, 64, 'MP', 128, 128, 'MP', 256, 256, 'MP', 512, 512, 'MP', 512, 512],
17 | # 'vgg16': [64, 64, 'MP', 128, 128, 'MP', 256, 256, 256, 'MP', 512, 512, 512, 'MP', 512, 512, 512],
18 | # 'vgg19': [64, 64, 'MP', 128, 128, 'MP', 256, 256, 256, 256, 'MP', 512, 512, 512, 512, 'MP', 512, 512, 512, 512],
19 | conv_layers: [64, 64, 'MP', 128, 128, 'MP', 256, 256, 256, 'MP', 512, 512, 512, 'MP', 512, 512, 512]
20 | use_bn: False
21 |
22 | # train:
23 | device: 'cuda:0'
24 | batch_size: 32
25 | num_workers: 0
26 | optimizer: adam
27 | betas: [0.9, 0.999]
28 | momentum: 0.9
29 | learning_rate: 3e-4
30 | weight_decay: 0.0001
31 | num_epochs: 100
32 | patience: 3
33 | logdir: './logs'
34 | cls_weights_in_loss: False
35 |
--------------------------------------------------------------------------------
/text_to_audio/Make_An_Audio/ldm/modules/losses_audio/vggishish/data/train_means_stds_melspec_10s_22050hz.txt:
--------------------------------------------------------------------------------
1 | 0.51234712 0.08187601
2 | 0.52630101 0.08393201
3 | 0.52002938 0.08533191
4 | 0.51831866 0.08651366
5 | 0.52457265 0.08795700
6 | 0.51129235 0.08924046
7 | 0.51403755 0.09011565
8 | 0.51189406 0.09138965
9 | 0.50142221 0.09215379
10 | 0.50632402 0.09251092
11 | 0.49724399 0.09356590
12 | 0.49062130 0.09333057
13 | 0.49971113 0.09446411
14 | 0.48442903 0.09400197
15 | 0.48598301 0.09477853
16 | 0.48681630 0.09487848
17 | 0.47436119 0.09424447
18 | 0.48031359 0.09417475
19 | 0.47422810 0.09498061
20 | 0.46369397 0.09362415
21 | 0.47413055 0.09453825
22 | 0.46062686 0.09481226
23 | 0.45677793 0.09359464
24 | 0.46135474 0.09437913
25 | 0.45246800 0.09384014
26 | 0.45232766 0.09438247
27 | 0.45208386 0.09419720
28 | 0.44351671 0.09340880
29 | 0.44667316 0.09423184
30 | 0.44447692 0.09344461
31 | 0.43676363 0.09265266
32 | 0.44002381 0.09307925
33 | 0.43772414 0.09299363
34 | 0.43061019 0.09236166
35 | 0.43120828 0.09156491
36 | 0.42603271 0.09161403
37 | 0.42863234 0.09150530
38 | 0.42296206 0.09102536
39 | 0.41733331 0.09048366
40 | 0.41804121 0.09025013
41 | 0.41605068 0.09078869
42 | 0.40875265 0.08985338
43 | 0.40666997 0.08877566
44 | 0.40407463 0.08961667
45 | 0.40353311 0.08859275
46 | 0.39708031 0.08827818
47 | 0.39375066 0.08833999
48 | 0.39301091 0.08760654
49 | 0.39047117 0.08812327
50 | 0.38461680 0.08782288
51 | 0.38145284 0.08645484
52 | 0.37985209 0.08718211
53 | 0.37419526 0.08644421
54 | 0.37080597 0.08532454
55 | 0.36786535 0.08592822
56 | 0.36569049 0.08452069
57 | 0.36336079 0.08474272
58 | 0.35775191 0.08476392
59 | 0.35504801 0.08334654
60 | 0.35284816 0.08412110
61 | 0.34594865 0.08367254
62 | 0.34112312 0.08252251
63 | 0.33784886 0.08320975
64 | 0.33095703 0.08257768
65 | 0.32559461 0.08171253
66 | 0.32003106 0.08204872
67 | 0.31506222 0.08098545
68 | 0.31138077 0.08152917
69 | 0.30403516 0.08209135
70 | 0.29969540 0.08073266
71 | 0.29578024 0.08225822
72 | 0.28861871 0.08324076
73 | 0.28581686 0.08058489
74 | 0.27922253 0.08515350
75 | 0.27444035 0.08355056
76 | 0.27339468 0.08067638
77 | 0.26571759 0.08536921
78 | 0.26280864 0.08107620
79 | 0.25664202 0.08357468
80 | 0.24853513 0.08556041
81 |
--------------------------------------------------------------------------------
/text_to_audio/Make_An_Audio/ldm/modules/losses_audio/vggishish/loss.py:
--------------------------------------------------------------------------------
1 | import torch
2 | import torch.nn as nn
3 | import torch.nn.functional as F
4 | import torch.optim as optim
5 |
6 | class WeightedCrossEntropy(nn.CrossEntropyLoss):
7 |
8 | def __init__(self, weights, **pytorch_ce_loss_args) -> None:
9 | super().__init__(reduction='none', **pytorch_ce_loss_args)
10 | self.weights = weights
11 |
12 | def __call__(self, outputs, targets, to_weight=True):
13 | loss = super().__call__(outputs, targets)
14 | if to_weight:
15 | return (loss * self.weights[targets]).sum() / self.weights[targets].sum()
16 | else:
17 | return loss.mean()
18 |
19 |
20 | if __name__ == '__main__':
21 | x = torch.randn(10, 5)
22 | target = torch.randint(0, 5, (10,))
23 | weights = torch.tensor([1., 2., 3., 4., 5.])
24 |
25 | # criterion_weighted = nn.CrossEntropyLoss(weight=weights)
26 | # loss_weighted = criterion_weighted(x, target)
27 |
28 | # criterion_weighted_manual = nn.CrossEntropyLoss(reduction='none')
29 | # loss_weighted_manual = criterion_weighted_manual(x, target)
30 | # print(loss_weighted, loss_weighted_manual.mean())
31 | # loss_weighted_manual = (loss_weighted_manual * weights[target]).sum() / weights[target].sum()
32 | # print(loss_weighted, loss_weighted_manual)
33 | # print(torch.allclose(loss_weighted, loss_weighted_manual))
34 |
35 | pytorch_weighted = nn.CrossEntropyLoss(weight=weights)
36 | pytorch_unweighted = nn.CrossEntropyLoss()
37 | custom = WeightedCrossEntropy(weights)
38 |
39 | assert torch.allclose(pytorch_weighted(x, target), custom(x, target, to_weight=True))
40 | assert torch.allclose(pytorch_unweighted(x, target), custom(x, target, to_weight=False))
41 | print(custom(x, target, to_weight=True), custom(x, target, to_weight=False))
42 |
--------------------------------------------------------------------------------
/text_to_audio/Make_An_Audio/ldm/modules/losses_audio/vggishish/metrics.py:
--------------------------------------------------------------------------------
1 | import logging
2 |
3 | import numpy as np
4 | import scipy
5 | import torch
6 | from sklearn.metrics import average_precision_score, roc_auc_score
7 |
8 | logger = logging.getLogger(f'main.{__name__}')
9 |
10 | def metrics(targets, outputs, topk=(1, 5)):
11 | """
12 | Adapted from https://github.com/hche11/VGGSound/blob/master/utils.py
13 |
14 | Calculate statistics including mAP, AUC, and d-prime.
15 | Args:
16 | output: 2d tensors, (dataset_size, classes_num) - before softmax
17 | target: 1d tensors, (dataset_size, )
18 | topk: tuple
19 | Returns:
20 | metric_dict: a dict of metrics
21 | """
22 | metrics_dict = dict()
23 |
24 | num_cls = outputs.shape[-1]
25 |
26 | # accuracy@k
27 | _, preds = torch.topk(outputs, k=max(topk), dim=1)
28 | correct_for_maxtopk = preds == targets.view(-1, 1).expand_as(preds)
29 | for k in topk:
30 | metrics_dict[f'accuracy_{k}'] = float(correct_for_maxtopk[:, :k].sum() / correct_for_maxtopk.shape[0])
31 |
32 | # avg precision, average roc_auc, and dprime
33 | targets = torch.nn.functional.one_hot(targets, num_classes=num_cls)
34 |
35 | # ids of the predicted classes (same as softmax)
36 | targets_pred = torch.softmax(outputs, dim=1)
37 |
38 | targets = targets.numpy()
39 | targets_pred = targets_pred.numpy()
40 |
41 | # one-vs-rest
42 | avg_p = [average_precision_score(targets[:, c], targets_pred[:, c], average=None) for c in range(num_cls)]
43 | try:
44 | roc_aucs = [roc_auc_score(targets[:, c], targets_pred[:, c], average=None) for c in range(num_cls)]
45 | except ValueError:
46 | logger.warning('Weird... Some classes never occured in targets. Do not trust the metrics.')
47 | roc_aucs = np.array([0.5])
48 | avg_p = np.array([0])
49 |
50 | metrics_dict['mAP'] = np.mean(avg_p)
51 | metrics_dict['mROCAUC'] = np.mean(roc_aucs)
52 | # Percent point function (ppf) (inverse of cdf — percentiles).
53 | metrics_dict['dprime'] = scipy.stats.norm().ppf(metrics_dict['mROCAUC']) * np.sqrt(2)
54 |
55 | return metrics_dict
56 |
57 |
58 | if __name__ == '__main__':
59 | targets = torch.tensor([3, 3, 1, 2, 1, 0])
60 | outputs = torch.tensor([
61 | [1.2, 1.3, 1.1, 1.5],
62 | [1.3, 1.4, 1.0, 1.1],
63 | [1.5, 1.1, 1.4, 1.3],
64 | [1.0, 1.2, 1.4, 1.5],
65 | [1.2, 1.3, 1.1, 1.1],
66 | [1.2, 1.1, 1.1, 1.1],
67 | ]).float()
68 | metrics_dict = metrics(targets, outputs, topk=(1, 3))
69 | print(metrics_dict)
70 |
--------------------------------------------------------------------------------
/text_to_audio/Make_An_Audio/ldm/modules/losses_audio/vggishish/model.py:
--------------------------------------------------------------------------------
1 | import torch
2 | import torch.nn as nn
3 |
4 |
5 | class VGGishish(nn.Module):
6 |
7 | def __init__(self, conv_layers, use_bn, num_classes):
8 | '''
9 | Mostly from
10 | https://pytorch.org/vision/0.8/_modules/torchvision/models/vgg.html
11 | '''
12 | super().__init__()
13 | layers = []
14 | in_channels = 1
15 |
16 | # a list of channels with 'MP' (maxpool) from config
17 | for v in conv_layers:
18 | if v == 'MP':
19 | layers += [nn.MaxPool2d(kernel_size=2, stride=2)]
20 | else:
21 | conv2d = nn.Conv2d(in_channels, v, kernel_size=3, padding=1, stride=1)
22 | if use_bn:
23 | layers += [conv2d, nn.BatchNorm2d(v), nn.ReLU(inplace=True)]
24 | else:
25 | layers += [conv2d, nn.ReLU(inplace=True)]
26 | in_channels = v
27 | self.features = nn.Sequential(*layers)
28 |
29 | self.avgpool = nn.AdaptiveAvgPool2d((5, 10))
30 |
31 | self.flatten = nn.Flatten()
32 | self.classifier = nn.Sequential(
33 | nn.Linear(512 * 5 * 10, 4096),
34 | nn.ReLU(True),
35 | nn.Linear(4096, 4096),
36 | nn.ReLU(True),
37 | nn.Linear(4096, num_classes)
38 | )
39 |
40 | # weight init
41 | self.reset_parameters()
42 |
43 | def forward(self, x):
44 | # adding channel dim for conv2d (B, 1, F, T) <-
45 | x = x.unsqueeze(1)
46 | # backbone (B, 1, 5, 53) <- (B, 1, 80, 860)
47 | x = self.features(x)
48 | # adaptive avg pooling (B, 1, 5, 10) <- (B, 1, 5, 53) – if no MP is used as the end of VGG
49 | x = self.avgpool(x)
50 | # flatten
51 | x = self.flatten(x)
52 | # classify
53 | x = self.classifier(x)
54 | return x
55 |
56 | def reset_parameters(self):
57 | for m in self.modules():
58 | if isinstance(m, nn.Conv2d):
59 | nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')
60 | if m.bias is not None:
61 | nn.init.constant_(m.bias, 0)
62 | elif isinstance(m, nn.BatchNorm2d):
63 | nn.init.constant_(m.weight, 1)
64 | nn.init.constant_(m.bias, 0)
65 | elif isinstance(m, nn.Linear):
66 | nn.init.normal_(m.weight, 0, 0.01)
67 | nn.init.constant_(m.bias, 0)
68 |
69 |
70 | if __name__ == '__main__':
71 | num_classes = 309
72 | inputs = torch.rand(3, 80, 848)
73 | conv_layers = [64, 64, 'MP', 128, 128, 'MP', 256, 256, 256, 'MP', 512, 512, 512, 'MP', 512, 512, 512]
74 | # conv_layers = [64, 'MP', 128, 'MP', 256, 256, 'MP', 512, 512, 'MP']
75 | model = VGGishish(conv_layers, use_bn=False, num_classes=num_classes)
76 | outputs = model(inputs)
77 | print(outputs.shape)
78 |
--------------------------------------------------------------------------------
/text_to_audio/Make_An_Audio/useful_ckpts/CLAP/config.yml:
--------------------------------------------------------------------------------
1 | # TEXT ENCODER CONFIG
2 | text_model: 'bert-base-uncased'
3 | text_len: 100
4 | transformer_embed_dim: 768
5 | freeze_text_encoder_weights: True
6 |
7 | # AUDIO ENCODER CONFIG
8 | audioenc_name: 'Cnn14'
9 | out_emb: 2048
10 | sampling_rate: 44100
11 | duration: 9
12 | fmin: 50
13 | fmax: 14000
14 | n_fft: 1028
15 | hop_size: 320
16 | mel_bins: 64
17 | window_size: 1024
18 |
19 | # PROJECTION SPACE CONFIG
20 | d_proj: 1024
21 | temperature: 0.003
22 |
23 | # TRAINING AND EVALUATION CONFIG
24 | num_classes: 527
25 | batch_size: 1024
26 | demo: False
27 |
--------------------------------------------------------------------------------
/text_to_audio/Make_An_Audio/vocoder/bigvgan/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/AIGC-Audio/AudioGPT/a674543c537bb3a60b8f521df76b97a627c1c379/text_to_audio/Make_An_Audio/vocoder/bigvgan/__init__.py
--------------------------------------------------------------------------------
/text_to_audio/Make_An_Audio/vocoder/bigvgan/alias_free_torch/__init__.py:
--------------------------------------------------------------------------------
1 | # Adapted from https://github.com/junjun3518/alias-free-torch under the Apache License 2.0
2 | # LICENSE is in incl_licenses directory.
3 |
4 | from .filter import *
5 | from .resample import *
6 | from .act import *
--------------------------------------------------------------------------------
/text_to_audio/Make_An_Audio/vocoder/bigvgan/alias_free_torch/act.py:
--------------------------------------------------------------------------------
1 | # Adapted from https://github.com/junjun3518/alias-free-torch under the Apache License 2.0
2 | # LICENSE is in incl_licenses directory.
3 |
4 | import torch.nn as nn
5 | from .resample import UpSample1d, DownSample1d
6 |
7 |
8 | class Activation1d(nn.Module):
9 | def __init__(self,
10 | activation,
11 | up_ratio: int = 2,
12 | down_ratio: int = 2,
13 | up_kernel_size: int = 12,
14 | down_kernel_size: int = 12):
15 | super().__init__()
16 | self.up_ratio = up_ratio
17 | self.down_ratio = down_ratio
18 | self.act = activation
19 | self.upsample = UpSample1d(up_ratio, up_kernel_size)
20 | self.downsample = DownSample1d(down_ratio, down_kernel_size)
21 |
22 | # x: [B,C,T]
23 | def forward(self, x):
24 | x = self.upsample(x)
25 | x = self.act(x)
26 | x = self.downsample(x)
27 |
28 | return x
--------------------------------------------------------------------------------
/text_to_audio/Make_An_Audio/vocoder/bigvgan/alias_free_torch/resample.py:
--------------------------------------------------------------------------------
1 | # Adapted from https://github.com/junjun3518/alias-free-torch under the Apache License 2.0
2 | # LICENSE is in incl_licenses directory.
3 |
4 | import torch.nn as nn
5 | from torch.nn import functional as F
6 | from .filter import LowPassFilter1d
7 | from .filter import kaiser_sinc_filter1d
8 |
9 |
10 | class UpSample1d(nn.Module):
11 | def __init__(self, ratio=2, kernel_size=None):
12 | super().__init__()
13 | self.ratio = ratio
14 | self.kernel_size = int(6 * ratio // 2) * 2 if kernel_size is None else kernel_size
15 | self.stride = ratio
16 | self.pad = self.kernel_size // ratio - 1
17 | self.pad_left = self.pad * self.stride + (self.kernel_size - self.stride) // 2
18 | self.pad_right = self.pad * self.stride + (self.kernel_size - self.stride + 1) // 2
19 | filter = kaiser_sinc_filter1d(cutoff=0.5 / ratio,
20 | half_width=0.6 / ratio,
21 | kernel_size=self.kernel_size)
22 | self.register_buffer("filter", filter)
23 |
24 | # x: [B, C, T]
25 | def forward(self, x):
26 | _, C, _ = x.shape
27 |
28 | x = F.pad(x, (self.pad, self.pad), mode='replicate')
29 | x = self.ratio * F.conv_transpose1d(
30 | x, self.filter.expand(C, -1, -1), stride=self.stride, groups=C)
31 | x = x[..., self.pad_left:-self.pad_right]
32 |
33 | return x
34 |
35 |
36 | class DownSample1d(nn.Module):
37 | def __init__(self, ratio=2, kernel_size=None):
38 | super().__init__()
39 | self.ratio = ratio
40 | self.kernel_size = int(6 * ratio // 2) * 2 if kernel_size is None else kernel_size
41 | self.lowpass = LowPassFilter1d(cutoff=0.5 / ratio,
42 | half_width=0.6 / ratio,
43 | stride=ratio,
44 | kernel_size=self.kernel_size)
45 |
46 | def forward(self, x):
47 | xx = self.lowpass(x)
48 |
49 | return xx
--------------------------------------------------------------------------------
/text_to_audio/Make_An_Audio/vocoder/logs/hifi_0127/args.yml:
--------------------------------------------------------------------------------
1 | adam_b1: 0.8
2 | adam_b2: 0.99
3 | batch_size: 24
4 | dist_config:
5 | dist_backend: nccl
6 | dist_url: tcp://localhost:54321
7 | world_size: 1
8 | fmax: 8000
9 | fmax_for_loss: null
10 | fmin: 0
11 | hop_size: 256
12 | learning_rate: 0.0002
13 | lr_decay: 0.999
14 | n_fft: 1024
15 | num_gpus: 0
16 | num_mels: 80
17 | num_workers: 4
18 | resblock: '1'
19 | resblock_dilation_sizes:
20 | - - 1
21 | - 3
22 | - 5
23 | - - 1
24 | - 3
25 | - 5
26 | - - 1
27 | - 3
28 | - 5
29 | resblock_kernel_sizes:
30 | - 3
31 | - 7
32 | - 11
33 | sampling_rate: 16000
34 | seed: 1234
35 | segment_size: 8192
36 | upsample_initial_channel: 512
37 | upsample_kernel_sizes:
38 | - 16
39 | - 16
40 | - 4
41 | - 4
42 | upsample_rates:
43 | - 8
44 | - 8
45 | - 2
46 | - 2
47 | win_size: 1024
48 |
--------------------------------------------------------------------------------
/text_to_audio/Make_An_Audio/wav_evaluation/models/__init__.py:
--------------------------------------------------------------------------------
1 | from . import clap
2 | from . import audio
3 | from . import utils
--------------------------------------------------------------------------------
/text_to_audio/Make_An_Audio/wav_evaluation/models/utils.py:
--------------------------------------------------------------------------------
1 | import argparse
2 | import yaml
3 | import sys
4 |
5 | def read_config_as_args(config_path,args=None,is_config_str=False):
6 | return_dict = {}
7 |
8 | if config_path is not None:
9 | if is_config_str:
10 | yml_config = yaml.load(config_path, Loader=yaml.FullLoader)
11 | else:
12 | with open(config_path, "r") as f:
13 | yml_config = yaml.load(f, Loader=yaml.FullLoader)
14 |
15 | if args != None:
16 | for k, v in yml_config.items():
17 | if k in args.__dict__:
18 | args.__dict__[k] = v
19 | else:
20 | sys.stderr.write("Ignored unknown parameter {} in yaml.\n".format(k))
21 | else:
22 | for k, v in yml_config.items():
23 | return_dict[k] = v
24 |
25 | args = args if args != None else return_dict
26 | return argparse.Namespace(**args)
27 |
--------------------------------------------------------------------------------