├── infer ├── modules │ ├── vc │ │ ├── __init__.py │ │ └── utils.py │ ├── gui │ │ ├── __init__.py │ │ └── utils.py │ ├── onnx │ │ └── export.py │ └── uvr5 │ │ └── modules.py └── lib │ ├── infer_pack │ └── modules │ │ └── F0Predictor │ │ ├── __init__.py │ │ ├── F0Predictor.py │ │ ├── HarvestF0Predictor.py │ │ ├── DioF0Predictor.py │ │ └── PMF0Predictor.py │ ├── jit │ ├── get_rmvpe.py │ └── get_synthesizer.py │ ├── uvr5_pack │ ├── lib_v5 │ │ ├── modelparams │ │ │ ├── 1band_sr44100_hl256.json │ │ │ ├── 1band_sr16000_hl512.json │ │ │ ├── 1band_sr32000_hl512.json │ │ │ ├── 1band_sr33075_hl384.json │ │ │ ├── 1band_sr44100_hl1024.json │ │ │ ├── 1band_sr44100_hl512.json │ │ │ ├── 1band_sr44100_hl512_cut.json │ │ │ ├── 2band_48000.json │ │ │ ├── 2band_32000.json │ │ │ ├── 2band_44100_lofi.json │ │ │ ├── 3band_44100.json │ │ │ ├── ensemble.json │ │ │ ├── 3band_44100_mid.json │ │ │ ├── 3band_44100_msb2.json │ │ │ ├── 4band_v2.json │ │ │ ├── 4band_v3.json │ │ │ ├── 4band_44100.json │ │ │ ├── 4band_44100_sw.json │ │ │ ├── 4band_44100_msb.json │ │ │ ├── 4band_44100_msb2.json │ │ │ ├── 4band_44100_reverse.json │ │ │ ├── 4band_v2_sn.json │ │ │ └── 4band_44100_mid.json │ │ ├── model_param_init.py │ │ ├── layers.py │ │ ├── layers_123812KB .py │ │ ├── layers_123821KB.py │ │ ├── nets.py │ │ ├── nets_123812KB.py │ │ ├── nets_123821KB.py │ │ ├── nets_33966KB.py │ │ ├── nets_61968KB.py │ │ ├── nets_537227KB.py │ │ ├── nets_537238KB.py │ │ ├── layers_new.py │ │ ├── layers_33966KB.py │ │ ├── layers_537227KB.py │ │ ├── layers_537238KB.py │ │ └── nets_new.py │ └── utils.py │ ├── audio.py │ └── train │ ├── losses.py │ └── mel_processing.py ├── assets ├── indices │ └── .gitignore ├── weights │ └── .gitignore ├── pretrained │ └── .gitignore ├── pretrained_v2 │ └── .gitignore ├── uvr5_weights │ └── .gitignore ├── rmvpe │ ├── .gitignore │ └── rmvpe_inputs.pth ├── hubert │ ├── .gitignore │ └── hubert_inputs.pth └── Synthesizer_inputs.pth ├── configs ├── inuse │ ├── v1 │ │ └── .gitignore │ ├── v2 │ │ └── .gitignore │ └── .gitignore ├── config.json ├── v1 │ ├── 32k.json │ ├── 40k.json │ └── 48k.json └── v2 │ ├── 32k.json │ └── 48k.json ├── go-realtime-gui.bat ├── .gitattributes ├── go-web.bat ├── go-realtime-gui-dml.bat ├── go-web-dml.bat ├── .gitignore ├── docker-compose.yml ├── CONTRIBUTING.md ├── README.md ├── requirements-win-for-realtime_vc_gui.txt ├── requirements-win-for-realtime_vc_gui-dml.txt ├── tools ├── onnx │ ├── onnx_inference_demo.py │ └── export_onnx.py ├── cmd │ ├── trans_weights.py │ ├── train-index.py │ ├── infer_cli.py │ ├── train-index-v2.py │ ├── infer_batch_rvc.py │ └── calc_rvc_model_similarity.py ├── download_models.py └── dlmodels.sh ├── i18n ├── i18n.py ├── locale_diff.py └── scan_i18n.py ├── requirements-dml.txt ├── requirements.txt ├── requirements-amd.txt ├── requirements-py311.txt ├── LICENSE ├── requirements-ipex.txt ├── MIT协议暨相关引用库协议 ├── run.sh ├── docs ├── jp │ ├── training_tips_ja.md │ ├── faiss_tips_ja.md │ └── Changelog_JA.md ├── kr │ ├── training_tips_ko.md │ ├── README.ko.han.md │ └── Changelog_KO.md ├── cn │ ├── Changelog_CN.md │ └── faq.md ├── en │ └── training_tips_en.md └── tr │ └── training_tips_tr.md ├── Dockerfile └── .env /infer/modules/vc/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /assets/indices/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | !.gitignore 3 | -------------------------------------------------------------------------------- /assets/weights/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | !.gitignore 3 | -------------------------------------------------------------------------------- /assets/pretrained/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | !.gitignore 3 | -------------------------------------------------------------------------------- /assets/pretrained_v2/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | !.gitignore 3 | -------------------------------------------------------------------------------- /assets/uvr5_weights/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | !.gitignore 3 | -------------------------------------------------------------------------------- /configs/inuse/v1/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | !.gitignore 3 | -------------------------------------------------------------------------------- /configs/inuse/v2/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | !.gitignore 3 | -------------------------------------------------------------------------------- /infer/lib/infer_pack/modules/F0Predictor/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /assets/rmvpe/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | !.gitignore 3 | !rmvpe_inputs.pth -------------------------------------------------------------------------------- /configs/inuse/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | !.gitignore 3 | !v1 4 | !v2 5 | -------------------------------------------------------------------------------- /assets/hubert/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | !.gitignore 3 | !hubert_inputs.pth -------------------------------------------------------------------------------- /go-realtime-gui.bat: -------------------------------------------------------------------------------- 1 | runtime\python.exe --nocheck gui_v1.py 2 | pause 3 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | -------------------------------------------------------------------------------- /assets/Synthesizer_inputs.pth: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NoorBayan/Ilqa/HEAD/assets/Synthesizer_inputs.pth -------------------------------------------------------------------------------- /assets/rmvpe/rmvpe_inputs.pth: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NoorBayan/Ilqa/HEAD/assets/rmvpe/rmvpe_inputs.pth -------------------------------------------------------------------------------- /go-web.bat: -------------------------------------------------------------------------------- 1 | runtime\python.exe infer-web.py --pycmd runtime\python.exe --nocheck --port 7897 2 | pause 3 | -------------------------------------------------------------------------------- /assets/hubert/hubert_inputs.pth: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NoorBayan/Ilqa/HEAD/assets/hubert/hubert_inputs.pth -------------------------------------------------------------------------------- /go-realtime-gui-dml.bat: -------------------------------------------------------------------------------- 1 | runtime\python.exe gui_v1.py --pycmd runtime\python.exe --nocheck --dml 2 | pause 3 | -------------------------------------------------------------------------------- /go-web-dml.bat: -------------------------------------------------------------------------------- 1 | runtime\python.exe infer-web.py --pycmd runtime\python.exe --nocheck --port 7897 --dml 2 | pause 3 | -------------------------------------------------------------------------------- /infer/lib/jit/get_rmvpe.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | 4 | def get_rmvpe(model_path="assets/rmvpe/rmvpe.pt", device=torch.device("cpu")): 5 | from infer.lib.rmvpe import E2E 6 | 7 | model = E2E(4, 1, (2, 2)) 8 | ckpt = torch.load(model_path, map_location=device) 9 | model.load_state_dict(ckpt) 10 | model.eval() 11 | model = model.to(device) 12 | return model 13 | -------------------------------------------------------------------------------- /infer/lib/uvr5_pack/lib_v5/modelparams/1band_sr44100_hl256.json: -------------------------------------------------------------------------------- 1 | { 2 | "bins": 256, 3 | "unstable_bins": 0, 4 | "reduction_bins": 0, 5 | "band": { 6 | "1": { 7 | "sr": 44100, 8 | "hl": 256, 9 | "n_fft": 512, 10 | "crop_start": 0, 11 | "crop_stop": 256, 12 | "hpf_start": -1, 13 | "res_type": "sinc_best" 14 | } 15 | }, 16 | "sr": 44100, 17 | "pre_filter_start": 256, 18 | "pre_filter_stop": 256 19 | } -------------------------------------------------------------------------------- /infer/lib/uvr5_pack/lib_v5/modelparams/1band_sr16000_hl512.json: -------------------------------------------------------------------------------- 1 | { 2 | "bins": 1024, 3 | "unstable_bins": 0, 4 | "reduction_bins": 0, 5 | "band": { 6 | "1": { 7 | "sr": 16000, 8 | "hl": 512, 9 | "n_fft": 2048, 10 | "crop_start": 0, 11 | "crop_stop": 1024, 12 | "hpf_start": -1, 13 | "res_type": "sinc_best" 14 | } 15 | }, 16 | "sr": 16000, 17 | "pre_filter_start": 1023, 18 | "pre_filter_stop": 1024 19 | } -------------------------------------------------------------------------------- /infer/lib/uvr5_pack/lib_v5/modelparams/1band_sr32000_hl512.json: -------------------------------------------------------------------------------- 1 | { 2 | "bins": 1024, 3 | "unstable_bins": 0, 4 | "reduction_bins": 0, 5 | "band": { 6 | "1": { 7 | "sr": 32000, 8 | "hl": 512, 9 | "n_fft": 2048, 10 | "crop_start": 0, 11 | "crop_stop": 1024, 12 | "hpf_start": -1, 13 | "res_type": "kaiser_fast" 14 | } 15 | }, 16 | "sr": 32000, 17 | "pre_filter_start": 1000, 18 | "pre_filter_stop": 1021 19 | } -------------------------------------------------------------------------------- /infer/lib/uvr5_pack/lib_v5/modelparams/1band_sr33075_hl384.json: -------------------------------------------------------------------------------- 1 | { 2 | "bins": 1024, 3 | "unstable_bins": 0, 4 | "reduction_bins": 0, 5 | "band": { 6 | "1": { 7 | "sr": 33075, 8 | "hl": 384, 9 | "n_fft": 2048, 10 | "crop_start": 0, 11 | "crop_stop": 1024, 12 | "hpf_start": -1, 13 | "res_type": "sinc_best" 14 | } 15 | }, 16 | "sr": 33075, 17 | "pre_filter_start": 1000, 18 | "pre_filter_stop": 1021 19 | } -------------------------------------------------------------------------------- /infer/lib/uvr5_pack/lib_v5/modelparams/1band_sr44100_hl1024.json: -------------------------------------------------------------------------------- 1 | { 2 | "bins": 1024, 3 | "unstable_bins": 0, 4 | "reduction_bins": 0, 5 | "band": { 6 | "1": { 7 | "sr": 44100, 8 | "hl": 1024, 9 | "n_fft": 2048, 10 | "crop_start": 0, 11 | "crop_stop": 1024, 12 | "hpf_start": -1, 13 | "res_type": "sinc_best" 14 | } 15 | }, 16 | "sr": 44100, 17 | "pre_filter_start": 1023, 18 | "pre_filter_stop": 1024 19 | } -------------------------------------------------------------------------------- /infer/lib/uvr5_pack/lib_v5/modelparams/1band_sr44100_hl512.json: -------------------------------------------------------------------------------- 1 | { 2 | "bins": 1024, 3 | "unstable_bins": 0, 4 | "reduction_bins": 0, 5 | "band": { 6 | "1": { 7 | "sr": 44100, 8 | "hl": 512, 9 | "n_fft": 2048, 10 | "crop_start": 0, 11 | "crop_stop": 1024, 12 | "hpf_start": -1, 13 | "res_type": "sinc_best" 14 | } 15 | }, 16 | "sr": 44100, 17 | "pre_filter_start": 1023, 18 | "pre_filter_stop": 1024 19 | } -------------------------------------------------------------------------------- /infer/lib/uvr5_pack/lib_v5/modelparams/1band_sr44100_hl512_cut.json: -------------------------------------------------------------------------------- 1 | { 2 | "bins": 1024, 3 | "unstable_bins": 0, 4 | "reduction_bins": 0, 5 | "band": { 6 | "1": { 7 | "sr": 44100, 8 | "hl": 512, 9 | "n_fft": 2048, 10 | "crop_start": 0, 11 | "crop_stop": 700, 12 | "hpf_start": -1, 13 | "res_type": "sinc_best" 14 | } 15 | }, 16 | "sr": 44100, 17 | "pre_filter_start": 1023, 18 | "pre_filter_stop": 700 19 | } -------------------------------------------------------------------------------- /infer/modules/gui/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | TorchGating is a PyTorch-based implementation of Spectral Gating 3 | ================================================ 4 | Author: Asaf Zorea 5 | 6 | Contents 7 | -------- 8 | torchgate imports all the functions from PyTorch, and in addition provides: 9 | TorchGating --- A PyTorch module that applies a spectral gate to an input signal 10 | 11 | """ 12 | 13 | from .torchgate import TorchGate 14 | -------------------------------------------------------------------------------- /configs/config.json: -------------------------------------------------------------------------------- 1 | {"pth_path": "assets/weights/kikiV1.pth", "index_path": "logs/kikiV1.index", "sg_hostapi": "MME", "sg_wasapi_exclusive": false, "sg_input_device": "VoiceMeeter Output (VB-Audio Vo", "sg_output_device": "VoiceMeeter Input (VB-Audio Voi", "sr_type": "sr_device", "threhold": -60.0, "pitch": 12.0, "rms_mix_rate": 0.5, "index_rate": 0.0, "block_time": 0.15, "crossfade_length": 0.08, "extra_time": 2.0, "n_cpu": 4.0, "use_jit": false, "use_pv": false, "f0method": "fcpe"} -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | __pycache__ 3 | /TEMP 4 | *.pyd 5 | .venv 6 | /opt 7 | tools/aria2c/ 8 | tools/flag.txt 9 | 10 | # Imported from huggingface.co/lj1995/VoiceConversionWebUI 11 | /pretrained 12 | /pretrained_v2 13 | /uvr5_weights 14 | hubert_base.pt 15 | rmvpe.onnx 16 | rmvpe.pt 17 | 18 | # Generated by RVC 19 | /logs 20 | /weights 21 | 22 | # To set a Python version for the project 23 | .tool-versions 24 | 25 | /runtime 26 | /assets/weights/* 27 | ffmpeg.* 28 | ffprobe.* -------------------------------------------------------------------------------- /infer/lib/infer_pack/modules/F0Predictor/F0Predictor.py: -------------------------------------------------------------------------------- 1 | class F0Predictor(object): 2 | def compute_f0(self, wav, p_len): 3 | """ 4 | input: wav:[signal_length] 5 | p_len:int 6 | output: f0:[signal_length//hop_length] 7 | """ 8 | pass 9 | 10 | def compute_f0_uv(self, wav, p_len): 11 | """ 12 | input: wav:[signal_length] 13 | p_len:int 14 | output: f0:[signal_length//hop_length],uv:[signal_length//hop_length] 15 | """ 16 | pass 17 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.8" 2 | services: 3 | rvc: 4 | build: 5 | context: . 6 | dockerfile: Dockerfile 7 | container_name: rvc 8 | volumes: 9 | - ./weights:/app/assets/weights 10 | - ./opt:/app/opt 11 | # - ./dataset:/app/dataset # you can use this folder in order to provide your dataset for model training 12 | ports: 13 | - 7865:7865 14 | deploy: 15 | resources: 16 | reservations: 17 | devices: 18 | - driver: nvidia 19 | count: 1 20 | capabilities: [gpu] -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # 贡献规则 2 | 1. 一般来说，作者`@RVC-Boss`将拒绝所有的算法更改，除非它是为了修复某个代码层面的错误或警告 3 | 2. 您可以贡献本仓库的其他位置，如翻译和WebUI，但请尽量作最小更改 4 | 3. 所有更改都需要由`@RVC-Boss`批准，因此您的PR可能会被搁置 5 | 4. 由此带来的不便请您谅解 6 | 7 | # Contributing Rules 8 | 1. Generally, the author `@RVC-Boss` will reject all algorithm changes unless what is to fix a code-level error or warning. 9 | 2. You can contribute to other parts of this repo like translations and WebUI, but please minimize your changes as much as possible. 10 | 3. All changes need to be approved by `@RVC-Boss`, so your PR may be put on hold. 11 | 4. Please accept our apologies for any inconvenience caused. 12 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Ilqa 2 | Ilqa is an AI-driven audio enhancement project designed to improve the quality of Quranic recitations and exegesis (tafsir) readings. Utilizing advanced speech processing and enhancement technologies, the project focuses on refining the clarity, tone, and accessibility of Quranic audio for educational purposes. The system works by enhancing the reciter's voice, ensuring that the recitation is clear, accurate, and easy to follow. This project aims to provide listeners with a high-quality audio experience of the Quran and its interpretations, supporting both learning and spiritual engagement. 3 | Files will be uploaded later. 4 | 5 | -------------------------------------------------------------------------------- /infer/lib/uvr5_pack/lib_v5/modelparams/2band_48000.json: -------------------------------------------------------------------------------- 1 | { 2 | "bins": 768, 3 | "unstable_bins": 7, 4 | "reduction_bins": 705, 5 | "band": { 6 | "1": { 7 | "sr": 6000, 8 | "hl": 66, 9 | "n_fft": 512, 10 | "crop_start": 0, 11 | "crop_stop": 240, 12 | "lpf_start": 60, 13 | "lpf_stop": 240, 14 | "res_type": "sinc_fastest" 15 | }, 16 | "2": { 17 | "sr": 48000, 18 | "hl": 528, 19 | "n_fft": 1536, 20 | "crop_start": 22, 21 | "crop_stop": 505, 22 | "hpf_start": 82, 23 | "hpf_stop": 22, 24 | "res_type": "sinc_medium" 25 | } 26 | }, 27 | "sr": 48000, 28 | "pre_filter_start": 710, 29 | "pre_filter_stop": 731 30 | } -------------------------------------------------------------------------------- /infer/lib/uvr5_pack/lib_v5/modelparams/2band_32000.json: -------------------------------------------------------------------------------- 1 | { 2 | "bins": 768, 3 | "unstable_bins": 7, 4 | "reduction_bins": 705, 5 | "band": { 6 | "1": { 7 | "sr": 6000, 8 | "hl": 66, 9 | "n_fft": 512, 10 | "crop_start": 0, 11 | "crop_stop": 240, 12 | "lpf_start": 60, 13 | "lpf_stop": 118, 14 | "res_type": "sinc_fastest" 15 | }, 16 | "2": { 17 | "sr": 32000, 18 | "hl": 352, 19 | "n_fft": 1024, 20 | "crop_start": 22, 21 | "crop_stop": 505, 22 | "hpf_start": 44, 23 | "hpf_stop": 23, 24 | "res_type": "sinc_medium" 25 | } 26 | }, 27 | "sr": 32000, 28 | "pre_filter_start": 710, 29 | "pre_filter_stop": 731 30 | } 31 | -------------------------------------------------------------------------------- /infer/lib/uvr5_pack/lib_v5/modelparams/2band_44100_lofi.json: -------------------------------------------------------------------------------- 1 | { 2 | "bins": 512, 3 | "unstable_bins": 7, 4 | "reduction_bins": 510, 5 | "band": { 6 | "1": { 7 | "sr": 11025, 8 | "hl": 160, 9 | "n_fft": 768, 10 | "crop_start": 0, 11 | "crop_stop": 192, 12 | "lpf_start": 41, 13 | "lpf_stop": 139, 14 | "res_type": "sinc_fastest" 15 | }, 16 | "2": { 17 | "sr": 44100, 18 | "hl": 640, 19 | "n_fft": 1024, 20 | "crop_start": 10, 21 | "crop_stop": 320, 22 | "hpf_start": 47, 23 | "hpf_stop": 15, 24 | "res_type": "sinc_medium" 25 | } 26 | }, 27 | "sr": 44100, 28 | "pre_filter_start": 510, 29 | "pre_filter_stop": 512 30 | } 31 | -------------------------------------------------------------------------------- /requirements-win-for-realtime_vc_gui.txt: -------------------------------------------------------------------------------- 1 | #1.Install torch from pytorch.org: 2 | #torch 2.0 with cuda 11.8 3 | #pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118 4 | #torch 1.11.0 with cuda 11.3 5 | #pip install torch==1.11.0+cu113 torchvision==0.12.0+cu113 torchaudio==0.11.0 --extra-index-url https://download.pytorch.org/whl/cu113 6 | einops 7 | fairseq 8 | flask 9 | flask_cors 10 | gin 11 | gin_config 12 | librosa 13 | local_attention 14 | matplotlib 15 | praat-parselmouth 16 | pyworld 17 | PyYAML 18 | resampy 19 | scikit_learn 20 | scipy 21 | SoundFile 22 | tensorboard 23 | tqdm 24 | wave 25 | FreeSimpleGUI 26 | sounddevice 27 | gradio 28 | noisereduce 29 | torchfcpe 30 | -------------------------------------------------------------------------------- /requirements-win-for-realtime_vc_gui-dml.txt: -------------------------------------------------------------------------------- 1 | #1.Install torch from pytorch.org: 2 | #torch 2.0 with cuda 11.8 3 | #pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118 4 | #torch 1.11.0 with cuda 11.3 5 | #pip install torch==1.11.0+cu113 torchvision==0.12.0+cu113 torchaudio==0.11.0 --extra-index-url https://download.pytorch.org/whl/cu113 6 | einops 7 | fairseq 8 | flask 9 | flask_cors 10 | gin 11 | gin_config 12 | librosa 13 | local_attention 14 | matplotlib 15 | praat-parselmouth 16 | pyworld 17 | PyYAML 18 | resampy 19 | scikit_learn 20 | scipy 21 | SoundFile 22 | tensorboard 23 | tqdm 24 | wave 25 | FreeSimpleGUI 26 | sounddevice 27 | gradio 28 | noisereduce 29 | onnxruntime-directml 30 | torchfcpe -------------------------------------------------------------------------------- /tools/onnx/onnx_inference_demo.py: -------------------------------------------------------------------------------- 1 | import soundfile 2 | 3 | from infer.lib.infer_pack.onnx_inference import OnnxRVC 4 | 5 | hop_size = 512 6 | sampling_rate = 40000 # 采样率 7 | f0_up_key = 0 # 升降调 8 | sid = 0 # 角色ID 9 | f0_method = "dio" # F0提取算法 10 | model_path = "ShirohaRVC.onnx" # 模型的完整路径 11 | vec_name = ( 12 | "vec-256-layer-9" # 内部自动补齐为 f"pretrained/{vec_name}.onnx" 需要onnx的vec模型 13 | ) 14 | wav_path = "123.wav" # 输入路径或ByteIO实例 15 | out_path = "out.wav" # 输出路径或ByteIO实例 16 | 17 | model = OnnxRVC( 18 | model_path, vec_path=vec_name, sr=sampling_rate, hop_size=hop_size, device="cuda" 19 | ) 20 | 21 | audio = model.inference(wav_path, sid, f0_method=f0_method, f0_up_key=f0_up_key) 22 | 23 | soundfile.write(out_path, audio, sampling_rate) 24 | -------------------------------------------------------------------------------- /tools/cmd/trans_weights.py: -------------------------------------------------------------------------------- 1 | import pdb 2 | 3 | import torch 4 | 5 | # a=torch.load(r"E:\codes\py39\vits_vc_gpu_train\logs\ft-mi-suc\G_1000.pth")["model"]#sim_nsf# 6 | # a=torch.load(r"E:\codes\py39\vits_vc_gpu_train\logs\ft-mi-freeze-vocoder-flow-enc_q\G_1000.pth")["model"]#sim_nsf# 7 | # a=torch.load(r"E:\codes\py39\vits_vc_gpu_train\logs\ft-mi-freeze-vocoder\G_1000.pth")["model"]#sim_nsf# 8 | # a=torch.load(r"E:\codes\py39\vits_vc_gpu_train\logs\ft-mi-test\G_1000.pth")["model"]#sim_nsf# 9 | a = torch.load( 10 | r"E:\codes\py39\vits_vc_gpu_train\logs\ft-mi-no_opt-no_dropout\G_1000.pth" 11 | )[ 12 | "model" 13 | ] # sim_nsf# 14 | for key in a.keys(): 15 | a[key] = a[key].half() 16 | # torch.save(a,"ft-mi-freeze-vocoder_true_1k.pt")# 17 | # torch.save(a,"ft-mi-sim1k.pt")# 18 | torch.save(a, "ft-mi-no_opt-no_dropout.pt") # 19 | -------------------------------------------------------------------------------- /i18n/i18n.py: -------------------------------------------------------------------------------- 1 | import json 2 | import locale 3 | import os 4 | 5 | 6 | def load_language_list(language): 7 | with open(f"./i18n/locale/{language}.json", "r", encoding="utf-8") as f: 8 | language_list = json.load(f) 9 | return language_list 10 | 11 | 12 | class I18nAuto: 13 | def __init__(self, language=None): 14 | if language in ["Auto", None]: 15 | language = locale.getdefaultlocale()[ 16 | 0 17 | ] # getlocale can't identify the system's language ((None, None)) 18 | if not os.path.exists(f"./i18n/locale/{language}.json"): 19 | language = "en_US" 20 | self.language = language 21 | self.language_map = load_language_list(language) 22 | 23 | def __call__(self, key): 24 | return self.language_map.get(key, key) 25 | 26 | def __repr__(self): 27 | return "Use Language: " + self.language 28 | -------------------------------------------------------------------------------- /requirements-dml.txt: -------------------------------------------------------------------------------- 1 | joblib>=1.1.0 2 | numba==0.56.4 3 | numpy==1.23.5 4 | scipy 5 | librosa==0.9.1 6 | llvmlite==0.39.0 7 | fairseq==0.12.2 8 | faiss-cpu==1.7.3 9 | gradio==4.23.0 10 | Cython 11 | pydub>=0.25.1 12 | soundfile>=0.12.1 13 | ffmpeg-python>=0.2.0 14 | tensorboardX 15 | Jinja2>=3.1.2 16 | json5 17 | Markdown 18 | matplotlib>=3.7.0 19 | matplotlib-inline>=0.1.3 20 | praat-parselmouth>=0.4.2 21 | Pillow>=9.1.1 22 | resampy>=0.4.2 23 | scikit-learn 24 | tensorboard 25 | tqdm>=4.63.1 26 | tornado>=6.1 27 | Werkzeug>=2.2.3 28 | uc-micro-py>=1.0.1 29 | sympy>=1.11.1 30 | tabulate>=0.8.10 31 | PyYAML>=6.0 32 | pyasn1>=0.4.8 33 | pyasn1-modules>=0.2.8 34 | fsspec>=2022.11.0 35 | absl-py>=1.2.0 36 | audioread 37 | uvicorn>=0.21.1 38 | colorama>=0.4.5 39 | pyworld==0.3.2 40 | httpx 41 | onnxruntime-directml 42 | torchcrepe==0.0.20 43 | fastapi 44 | ffmpy==0.3.1 45 | python-dotenv>=1.0.0 46 | av 47 | torchfcpe 48 | -------------------------------------------------------------------------------- /infer/lib/uvr5_pack/lib_v5/modelparams/3band_44100.json: -------------------------------------------------------------------------------- 1 | { 2 | "bins": 768, 3 | "unstable_bins": 5, 4 | "reduction_bins": 733, 5 | "band": { 6 | "1": { 7 | "sr": 11025, 8 | "hl": 128, 9 | "n_fft": 768, 10 | "crop_start": 0, 11 | "crop_stop": 278, 12 | "lpf_start": 28, 13 | "lpf_stop": 140, 14 | "res_type": "polyphase" 15 | }, 16 | "2": { 17 | "sr": 22050, 18 | "hl": 256, 19 | "n_fft": 768, 20 | "crop_start": 14, 21 | "crop_stop": 322, 22 | "hpf_start": 70, 23 | "hpf_stop": 14, 24 | "lpf_start": 283, 25 | "lpf_stop": 314, 26 | "res_type": "polyphase" 27 | }, 28 | "3": { 29 | "sr": 44100, 30 | "hl": 512, 31 | "n_fft": 768, 32 | "crop_start": 131, 33 | "crop_stop": 313, 34 | "hpf_start": 154, 35 | "hpf_stop": 141, 36 | "res_type": "sinc_medium" 37 | } 38 | }, 39 | "sr": 44100, 40 | "pre_filter_start": 757, 41 | "pre_filter_stop": 768 42 | } 43 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | joblib>=1.1.0 2 | numba 3 | numpy==1.23.5 4 | scipy 5 | librosa==0.9.1 6 | llvmlite 7 | fairseq 8 | faiss-cpu 9 | gradio==4.23.0 10 | Cython 11 | pydub>=0.25.1 12 | soundfile>=0.12.1 13 | ffmpeg-python>=0.2.0 14 | tensorboardX 15 | Jinja2>=3.1.2 16 | json5 17 | Markdown 18 | matplotlib==3.5 19 | matplotlib-inline>=0.1.3 20 | praat-parselmouth>=0.4.2 21 | Pillow>=9.1.1 22 | resampy>=0.4.2 23 | scikit-learn 24 | tensorboard 25 | tqdm>=4.63.1 26 | tornado>=6.1 27 | Werkzeug>=2.2.3 28 | uc-micro-py>=1.0.1 29 | sympy>=1.11.1 30 | tabulate>=0.8.10 31 | PyYAML>=6.0 32 | pyasn1>=0.4.8 33 | pyasn1-modules>=0.2.8 34 | fsspec>=2022.11.0 35 | absl-py>=1.2.0 36 | audioread 37 | uvicorn>=0.21.1 38 | colorama>=0.4.5 39 | pyworld==0.3.2 40 | httpx 41 | onnxruntime; sys_platform == 'darwin' 42 | onnxruntime-gpu; sys_platform != 'darwin' 43 | torchcrepe==0.0.20 44 | fastapi 45 | torchfcpe 46 | ffmpy==0.3.1 47 | python-dotenv>=1.0.0 48 | av 49 | -------------------------------------------------------------------------------- /requirements-amd.txt: -------------------------------------------------------------------------------- 1 | tensorflow-rocm 2 | joblib>=1.1.0 3 | numba==0.56.4 4 | numpy==1.23.5 5 | scipy 6 | librosa==0.9.1 7 | llvmlite==0.39.0 8 | fairseq==0.12.2 9 | faiss-cpu==1.7.3 10 | gradio==4.23.0 11 | Cython 12 | pydub>=0.25.1 13 | soundfile>=0.12.1 14 | ffmpeg-python>=0.2.0 15 | tensorboardX 16 | Jinja2>=3.1.2 17 | json5 18 | Markdown 19 | matplotlib>=3.7.0 20 | matplotlib-inline>=0.1.3 21 | praat-parselmouth>=0.4.2 22 | Pillow>=9.1.1 23 | resampy>=0.4.2 24 | scikit-learn 25 | tensorboard 26 | tqdm>=4.63.1 27 | tornado>=6.1 28 | Werkzeug>=2.2.3 29 | uc-micro-py>=1.0.1 30 | sympy>=1.11.1 31 | tabulate>=0.8.10 32 | PyYAML>=6.0 33 | pyasn1>=0.4.8 34 | pyasn1-modules>=0.2.8 35 | fsspec>=2022.11.0 36 | absl-py>=1.2.0 37 | audioread 38 | uvicorn>=0.21.1 39 | colorama>=0.4.5 40 | pyworld==0.3.2 41 | httpx 42 | onnxruntime 43 | onnxruntime-gpu 44 | torchcrepe==0.0.20 45 | fastapi 46 | ffmpy==0.3.1 47 | python-dotenv>=1.0.0 48 | av 49 | torchfcpe 50 | -------------------------------------------------------------------------------- /infer/lib/uvr5_pack/lib_v5/modelparams/ensemble.json: -------------------------------------------------------------------------------- 1 | { 2 | "mid_side_b2": true, 3 | "bins": 1280, 4 | "unstable_bins": 7, 5 | "reduction_bins": 565, 6 | "band": { 7 | "1": { 8 | "sr": 11025, 9 | "hl": 108, 10 | "n_fft": 2048, 11 | "crop_start": 0, 12 | "crop_stop": 374, 13 | "lpf_start": 92, 14 | "lpf_stop": 186, 15 | "res_type": "polyphase" 16 | }, 17 | "2": { 18 | "sr": 22050, 19 | "hl": 216, 20 | "n_fft": 1536, 21 | "crop_start": 0, 22 | "crop_stop": 424, 23 | "hpf_start": 68, 24 | "hpf_stop": 34, 25 | "lpf_start": 348, 26 | "lpf_stop": 418, 27 | "res_type": "polyphase" 28 | }, 29 | "3": { 30 | "sr": 44100, 31 | "hl": 432, 32 | "n_fft": 1280, 33 | "crop_start": 132, 34 | "crop_stop": 614, 35 | "hpf_start": 172, 36 | "hpf_stop": 144, 37 | "res_type": "polyphase" 38 | } 39 | }, 40 | "sr": 44100, 41 | "pre_filter_start": 1280, 42 | "pre_filter_stop": 1280 43 | } -------------------------------------------------------------------------------- /infer/lib/uvr5_pack/lib_v5/modelparams/3band_44100_mid.json: -------------------------------------------------------------------------------- 1 | { 2 | "mid_side": true, 3 | "bins": 768, 4 | "unstable_bins": 5, 5 | "reduction_bins": 733, 6 | "band": { 7 | "1": { 8 | "sr": 11025, 9 | "hl": 128, 10 | "n_fft": 768, 11 | "crop_start": 0, 12 | "crop_stop": 278, 13 | "lpf_start": 28, 14 | "lpf_stop": 140, 15 | "res_type": "polyphase" 16 | }, 17 | "2": { 18 | "sr": 22050, 19 | "hl": 256, 20 | "n_fft": 768, 21 | "crop_start": 14, 22 | "crop_stop": 322, 23 | "hpf_start": 70, 24 | "hpf_stop": 14, 25 | "lpf_start": 283, 26 | "lpf_stop": 314, 27 | "res_type": "polyphase" 28 | }, 29 | "3": { 30 | "sr": 44100, 31 | "hl": 512, 32 | "n_fft": 768, 33 | "crop_start": 131, 34 | "crop_stop": 313, 35 | "hpf_start": 154, 36 | "hpf_stop": 141, 37 | "res_type": "sinc_medium" 38 | } 39 | }, 40 | "sr": 44100, 41 | "pre_filter_start": 757, 42 | "pre_filter_stop": 768 43 | } 44 | -------------------------------------------------------------------------------- /infer/lib/uvr5_pack/lib_v5/modelparams/3band_44100_msb2.json: -------------------------------------------------------------------------------- 1 | { 2 | "mid_side_b2": true, 3 | "bins": 640, 4 | "unstable_bins": 7, 5 | "reduction_bins": 565, 6 | "band": { 7 | "1": { 8 | "sr": 11025, 9 | "hl": 108, 10 | "n_fft": 1024, 11 | "crop_start": 0, 12 | "crop_stop": 187, 13 | "lpf_start": 92, 14 | "lpf_stop": 186, 15 | "res_type": "polyphase" 16 | }, 17 | "2": { 18 | "sr": 22050, 19 | "hl": 216, 20 | "n_fft": 768, 21 | "crop_start": 0, 22 | "crop_stop": 212, 23 | "hpf_start": 68, 24 | "hpf_stop": 34, 25 | "lpf_start": 174, 26 | "lpf_stop": 209, 27 | "res_type": "polyphase" 28 | }, 29 | "3": { 30 | "sr": 44100, 31 | "hl": 432, 32 | "n_fft": 640, 33 | "crop_start": 66, 34 | "crop_stop": 307, 35 | "hpf_start": 86, 36 | "hpf_stop": 72, 37 | "res_type": "kaiser_fast" 38 | } 39 | }, 40 | "sr": 44100, 41 | "pre_filter_start": 639, 42 | "pre_filter_stop": 640 43 | } 44 | -------------------------------------------------------------------------------- /requirements-py311.txt: -------------------------------------------------------------------------------- 1 | joblib>=1.1.0 2 | numba 3 | numpy 4 | scipy 5 | librosa==0.9.1 6 | llvmlite 7 | fairseq @ git+https://github.com/One-sixth/fairseq.git 8 | faiss-cpu 9 | gradio==4.23.0 10 | Cython 11 | pydub>=0.25.1 12 | soundfile>=0.12.1 13 | ffmpeg-python>=0.2.0 14 | tensorboardX 15 | Jinja2>=3.1.2 16 | json5 17 | Markdown 18 | matplotlib>=3.7.0 19 | matplotlib-inline>=0.1.3 20 | praat-parselmouth>=0.4.2 21 | Pillow>=9.1.1 22 | resampy>=0.4.2 23 | scikit-learn 24 | tensorboard 25 | tqdm>=4.63.1 26 | tornado>=6.1 27 | Werkzeug>=2.2.3 28 | uc-micro-py>=1.0.1 29 | sympy>=1.11.1 30 | tabulate>=0.8.10 31 | PyYAML>=6.0 32 | pyasn1>=0.4.8 33 | pyasn1-modules>=0.2.8 34 | fsspec>=2022.11.0 35 | absl-py>=1.2.0 36 | audioread 37 | uvicorn>=0.21.1 38 | colorama>=0.4.5 39 | pyworld==0.3.2 40 | httpx 41 | onnxruntime; sys_platform == 'darwin' 42 | onnxruntime-gpu; sys_platform != 'darwin' 43 | torchcrepe==0.0.20 44 | fastapi 45 | torchfcpe 46 | ffmpy==0.3.1 47 | python-dotenv>=1.0.0 48 | av 49 | -------------------------------------------------------------------------------- /infer/modules/vc/utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from fairseq import checkpoint_utils 4 | 5 | 6 | def get_index_path_from_model(sid): 7 | return next( 8 | ( 9 | f 10 | for f in [ 11 | os.path.join(root, name) 12 | for root, _, files in os.walk(os.getenv("index_root"), topdown=False) 13 | for name in files 14 | if name.endswith(".index") and "trained" not in name 15 | ] 16 | if sid.split(".")[0] in f 17 | ), 18 | "", 19 | ) 20 | 21 | 22 | def load_hubert(config): 23 | models, _, _ = checkpoint_utils.load_model_ensemble_and_task( 24 | ["assets/hubert/hubert_base.pt"], 25 | suffix="", 26 | ) 27 | hubert_model = models[0] 28 | hubert_model = hubert_model.to(config.device) 29 | if config.is_half: 30 | hubert_model = hubert_model.half() 31 | else: 32 | hubert_model = hubert_model.float() 33 | return hubert_model.eval() 34 | -------------------------------------------------------------------------------- /infer/lib/uvr5_pack/lib_v5/modelparams/4band_v2.json: -------------------------------------------------------------------------------- 1 | { 2 | "bins": 672, 3 | "unstable_bins": 8, 4 | "reduction_bins": 637, 5 | "band": { 6 | "1": { 7 | "sr": 7350, 8 | "hl": 80, 9 | "n_fft": 640, 10 | "crop_start": 0, 11 | "crop_stop": 85, 12 | "lpf_start": 25, 13 | "lpf_stop": 53, 14 | "res_type": "polyphase" 15 | }, 16 | "2": { 17 | "sr": 7350, 18 | "hl": 80, 19 | "n_fft": 320, 20 | "crop_start": 4, 21 | "crop_stop": 87, 22 | "hpf_start": 25, 23 | "hpf_stop": 12, 24 | "lpf_start": 31, 25 | "lpf_stop": 62, 26 | "res_type": "polyphase" 27 | }, 28 | "3": { 29 | "sr": 14700, 30 | "hl": 160, 31 | "n_fft": 512, 32 | "crop_start": 17, 33 | "crop_stop": 216, 34 | "hpf_start": 48, 35 | "hpf_stop": 24, 36 | "lpf_start": 139, 37 | "lpf_stop": 210, 38 | "res_type": "polyphase" 39 | }, 40 | "4": { 41 | "sr": 44100, 42 | "hl": 480, 43 | "n_fft": 960, 44 | "crop_start": 78, 45 | "crop_stop": 383, 46 | "hpf_start": 130, 47 | "hpf_stop": 86, 48 | "res_type": "kaiser_fast" 49 | } 50 | }, 51 | "sr": 44100, 52 | "pre_filter_start": 668, 53 | "pre_filter_stop": 672 54 | } -------------------------------------------------------------------------------- /infer/lib/uvr5_pack/lib_v5/modelparams/4band_v3.json: -------------------------------------------------------------------------------- 1 | { 2 | "bins": 672, 3 | "unstable_bins": 8, 4 | "reduction_bins": 530, 5 | "band": { 6 | "1": { 7 | "sr": 7350, 8 | "hl": 80, 9 | "n_fft": 640, 10 | "crop_start": 0, 11 | "crop_stop": 85, 12 | "lpf_start": 25, 13 | "lpf_stop": 53, 14 | "res_type": "polyphase" 15 | }, 16 | "2": { 17 | "sr": 7350, 18 | "hl": 80, 19 | "n_fft": 320, 20 | "crop_start": 4, 21 | "crop_stop": 87, 22 | "hpf_start": 25, 23 | "hpf_stop": 12, 24 | "lpf_start": 31, 25 | "lpf_stop": 62, 26 | "res_type": "polyphase" 27 | }, 28 | "3": { 29 | "sr": 14700, 30 | "hl": 160, 31 | "n_fft": 512, 32 | "crop_start": 17, 33 | "crop_stop": 216, 34 | "hpf_start": 48, 35 | "hpf_stop": 24, 36 | "lpf_start": 139, 37 | "lpf_stop": 210, 38 | "res_type": "polyphase" 39 | }, 40 | "4": { 41 | "sr": 44100, 42 | "hl": 480, 43 | "n_fft": 960, 44 | "crop_start": 78, 45 | "crop_stop": 383, 46 | "hpf_start": 130, 47 | "hpf_stop": 86, 48 | "res_type": "kaiser_fast" 49 | } 50 | }, 51 | "sr": 44100, 52 | "pre_filter_start": 668, 53 | "pre_filter_stop": 672 54 | } -------------------------------------------------------------------------------- /infer/lib/uvr5_pack/lib_v5/modelparams/4band_44100.json: -------------------------------------------------------------------------------- 1 | { 2 | "bins": 768, 3 | "unstable_bins": 7, 4 | "reduction_bins": 668, 5 | "band": { 6 | "1": { 7 | "sr": 11025, 8 | "hl": 128, 9 | "n_fft": 1024, 10 | "crop_start": 0, 11 | "crop_stop": 186, 12 | "lpf_start": 37, 13 | "lpf_stop": 73, 14 | "res_type": "polyphase" 15 | }, 16 | "2": { 17 | "sr": 11025, 18 | "hl": 128, 19 | "n_fft": 512, 20 | "crop_start": 4, 21 | "crop_stop": 185, 22 | "hpf_start": 36, 23 | "hpf_stop": 18, 24 | "lpf_start": 93, 25 | "lpf_stop": 185, 26 | "res_type": "polyphase" 27 | }, 28 | "3": { 29 | "sr": 22050, 30 | "hl": 256, 31 | "n_fft": 512, 32 | "crop_start": 46, 33 | "crop_stop": 186, 34 | "hpf_start": 93, 35 | "hpf_stop": 46, 36 | "lpf_start": 164, 37 | "lpf_stop": 186, 38 | "res_type": "polyphase" 39 | }, 40 | "4": { 41 | "sr": 44100, 42 | "hl": 512, 43 | "n_fft": 768, 44 | "crop_start": 121, 45 | "crop_stop": 382, 46 | "hpf_start": 138, 47 | "hpf_stop": 123, 48 | "res_type": "sinc_medium" 49 | } 50 | }, 51 | "sr": 44100, 52 | "pre_filter_start": 740, 53 | "pre_filter_stop": 768 54 | } 55 | -------------------------------------------------------------------------------- /infer/lib/uvr5_pack/lib_v5/modelparams/4band_44100_sw.json: -------------------------------------------------------------------------------- 1 | { 2 | "stereo_w": true, 3 | "bins": 768, 4 | "unstable_bins": 7, 5 | "reduction_bins": 668, 6 | "band": { 7 | "1": { 8 | "sr": 11025, 9 | "hl": 128, 10 | "n_fft": 1024, 11 | "crop_start": 0, 12 | "crop_stop": 186, 13 | "lpf_start": 37, 14 | "lpf_stop": 73, 15 | "res_type": "polyphase" 16 | }, 17 | "2": { 18 | "sr": 11025, 19 | "hl": 128, 20 | "n_fft": 512, 21 | "crop_start": 4, 22 | "crop_stop": 185, 23 | "hpf_start": 36, 24 | "hpf_stop": 18, 25 | "lpf_start": 93, 26 | "lpf_stop": 185, 27 | "res_type": "polyphase" 28 | }, 29 | "3": { 30 | "sr": 22050, 31 | "hl": 256, 32 | "n_fft": 512, 33 | "crop_start": 46, 34 | "crop_stop": 186, 35 | "hpf_start": 93, 36 | "hpf_stop": 46, 37 | "lpf_start": 164, 38 | "lpf_stop": 186, 39 | "res_type": "polyphase" 40 | }, 41 | "4": { 42 | "sr": 44100, 43 | "hl": 512, 44 | "n_fft": 768, 45 | "crop_start": 121, 46 | "crop_stop": 382, 47 | "hpf_start": 138, 48 | "hpf_stop": 123, 49 | "res_type": "sinc_medium" 50 | } 51 | }, 52 | "sr": 44100, 53 | "pre_filter_start": 740, 54 | "pre_filter_stop": 768 55 | } -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023-2024 liujing04 4 | Copyright (c) 2023-2024 fumiama 5 | Copyright (c) 2023-2024 Ftps 6 | 7 | Permission is hereby granted, free of charge, to any person obtaining a copy 8 | of this software and associated documentation files (the "Software"), to deal 9 | in the Software without restriction, including without limitation the rights 10 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 11 | copies of the Software, and to permit persons to whom the Software is 12 | furnished to do so, subject to the following conditions: 13 | 14 | The above copyright notice and this permission notice shall be included in all 15 | copies or substantial portions of the Software. 16 | 17 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 18 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 19 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 20 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 22 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 23 | SOFTWARE. 24 | -------------------------------------------------------------------------------- /infer/lib/uvr5_pack/lib_v5/modelparams/4band_44100_msb.json: -------------------------------------------------------------------------------- 1 | { 2 | "mid_side_b": true, 3 | "bins": 768, 4 | "unstable_bins": 7, 5 | "reduction_bins": 668, 6 | "band": { 7 | "1": { 8 | "sr": 11025, 9 | "hl": 128, 10 | "n_fft": 1024, 11 | "crop_start": 0, 12 | "crop_stop": 186, 13 | "lpf_start": 37, 14 | "lpf_stop": 73, 15 | "res_type": "polyphase" 16 | }, 17 | "2": { 18 | "sr": 11025, 19 | "hl": 128, 20 | "n_fft": 512, 21 | "crop_start": 4, 22 | "crop_stop": 185, 23 | "hpf_start": 36, 24 | "hpf_stop": 18, 25 | "lpf_start": 93, 26 | "lpf_stop": 185, 27 | "res_type": "polyphase" 28 | }, 29 | "3": { 30 | "sr": 22050, 31 | "hl": 256, 32 | "n_fft": 512, 33 | "crop_start": 46, 34 | "crop_stop": 186, 35 | "hpf_start": 93, 36 | "hpf_stop": 46, 37 | "lpf_start": 164, 38 | "lpf_stop": 186, 39 | "res_type": "polyphase" 40 | }, 41 | "4": { 42 | "sr": 44100, 43 | "hl": 512, 44 | "n_fft": 768, 45 | "crop_start": 121, 46 | "crop_stop": 382, 47 | "hpf_start": 138, 48 | "hpf_stop": 123, 49 | "res_type": "sinc_medium" 50 | } 51 | }, 52 | "sr": 44100, 53 | "pre_filter_start": 740, 54 | "pre_filter_stop": 768 55 | } -------------------------------------------------------------------------------- /infer/lib/uvr5_pack/lib_v5/modelparams/4band_44100_msb2.json: -------------------------------------------------------------------------------- 1 | { 2 | "mid_side_b": true, 3 | "bins": 768, 4 | "unstable_bins": 7, 5 | "reduction_bins": 668, 6 | "band": { 7 | "1": { 8 | "sr": 11025, 9 | "hl": 128, 10 | "n_fft": 1024, 11 | "crop_start": 0, 12 | "crop_stop": 186, 13 | "lpf_start": 37, 14 | "lpf_stop": 73, 15 | "res_type": "polyphase" 16 | }, 17 | "2": { 18 | "sr": 11025, 19 | "hl": 128, 20 | "n_fft": 512, 21 | "crop_start": 4, 22 | "crop_stop": 185, 23 | "hpf_start": 36, 24 | "hpf_stop": 18, 25 | "lpf_start": 93, 26 | "lpf_stop": 185, 27 | "res_type": "polyphase" 28 | }, 29 | "3": { 30 | "sr": 22050, 31 | "hl": 256, 32 | "n_fft": 512, 33 | "crop_start": 46, 34 | "crop_stop": 186, 35 | "hpf_start": 93, 36 | "hpf_stop": 46, 37 | "lpf_start": 164, 38 | "lpf_stop": 186, 39 | "res_type": "polyphase" 40 | }, 41 | "4": { 42 | "sr": 44100, 43 | "hl": 512, 44 | "n_fft": 768, 45 | "crop_start": 121, 46 | "crop_stop": 382, 47 | "hpf_start": 138, 48 | "hpf_stop": 123, 49 | "res_type": "sinc_medium" 50 | } 51 | }, 52 | "sr": 44100, 53 | "pre_filter_start": 740, 54 | "pre_filter_stop": 768 55 | } -------------------------------------------------------------------------------- /infer/lib/uvr5_pack/lib_v5/modelparams/4band_44100_reverse.json: -------------------------------------------------------------------------------- 1 | { 2 | "reverse": true, 3 | "bins": 768, 4 | "unstable_bins": 7, 5 | "reduction_bins": 668, 6 | "band": { 7 | "1": { 8 | "sr": 11025, 9 | "hl": 128, 10 | "n_fft": 1024, 11 | "crop_start": 0, 12 | "crop_stop": 186, 13 | "lpf_start": 37, 14 | "lpf_stop": 73, 15 | "res_type": "polyphase" 16 | }, 17 | "2": { 18 | "sr": 11025, 19 | "hl": 128, 20 | "n_fft": 512, 21 | "crop_start": 4, 22 | "crop_stop": 185, 23 | "hpf_start": 36, 24 | "hpf_stop": 18, 25 | "lpf_start": 93, 26 | "lpf_stop": 185, 27 | "res_type": "polyphase" 28 | }, 29 | "3": { 30 | "sr": 22050, 31 | "hl": 256, 32 | "n_fft": 512, 33 | "crop_start": 46, 34 | "crop_stop": 186, 35 | "hpf_start": 93, 36 | "hpf_stop": 46, 37 | "lpf_start": 164, 38 | "lpf_stop": 186, 39 | "res_type": "polyphase" 40 | }, 41 | "4": { 42 | "sr": 44100, 43 | "hl": 512, 44 | "n_fft": 768, 45 | "crop_start": 121, 46 | "crop_stop": 382, 47 | "hpf_start": 138, 48 | "hpf_stop": 123, 49 | "res_type": "sinc_medium" 50 | } 51 | }, 52 | "sr": 44100, 53 | "pre_filter_start": 740, 54 | "pre_filter_stop": 768 55 | } -------------------------------------------------------------------------------- /infer/lib/uvr5_pack/lib_v5/modelparams/4band_v2_sn.json: -------------------------------------------------------------------------------- 1 | { 2 | "bins": 672, 3 | "unstable_bins": 8, 4 | "reduction_bins": 637, 5 | "band": { 6 | "1": { 7 | "sr": 7350, 8 | "hl": 80, 9 | "n_fft": 640, 10 | "crop_start": 0, 11 | "crop_stop": 85, 12 | "lpf_start": 25, 13 | "lpf_stop": 53, 14 | "res_type": "polyphase" 15 | }, 16 | "2": { 17 | "sr": 7350, 18 | "hl": 80, 19 | "n_fft": 320, 20 | "crop_start": 4, 21 | "crop_stop": 87, 22 | "hpf_start": 25, 23 | "hpf_stop": 12, 24 | "lpf_start": 31, 25 | "lpf_stop": 62, 26 | "res_type": "polyphase" 27 | }, 28 | "3": { 29 | "sr": 14700, 30 | "hl": 160, 31 | "n_fft": 512, 32 | "crop_start": 17, 33 | "crop_stop": 216, 34 | "hpf_start": 48, 35 | "hpf_stop": 24, 36 | "lpf_start": 139, 37 | "lpf_stop": 210, 38 | "res_type": "polyphase" 39 | }, 40 | "4": { 41 | "sr": 44100, 42 | "hl": 480, 43 | "n_fft": 960, 44 | "crop_start": 78, 45 | "crop_stop": 383, 46 | "hpf_start": 130, 47 | "hpf_stop": 86, 48 | "convert_channels": "stereo_n", 49 | "res_type": "kaiser_fast" 50 | } 51 | }, 52 | "sr": 44100, 53 | "pre_filter_start": 668, 54 | "pre_filter_stop": 672 55 | } -------------------------------------------------------------------------------- /infer/lib/uvr5_pack/lib_v5/modelparams/4band_44100_mid.json: -------------------------------------------------------------------------------- 1 | { 2 | "bins": 768, 3 | "unstable_bins": 7, 4 | "mid_side": true, 5 | "reduction_bins": 668, 6 | "band": { 7 | "1": { 8 | "sr": 11025, 9 | "hl": 128, 10 | "n_fft": 1024, 11 | "crop_start": 0, 12 | "crop_stop": 186, 13 | "lpf_start": 37, 14 | "lpf_stop": 73, 15 | "res_type": "polyphase" 16 | }, 17 | "2": { 18 | "sr": 11025, 19 | "hl": 128, 20 | "n_fft": 512, 21 | "crop_start": 4, 22 | "crop_stop": 185, 23 | "hpf_start": 36, 24 | "hpf_stop": 18, 25 | "lpf_start": 93, 26 | "lpf_stop": 185, 27 | "res_type": "polyphase" 28 | }, 29 | "3": { 30 | "sr": 22050, 31 | "hl": 256, 32 | "n_fft": 512, 33 | "crop_start": 46, 34 | "crop_stop": 186, 35 | "hpf_start": 93, 36 | "hpf_stop": 46, 37 | "lpf_start": 164, 38 | "lpf_stop": 186, 39 | "res_type": "polyphase" 40 | }, 41 | "4": { 42 | "sr": 44100, 43 | "hl": 512, 44 | "n_fft": 768, 45 | "crop_start": 121, 46 | "crop_stop": 382, 47 | "hpf_start": 138, 48 | "hpf_stop": 123, 49 | "res_type": "sinc_medium" 50 | } 51 | }, 52 | "sr": 44100, 53 | "pre_filter_start": 740, 54 | "pre_filter_stop": 768 55 | } 56 | -------------------------------------------------------------------------------- /configs/v1/32k.json: -------------------------------------------------------------------------------- 1 | { 2 | "train": { 3 | "log_interval": 200, 4 | "seed": 1234, 5 | "epochs": 20000, 6 | "learning_rate": 1e-4, 7 | "betas": [0.8, 0.99], 8 | "eps": 1e-9, 9 | "batch_size": 4, 10 | "fp16_run": true, 11 | "lr_decay": 0.999875, 12 | "segment_size": 12800, 13 | "init_lr_ratio": 1, 14 | "warmup_epochs": 0, 15 | "c_mel": 45, 16 | "c_kl": 1.0 17 | }, 18 | "data": { 19 | "max_wav_value": 32768.0, 20 | "sampling_rate": 32000, 21 | "filter_length": 1024, 22 | "hop_length": 320, 23 | "win_length": 1024, 24 | "n_mel_channels": 80, 25 | "mel_fmin": 0.0, 26 | "mel_fmax": null 27 | }, 28 | "model": { 29 | "inter_channels": 192, 30 | "hidden_channels": 192, 31 | "filter_channels": 768, 32 | "n_heads": 2, 33 | "n_layers": 6, 34 | "kernel_size": 3, 35 | "p_dropout": 0, 36 | "resblock": "1", 37 | "resblock_kernel_sizes": [3,7,11], 38 | "resblock_dilation_sizes": [[1,3,5], [1,3,5], [1,3,5]], 39 | "upsample_rates": [10,4,2,2,2], 40 | "upsample_initial_channel": 512, 41 | "upsample_kernel_sizes": [16,16,4,4,4], 42 | "use_spectral_norm": false, 43 | "gin_channels": 256, 44 | "spk_embed_dim": 109 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /configs/v1/40k.json: -------------------------------------------------------------------------------- 1 | { 2 | "train": { 3 | "log_interval": 200, 4 | "seed": 1234, 5 | "epochs": 20000, 6 | "learning_rate": 1e-4, 7 | "betas": [0.8, 0.99], 8 | "eps": 1e-9, 9 | "batch_size": 4, 10 | "fp16_run": true, 11 | "lr_decay": 0.999875, 12 | "segment_size": 12800, 13 | "init_lr_ratio": 1, 14 | "warmup_epochs": 0, 15 | "c_mel": 45, 16 | "c_kl": 1.0 17 | }, 18 | "data": { 19 | "max_wav_value": 32768.0, 20 | "sampling_rate": 40000, 21 | "filter_length": 2048, 22 | "hop_length": 400, 23 | "win_length": 2048, 24 | "n_mel_channels": 125, 25 | "mel_fmin": 0.0, 26 | "mel_fmax": null 27 | }, 28 | "model": { 29 | "inter_channels": 192, 30 | "hidden_channels": 192, 31 | "filter_channels": 768, 32 | "n_heads": 2, 33 | "n_layers": 6, 34 | "kernel_size": 3, 35 | "p_dropout": 0, 36 | "resblock": "1", 37 | "resblock_kernel_sizes": [3,7,11], 38 | "resblock_dilation_sizes": [[1,3,5], [1,3,5], [1,3,5]], 39 | "upsample_rates": [10,10,2,2], 40 | "upsample_initial_channel": 512, 41 | "upsample_kernel_sizes": [16,16,4,4], 42 | "use_spectral_norm": false, 43 | "gin_channels": 256, 44 | "spk_embed_dim": 109 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /configs/v2/32k.json: -------------------------------------------------------------------------------- 1 | { 2 | "train": { 3 | "log_interval": 200, 4 | "seed": 1234, 5 | "epochs": 20000, 6 | "learning_rate": 1e-4, 7 | "betas": [0.8, 0.99], 8 | "eps": 1e-9, 9 | "batch_size": 4, 10 | "fp16_run": true, 11 | "lr_decay": 0.999875, 12 | "segment_size": 12800, 13 | "init_lr_ratio": 1, 14 | "warmup_epochs": 0, 15 | "c_mel": 45, 16 | "c_kl": 1.0 17 | }, 18 | "data": { 19 | "max_wav_value": 32768.0, 20 | "sampling_rate": 32000, 21 | "filter_length": 1024, 22 | "hop_length": 320, 23 | "win_length": 1024, 24 | "n_mel_channels": 80, 25 | "mel_fmin": 0.0, 26 | "mel_fmax": null 27 | }, 28 | "model": { 29 | "inter_channels": 192, 30 | "hidden_channels": 192, 31 | "filter_channels": 768, 32 | "n_heads": 2, 33 | "n_layers": 6, 34 | "kernel_size": 3, 35 | "p_dropout": 0, 36 | "resblock": "1", 37 | "resblock_kernel_sizes": [3,7,11], 38 | "resblock_dilation_sizes": [[1,3,5], [1,3,5], [1,3,5]], 39 | "upsample_rates": [10,8,2,2], 40 | "upsample_initial_channel": 512, 41 | "upsample_kernel_sizes": [20,16,4,4], 42 | "use_spectral_norm": false, 43 | "gin_channels": 256, 44 | "spk_embed_dim": 109 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /configs/v2/48k.json: -------------------------------------------------------------------------------- 1 | { 2 | "train": { 3 | "log_interval": 200, 4 | "seed": 1234, 5 | "epochs": 20000, 6 | "learning_rate": 1e-4, 7 | "betas": [0.8, 0.99], 8 | "eps": 1e-9, 9 | "batch_size": 4, 10 | "fp16_run": true, 11 | "lr_decay": 0.999875, 12 | "segment_size": 17280, 13 | "init_lr_ratio": 1, 14 | "warmup_epochs": 0, 15 | "c_mel": 45, 16 | "c_kl": 1.0 17 | }, 18 | "data": { 19 | "max_wav_value": 32768.0, 20 | "sampling_rate": 48000, 21 | "filter_length": 2048, 22 | "hop_length": 480, 23 | "win_length": 2048, 24 | "n_mel_channels": 128, 25 | "mel_fmin": 0.0, 26 | "mel_fmax": null 27 | }, 28 | "model": { 29 | "inter_channels": 192, 30 | "hidden_channels": 192, 31 | "filter_channels": 768, 32 | "n_heads": 2, 33 | "n_layers": 6, 34 | "kernel_size": 3, 35 | "p_dropout": 0, 36 | "resblock": "1", 37 | "resblock_kernel_sizes": [3,7,11], 38 | "resblock_dilation_sizes": [[1,3,5], [1,3,5], [1,3,5]], 39 | "upsample_rates": [12,10,2,2], 40 | "upsample_initial_channel": 512, 41 | "upsample_kernel_sizes": [24,20,4,4], 42 | "use_spectral_norm": false, 43 | "gin_channels": 256, 44 | "spk_embed_dim": 109 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /configs/v1/48k.json: -------------------------------------------------------------------------------- 1 | { 2 | "train": { 3 | "log_interval": 200, 4 | "seed": 1234, 5 | "epochs": 20000, 6 | "learning_rate": 1e-4, 7 | "betas": [0.8, 0.99], 8 | "eps": 1e-9, 9 | "batch_size": 4, 10 | "fp16_run": true, 11 | "lr_decay": 0.999875, 12 | "segment_size": 11520, 13 | "init_lr_ratio": 1, 14 | "warmup_epochs": 0, 15 | "c_mel": 45, 16 | "c_kl": 1.0 17 | }, 18 | "data": { 19 | "max_wav_value": 32768.0, 20 | "sampling_rate": 48000, 21 | "filter_length": 2048, 22 | "hop_length": 480, 23 | "win_length": 2048, 24 | "n_mel_channels": 128, 25 | "mel_fmin": 0.0, 26 | "mel_fmax": null 27 | }, 28 | "model": { 29 | "inter_channels": 192, 30 | "hidden_channels": 192, 31 | "filter_channels": 768, 32 | "n_heads": 2, 33 | "n_layers": 6, 34 | "kernel_size": 3, 35 | "p_dropout": 0, 36 | "resblock": "1", 37 | "resblock_kernel_sizes": [3,7,11], 38 | "resblock_dilation_sizes": [[1,3,5], [1,3,5], [1,3,5]], 39 | "upsample_rates": [10,6,2,2,2], 40 | "upsample_initial_channel": 512, 41 | "upsample_kernel_sizes": [16,16,4,4,4], 42 | "use_spectral_norm": false, 43 | "gin_channels": 256, 44 | "spk_embed_dim": 109 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /requirements-ipex.txt: -------------------------------------------------------------------------------- 1 | torch==2.0.1a0 2 | intel_extension_for_pytorch==2.0.110+xpu 3 | torchvision==0.15.2a0 4 | https://github.com/Disty0/Retrieval-based-Voice-Conversion-WebUI/releases/download/torchaudio_wheels_for_ipex/torchaudio-2.0.2+31de77d-cp310-cp310-linux_x86_64.whl 5 | --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/us/ 6 | joblib>=1.1.0 7 | numba==0.56.4 8 | numpy==1.23.5 9 | scipy 10 | librosa==0.9.1 11 | llvmlite==0.39.0 12 | fairseq==0.12.2 13 | faiss-cpu==1.7.3 14 | gradio==4.23.0 15 | Cython 16 | pydub>=0.25.1 17 | soundfile>=0.12.1 18 | ffmpeg-python>=0.2.0 19 | tensorboardX 20 | Jinja2>=3.1.2 21 | json5 22 | Markdown 23 | matplotlib>=3.7.0 24 | matplotlib-inline>=0.1.3 25 | praat-parselmouth>=0.4.2 26 | Pillow>=9.1.1 27 | resampy>=0.4.2 28 | scikit-learn 29 | tensorboard 30 | tqdm>=4.63.1 31 | tornado>=6.1 32 | Werkzeug>=2.2.3 33 | uc-micro-py>=1.0.1 34 | sympy>=1.11.1 35 | tabulate>=0.8.10 36 | PyYAML>=6.0 37 | pyasn1>=0.4.8 38 | pyasn1-modules>=0.2.8 39 | fsspec>=2022.11.0 40 | absl-py>=1.2.0 41 | audioread 42 | uvicorn>=0.21.1 43 | colorama>=0.4.5 44 | pyworld==0.3.2 45 | httpx 46 | onnxruntime; sys_platform == 'darwin' 47 | onnxruntime-gpu; sys_platform != 'darwin' 48 | torchcrepe==0.0.20 49 | fastapi 50 | ffmpy==0.3.1 51 | python-dotenv>=1.0.0 52 | av 53 | FreeSimpleGUI 54 | sounddevice 55 | torchfcpe 56 | -------------------------------------------------------------------------------- /tools/cmd/train-index.py: -------------------------------------------------------------------------------- 1 | """ 2 | 格式：直接cid为自带的index位；aid放不下了，通过字典来查，反正就5w个 3 | """ 4 | 5 | import os 6 | import logging 7 | 8 | logger = logging.getLogger(__name__) 9 | 10 | import faiss 11 | import numpy as np 12 | 13 | # ###########如果是原始特征要先写save 14 | inp_root = r"E:\codes\py39\dataset\mi\2-co256" 15 | npys = [] 16 | for name in sorted(list(os.listdir(inp_root))): 17 | phone = np.load("%s/%s" % (inp_root, name)) 18 | npys.append(phone) 19 | big_npy = np.concatenate(npys, 0) 20 | logger.debug(big_npy.shape) # (6196072, 192)#fp32#4.43G 21 | np.save("infer/big_src_feature_mi.npy", big_npy) 22 | 23 | ##################train+add 24 | # big_npy=np.load("/bili-coeus/jupyter/jupyterhub-liujing04/vits_ch/inference_f0/big_src_feature_mi.npy") 25 | logger.debug(big_npy.shape) 26 | index = faiss.index_factory(256, "IVF512,Flat") # mi 27 | logger.info("Training...") 28 | index_ivf = faiss.extract_index_ivf(index) # 29 | index_ivf.nprobe = 9 30 | index.train(big_npy) 31 | faiss.write_index(index, "infer/trained_IVF512_Flat_mi_baseline_src_feat.index") 32 | logger.info("Adding...") 33 | index.add(big_npy) 34 | faiss.write_index(index, "infer/added_IVF512_Flat_mi_baseline_src_feat.index") 35 | """ 36 | 大小（都是FP32） 37 | big_src_feature 2.95G 38 | (3098036, 256) 39 | big_emb 4.43G 40 | (6196072, 192) 41 | big_emb双倍是因为求特征要repeat后再加pitch 42 | 43 | """ 44 | -------------------------------------------------------------------------------- /infer/lib/jit/get_synthesizer.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | 4 | def get_synthesizer(pth_path, device=torch.device("cpu")): 5 | from infer.lib.infer_pack.models import ( 6 | SynthesizerTrnMs256NSFsid, 7 | SynthesizerTrnMs256NSFsid_nono, 8 | SynthesizerTrnMs768NSFsid, 9 | SynthesizerTrnMs768NSFsid_nono, 10 | ) 11 | 12 | cpt = torch.load(pth_path, map_location=torch.device("cpu")) 13 | # tgt_sr = cpt["config"][-1] 14 | cpt["config"][-3] = cpt["weight"]["emb_g.weight"].shape[0] 15 | if_f0 = cpt.get("f0", 1) 16 | version = cpt.get("version", "v1") 17 | if version == "v1": 18 | if if_f0 == 1: 19 | net_g = SynthesizerTrnMs256NSFsid(*cpt["config"], is_half=False) 20 | else: 21 | net_g = SynthesizerTrnMs256NSFsid_nono(*cpt["config"]) 22 | elif version == "v2": 23 | if if_f0 == 1: 24 | net_g = SynthesizerTrnMs768NSFsid(*cpt["config"], is_half=False) 25 | else: 26 | net_g = SynthesizerTrnMs768NSFsid_nono(*cpt["config"]) 27 | del net_g.enc_q 28 | # net_g.forward = net_g.infer 29 | # ckpt = {} 30 | # ckpt["config"] = cpt["config"] 31 | # ckpt["f0"] = if_f0 32 | # ckpt["version"] = version 33 | # ckpt["info"] = cpt.get("info", "0epoch") 34 | net_g.load_state_dict(cpt["weight"], strict=False) 35 | net_g = net_g.float() 36 | net_g.eval().to(device) 37 | net_g.remove_weight_norm() 38 | return net_g, cpt 39 | -------------------------------------------------------------------------------- /MIT协议暨相关引用库协议: -------------------------------------------------------------------------------- 1 | 本软件及其相关代码以MIT协议开源，作者不对软件具备任何控制力，使用软件者、传播软件导出的声音者自负全责。 2 | 如不认可该条款，则不能使用或引用软件包内任何代码和文件。 3 | 4 | 特此授予任何获得本软件和相关文档文件（以下简称“软件”）副本的人免费使用、复制、修改、合并、出版、分发、再授权和/或销售本软件的权利，以及授予本软件所提供的人使用本软件的权利，但须符合以下条件： 5 | 上述版权声明和本许可声明应包含在软件的所有副本或实质部分中。 6 | 软件是“按原样”提供的，没有任何明示或暗示的保证，包括但不限于适销性、适用于特定目的和不侵权的保证。在任何情况下，作者或版权持有人均不承担因软件或软件的使用或其他交易而产生、产生或与之相关的任何索赔、损害赔偿或其他责任，无论是在合同诉讼、侵权诉讼还是其他诉讼中。 7 | 8 | 9 | The LICENCEs for related libraries are as follows. 10 | 相关引用库协议如下： 11 | 12 | ContentVec 13 | https://github.com/auspicious3000/contentvec/blob/main/LICENSE 14 | MIT License 15 | 16 | VITS 17 | https://github.com/jaywalnut310/vits/blob/main/LICENSE 18 | MIT License 19 | 20 | HIFIGAN 21 | https://github.com/jik876/hifi-gan/blob/master/LICENSE 22 | MIT License 23 | 24 | gradio 25 | https://github.com/gradio-app/gradio/blob/main/LICENSE 26 | Apache License 2.0 27 | 28 | ffmpeg 29 | https://github.com/FFmpeg/FFmpeg/blob/master/COPYING.LGPLv3 30 | https://github.com/BtbN/FFmpeg-Builds/releases/download/autobuild-2021-02-28-12-32/ffmpeg-n4.3.2-160-gfbb9368226-win64-lgpl-4.3.zip 31 | LPGLv3 License 32 | MIT License 33 | 34 | ultimatevocalremovergui 35 | https://github.com/Anjok07/ultimatevocalremovergui/blob/master/LICENSE 36 | https://github.com/yang123qwe/vocal_separation_by_uvr5 37 | MIT License 38 | 39 | audio-slicer 40 | https://github.com/openvpi/audio-slicer/blob/main/LICENSE 41 | MIT License 42 | 43 | FreeSimpleGUI 44 | https://github.com/spyoungtech/FreeSimpleGUI/blob/master/license.txt 45 | LPGLv3 License 46 | -------------------------------------------------------------------------------- /infer/lib/audio.py: -------------------------------------------------------------------------------- 1 | import platform 2 | import ffmpeg 3 | import numpy as np 4 | import av 5 | 6 | 7 | def wav2(i, o, format): 8 | inp = av.open(i, "rb") 9 | if format == "m4a": 10 | format = "mp4" 11 | out = av.open(o, "wb", format=format) 12 | if format == "ogg": 13 | format = "libvorbis" 14 | if format == "mp4": 15 | format = "aac" 16 | 17 | ostream = out.add_stream(format) 18 | 19 | for frame in inp.decode(audio=0): 20 | for p in ostream.encode(frame): 21 | out.mux(p) 22 | 23 | for p in ostream.encode(None): 24 | out.mux(p) 25 | 26 | out.close() 27 | inp.close() 28 | 29 | 30 | def load_audio(file, sr): 31 | try: 32 | # https://github.com/openai/whisper/blob/main/whisper/audio.py#L26 33 | # This launches a subprocess to decode audio while down-mixing and resampling as necessary. 34 | # Requires the ffmpeg CLI and `ffmpeg-python` package to be installed. 35 | file = clean_path(file) # 防止小白拷路径头尾带了空格和"和回车 36 | out, _ = ( 37 | ffmpeg.input(file, threads=0) 38 | .output("-", format="f32le", acodec="pcm_f32le", ac=1, ar=sr) 39 | .run(cmd=["ffmpeg", "-nostdin"], capture_stdout=True, capture_stderr=True) 40 | ) 41 | except Exception as e: 42 | raise RuntimeError(f"Failed to load audio: {e}") 43 | 44 | return np.frombuffer(out, np.float32).flatten() 45 | 46 | 47 | def clean_path(path_str): 48 | if platform.system() == "Windows": 49 | path_str = path_str.replace("/", "\\") 50 | return path_str.strip(" ").strip('"').strip("\n").strip('"').strip(" ") 51 | -------------------------------------------------------------------------------- /infer/lib/train/losses.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | 4 | def feature_loss(fmap_r, fmap_g): 5 | loss = 0 6 | for dr, dg in zip(fmap_r, fmap_g): 7 | for rl, gl in zip(dr, dg): 8 | rl = rl.float().detach() 9 | gl = gl.float() 10 | loss += torch.mean(torch.abs(rl - gl)) 11 | 12 | return loss * 2 13 | 14 | 15 | def discriminator_loss(disc_real_outputs, disc_generated_outputs): 16 | loss = 0 17 | r_losses = [] 18 | g_losses = [] 19 | for dr, dg in zip(disc_real_outputs, disc_generated_outputs): 20 | dr = dr.float() 21 | dg = dg.float() 22 | r_loss = torch.mean((1 - dr) ** 2) 23 | g_loss = torch.mean(dg**2) 24 | loss += r_loss + g_loss 25 | r_losses.append(r_loss.item()) 26 | g_losses.append(g_loss.item()) 27 | 28 | return loss, r_losses, g_losses 29 | 30 | 31 | def generator_loss(disc_outputs): 32 | loss = 0 33 | gen_losses = [] 34 | for dg in disc_outputs: 35 | dg = dg.float() 36 | l = torch.mean((1 - dg) ** 2) 37 | gen_losses.append(l) 38 | loss += l 39 | 40 | return loss, gen_losses 41 | 42 | 43 | def kl_loss(z_p, logs_q, m_p, logs_p, z_mask): 44 | """ 45 | z_p, logs_q: [b, h, t_t] 46 | m_p, logs_p: [b, h, t_t] 47 | """ 48 | z_p = z_p.float() 49 | logs_q = logs_q.float() 50 | m_p = m_p.float() 51 | logs_p = logs_p.float() 52 | z_mask = z_mask.float() 53 | 54 | kl = logs_p - logs_q - 0.5 55 | kl += 0.5 * ((z_p - m_p) ** 2) * torch.exp(-2.0 * logs_p) 56 | kl = torch.sum(kl * z_mask) 57 | l = kl / torch.sum(z_mask) 58 | return l 59 | -------------------------------------------------------------------------------- /i18n/locale_diff.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | from collections import OrderedDict 4 | 5 | # Define the standard file name 6 | standard_file = "locale/zh_CN.json" 7 | 8 | # Find all JSON files in the directory 9 | dir_path = "locale/" 10 | languages = [ 11 | os.path.join(dir_path, f) 12 | for f in os.listdir(dir_path) 13 | if f.endswith(".json") and f != standard_file 14 | ] 15 | 16 | # Load the standard file 17 | with open(standard_file, "r", encoding="utf-8") as f: 18 | standard_data = json.load(f, object_pairs_hook=OrderedDict) 19 | 20 | # Loop through each language file 21 | for lang_file in languages: 22 | # Load the language file 23 | with open(lang_file, "r", encoding="utf-8") as f: 24 | lang_data = json.load(f, object_pairs_hook=OrderedDict) 25 | 26 | # Find the difference between the language file and the standard file 27 | diff = set(standard_data.keys()) - set(lang_data.keys()) 28 | 29 | miss = set(lang_data.keys()) - set(standard_data.keys()) 30 | 31 | # Add any missing keys to the language file 32 | for key in diff: 33 | lang_data[key] = key 34 | 35 | # Del any extra keys to the language file 36 | for key in miss: 37 | del lang_data[key] 38 | 39 | # Sort the keys of the language file to match the order of the standard file 40 | lang_data = OrderedDict( 41 | sorted(lang_data.items(), key=lambda x: list(standard_data.keys()).index(x[0])) 42 | ) 43 | 44 | # Save the updated language file 45 | with open(lang_file, "w", encoding="utf-8") as f: 46 | json.dump(lang_data, f, ensure_ascii=False, indent=4, sort_keys=True) 47 | f.write("\n") 48 | -------------------------------------------------------------------------------- /run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | if [ "$(uname)" = "Darwin" ]; then 4 | # macOS specific env: 5 | export PYTORCH_ENABLE_MPS_FALLBACK=1 6 | export PYTORCH_MPS_HIGH_WATERMARK_RATIO=0.0 7 | fi 8 | 9 | if [ -d ".venv" ]; then 10 | echo "Activate venv..." 11 | . .venv/bin/activate 12 | else 13 | echo "Create venv..." 14 | requirements_file="requirements.txt" 15 | 16 | # Check if Python 3.8 is installed 17 | if ! command -v python3.8 >/dev/null 2>&1 || pyenv versions --bare | grep -q "3.8"; then 18 | echo "Python 3 not found. Attempting to install 3.8..." 19 | if [ "$(uname)" = "Darwin" ] && command -v brew >/dev/null 2>&1; then 20 | brew install python@3.8 21 | elif [ "$(uname)" = "Linux" ] && command -v apt-get >/dev/null 2>&1; then 22 | sudo apt update 23 | sudo apt install -y python3.8 24 | else 25 | echo "Please install Python 3.8 manually." 26 | exit 1 27 | fi 28 | fi 29 | 30 | python3.8 -m venv .venv 31 | . .venv/bin/activate 32 | 33 | # Check if required packages are installed and install them if not 34 | if [ -f "${requirements_file}" ]; then 35 | installed_packages=$(python3.8 -m pip freeze) 36 | while IFS= read -r package; do 37 | expr "${package}" : "^#.*" > /dev/null && continue 38 | package_name=$(echo "${package}" | sed 's/[<>=!].*//') 39 | if ! echo "${installed_packages}" | grep -q "${package_name}"; then 40 | echo "${package_name} not found. Attempting to install..." 41 | python3.8 -m pip install --upgrade "${package}" 42 | fi 43 | done < "${requirements_file}" 44 | else 45 | echo "${requirements_file} not found. Please ensure the requirements file with required packages exists." 46 | exit 1 47 | fi 48 | fi 49 | 50 | # Run the main script 51 | python3 infer-web.py --pycmd python3 52 | -------------------------------------------------------------------------------- /infer/modules/onnx/export.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | from infer.lib.infer_pack.models_onnx import SynthesizerTrnMsNSFsidM 4 | 5 | 6 | def export_onnx(ModelPath, ExportedPath): 7 | cpt = torch.load(ModelPath, map_location="cpu") 8 | cpt["config"][-3] = cpt["weight"]["emb_g.weight"].shape[0] 9 | vec_channels = 256 if cpt.get("version", "v1") == "v1" else 768 10 | 11 | test_phone = torch.rand(1, 200, vec_channels) # hidden unit 12 | test_phone_lengths = torch.tensor([200]).long() # hidden unit 长度（貌似没啥用） 13 | test_pitch = torch.randint(size=(1, 200), low=5, high=255) # 基频（单位赫兹） 14 | test_pitchf = torch.rand(1, 200) # nsf基频 15 | test_ds = torch.LongTensor([0]) # 说话人ID 16 | test_rnd = torch.rand(1, 192, 200) # 噪声（加入随机因子） 17 | 18 | device = "cpu" # 导出时设备（不影响使用模型） 19 | 20 | net_g = SynthesizerTrnMsNSFsidM( 21 | *cpt["config"], is_half=False, encoder_dim=vec_channels 22 | ) # fp32导出（C++要支持fp16必须手动将内存重新排列所以暂时不用fp16） 23 | net_g.load_state_dict(cpt["weight"], strict=False) 24 | input_names = ["phone", "phone_lengths", "pitch", "pitchf", "ds", "rnd"] 25 | output_names = [ 26 | "audio", 27 | ] 28 | # net_g.construct_spkmixmap(n_speaker) 多角色混合轨道导出 29 | torch.onnx.export( 30 | net_g, 31 | ( 32 | test_phone.to(device), 33 | test_phone_lengths.to(device), 34 | test_pitch.to(device), 35 | test_pitchf.to(device), 36 | test_ds.to(device), 37 | test_rnd.to(device), 38 | ), 39 | ExportedPath, 40 | dynamic_axes={ 41 | "phone": [1], 42 | "pitch": [1], 43 | "pitchf": [1], 44 | "rnd": [2], 45 | }, 46 | do_constant_folding=False, 47 | opset_version=18, 48 | verbose=True, 49 | input_names=input_names, 50 | output_names=output_names, 51 | ) 52 | return "Finished" 53 | -------------------------------------------------------------------------------- /infer/lib/uvr5_pack/lib_v5/model_param_init.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import pathlib 4 | 5 | default_param = {} 6 | default_param["bins"] = 768 7 | default_param["unstable_bins"] = 9 # training only 8 | default_param["reduction_bins"] = 762 # training only 9 | default_param["sr"] = 44100 10 | default_param["pre_filter_start"] = 757 11 | default_param["pre_filter_stop"] = 768 12 | default_param["band"] = {} 13 | 14 | 15 | default_param["band"][1] = { 16 | "sr": 11025, 17 | "hl": 128, 18 | "n_fft": 960, 19 | "crop_start": 0, 20 | "crop_stop": 245, 21 | "lpf_start": 61, # inference only 22 | "res_type": "polyphase", 23 | } 24 | 25 | default_param["band"][2] = { 26 | "sr": 44100, 27 | "hl": 512, 28 | "n_fft": 1536, 29 | "crop_start": 24, 30 | "crop_stop": 547, 31 | "hpf_start": 81, # inference only 32 | "res_type": "sinc_best", 33 | } 34 | 35 | 36 | def int_keys(d): 37 | r = {} 38 | for k, v in d: 39 | if k.isdigit(): 40 | k = int(k) 41 | r[k] = v 42 | return r 43 | 44 | 45 | class ModelParameters(object): 46 | def __init__(self, config_path=""): 47 | if ".pth" == pathlib.Path(config_path).suffix: 48 | import zipfile 49 | 50 | with zipfile.ZipFile(config_path, "r") as zip: 51 | self.param = json.loads( 52 | zip.read("param.json"), object_pairs_hook=int_keys 53 | ) 54 | elif ".json" == pathlib.Path(config_path).suffix: 55 | with open(config_path, "r") as f: 56 | self.param = json.loads(f.read(), object_pairs_hook=int_keys) 57 | else: 58 | self.param = default_param 59 | 60 | for k in [ 61 | "mid_side", 62 | "mid_side_b", 63 | "mid_side_b2", 64 | "stereo_w", 65 | "stereo_n", 66 | "reverse", 67 | ]: 68 | if not k in self.param: 69 | self.param[k] = False 70 | -------------------------------------------------------------------------------- /tools/onnx/export_onnx.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from infer.lib.infer_pack.models_onnx import SynthesizerTrnMsNSFsidM 3 | 4 | if __name__ == "__main__": 5 | MoeVS = True # 模型是否为MoeVoiceStudio（原MoeSS）使用 6 | 7 | ModelPath = "Shiroha/shiroha.pth" # 模型路径 8 | ExportedPath = "model.onnx" # 输出路径 9 | encoder_dim = 256 # encoder_dim 10 | cpt = torch.load(ModelPath, map_location="cpu") 11 | cpt["config"][-3] = cpt["weight"]["emb_g.weight"].shape[0] # n_spk 12 | print(*cpt["config"]) 13 | 14 | test_phone = torch.rand(1, 200, encoder_dim) # hidden unit 15 | test_phone_lengths = torch.tensor([200]).long() # hidden unit 长度（貌似没啥用） 16 | test_pitch = torch.randint(size=(1, 200), low=5, high=255) # 基频（单位赫兹） 17 | test_pitchf = torch.rand(1, 200) # nsf基频 18 | test_ds = torch.LongTensor([0]) # 说话人ID 19 | test_rnd = torch.rand(1, 192, 200) # 噪声（加入随机因子） 20 | 21 | device = "cpu" # 导出时设备（不影响使用模型） 22 | 23 | net_g = SynthesizerTrnMsNSFsidM( 24 | *cpt["config"], is_half=False, encoder_dim=encoder_dim 25 | ) # fp32导出（C++要支持fp16必须手动将内存重新排列所以暂时不用fp16） 26 | net_g.load_state_dict(cpt["weight"], strict=False) 27 | input_names = ["phone", "phone_lengths", "pitch", "pitchf", "ds", "rnd"] 28 | output_names = [ 29 | "audio", 30 | ] 31 | # net_g.construct_spkmixmap(n_speaker) 多角色混合轨道导出 32 | torch.onnx.export( 33 | net_g, 34 | ( 35 | test_phone.to(device), 36 | test_phone_lengths.to(device), 37 | test_pitch.to(device), 38 | test_pitchf.to(device), 39 | test_ds.to(device), 40 | test_rnd.to(device), 41 | ), 42 | ExportedPath, 43 | dynamic_axes={ 44 | "phone": [1], 45 | "pitch": [1], 46 | "pitchf": [1], 47 | "rnd": [2], 48 | }, 49 | do_constant_folding=False, 50 | opset_version=18, 51 | verbose=False, 52 | input_names=input_names, 53 | output_names=output_names, 54 | ) 55 | -------------------------------------------------------------------------------- /i18n/scan_i18n.py: -------------------------------------------------------------------------------- 1 | import ast 2 | import glob 3 | import json 4 | from collections import OrderedDict 5 | 6 | 7 | def extract_i18n_strings(node): 8 | i18n_strings = [] 9 | 10 | if ( 11 | isinstance(node, ast.Call) 12 | and isinstance(node.func, ast.Name) 13 | and node.func.id == "i18n" 14 | ): 15 | for arg in node.args: 16 | if isinstance(arg, ast.Str): 17 | i18n_strings.append(arg.s) 18 | 19 | for child_node in ast.iter_child_nodes(node): 20 | i18n_strings.extend(extract_i18n_strings(child_node)) 21 | 22 | return i18n_strings 23 | 24 | 25 | # scan the directory for all .py files (recursively) 26 | # for each file, parse the code into an AST 27 | # for each AST, extract the i18n strings 28 | 29 | strings = [] 30 | for filename in glob.iglob("**/*.py", recursive=True): 31 | with open(filename, "r") as f: 32 | code = f.read() 33 | if "I18nAuto" in code: 34 | tree = ast.parse(code) 35 | i18n_strings = extract_i18n_strings(tree) 36 | print(filename, len(i18n_strings)) 37 | strings.extend(i18n_strings) 38 | code_keys = set(strings) 39 | """ 40 | n_i18n.py 41 | gui_v1.py 26 42 | app.py 16 43 | infer-web.py 147 44 | scan_i18n.py 0 45 | i18n.py 0 46 | lib/train/process_ckpt.py 1 47 | """ 48 | print() 49 | print("Total unique:", len(code_keys)) 50 | 51 | 52 | standard_file = "i18n/locale/zh_CN.json" 53 | with open(standard_file, "r", encoding="utf-8") as f: 54 | standard_data = json.load(f, object_pairs_hook=OrderedDict) 55 | standard_keys = set(standard_data.keys()) 56 | 57 | # Define the standard file name 58 | unused_keys = standard_keys - code_keys 59 | print("Unused keys:", len(unused_keys)) 60 | for unused_key in unused_keys: 61 | print("\t", unused_key) 62 | 63 | missing_keys = code_keys - standard_keys 64 | print("Missing keys:", len(missing_keys)) 65 | for missing_key in missing_keys: 66 | print("\t", missing_key) 67 | 68 | code_keys_dict = OrderedDict() 69 | for s in strings: 70 | code_keys_dict[s] = s 71 | 72 | # write back 73 | with open(standard_file, "w", encoding="utf-8") as f: 74 | json.dump(code_keys_dict, f, ensure_ascii=False, indent=4, sort_keys=True) 75 | f.write("\n") 76 | -------------------------------------------------------------------------------- /tools/cmd/infer_cli.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | import sys 4 | 5 | now_dir = os.getcwd() 6 | sys.path.append(now_dir) 7 | from dotenv import load_dotenv 8 | from scipy.io import wavfile 9 | 10 | from configs.config import Config 11 | from infer.modules.vc.modules import VC 12 | 13 | #### 14 | # USAGE 15 | # 16 | # In your Terminal or CMD or whatever 17 | 18 | 19 | def arg_parse() -> tuple: 20 | parser = argparse.ArgumentParser() 21 | parser.add_argument("--f0up_key", type=int, default=0) 22 | parser.add_argument("--input_path", type=str, help="input path") 23 | parser.add_argument("--index_path", type=str, help="index path") 24 | parser.add_argument("--f0method", type=str, default="harvest", help="harvest or pm") 25 | parser.add_argument("--opt_path", type=str, help="opt path") 26 | parser.add_argument("--model_name", type=str, help="store in assets/weight_root") 27 | parser.add_argument("--index_rate", type=float, default=0.66, help="index rate") 28 | parser.add_argument("--device", type=str, help="device") 29 | parser.add_argument("--is_half", type=bool, help="use half -> True") 30 | parser.add_argument("--filter_radius", type=int, default=3, help="filter radius") 31 | parser.add_argument("--resample_sr", type=int, default=0, help="resample sr") 32 | parser.add_argument("--rms_mix_rate", type=float, default=1, help="rms mix rate") 33 | parser.add_argument("--protect", type=float, default=0.33, help="protect") 34 | 35 | args = parser.parse_args() 36 | sys.argv = sys.argv[:1] 37 | 38 | return args 39 | 40 | 41 | def main(): 42 | load_dotenv() 43 | args = arg_parse() 44 | config = Config() 45 | config.device = args.device if args.device else config.device 46 | config.is_half = args.is_half if args.is_half else config.is_half 47 | vc = VC(config) 48 | vc.get_vc(args.model_name) 49 | _, wav_opt = vc.vc_single( 50 | 0, 51 | args.input_path, 52 | args.f0up_key, 53 | None, 54 | args.f0method, 55 | args.index_path, 56 | None, 57 | args.index_rate, 58 | args.filter_radius, 59 | args.resample_sr, 60 | args.rms_mix_rate, 61 | args.protect, 62 | ) 63 | wavfile.write(args.opt_path, wav_opt[0], wav_opt[1]) 64 | 65 | 66 | if __name__ == "__main__": 67 | main() 68 | -------------------------------------------------------------------------------- /tools/cmd/train-index-v2.py: -------------------------------------------------------------------------------- 1 | """ 2 | 格式：直接cid为自带的index位；aid放不下了，通过字典来查，反正就5w个 3 | """ 4 | 5 | import os 6 | import traceback 7 | import logging 8 | 9 | logger = logging.getLogger(__name__) 10 | 11 | from multiprocessing import cpu_count 12 | 13 | import faiss 14 | import numpy as np 15 | from sklearn.cluster import MiniBatchKMeans 16 | 17 | # ###########如果是原始特征要先写save 18 | n_cpu = 0 19 | if n_cpu == 0: 20 | n_cpu = cpu_count() 21 | inp_root = r"./logs/anz/3_feature768" 22 | npys = [] 23 | listdir_res = list(os.listdir(inp_root)) 24 | for name in sorted(listdir_res): 25 | phone = np.load("%s/%s" % (inp_root, name)) 26 | npys.append(phone) 27 | big_npy = np.concatenate(npys, 0) 28 | big_npy_idx = np.arange(big_npy.shape[0]) 29 | np.random.shuffle(big_npy_idx) 30 | big_npy = big_npy[big_npy_idx] 31 | logger.debug(big_npy.shape) # (6196072, 192)#fp32#4.43G 32 | if big_npy.shape[0] > 2e5: 33 | # if(1): 34 | info = "Trying doing kmeans %s shape to 10k centers." % big_npy.shape[0] 35 | logger.info(info) 36 | try: 37 | big_npy = ( 38 | MiniBatchKMeans( 39 | n_clusters=10000, 40 | verbose=True, 41 | batch_size=256 * n_cpu, 42 | compute_labels=False, 43 | init="random", 44 | ) 45 | .fit(big_npy) 46 | .cluster_centers_ 47 | ) 48 | except: 49 | info = traceback.format_exc() 50 | logger.warning(info) 51 | 52 | np.save("tools/infer/big_src_feature_mi.npy", big_npy) 53 | 54 | ##################train+add 55 | # big_npy=np.load("/bili-coeus/jupyter/jupyterhub-liujing04/vits_ch/inference_f0/big_src_feature_mi.npy") 56 | n_ivf = min(int(16 * np.sqrt(big_npy.shape[0])), big_npy.shape[0] // 39) 57 | index = faiss.index_factory(768, "IVF%s,Flat" % n_ivf) # mi 58 | logger.info("Training...") 59 | index_ivf = faiss.extract_index_ivf(index) # 60 | index_ivf.nprobe = 1 61 | index.train(big_npy) 62 | faiss.write_index( 63 | index, "tools/infer/trained_IVF%s_Flat_baseline_src_feat_v2.index" % (n_ivf) 64 | ) 65 | logger.info("Adding...") 66 | batch_size_add = 8192 67 | for i in range(0, big_npy.shape[0], batch_size_add): 68 | index.add(big_npy[i : i + batch_size_add]) 69 | faiss.write_index( 70 | index, "tools/infer/added_IVF%s_Flat_mi_baseline_src_feat.index" % (n_ivf) 71 | ) 72 | """ 73 | 大小（都是FP32） 74 | big_src_feature 2.95G 75 | (3098036, 256) 76 | big_emb 4.43G 77 | (6196072, 192) 78 | big_emb双倍是因为求特征要repeat后再加pitch 79 | 80 | """ 81 | -------------------------------------------------------------------------------- /docs/jp/training_tips_ja.md: -------------------------------------------------------------------------------- 1 | RVCの訓練における説明、およびTIPS 2 | =============================== 3 | 本TIPSではどのようにデータの訓練が行われているかを説明します。 4 | 5 | # 訓練の流れ 6 | GUIの訓練タブのstepに沿って説明します。 7 | 8 | ## step1 9 | 実験名の設定を行います。 10 | 11 | また、モデルに音高ガイド(ピッチ)を考慮させるかもここで設定できます。考慮させない場合はモデルは軽量になりますが、歌唱には向かなくなります。 12 | 13 | 各実験のデータは`/logs/実験名/`に配置されます。 14 | 15 | ## step2a 16 | 音声の読み込みと前処理を行います。 17 | 18 | ### load audio 19 | 音声のあるフォルダを指定すると、そのフォルダ内にある音声ファイルを自動で読み込みます。 20 | 例えば`C:Users\hoge\voices`を指定した場合、`C:Users\hoge\voices\voice.mp3`は読み込まれますが、`C:Users\hoge\voices\dir\voice.mp3`は読み込まれません。 21 | 22 | 音声の読み込みには内部でffmpegを利用しているので、ffmpegで対応している拡張子であれば自動的に読み込まれます。 23 | ffmpegでint16に変換した後、float32に変換し、-1 ~ 1の間に正規化されます。 24 | 25 | ### denoising 26 | 音声についてscipyのfiltfiltによる平滑化を行います。 27 | 28 | ### 音声の分割 29 | 入力した音声はまず、一定期間(max_sil_kept=5秒?)より長く無音が続く部分を検知して音声を分割します。無音で音声を分割した後は、0.3秒のoverlapを含む4秒ごとに音声を分割します。4秒以内に区切られた音声は、音量の正規化を行った後wavファイルを`/logs/実験名/0_gt_wavs`に、そこから16kのサンプリングレートに変換して`/logs/実験名/1_16k_wavs`にwavファイルで保存します。 30 | 31 | ## step2b 32 | ### ピッチの抽出 33 | wavファイルからピッチ(音の高低)の情報を抽出します。parselmouthやpyworldに内蔵されている手法でピッチ情報(=f0)を抽出し、`/logs/実験名/2a_f0`に保存します。その後、ピッチ情報を対数で変換して1~255の整数に変換し、`/logs/実験名/2b-f0nsf`に保存します。 34 | 35 | ### feature_printの抽出 36 | HuBERTを用いてwavファイルを事前にembeddingに変換します。`/logs/実験名/1_16k_wavs`に保存したwavファイルを読み込み、HuBERTでwavファイルを256次元の特徴量に変換し、npy形式で`/logs/実験名/3_feature256`に保存します。 37 | 38 | ## step3 39 | モデルのトレーニングを行います。 40 | ### 初心者向け用語解説 41 | 深層学習ではデータセットを分割し、少しずつ学習を進めていきます。一回のモデルの更新(step)では、batch_size個のデータを取り出し予測と誤差の修正を行います。これをデータセットに対して一通り行うと一epochと数えます。 42 | 43 | そのため、学習時間は 1step当たりの学習時間 x (データセット内のデータ数 ÷ バッチサイズ) x epoch数 かかります。一般にバッチサイズを大きくするほど学習は安定し、(1step当たりの学習時間÷バッチサイズ)は小さくなりますが、その分GPUのメモリを多く使用します。GPUのRAMはnvidia-smiコマンド等で確認できます。実行環境のマシンに合わせてバッチサイズをできるだけ大きくするとより短時間で学習が可能です。 44 | 45 | ### pretrained modelの指定 46 | RVCではモデルの訓練を0からではなく、事前学習済みの重みから開始するため、少ないデータセットで学習を行えます。 47 | 48 | デフォルトでは 49 | 50 | - 音高ガイドを考慮する場合、`RVCのある場所/pretrained/f0G40k.pth`と`RVCのある場所/pretrained/f0D40k.pth`を読み込みます。 51 | - 音高ガイドを考慮しない場合、`RVCのある場所/pretrained/G40k.pth`と`RVCのある場所/pretrained/D40k.pth`を読み込みます。 52 | 53 | 学習時はsave_every_epochごとにモデルのパラメータが`logs/実験名/G_{}.pth`と`logs/実験名/D_{}.pth`に保存されますが、このパスを指定することで学習を再開したり、もしくは違う実験で学習したモデルの重みから学習を開始できます。 54 | 55 | ### indexの学習 56 | RVCでは学習時に使われたHuBERTの特徴量を保存し、推論時は学習時の特徴量から近い特徴量を探してきて推論を行います。この検索を高速に行うために事前にindexの学習を行います。 57 | indexの学習には近似近傍探索ライブラリのfaissを用います。`/logs/実験名/3_feature256`の特徴量を読み込み、それを用いて学習したindexを`/logs/実験名/add_XXX.index`として保存します。 58 | (20230428updateよりtotal_fea.npyはindexから読み込むので不要になりました。) 59 | 60 | ### ボタンの説明 61 | - モデルのトレーニング: step2bまでを実行した後、このボタンを押すとモデルの学習を行います。 62 | - 特徴インデックスのトレーニング: モデルのトレーニング後、indexの学習を行います。 63 | - ワンクリックトレーニング: step2bまでとモデルのトレーニング、特徴インデックスのトレーニングを一括で行います。 64 | 65 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # syntax=docker/dockerfile:1 2 | 3 | FROM nvidia/cuda:11.6.2-cudnn8-runtime-ubuntu20.04 4 | 5 | EXPOSE 7865 6 | 7 | WORKDIR /app 8 | 9 | # Install dependenceis to add PPAs 10 | RUN apt-get update && \ 11 | apt-get install -y -qq ffmpeg aria2 && apt clean && \ 12 | apt-get install -y software-properties-common && \ 13 | apt-get clean && \ 14 | rm -rf /var/lib/apt/lists/* 15 | # Add the deadsnakes PPA to get Python 3.9 16 | RUN add-apt-repository ppa:deadsnakes/ppa 17 | 18 | # Install Python 3.9 and pip 19 | RUN apt-get update && \ 20 | apt-get install -y build-essential python-dev python3-dev python3.9-distutils python3.9-dev python3.9 curl && \ 21 | apt-get clean && \ 22 | update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.9 1 && \ 23 | curl https://bootstrap.pypa.io/get-pip.py | python3.9 24 | 25 | # Set Python 3.9 as the default 26 | RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.9 1 27 | 28 | COPY . . 29 | 30 | RUN python3 -m pip install --no-cache-dir -r requirements.txt 31 | 32 | RUN aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/lj1995/VoiceConversionWebUI/resolve/main/pretrained_v2/D40k.pth -d assets/pretrained_v2/ -o D40k.pth 33 | RUN aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/lj1995/VoiceConversionWebUI/resolve/main/pretrained_v2/G40k.pth -d assets/pretrained_v2/ -o G40k.pth 34 | RUN aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/lj1995/VoiceConversionWebUI/resolve/main/pretrained_v2/f0D40k.pth -d assets/pretrained_v2/ -o f0D40k.pth 35 | RUN aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/lj1995/VoiceConversionWebUI/resolve/main/pretrained_v2/f0G40k.pth -d assets/pretrained_v2/ -o f0G40k.pth 36 | 37 | RUN aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/lj1995/VoiceConversionWebUI/resolve/main/uvr5_weights/HP2-人声vocals+非人声instrumentals.pth -d assets/uvr5_weights/ -o HP2-人声vocals+非人声instrumentals.pth 38 | RUN aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/lj1995/VoiceConversionWebUI/resolve/main/uvr5_weights/HP5-主旋律人声vocals+其他instrumentals.pth -d assets/uvr5_weights/ -o HP5-主旋律人声vocals+其他instrumentals.pth 39 | 40 | RUN aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/lj1995/VoiceConversionWebUI/resolve/main/hubert_base.pt -d assets/hubert -o hubert_base.pt 41 | 42 | RUN aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/lj1995/VoiceConversionWebUI/resolve/main/rmvpe.pt -d assets/rmvpe -o rmvpe.pt 43 | 44 | VOLUME [ "/app/weights", "/app/opt" ] 45 | 46 | CMD ["python3", "infer-web.py"] 47 | -------------------------------------------------------------------------------- /tools/cmd/infer_batch_rvc.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | import sys 4 | 5 | print("Command-line arguments:", sys.argv) 6 | 7 | now_dir = os.getcwd() 8 | sys.path.append(now_dir) 9 | import sys 10 | 11 | import tqdm as tq 12 | from dotenv import load_dotenv 13 | from scipy.io import wavfile 14 | 15 | from configs.config import Config 16 | from infer.modules.vc.modules import VC 17 | 18 | 19 | def arg_parse() -> tuple: 20 | parser = argparse.ArgumentParser() 21 | parser.add_argument("--f0up_key", type=int, default=0) 22 | parser.add_argument("--input_path", type=str, help="input path") 23 | parser.add_argument("--index_path", type=str, help="index path") 24 | parser.add_argument("--f0method", type=str, default="harvest", help="harvest or pm") 25 | parser.add_argument("--opt_path", type=str, help="opt path") 26 | parser.add_argument("--model_name", type=str, help="store in assets/weight_root") 27 | parser.add_argument("--index_rate", type=float, default=0.66, help="index rate") 28 | parser.add_argument("--device", type=str, help="device") 29 | parser.add_argument("--is_half", type=bool, help="use half -> True") 30 | parser.add_argument("--filter_radius", type=int, default=3, help="filter radius") 31 | parser.add_argument("--resample_sr", type=int, default=0, help="resample sr") 32 | parser.add_argument("--rms_mix_rate", type=float, default=1, help="rms mix rate") 33 | parser.add_argument("--protect", type=float, default=0.33, help="protect") 34 | 35 | args = parser.parse_args() 36 | sys.argv = sys.argv[:1] 37 | 38 | return args 39 | 40 | 41 | def main(): 42 | load_dotenv() 43 | args = arg_parse() 44 | config = Config() 45 | config.device = args.device if args.device else config.device 46 | config.is_half = args.is_half if args.is_half else config.is_half 47 | vc = VC(config) 48 | vc.get_vc(args.model_name) 49 | audios = os.listdir(args.input_path) 50 | for file in tq.tqdm(audios): 51 | if file.endswith(".wav"): 52 | file_path = os.path.join(args.input_path, file) 53 | _, wav_opt = vc.vc_single( 54 | 0, 55 | file_path, 56 | args.f0up_key, 57 | None, 58 | args.f0method, 59 | args.index_path, 60 | None, 61 | args.index_rate, 62 | args.filter_radius, 63 | args.resample_sr, 64 | args.rms_mix_rate, 65 | args.protect, 66 | ) 67 | out_path = os.path.join(args.opt_path, file) 68 | wavfile.write(out_path, wav_opt[0], wav_opt[1]) 69 | 70 | 71 | if __name__ == "__main__": 72 | main() 73 | -------------------------------------------------------------------------------- /docs/kr/training_tips_ko.md: -------------------------------------------------------------------------------- 1 | RVC 훈련에 대한 설명과 팁들 2 | ====================================== 3 | 본 팁에서는 어떻게 데이터 훈련이 이루어지고 있는지 설명합니다. 4 | 5 | # 훈련의 흐름 6 | GUI의 훈련 탭의 단계를 따라 설명합니다. 7 | 8 | ## step1 9 | 실험 이름을 지정합니다. 또한, 모델이 피치(소리의 높낮이)를 고려해야 하는지 여부를 여기에서 설정할 수도 있습니다.. 10 | 각 실험을 위한 데이터는 `/logs/experiment name/`에 배치됩니다.. 11 | 12 | ## step2a 13 | 음성 파일을 불러오고 전처리합니다. 14 | 15 | ### 음성 파일 불러오기 16 | 음성 파일이 있는 폴더를 지정하면 해당 폴더에 있는 음성 파일이 자동으로 가져와집니다. 17 | 예를 들어 `C:Users\hoge\voices`를 지정하면 `C:Users\hoge\voices\voice.mp3`가 읽히지만 `C:Users\hoge\voices\dir\voice.mp3`는 읽히지 않습니다. 18 | 19 | 음성 로드에는 내부적으로 ffmpeg를 이용하고 있으므로, ffmpeg로 대응하고 있는 확장자라면 자동적으로 읽힙니다. 20 | ffmpeg에서 int16으로 변환한 후 float32로 변환하고 -1과 1 사이에 정규화됩니다. 21 | 22 | ### 잡음 제거 23 | 음성 파일에 대해 scipy의 filtfilt를 이용하여 잡음을 처리합니다. 24 | 25 | ### 음성 분할 26 | 입력한 음성 파일은 먼저 일정 기간(max_sil_kept=5초?)보다 길게 무음이 지속되는 부분을 감지하여 음성을 분할합니다.무음으로 음성을 분할한 후에는 0.3초의 overlap을 포함하여 4초마다 음성을 분할합니다.4초 이내에 구분된 음성은 음량의 정규화를 실시한 후 wav 파일을 `/logs/실험명/0_gt_wavs`로, 거기에서 16k의 샘플링 레이트로 변환해 `/logs/실험명/1_16k_wavs`에 wav 파일로 저장합니다. 27 | 28 | ## step2b 29 | ### 피치 추출 30 | wav 파일에서 피치(소리의 높낮이) 정보를 추출합니다. parselmouth나 pyworld에 내장되어 있는 메서드으로 피치 정보(=f0)를 추출해, `/logs/실험명/2a_f0`에 저장합니다. 그 후 피치 정보를 로그로 변환하여 1~255 정수로 변환하고 `/logs/실험명/2b-f0nsf`에 저장합니다. 31 | 32 | ### feature_print 추출 33 | HuBERT를 이용하여 wav 파일을 미리 embedding으로 변환합니다. `/logs/실험명/1_16k_wavs`에 저장한 wav 파일을 읽고 HuBERT에서 wav 파일을 256차원 feature들로 변환한 후 npy 형식으로 `/logs/실험명/3_feature256`에 저장합니다. 34 | 35 | ## step3 36 | 모델의 훈련을 진행합니다. 37 | 38 | ### 초보자용 용어 해설 39 | 심층학습(딥러닝)에서는 데이터셋을 분할하여 조금씩 학습을 진행합니다.한 번의 모델 업데이트(step) 단계 당 batch_size개의 데이터를 탐색하여 예측과 오차를 수정합니다. 데이터셋 전부에 대해 이 작업을 한 번 수행하는 이를 하나의 epoch라고 계산합니다. 40 | 41 | 따라서 학습 시간은 단계당 학습 시간 x (데이터셋 내 데이터의 수 / batch size) x epoch 수가 소요됩니다. 일반적으로 batch size가 클수록 학습이 안정적이게 됩니다. (step당 학습 시간 ÷ batch size)는 작아지지만 GPU 메모리를 더 많이 사용합니다. GPU RAM은 nvidia-smi 명령어를 통해 확인할 수 있습니다. 실행 환경에 따라 배치 크기를 최대한 늘리면 짧은 시간 내에 학습이 가능합니다. 42 | 43 | ### 사전 학습된 모델 지정 44 | RVC는 적은 데이터셋으로도 훈련이 가능하도록 사전 훈련된 가중치에서 모델 훈련을 시작합니다. 기본적으로 `rvc-location/pretrained/f0G40k.pth` 및 `rvc-location/pretrained/f0D40k.pth`를 불러옵니다. 학습을 할 시에, 모델 파라미터는 각 save_every_epoch별로 `logs/experiment name/G_{}.pth` 와 `logs/experiment name/D_{}.pth`로 저장이 되는데, 이 경로를 지정함으로써 학습을 재개하거나, 다른 실험에서 학습한 모델의 가중치에서 학습을 시작할 수 있습니다. 45 | 46 | ### index의 학습 47 | RVC에서는 학습시에 사용된 HuBERT의 feature값을 저장하고, 추론 시에는 학습 시 사용한 feature값과 유사한 feature 값을 탐색해 추론을 진행합니다. 이 탐색을 고속으로 수행하기 위해 사전에 index을 학습하게 됩니다. 48 | Index 학습에는 근사 근접 탐색법 라이브러리인 Faiss를 사용하게 됩니다. `/logs/실험명/3_feature256`의 feature값을 불러와, 이를 모두 결합시킨 feature값을 `/logs/실험명/total_fea.npy`로서 저장, 그것을 사용해 학습한 index를`/logs/실험명/add_XXX.index`로 저장합니다. 49 | 50 | ### 버튼 설명 51 | - モデルのトレーニング (모델 학습): step2b까지 실행한 후, 이 버튼을 눌러 모델을 학습합니다. 52 | - 特徴インデックスのトレーニング (특징 지수 훈련): 모델의 훈련 후, index를 학습합니다. 53 | - ワンクリックトレーニング (원클릭 트레이닝): step2b까지의 모델 훈련, feature index 훈련을 일괄로 실시합니다. -------------------------------------------------------------------------------- /tools/download_models.py: -------------------------------------------------------------------------------- 1 | import os 2 | from pathlib import Path 3 | import requests 4 | 5 | RVC_DOWNLOAD_LINK = "https://huggingface.co/lj1995/VoiceConversionWebUI/resolve/main/" 6 | 7 | BASE_DIR = Path(__file__).resolve().parent.parent 8 | 9 | 10 | def dl_model(link, model_name, dir_name): 11 | with requests.get(f"{link}{model_name}") as r: 12 | r.raise_for_status() 13 | os.makedirs(os.path.dirname(dir_name / model_name), exist_ok=True) 14 | with open(dir_name / model_name, "wb") as f: 15 | for chunk in r.iter_content(chunk_size=8192): 16 | f.write(chunk) 17 | 18 | 19 | if __name__ == "__main__": 20 | print("Downloading hubert_base.pt...") 21 | dl_model(RVC_DOWNLOAD_LINK, "hubert_base.pt", BASE_DIR / "assets/hubert") 22 | print("Downloading rmvpe.pt...") 23 | dl_model(RVC_DOWNLOAD_LINK, "rmvpe.pt", BASE_DIR / "assets/rmvpe") 24 | print("Downloading vocals.onnx...") 25 | dl_model( 26 | RVC_DOWNLOAD_LINK + "uvr5_weights/onnx_dereverb_By_FoxJoy/", 27 | "vocals.onnx", 28 | BASE_DIR / "assets/uvr5_weights/onnx_dereverb_By_FoxJoy", 29 | ) 30 | 31 | rvc_models_dir = BASE_DIR / "assets/pretrained" 32 | 33 | print("Downloading pretrained models:") 34 | 35 | model_names = [ 36 | "D32k.pth", 37 | "D40k.pth", 38 | "D48k.pth", 39 | "G32k.pth", 40 | "G40k.pth", 41 | "G48k.pth", 42 | "f0D32k.pth", 43 | "f0D40k.pth", 44 | "f0D48k.pth", 45 | "f0G32k.pth", 46 | "f0G40k.pth", 47 | "f0G48k.pth", 48 | ] 49 | for model in model_names: 50 | print(f"Downloading {model}...") 51 | dl_model(RVC_DOWNLOAD_LINK + "pretrained/", model, rvc_models_dir) 52 | 53 | rvc_models_dir = BASE_DIR / "assets/pretrained_v2" 54 | 55 | print("Downloading pretrained models v2:") 56 | 57 | for model in model_names: 58 | print(f"Downloading {model}...") 59 | dl_model(RVC_DOWNLOAD_LINK + "pretrained_v2/", model, rvc_models_dir) 60 | 61 | print("Downloading uvr5_weights:") 62 | 63 | rvc_models_dir = BASE_DIR / "assets/uvr5_weights" 64 | 65 | model_names = [ 66 | "HP2-%E4%BA%BA%E5%A3%B0vocals%2B%E9%9D%9E%E4%BA%BA%E5%A3%B0instrumentals.pth", 67 | "HP2_all_vocals.pth", 68 | "HP3_all_vocals.pth", 69 | "HP5-%E4%B8%BB%E6%97%8B%E5%BE%8B%E4%BA%BA%E5%A3%B0vocals%2B%E5%85%B6%E4%BB%96instrumentals.pth", 70 | "HP5_only_main_vocal.pth", 71 | "VR-DeEchoAggressive.pth", 72 | "VR-DeEchoDeReverb.pth", 73 | "VR-DeEchoNormal.pth", 74 | ] 75 | for model in model_names: 76 | print(f"Downloading {model}...") 77 | dl_model(RVC_DOWNLOAD_LINK + "uvr5_weights/", model, rvc_models_dir) 78 | 79 | print("All models downloaded!") 80 | -------------------------------------------------------------------------------- /infer/modules/gui/utils.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch.types import Number 3 | 4 | 5 | @torch.no_grad() 6 | def amp_to_db( 7 | x: torch.Tensor, eps=torch.finfo(torch.float64).eps, top_db=40 8 | ) -> torch.Tensor: 9 | """ 10 | Convert the input tensor from amplitude to decibel scale. 11 | 12 | Arguments: 13 | x {[torch.Tensor]} -- [Input tensor.] 14 | 15 | Keyword Arguments: 16 | eps {[float]} -- [Small value to avoid numerical instability.] 17 | (default: {torch.finfo(torch.float64).eps}) 18 | top_db {[float]} -- [threshold the output at ``top_db`` below the peak] 19 | ` (default: {40}) 20 | 21 | Returns: 22 | [torch.Tensor] -- [Output tensor in decibel scale.] 23 | """ 24 | x_db = 20 * torch.log10(x.abs() + eps) 25 | return torch.max(x_db, (x_db.max(-1).values - top_db).unsqueeze(-1)) 26 | 27 | 28 | @torch.no_grad() 29 | def temperature_sigmoid(x: torch.Tensor, x0: float, temp_coeff: float) -> torch.Tensor: 30 | """ 31 | Apply a sigmoid function with temperature scaling. 32 | 33 | Arguments: 34 | x {[torch.Tensor]} -- [Input tensor.] 35 | x0 {[float]} -- [Parameter that controls the threshold of the sigmoid.] 36 | temp_coeff {[float]} -- [Parameter that controls the slope of the sigmoid.] 37 | 38 | Returns: 39 | [torch.Tensor] -- [Output tensor after applying the sigmoid with temperature scaling.] 40 | """ 41 | return torch.sigmoid((x - x0) / temp_coeff) 42 | 43 | 44 | @torch.no_grad() 45 | def linspace( 46 | start: Number, stop: Number, num: int = 50, endpoint: bool = True, **kwargs 47 | ) -> torch.Tensor: 48 | """ 49 | Generate a linearly spaced 1-D tensor. 50 | 51 | Arguments: 52 | start {[Number]} -- [The starting value of the sequence.] 53 | stop {[Number]} -- [The end value of the sequence, unless `endpoint` is set to False. 54 | In that case, the sequence consists of all but the last of ``num + 1`` 55 | evenly spaced samples, so that `stop` is excluded. Note that the step 56 | size changes when `endpoint` is False.] 57 | 58 | Keyword Arguments: 59 | num {[int]} -- [Number of samples to generate. Default is 50. Must be non-negative.] 60 | endpoint {[bool]} -- [If True, `stop` is the last sample. Otherwise, it is not included. 61 | Default is True.] 62 | **kwargs -- [Additional arguments to be passed to the underlying PyTorch `linspace` function.] 63 | 64 | Returns: 65 | [torch.Tensor] -- [1-D tensor of `num` equally spaced samples from `start` to `stop`.] 66 | """ 67 | if endpoint: 68 | return torch.linspace(start, stop, num, **kwargs) 69 | else: 70 | return torch.linspace(start, stop, num + 1, **kwargs)[:-1] 71 | -------------------------------------------------------------------------------- /tools/dlmodels.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | printf "working dir is %s\n" "$PWD" 4 | echo "downloading requirement aria2 check." 5 | 6 | if command -v aria2c > /dev/null 2>&1 7 | then 8 | echo "aria2 command found" 9 | else 10 | echo "failed. please install aria2" 11 | exit 1 12 | fi 13 | 14 | echo "dir check start." 15 | 16 | check_dir() { 17 | [ -d "$1" ] && printf "dir %s checked\n" "$1" || \ 18 | printf "failed. generating dir %s\n" "$1" && mkdir -p "$1" 19 | } 20 | 21 | check_dir "./assets/pretrained" 22 | check_dir "./assets/pretrained_v2" 23 | check_dir "./assets/uvr5_weights" 24 | check_dir "./assets/uvr5_weights/onnx_dereverb_By_FoxJoy" 25 | 26 | echo "dir check finished." 27 | 28 | echo "required files check start." 29 | check_file_pretrained() { 30 | printf "checking %s\n" "$2" 31 | if [ -f "./assets/""$1""/""$2""" ]; then 32 | printf "%s in ./assets/%s checked.\n" "$2" "$1" 33 | else 34 | echo failed. starting download from huggingface. 35 | if command -v aria2c > /dev/null 2>&1; then 36 | aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/lj1995/VoiceConversionWebUI/resolve/main/"$1"/"$2" -d ./assets/"$1" -o "$2" 37 | [ -f "./assets/""$1""/""$2""" ] && echo "download successful." || echo "please try again!" && exit 1 38 | else 39 | echo "aria2c command not found. Please install aria2c and try again." 40 | exit 1 41 | fi 42 | fi 43 | } 44 | 45 | check_file_special() { 46 | printf "checking %s\n" "$2" 47 | if [ -f "./assets/""$1""/""$2""" ]; then 48 | printf "%s in ./assets/%s checked.\n" "$2" "$1" 49 | else 50 | echo failed. starting download from huggingface. 51 | if command -v aria2c > /dev/null 2>&1; then 52 | aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/lj1995/VoiceConversionWebUI/resolve/main/"$2" -d ./assets/"$1" -o "$2" 53 | [ -f "./assets/""$1""/""$2""" ] && echo "download successful." || echo "please try again!" && exit 1 54 | else 55 | echo "aria2c command not found. Please install aria2c and try again." 56 | exit 1 57 | fi 58 | fi 59 | } 60 | 61 | check_file_pretrained pretrained D32k.pth 62 | check_file_pretrained pretrained D40k.pth 63 | check_file_pretrained pretrained D48k.pth 64 | check_file_pretrained pretrained G32k.pth 65 | check_file_pretrained pretrained G40k.pth 66 | check_file_pretrained pretrained G48k.pth 67 | check_file_pretrained pretrained_v2 f0D40k.pth 68 | check_file_pretrained pretrained_v2 f0G40k.pth 69 | check_file_pretrained pretrained_v2 D40k.pth 70 | check_file_pretrained pretrained_v2 G40k.pth 71 | check_file_pretrained uvr5_weights HP2_all_vocals.pth 72 | check_file_pretrained uvr5_weights HP3_all_vocals.pth 73 | check_file_pretrained uvr5_weights HP5_only_main_vocal.pth 74 | check_file_pretrained uvr5_weights VR-DeEchoAggressive.pth 75 | check_file_pretrained uvr5_weights VR-DeEchoDeReverb.pth 76 | check_file_pretrained uvr5_weights VR-DeEchoNormal.pth 77 | check_file_pretrained uvr5_weights "onnx_dereverb_By_FoxJoy/vocals.onnx" 78 | check_file_special rmvpe rmvpe.pt 79 | check_file_special hubert hubert_base.pt 80 | 81 | echo "required files check finished." 82 | -------------------------------------------------------------------------------- /infer/lib/infer_pack/modules/F0Predictor/HarvestF0Predictor.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pyworld 3 | 4 | from infer.lib.infer_pack.modules.F0Predictor.F0Predictor import F0Predictor 5 | 6 | 7 | class HarvestF0Predictor(F0Predictor): 8 | def __init__(self, hop_length=512, f0_min=50, f0_max=1100, sampling_rate=44100): 9 | self.hop_length = hop_length 10 | self.f0_min = f0_min 11 | self.f0_max = f0_max 12 | self.sampling_rate = sampling_rate 13 | 14 | def interpolate_f0(self, f0): 15 | """ 16 | 对F0进行插值处理 17 | """ 18 | 19 | data = np.reshape(f0, (f0.size, 1)) 20 | 21 | vuv_vector = np.zeros((data.size, 1), dtype=np.float32) 22 | vuv_vector[data > 0.0] = 1.0 23 | vuv_vector[data <= 0.0] = 0.0 24 | 25 | ip_data = data 26 | 27 | frame_number = data.size 28 | last_value = 0.0 29 | for i in range(frame_number): 30 | if data[i] <= 0.0: 31 | j = i + 1 32 | for j in range(i + 1, frame_number): 33 | if data[j] > 0.0: 34 | break 35 | if j < frame_number - 1: 36 | if last_value > 0.0: 37 | step = (data[j] - data[i - 1]) / float(j - i) 38 | for k in range(i, j): 39 | ip_data[k] = data[i - 1] + step * (k - i + 1) 40 | else: 41 | for k in range(i, j): 42 | ip_data[k] = data[j] 43 | else: 44 | for k in range(i, frame_number): 45 | ip_data[k] = last_value 46 | else: 47 | ip_data[i] = data[i] # 这里可能存在一个没有必要的拷贝 48 | last_value = data[i] 49 | 50 | return ip_data[:, 0], vuv_vector[:, 0] 51 | 52 | def resize_f0(self, x, target_len): 53 | source = np.array(x) 54 | source[source < 0.001] = np.nan 55 | target = np.interp( 56 | np.arange(0, len(source) * target_len, len(source)) / target_len, 57 | np.arange(0, len(source)), 58 | source, 59 | ) 60 | res = np.nan_to_num(target) 61 | return res 62 | 63 | def compute_f0(self, wav, p_len=None): 64 | if p_len is None: 65 | p_len = wav.shape[0] // self.hop_length 66 | f0, t = pyworld.harvest( 67 | wav.astype(np.double), 68 | fs=self.sampling_rate, 69 | f0_ceil=self.f0_max, 70 | f0_floor=self.f0_min, 71 | frame_period=1000 * self.hop_length / self.sampling_rate, 72 | ) 73 | f0 = pyworld.stonemask(wav.astype(np.double), f0, t, self.fs) 74 | return self.interpolate_f0(self.resize_f0(f0, p_len))[0] 75 | 76 | def compute_f0_uv(self, wav, p_len=None): 77 | if p_len is None: 78 | p_len = wav.shape[0] // self.hop_length 79 | f0, t = pyworld.harvest( 80 | wav.astype(np.double), 81 | fs=self.sampling_rate, 82 | f0_floor=self.f0_min, 83 | f0_ceil=self.f0_max, 84 | frame_period=1000 * self.hop_length / self.sampling_rate, 85 | ) 86 | f0 = pyworld.stonemask(wav.astype(np.double), f0, t, self.sampling_rate) 87 | return self.interpolate_f0(self.resize_f0(f0, p_len)) 88 | -------------------------------------------------------------------------------- /tools/cmd/calc_rvc_model_similarity.py: -------------------------------------------------------------------------------- 1 | # This code references https://huggingface.co/JosephusCheung/ASimilarityCalculatior/blob/main/qwerty.py 2 | # Fill in the path of the model to be queried and the root directory of the reference models, and this script will return the similarity between the model to be queried and all reference models. 3 | import os 4 | import logging 5 | 6 | logger = logging.getLogger(__name__) 7 | 8 | import torch 9 | import torch.nn as nn 10 | import torch.nn.functional as F 11 | 12 | 13 | def cal_cross_attn(to_q, to_k, to_v, rand_input): 14 | hidden_dim, embed_dim = to_q.shape 15 | attn_to_q = nn.Linear(hidden_dim, embed_dim, bias=False) 16 | attn_to_k = nn.Linear(hidden_dim, embed_dim, bias=False) 17 | attn_to_v = nn.Linear(hidden_dim, embed_dim, bias=False) 18 | attn_to_q.load_state_dict({"weight": to_q}) 19 | attn_to_k.load_state_dict({"weight": to_k}) 20 | attn_to_v.load_state_dict({"weight": to_v}) 21 | 22 | return torch.einsum( 23 | "ik, jk -> ik", 24 | F.softmax( 25 | torch.einsum("ij, kj -> ik", attn_to_q(rand_input), attn_to_k(rand_input)), 26 | dim=-1, 27 | ), 28 | attn_to_v(rand_input), 29 | ) 30 | 31 | 32 | def model_hash(filename): 33 | try: 34 | with open(filename, "rb") as file: 35 | import hashlib 36 | 37 | m = hashlib.sha256() 38 | 39 | file.seek(0x100000) 40 | m.update(file.read(0x10000)) 41 | return m.hexdigest()[0:8] 42 | except FileNotFoundError: 43 | return "NOFILE" 44 | 45 | 46 | def eval(model, n, input): 47 | qk = f"enc_p.encoder.attn_layers.{n}.conv_q.weight" 48 | uk = f"enc_p.encoder.attn_layers.{n}.conv_k.weight" 49 | vk = f"enc_p.encoder.attn_layers.{n}.conv_v.weight" 50 | atoq, atok, atov = model[qk][:, :, 0], model[uk][:, :, 0], model[vk][:, :, 0] 51 | 52 | attn = cal_cross_attn(atoq, atok, atov, input) 53 | return attn 54 | 55 | 56 | def main(path, root): 57 | torch.manual_seed(114514) 58 | model_a = torch.load(path, map_location="cpu")["weight"] 59 | 60 | logger.info("Query:\t\t%s\t%s" % (path, model_hash(path))) 61 | 62 | map_attn_a = {} 63 | map_rand_input = {} 64 | for n in range(6): 65 | hidden_dim, embed_dim, _ = model_a[ 66 | f"enc_p.encoder.attn_layers.{n}.conv_v.weight" 67 | ].shape 68 | rand_input = torch.randn([embed_dim, hidden_dim]) 69 | 70 | map_attn_a[n] = eval(model_a, n, rand_input) 71 | map_rand_input[n] = rand_input 72 | 73 | del model_a 74 | 75 | for name in sorted(list(os.listdir(root))): 76 | path = "%s/%s" % (root, name) 77 | model_b = torch.load(path, map_location="cpu")["weight"] 78 | 79 | sims = [] 80 | for n in range(6): 81 | attn_a = map_attn_a[n] 82 | attn_b = eval(model_b, n, map_rand_input[n]) 83 | 84 | sim = torch.mean(torch.cosine_similarity(attn_a, attn_b)) 85 | sims.append(sim) 86 | 87 | logger.info( 88 | "Reference:\t%s\t%s\t%s" 89 | % (path, model_hash(path), f"{torch.mean(torch.stack(sims)) * 1e2:.2f}%") 90 | ) 91 | 92 | 93 | if __name__ == "__main__": 94 | query_path = r"assets\weights\mi v3.pth" 95 | reference_root = r"assets\weights" 96 | main(query_path, reference_root) 97 | -------------------------------------------------------------------------------- /docs/cn/Changelog_CN.md: -------------------------------------------------------------------------------- 1 | ### 20231006更新 2 | 3 | 我们制作了一个用于实时变声的界面go-realtime-gui.bat/gui_v1.py（事实上早就存在了），本次更新重点也优化了实时变声的性能。对比0813版： 4 | - 1、优优化界面操作：参数热更新（调整参数不需要中止再启动），懒加载模型（已加载过的模型不需要重新加载），增加响度因子参数（响度向输入音频靠近） 5 | - 2、优化自带降噪效果与速度 6 | - 3、大幅优化推理速度 7 | 8 | 注意输入输出设备应该选择同种类型，例如都选MME类型。 9 | 10 | 1006版本整体的更新为： 11 | - 1、继续提升rmvpe音高提取算法效果，对于男低音有更大的提升 12 | - 2、优化推理界面布局 13 | 14 | ### 20230813更新 15 | 1-常规bug修复 16 | - 保存频率总轮数最低改为1 总轮数最低改为2 17 | - 修复无pretrain模型训练报错 18 | - 增加伴奏人声分离完毕清理显存 19 | - faiss保存路径绝对路径改为相对路径 20 | - 支持路径包含空格（训练集路径+实验名称均支持，不再会报错） 21 | - filelist取消强制utf8编码 22 | - 解决实时变声中开启索引导致的CPU极大占用问题 23 | 24 | 2-重点更新 25 | - 训练出当前最强开源人声音高提取模型RMVPE，并用于RVC的训练、离线/实时推理，支持pytorch/onnx/DirectML 26 | - 通过pytorch-dml支持A卡和I卡的 27 | （1）实时变声（2）推理（3）人声伴奏分离（4）训练暂未支持，会切换至CPU训练；通过onnx_dml支持rmvpe_gpu的推理 28 | 29 | ### 20230618更新 30 | - v2增加32k和48k两个新预训练模型 31 | - 修复非f0模型推理报错 32 | - 对于超过一小时的训练集的索引建立环节，自动kmeans缩小特征处理以加速索引训练、加入和查询 33 | - 附送一个人声转吉他玩具仓库 34 | - 数据处理剔除异常值切片 35 | - onnx导出选项卡 36 | 37 | 失败的实验： 38 | - ~~特征检索增加时序维度：寄，没啥效果~~ 39 | - ~~特征检索增加PCAR降维可选项：寄，数据大用kmeans缩小数据量，数据小降维操作耗时比省下的匹配耗时还多~~ 40 | - ~~支持onnx推理（附带仅推理的小压缩包）：寄，生成nsf还是需要pytorch~~ 41 | - ~~训练时在音高、gender、eq、噪声等方面对输入进行随机增强：寄，没啥效果~~ 42 | - ~~接入小型声码器调研:寄，效果变差~~ 43 | 44 | todolist： 45 | - ~~训练集音高识别支持crepe：已经被RMVPE取代，不需要~~ 46 | - ~~多进程harvest推理：已经被RMVPE取代，不需要~~ 47 | - ~~crepe的精度支持和RVC-config同步：已经被RMVPE取代，不需要。支持这个还要同步torchcrepe的库，麻烦~~ 48 | - 对接F0编辑器 49 | 50 | 51 | ### 20230528更新 52 | - 增加v2的jupyter notebook，韩文changelog，增加一些环境依赖 53 | - 增加呼吸、清辅音、齿音保护模式 54 | - 支持crepe-full推理 55 | - UVR5人声伴奏分离加上3个去延迟模型和MDX-Net去混响模型，增加HP3人声提取模型 56 | - 索引名称增加版本和实验名称 57 | - 人声伴奏分离、推理批量导出增加音频导出格式选项 58 | - 废弃32k模型的训练 59 | 60 | ### 20230513更新 61 | - 清除一键包内部老版本runtime内残留的lib.infer_pack和uvr5_pack 62 | - 修复训练集预处理伪多进程的bug 63 | - 增加harvest识别音高可选通过中值滤波削弱哑音现象，可调整中值滤波半径 64 | - 导出音频增加后处理重采样 65 | - 训练n_cpu进程数从"仅调整f0提取"改为"调整数据预处理和f0提取" 66 | - 自动检测logs文件夹下的index路径，提供下拉列表功能 67 | - tab页增加"常见问题解答"（也可参考github-rvc-wiki） 68 | - 相同路径的输入音频推理增加了音高缓存（用途：使用harvest音高提取，整个pipeline会经历漫长且重复的音高提取过程，如果不使用缓存，实验不同音色、索引、音高中值滤波半径参数的用户在第一次测试后的等待结果会非常痛苦） 69 | 70 | ### 20230514更新 71 | - 音量包络对齐输入混合（可以缓解“输入静音输出小幅度噪声”的问题。如果输入音频背景底噪大则不建议开启，默认不开启（值为1可视为不开启）） 72 | - 支持按照指定频率保存提取的小模型（假如你想尝试不同epoch下的推理效果，但是不想保存所有大checkpoint并且每次都要ckpt手工处理提取小模型，这项功能会非常实用） 73 | - 通过设置环境变量解决服务端开了系统全局代理导致浏览器连接错误的问题 74 | - 支持v2预训练模型（目前只公开了40k版本进行测试，另外2个采样率还没有训练完全） 75 | - 推理前限制超过1的过大音量 76 | - 微调数据预处理参数 77 | 78 | 79 | ### 20230409更新 80 | - 修正训练参数，提升显卡平均利用率，A100最高从25%提升至90%左右，V100:50%->90%左右，2060S:60%->85%左右，P40:25%->95%左右，训练速度显著提升 81 | - 修正参数：总batch_size改为每张卡的batch_size 82 | - 修正total_epoch：最大限制100解锁至1000；默认10提升至默认20 83 | - 修复ckpt提取识别是否带音高错误导致推理异常的问题 84 | - 修复分布式训练每个rank都保存一次ckpt的问题 85 | - 特征提取进行nan特征过滤 86 | - 修复静音输入输出随机辅音or噪声的问题（老版模型需要重做训练集重训） 87 | 88 | ### 20230416更新 89 | - 新增本地实时变声迷你GUI，双击go-realtime-gui.bat启动 90 | - 训练推理均对<50Hz的频段进行滤波过滤 91 | - 训练推理音高提取pyworld最低音高从默认80下降至50,50-80hz间的男声低音不会哑 92 | - WebUI支持根据系统区域变更语言（现支持en_US，ja_JP，zh_CN，zh_HK，zh_SG，zh_TW，不支持的默认en_US） 93 | - 修正部分显卡识别（例如V100-16G识别失败，P4识别失败） 94 | 95 | ### 20230428更新 96 | - 升级faiss索引设置，速度更快，质量更高 97 | - 取消total_npy依赖，后续分享模型不再需要填写total_npy 98 | - 解锁16系限制。4G显存GPU给到4G的推理设置。 99 | - 修复部分音频格式下UVR5人声伴奏分离的bug 100 | - 实时变声迷你gui增加对非40k与不懈怠音高模型的支持 101 | 102 | ### 后续计划： 103 | 功能： 104 | - 支持多人训练选项卡（至多4人） 105 | 106 | 底模： 107 | - 收集呼吸wav加入训练集修正呼吸变声电音的问题 108 | - 我们正在训练增加了歌声训练集的底模，未来会公开 109 | 110 | -------------------------------------------------------------------------------- /infer/lib/infer_pack/modules/F0Predictor/DioF0Predictor.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pyworld 3 | 4 | from infer.lib.infer_pack.modules.F0Predictor.F0Predictor import F0Predictor 5 | 6 | 7 | class DioF0Predictor(F0Predictor): 8 | def __init__(self, hop_length=512, f0_min=50, f0_max=1100, sampling_rate=44100): 9 | self.hop_length = hop_length 10 | self.f0_min = f0_min 11 | self.f0_max = f0_max 12 | self.sampling_rate = sampling_rate 13 | 14 | def interpolate_f0(self, f0): 15 | """ 16 | 对F0进行插值处理 17 | """ 18 | 19 | data = np.reshape(f0, (f0.size, 1)) 20 | 21 | vuv_vector = np.zeros((data.size, 1), dtype=np.float32) 22 | vuv_vector[data > 0.0] = 1.0 23 | vuv_vector[data <= 0.0] = 0.0 24 | 25 | ip_data = data 26 | 27 | frame_number = data.size 28 | last_value = 0.0 29 | for i in range(frame_number): 30 | if data[i] <= 0.0: 31 | j = i + 1 32 | for j in range(i + 1, frame_number): 33 | if data[j] > 0.0: 34 | break 35 | if j < frame_number - 1: 36 | if last_value > 0.0: 37 | step = (data[j] - data[i - 1]) / float(j - i) 38 | for k in range(i, j): 39 | ip_data[k] = data[i - 1] + step * (k - i + 1) 40 | else: 41 | for k in range(i, j): 42 | ip_data[k] = data[j] 43 | else: 44 | for k in range(i, frame_number): 45 | ip_data[k] = last_value 46 | else: 47 | ip_data[i] = data[i] # 这里可能存在一个没有必要的拷贝 48 | last_value = data[i] 49 | 50 | return ip_data[:, 0], vuv_vector[:, 0] 51 | 52 | def resize_f0(self, x, target_len): 53 | source = np.array(x) 54 | source[source < 0.001] = np.nan 55 | target = np.interp( 56 | np.arange(0, len(source) * target_len, len(source)) / target_len, 57 | np.arange(0, len(source)), 58 | source, 59 | ) 60 | res = np.nan_to_num(target) 61 | return res 62 | 63 | def compute_f0(self, wav, p_len=None): 64 | if p_len is None: 65 | p_len = wav.shape[0] // self.hop_length 66 | f0, t = pyworld.dio( 67 | wav.astype(np.double), 68 | fs=self.sampling_rate, 69 | f0_floor=self.f0_min, 70 | f0_ceil=self.f0_max, 71 | frame_period=1000 * self.hop_length / self.sampling_rate, 72 | ) 73 | f0 = pyworld.stonemask(wav.astype(np.double), f0, t, self.sampling_rate) 74 | for index, pitch in enumerate(f0): 75 | f0[index] = round(pitch, 1) 76 | return self.interpolate_f0(self.resize_f0(f0, p_len))[0] 77 | 78 | def compute_f0_uv(self, wav, p_len=None): 79 | if p_len is None: 80 | p_len = wav.shape[0] // self.hop_length 81 | f0, t = pyworld.dio( 82 | wav.astype(np.double), 83 | fs=self.sampling_rate, 84 | f0_floor=self.f0_min, 85 | f0_ceil=self.f0_max, 86 | frame_period=1000 * self.hop_length / self.sampling_rate, 87 | ) 88 | f0 = pyworld.stonemask(wav.astype(np.double), f0, t, self.sampling_rate) 89 | for index, pitch in enumerate(f0): 90 | f0[index] = round(pitch, 1) 91 | return self.interpolate_f0(self.resize_f0(f0, p_len)) 92 | -------------------------------------------------------------------------------- /docs/jp/faiss_tips_ja.md: -------------------------------------------------------------------------------- 1 | faiss tuning TIPS 2 | ================== 3 | # about faiss 4 | faissはfacebook researchの開発する、密なベクトルに対する近傍探索をまとめたライブラリで、多くの近似近傍探索の手法を効率的に実装しています。 5 | 近似近傍探索はある程度精度を犠牲にしながら高速に類似するベクトルを探します。 6 | 7 | ## faiss in RVC 8 | RVCではHuBERTで変換した特徴量のEmbeddingに対し、学習データから生成されたEmbeddingと類似するものを検索し、混ぜることでより元の音声に近い変換を実現しています。ただ、この検索は愚直に行うと時間がかかるため、近似近傍探索を用いることで高速な変換を実現しています。 9 | 10 | # 実装のoverview 11 | モデルが配置されている '/logs/your-experiment/3_feature256'には各音声データからHuBERTで抽出された特徴量が配置されています。 12 | ここからnpyファイルをファイル名でソートした順番で読み込み、ベクトルを連結してbig_npyを作成しfaissを学習させます。(このベクトルのshapeは[N, 256]です。) 13 | 14 | 本Tipsではまずこれらのパラメータの意味を解説します。 15 | 16 | # 手法の解説 17 | ## index factory 18 | index factoryは複数の近似近傍探索の手法を繋げるパイプラインをstringで表記するfaiss独自の記法です。 19 | これにより、index factoryの文字列を変更するだけで様々な近似近傍探索の手法を試せます。 20 | RVCでは以下のように使われています。 21 | 22 | ```python 23 | index = faiss.index_factory(256, "IVF%s,Flat" % n_ivf) 24 | ``` 25 | index_factoryの引数のうち、1つ目はベクトルの次元数、2つ目はindex factoryの文字列で、3つ目には用いる距離を指定することができます。 26 | 27 | より詳細な記法については 28 | https://github.com/facebookresearch/faiss/wiki/The-index-factory 29 | 30 | ## 距離指標 31 | embeddingの類似度として用いられる代表的な指標として以下の二つがあります。 32 | 33 | - ユークリッド距離(METRIC_L2) 34 | - 内積(METRIC_INNER_PRODUCT) 35 | 36 | ユークリッド距離では各次元において二乗の差をとり、全次元の差を足してから平方根をとります。これは日常的に用いる2次元、3次元での距離と同じです。 37 | 内積はこのままでは類似度の指標として用いず、一般的にはL2ノルムで正規化してから内積をとるコサイン類似度を用います。 38 | 39 | どちらがよいかは場合によりますが、word2vec等で得られるembeddingやArcFace等で学習した類似画像検索のモデルではコサイン類似度が用いられることが多いです。ベクトルXに対してl2正規化をnumpyで行う場合は、0 divisionを避けるために十分に小さな値をepsとして以下のコードで可能です。 40 | 41 | ```python 42 | X_normed = X / np.maximum(eps, np.linalg.norm(X, ord=2, axis=-1, keepdims=True)) 43 | ``` 44 | 45 | また、index factoryには第3引数に渡す値を選ぶことで計算に用いる距離指標を変更できます。 46 | 47 | ```python 48 | index = faiss.index_factory(dimention, text, faiss.METRIC_INNER_PRODUCT) 49 | ``` 50 | 51 | ## IVF 52 | IVF(Inverted file indexes)は全文検索における転置インデックスと似たようなアルゴリズムです。 53 | 学習時には検索対象に対してkmeansでクラスタリングを行い、クラスタ中心を用いてボロノイ分割を行います。各データ点には一つずつクラスタが割り当てられるので、クラスタからデータ点を逆引きする辞書を作成します。 54 | 55 | 例えば以下のようにクラスタが割り当てられた場合 56 | |index|クラスタ| 57 | |-----|-------| 58 | |1|A| 59 | |2|B| 60 | |3|A| 61 | |4|C| 62 | |5|B| 63 | 64 | 作成される転置インデックスは以下のようになります。 65 | 66 | |クラスタ|index| 67 | |-------|-----| 68 | |A|1, 3| 69 | |B|2, 5| 70 | |C|4| 71 | 72 | 検索時にはまずクラスタからn_probe個のクラスタを検索し、次にそれぞれのクラスタに属するデータ点について距離を計算します。 73 | 74 | # 推奨されるパラメータ 75 | indexの選び方については公式にガイドラインがあるので、それに準じて説明します。 76 | https://github.com/facebookresearch/faiss/wiki/Guidelines-to-choose-an-index 77 | 78 | 1M以下のデータセットにおいては4bit-PQが2023年4月時点ではfaissで利用できる最も効率的な手法です。 79 | これをIVFと組み合わせ、4bit-PQで候補を絞り、最後に正確な指標で距離を再計算するには以下のindex factoryを用いることで記載できます。 80 | 81 | ```python 82 | index = faiss.index_factory(256, "IVF1024,PQ128x4fs,RFlat") 83 | ``` 84 | 85 | ## IVFの推奨パラメータ 86 | IVFの数が多すぎる場合、たとえばデータ数の数だけIVFによる粗量子化を行うと、これは愚直な全探索と同じになり効率が悪いです。 87 | 1M以下の場合ではIVFの値はデータ点の数Nに対して4*sqrt(N) ~ 16*sqrt(N)に推奨しています。 88 | 89 | n_probeはn_probeの数に比例して計算時間が増えるので、精度と相談して適切に選んでください。個人的にはRVCにおいてそこまで精度は必要ないと思うのでn_probe = 1で良いと思います。 90 | 91 | ## FastScan 92 | FastScanは直積量子化で大まかに距離を近似するのを、レジスタ内で行うことにより高速に行うようにした手法です。 93 | 直積量子化は学習時にd次元ごと(通常はd=2)に独立してクラスタリングを行い、クラスタ同士の距離を事前計算してlookup tableを作成します。予測時はlookup tableを見ることで各次元の距離をO(1)で計算できます。 94 | そのため、PQの次に指定する数字は通常ベクトルの半分の次元を指定します。 95 | 96 | FastScanに関するより詳細な説明は公式のドキュメントを参照してください。 97 | https://github.com/facebookresearch/faiss/wiki/Fast-accumulation-of-PQ-and-AQ-codes-(FastScan) 98 | 99 | ## RFlat 100 | RFlatはFastScanで計算した大まかな距離を、index factoryの第三引数で指定した正確な距離で再計算する指示です。 101 | k個の近傍を取得する際は、k*k_factor個の点について再計算が行われます。 102 | -------------------------------------------------------------------------------- /.env: -------------------------------------------------------------------------------- 1 | OPENBLAS_NUM_THREADS = 1 2 | no_proxy = localhost, 127.0.0.1, ::1 3 | 4 | # You can change the location of the model, etc. by changing here 5 | weight_root = assets/weights 6 | weight_uvr5_root = assets/uvr5_weights 7 | index_root = logs 8 | outside_index_root = assets/indices 9 | rmvpe_root = assets/rmvpe 10 | 11 | sha256_hubert_base_pt = f54b40fd2802423a5643779c4861af1e9ee9c1564dc9d32f54f20b5ffba7db96 12 | sha256_rmvpe_pt = 6d62215f4306e3ca278246188607209f09af3dc77ed4232efdd069798c4ec193 13 | sha256_rmvpe_onnx = 5370e71ac80af8b4b7c793d27efd51fd8bf962de3a7ede0766dac0befa3660fd 14 | 15 | sha256_v1_D32k_pth = 2ab20645829460fdad0d3c44254f1ab53c32cae50c22a66c926ae5aa30abda6f 16 | sha256_v1_D40k_pth = 547f66dbbcd9023b9051ed244d12ab043ba8a4e854b154cc28761ac7c002909b 17 | sha256_v1_D48k_pth = 8cc013fa60ed9c3f902f5bd99f48c7e3b9352d763d4d3cd6bc241c37b0bfd9ad 18 | sha256_v1_G32k_pth = 81817645cde7ed2e2d83f23ef883f33dda564924b497e84d792743912eca4c23 19 | sha256_v1_G40k_pth = e428573bda1124b0ae0ae843fd8dcded6027d3993444790b3e9b0100938b2113 20 | sha256_v1_G48k_pth = 3862a67ea6313e8ffefc05cee6bee656ef3e089442e9ecf4a6618d60721f3e95 21 | sha256_v1_f0D32k_pth = 294db3087236e2c75260d6179056791c9231245daf5d0485545d9e54c4057c77 22 | sha256_v1_f0D40k_pth = 7d4f5a441594b470d67579958b2fd4c6b992852ded28ff9e72eda67abcebe423 23 | sha256_v1_f0D48k_pth = 1b84c8bf347ad1e539c842e8f2a4c36ecd9e7fb23c16041189e4877e9b07925c 24 | sha256_v1_f0G32k_pth = 285f524bf48bb692c76ad7bd0bc654c12bd9e5edeb784dddf7f61a789a608574 25 | sha256_v1_f0G40k_pth = 9115654aeef1995f7dd3c6fc4140bebbef0ca9760bed798105a2380a34299831 26 | sha256_v1_f0G48k_pth = 78bc9cab27e34bcfc194f93029374d871d8b3e663ddedea32a9709e894cc8fe8 27 | 28 | sha256_v2_D32k_pth = d8043378cc6619083d385f5a045de09b83fb3bf8de45c433ca863b71723ac3ca 29 | sha256_v2_D40k_pth = 471378e894e7191f89a94eda8288c5947b16bbe0b10c3f1f17efdb7a1d998242 30 | sha256_v2_D48k_pth = db01094a93c09868a278e03dafe8bb781bfcc1a5ba8df168c948bf9168c84d82 31 | sha256_v2_G32k_pth = 869b26a47f75168d6126f64ac39e6de5247017a8658cfd68aca600f7323efb9f 32 | sha256_v2_G40k_pth = a3843da7fde33db1dab176146c70d6c2df06eafe9457f4e3aa10024e9c6a4b69 33 | sha256_v2_G48k_pth = 2e2b1581a436d07a76b10b9d38765f64aa02836dc65c7dee1ce4140c11ea158b 34 | sha256_v2_f0D32k_pth = bd7134e7793674c85474d5145d2d982e3c5d8124fc7bb6c20f710ed65808fa8a 35 | sha256_v2_f0D40k_pth = 6b6ab091e70801b28e3f41f335f2fc5f3f35c75b39ae2628d419644ec2b0fa09 36 | sha256_v2_f0D48k_pth = 2269b73c7a4cf34da09aea99274dabf99b2ddb8a42cbfb065fb3c0aa9a2fc748 37 | sha256_v2_f0G32k_pth = 2332611297b8d88c7436de8f17ef5f07a2119353e962cd93cda5806d59a1133d 38 | sha256_v2_f0G40k_pth = 3b2c44035e782c4b14ddc0bede9e2f4a724d025cd073f736d4f43708453adfcb 39 | sha256_v2_f0G48k_pth = b5d51f589cc3632d4eae36a315b4179397695042edc01d15312e1bddc2b764a4 40 | 41 | sha256_uvr5_HP2-人声vocals+非人声instrumentals_pth = 39796caa5db18d7f9382d8ac997ac967bfd85f7761014bb807d2543cc844ef05 42 | sha256_uvr5_HP2_all_vocals_pth = 39796caa5db18d7f9382d8ac997ac967bfd85f7761014bb807d2543cc844ef05 43 | sha256_uvr5_HP3_all_vocals_pth = 45e6b65199e781b4a6542002699be9f19cd3d1cb7d1558bc2bfbcd84674dfe28 44 | sha256_uvr5_HP5-主旋律人声vocals+其他instrumentals_pth = 5908891829634926119720241e8573d97cbeb8277110a7512bdb0bd7563258ee 45 | sha256_uvr5_HP5_only_main_vocal_pth = 5908891829634926119720241e8573d97cbeb8277110a7512bdb0bd7563258ee 46 | sha256_uvr5_VR-DeEchoAggressive_pth = 8c8fd1582f9aabc363e47af62ddb88df6cae7e064cae75bbf041a067a5e0aee2 47 | sha256_uvr5_VR-DeEchoDeReverb_pth = 01376dd2a571bf3cb9cced680732726d2d732609d09216a610b0d110f133febe 48 | sha256_uvr5_VR-DeEchoNormal_pth = 56aba59db3bcdd14a14464e62f3129698ecdea62eee0f003b9360923eb3ac79e 49 | sha256_uvr5_vocals_onnx = 233bb5c6aaa365e568659a0a81211746fa881f8f47f82d9e864fce1f7692db80 50 | -------------------------------------------------------------------------------- /infer/lib/infer_pack/modules/F0Predictor/PMF0Predictor.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import parselmouth 3 | 4 | from infer.lib.infer_pack.modules.F0Predictor.F0Predictor import F0Predictor 5 | 6 | 7 | class PMF0Predictor(F0Predictor): 8 | def __init__(self, hop_length=512, f0_min=50, f0_max=1100, sampling_rate=44100): 9 | self.hop_length = hop_length 10 | self.f0_min = f0_min 11 | self.f0_max = f0_max 12 | self.sampling_rate = sampling_rate 13 | 14 | def interpolate_f0(self, f0): 15 | """ 16 | 对F0进行插值处理 17 | """ 18 | 19 | data = np.reshape(f0, (f0.size, 1)) 20 | 21 | vuv_vector = np.zeros((data.size, 1), dtype=np.float32) 22 | vuv_vector[data > 0.0] = 1.0 23 | vuv_vector[data <= 0.0] = 0.0 24 | 25 | ip_data = data 26 | 27 | frame_number = data.size 28 | last_value = 0.0 29 | for i in range(frame_number): 30 | if data[i] <= 0.0: 31 | j = i + 1 32 | for j in range(i + 1, frame_number): 33 | if data[j] > 0.0: 34 | break 35 | if j < frame_number - 1: 36 | if last_value > 0.0: 37 | step = (data[j] - data[i - 1]) / float(j - i) 38 | for k in range(i, j): 39 | ip_data[k] = data[i - 1] + step * (k - i + 1) 40 | else: 41 | for k in range(i, j): 42 | ip_data[k] = data[j] 43 | else: 44 | for k in range(i, frame_number): 45 | ip_data[k] = last_value 46 | else: 47 | ip_data[i] = data[i] # 这里可能存在一个没有必要的拷贝 48 | last_value = data[i] 49 | 50 | return ip_data[:, 0], vuv_vector[:, 0] 51 | 52 | def compute_f0(self, wav, p_len=None): 53 | x = wav 54 | if p_len is None: 55 | p_len = x.shape[0] // self.hop_length 56 | else: 57 | assert abs(p_len - x.shape[0] // self.hop_length) < 4, "pad length error" 58 | time_step = self.hop_length / self.sampling_rate * 1000 59 | f0 = ( 60 | parselmouth.Sound(x, self.sampling_rate) 61 | .to_pitch_ac( 62 | time_step=time_step / 1000, 63 | voicing_threshold=0.6, 64 | pitch_floor=self.f0_min, 65 | pitch_ceiling=self.f0_max, 66 | ) 67 | .selected_array["frequency"] 68 | ) 69 | 70 | pad_size = (p_len - len(f0) + 1) // 2 71 | if pad_size > 0 or p_len - len(f0) - pad_size > 0: 72 | f0 = np.pad(f0, [[pad_size, p_len - len(f0) - pad_size]], mode="constant") 73 | f0, uv = self.interpolate_f0(f0) 74 | return f0 75 | 76 | def compute_f0_uv(self, wav, p_len=None): 77 | x = wav 78 | if p_len is None: 79 | p_len = x.shape[0] // self.hop_length 80 | else: 81 | assert abs(p_len - x.shape[0] // self.hop_length) < 4, "pad length error" 82 | time_step = self.hop_length / self.sampling_rate * 1000 83 | f0 = ( 84 | parselmouth.Sound(x, self.sampling_rate) 85 | .to_pitch_ac( 86 | time_step=time_step / 1000, 87 | voicing_threshold=0.6, 88 | pitch_floor=self.f0_min, 89 | pitch_ceiling=self.f0_max, 90 | ) 91 | .selected_array["frequency"] 92 | ) 93 | 94 | pad_size = (p_len - len(f0) + 1) // 2 95 | if pad_size > 0 or p_len - len(f0) - pad_size > 0: 96 | f0 = np.pad(f0, [[pad_size, p_len - len(f0) - pad_size]], mode="constant") 97 | f0, uv = self.interpolate_f0(f0) 98 | return f0, uv 99 | -------------------------------------------------------------------------------- /infer/lib/uvr5_pack/utils.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | import numpy as np 4 | import torch 5 | from tqdm import tqdm 6 | 7 | 8 | def load_data(file_name: str = "./infer/lib/uvr5_pack/name_params.json") -> dict: 9 | with open(file_name, "r") as f: 10 | data = json.load(f) 11 | 12 | return data 13 | 14 | 15 | def make_padding(width, cropsize, offset): 16 | left = offset 17 | roi_size = cropsize - left * 2 18 | if roi_size == 0: 19 | roi_size = cropsize 20 | right = roi_size - (width % roi_size) + left 21 | 22 | return left, right, roi_size 23 | 24 | 25 | def inference(X_spec, device, model, aggressiveness, data): 26 | """ 27 | data ： dic configs 28 | """ 29 | 30 | def _execute( 31 | X_mag_pad, roi_size, n_window, device, model, aggressiveness, is_half=True 32 | ): 33 | model.eval() 34 | with torch.no_grad(): 35 | preds = [] 36 | 37 | iterations = [n_window] 38 | 39 | total_iterations = sum(iterations) 40 | for i in tqdm(range(n_window)): 41 | start = i * roi_size 42 | X_mag_window = X_mag_pad[ 43 | None, :, :, start : start + data["window_size"] 44 | ] 45 | X_mag_window = torch.from_numpy(X_mag_window) 46 | if is_half: 47 | X_mag_window = X_mag_window.half() 48 | X_mag_window = X_mag_window.to(device) 49 | 50 | pred = model.predict(X_mag_window, aggressiveness) 51 | 52 | pred = pred.detach().cpu().numpy() 53 | preds.append(pred[0]) 54 | 55 | pred = np.concatenate(preds, axis=2) 56 | return pred 57 | 58 | def preprocess(X_spec): 59 | X_mag = np.abs(X_spec) 60 | X_phase = np.angle(X_spec) 61 | 62 | return X_mag, X_phase 63 | 64 | X_mag, X_phase = preprocess(X_spec) 65 | 66 | coef = X_mag.max() 67 | X_mag_pre = X_mag / coef 68 | 69 | n_frame = X_mag_pre.shape[2] 70 | pad_l, pad_r, roi_size = make_padding(n_frame, data["window_size"], model.offset) 71 | n_window = int(np.ceil(n_frame / roi_size)) 72 | 73 | X_mag_pad = np.pad(X_mag_pre, ((0, 0), (0, 0), (pad_l, pad_r)), mode="constant") 74 | 75 | if list(model.state_dict().values())[0].dtype == torch.float16: 76 | is_half = True 77 | else: 78 | is_half = False 79 | pred = _execute( 80 | X_mag_pad, roi_size, n_window, device, model, aggressiveness, is_half 81 | ) 82 | pred = pred[:, :, :n_frame] 83 | 84 | if data["tta"]: 85 | pad_l += roi_size // 2 86 | pad_r += roi_size // 2 87 | n_window += 1 88 | 89 | X_mag_pad = np.pad(X_mag_pre, ((0, 0), (0, 0), (pad_l, pad_r)), mode="constant") 90 | 91 | pred_tta = _execute( 92 | X_mag_pad, roi_size, n_window, device, model, aggressiveness, is_half 93 | ) 94 | pred_tta = pred_tta[:, :, roi_size // 2 :] 95 | pred_tta = pred_tta[:, :, :n_frame] 96 | 97 | return (pred + pred_tta) * 0.5 * coef, X_mag, np.exp(1.0j * X_phase) 98 | else: 99 | return pred * coef, X_mag, np.exp(1.0j * X_phase) 100 | 101 | 102 | def _get_name_params(model_path, model_hash): 103 | data = load_data() 104 | flag = False 105 | ModelName = model_path 106 | for type in list(data): 107 | for model in list(data[type][0]): 108 | for i in range(len(data[type][0][model])): 109 | if str(data[type][0][model][i]["hash_name"]) == model_hash: 110 | flag = True 111 | elif str(data[type][0][model][i]["hash_name"]) in ModelName: 112 | flag = True 113 | 114 | if flag: 115 | model_params_auto = data[type][0][model][i]["model_params"] 116 | param_name_auto = data[type][0][model][i]["param_name"] 117 | if type == "equivalent": 118 | return param_name_auto, model_params_auto 119 | else: 120 | flag = False 121 | return param_name_auto, model_params_auto 122 | -------------------------------------------------------------------------------- /infer/lib/uvr5_pack/lib_v5/layers.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn.functional as F 3 | from torch import nn 4 | 5 | from . import spec_utils 6 | 7 | 8 | class Conv2DBNActiv(nn.Module): 9 | def __init__(self, nin, nout, ksize=3, stride=1, pad=1, dilation=1, activ=nn.ReLU): 10 | super(Conv2DBNActiv, self).__init__() 11 | self.conv = nn.Sequential( 12 | nn.Conv2d( 13 | nin, 14 | nout, 15 | kernel_size=ksize, 16 | stride=stride, 17 | padding=pad, 18 | dilation=dilation, 19 | bias=False, 20 | ), 21 | nn.BatchNorm2d(nout), 22 | activ(), 23 | ) 24 | 25 | def __call__(self, x): 26 | return self.conv(x) 27 | 28 | 29 | class SeperableConv2DBNActiv(nn.Module): 30 | def __init__(self, nin, nout, ksize=3, stride=1, pad=1, dilation=1, activ=nn.ReLU): 31 | super(SeperableConv2DBNActiv, self).__init__() 32 | self.conv = nn.Sequential( 33 | nn.Conv2d( 34 | nin, 35 | nin, 36 | kernel_size=ksize, 37 | stride=stride, 38 | padding=pad, 39 | dilation=dilation, 40 | groups=nin, 41 | bias=False, 42 | ), 43 | nn.Conv2d(nin, nout, kernel_size=1, bias=False), 44 | nn.BatchNorm2d(nout), 45 | activ(), 46 | ) 47 | 48 | def __call__(self, x): 49 | return self.conv(x) 50 | 51 | 52 | class Encoder(nn.Module): 53 | def __init__(self, nin, nout, ksize=3, stride=1, pad=1, activ=nn.LeakyReLU): 54 | super(Encoder, self).__init__() 55 | self.conv1 = Conv2DBNActiv(nin, nout, ksize, 1, pad, activ=activ) 56 | self.conv2 = Conv2DBNActiv(nout, nout, ksize, stride, pad, activ=activ) 57 | 58 | def __call__(self, x): 59 | skip = self.conv1(x) 60 | h = self.conv2(skip) 61 | 62 | return h, skip 63 | 64 | 65 | class Decoder(nn.Module): 66 | def __init__( 67 | self, nin, nout, ksize=3, stride=1, pad=1, activ=nn.ReLU, dropout=False 68 | ): 69 | super(Decoder, self).__init__() 70 | self.conv = Conv2DBNActiv(nin, nout, ksize, 1, pad, activ=activ) 71 | self.dropout = nn.Dropout2d(0.1) if dropout else None 72 | 73 | def __call__(self, x, skip=None): 74 | x = F.interpolate(x, scale_factor=2, mode="bilinear", align_corners=True) 75 | if skip is not None: 76 | skip = spec_utils.crop_center(skip, x) 77 | x = torch.cat([x, skip], dim=1) 78 | h = self.conv(x) 79 | 80 | if self.dropout is not None: 81 | h = self.dropout(h) 82 | 83 | return h 84 | 85 | 86 | class ASPPModule(nn.Module): 87 | def __init__(self, nin, nout, dilations=(4, 8, 16), activ=nn.ReLU): 88 | super(ASPPModule, self).__init__() 89 | self.conv1 = nn.Sequential( 90 | nn.AdaptiveAvgPool2d((1, None)), 91 | Conv2DBNActiv(nin, nin, 1, 1, 0, activ=activ), 92 | ) 93 | self.conv2 = Conv2DBNActiv(nin, nin, 1, 1, 0, activ=activ) 94 | self.conv3 = SeperableConv2DBNActiv( 95 | nin, nin, 3, 1, dilations[0], dilations[0], activ=activ 96 | ) 97 | self.conv4 = SeperableConv2DBNActiv( 98 | nin, nin, 3, 1, dilations[1], dilations[1], activ=activ 99 | ) 100 | self.conv5 = SeperableConv2DBNActiv( 101 | nin, nin, 3, 1, dilations[2], dilations[2], activ=activ 102 | ) 103 | self.bottleneck = nn.Sequential( 104 | Conv2DBNActiv(nin * 5, nout, 1, 1, 0, activ=activ), nn.Dropout2d(0.1) 105 | ) 106 | 107 | def forward(self, x): 108 | _, _, h, w = x.size() 109 | feat1 = F.interpolate( 110 | self.conv1(x), size=(h, w), mode="bilinear", align_corners=True 111 | ) 112 | feat2 = self.conv2(x) 113 | feat3 = self.conv3(x) 114 | feat4 = self.conv4(x) 115 | feat5 = self.conv5(x) 116 | out = torch.cat((feat1, feat2, feat3, feat4, feat5), dim=1) 117 | bottle = self.bottleneck(out) 118 | return bottle 119 | -------------------------------------------------------------------------------- /infer/lib/uvr5_pack/lib_v5/layers_123812KB .py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn.functional as F 3 | from torch import nn 4 | 5 | from . import spec_utils 6 | 7 | 8 | class Conv2DBNActiv(nn.Module): 9 | def __init__(self, nin, nout, ksize=3, stride=1, pad=1, dilation=1, activ=nn.ReLU): 10 | super(Conv2DBNActiv, self).__init__() 11 | self.conv = nn.Sequential( 12 | nn.Conv2d( 13 | nin, 14 | nout, 15 | kernel_size=ksize, 16 | stride=stride, 17 | padding=pad, 18 | dilation=dilation, 19 | bias=False, 20 | ), 21 | nn.BatchNorm2d(nout), 22 | activ(), 23 | ) 24 | 25 | def __call__(self, x): 26 | return self.conv(x) 27 | 28 | 29 | class SeperableConv2DBNActiv(nn.Module): 30 | def __init__(self, nin, nout, ksize=3, stride=1, pad=1, dilation=1, activ=nn.ReLU): 31 | super(SeperableConv2DBNActiv, self).__init__() 32 | self.conv = nn.Sequential( 33 | nn.Conv2d( 34 | nin, 35 | nin, 36 | kernel_size=ksize, 37 | stride=stride, 38 | padding=pad, 39 | dilation=dilation, 40 | groups=nin, 41 | bias=False, 42 | ), 43 | nn.Conv2d(nin, nout, kernel_size=1, bias=False), 44 | nn.BatchNorm2d(nout), 45 | activ(), 46 | ) 47 | 48 | def __call__(self, x): 49 | return self.conv(x) 50 | 51 | 52 | class Encoder(nn.Module): 53 | def __init__(self, nin, nout, ksize=3, stride=1, pad=1, activ=nn.LeakyReLU): 54 | super(Encoder, self).__init__() 55 | self.conv1 = Conv2DBNActiv(nin, nout, ksize, 1, pad, activ=activ) 56 | self.conv2 = Conv2DBNActiv(nout, nout, ksize, stride, pad, activ=activ) 57 | 58 | def __call__(self, x): 59 | skip = self.conv1(x) 60 | h = self.conv2(skip) 61 | 62 | return h, skip 63 | 64 | 65 | class Decoder(nn.Module): 66 | def __init__( 67 | self, nin, nout, ksize=3, stride=1, pad=1, activ=nn.ReLU, dropout=False 68 | ): 69 | super(Decoder, self).__init__() 70 | self.conv = Conv2DBNActiv(nin, nout, ksize, 1, pad, activ=activ) 71 | self.dropout = nn.Dropout2d(0.1) if dropout else None 72 | 73 | def __call__(self, x, skip=None): 74 | x = F.interpolate(x, scale_factor=2, mode="bilinear", align_corners=True) 75 | if skip is not None: 76 | skip = spec_utils.crop_center(skip, x) 77 | x = torch.cat([x, skip], dim=1) 78 | h = self.conv(x) 79 | 80 | if self.dropout is not None: 81 | h = self.dropout(h) 82 | 83 | return h 84 | 85 | 86 | class ASPPModule(nn.Module): 87 | def __init__(self, nin, nout, dilations=(4, 8, 16), activ=nn.ReLU): 88 | super(ASPPModule, self).__init__() 89 | self.conv1 = nn.Sequential( 90 | nn.AdaptiveAvgPool2d((1, None)), 91 | Conv2DBNActiv(nin, nin, 1, 1, 0, activ=activ), 92 | ) 93 | self.conv2 = Conv2DBNActiv(nin, nin, 1, 1, 0, activ=activ) 94 | self.conv3 = SeperableConv2DBNActiv( 95 | nin, nin, 3, 1, dilations[0], dilations[0], activ=activ 96 | ) 97 | self.conv4 = SeperableConv2DBNActiv( 98 | nin, nin, 3, 1, dilations[1], dilations[1], activ=activ 99 | ) 100 | self.conv5 = SeperableConv2DBNActiv( 101 | nin, nin, 3, 1, dilations[2], dilations[2], activ=activ 102 | ) 103 | self.bottleneck = nn.Sequential( 104 | Conv2DBNActiv(nin * 5, nout, 1, 1, 0, activ=activ), nn.Dropout2d(0.1) 105 | ) 106 | 107 | def forward(self, x): 108 | _, _, h, w = x.size() 109 | feat1 = F.interpolate( 110 | self.conv1(x), size=(h, w), mode="bilinear", align_corners=True 111 | ) 112 | feat2 = self.conv2(x) 113 | feat3 = self.conv3(x) 114 | feat4 = self.conv4(x) 115 | feat5 = self.conv5(x) 116 | out = torch.cat((feat1, feat2, feat3, feat4, feat5), dim=1) 117 | bottle = self.bottleneck(out) 118 | return bottle 119 | -------------------------------------------------------------------------------- /infer/lib/uvr5_pack/lib_v5/layers_123821KB.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn.functional as F 3 | from torch import nn 4 | 5 | from . import spec_utils 6 | 7 | 8 | class Conv2DBNActiv(nn.Module): 9 | def __init__(self, nin, nout, ksize=3, stride=1, pad=1, dilation=1, activ=nn.ReLU): 10 | super(Conv2DBNActiv, self).__init__() 11 | self.conv = nn.Sequential( 12 | nn.Conv2d( 13 | nin, 14 | nout, 15 | kernel_size=ksize, 16 | stride=stride, 17 | padding=pad, 18 | dilation=dilation, 19 | bias=False, 20 | ), 21 | nn.BatchNorm2d(nout), 22 | activ(), 23 | ) 24 | 25 | def __call__(self, x): 26 | return self.conv(x) 27 | 28 | 29 | class SeperableConv2DBNActiv(nn.Module): 30 | def __init__(self, nin, nout, ksize=3, stride=1, pad=1, dilation=1, activ=nn.ReLU): 31 | super(SeperableConv2DBNActiv, self).__init__() 32 | self.conv = nn.Sequential( 33 | nn.Conv2d( 34 | nin, 35 | nin, 36 | kernel_size=ksize, 37 | stride=stride, 38 | padding=pad, 39 | dilation=dilation, 40 | groups=nin, 41 | bias=False, 42 | ), 43 | nn.Conv2d(nin, nout, kernel_size=1, bias=False), 44 | nn.BatchNorm2d(nout), 45 | activ(), 46 | ) 47 | 48 | def __call__(self, x): 49 | return self.conv(x) 50 | 51 | 52 | class Encoder(nn.Module): 53 | def __init__(self, nin, nout, ksize=3, stride=1, pad=1, activ=nn.LeakyReLU): 54 | super(Encoder, self).__init__() 55 | self.conv1 = Conv2DBNActiv(nin, nout, ksize, 1, pad, activ=activ) 56 | self.conv2 = Conv2DBNActiv(nout, nout, ksize, stride, pad, activ=activ) 57 | 58 | def __call__(self, x): 59 | skip = self.conv1(x) 60 | h = self.conv2(skip) 61 | 62 | return h, skip 63 | 64 | 65 | class Decoder(nn.Module): 66 | def __init__( 67 | self, nin, nout, ksize=3, stride=1, pad=1, activ=nn.ReLU, dropout=False 68 | ): 69 | super(Decoder, self).__init__() 70 | self.conv = Conv2DBNActiv(nin, nout, ksize, 1, pad, activ=activ) 71 | self.dropout = nn.Dropout2d(0.1) if dropout else None 72 | 73 | def __call__(self, x, skip=None): 74 | x = F.interpolate(x, scale_factor=2, mode="bilinear", align_corners=True) 75 | if skip is not None: 76 | skip = spec_utils.crop_center(skip, x) 77 | x = torch.cat([x, skip], dim=1) 78 | h = self.conv(x) 79 | 80 | if self.dropout is not None: 81 | h = self.dropout(h) 82 | 83 | return h 84 | 85 | 86 | class ASPPModule(nn.Module): 87 | def __init__(self, nin, nout, dilations=(4, 8, 16), activ=nn.ReLU): 88 | super(ASPPModule, self).__init__() 89 | self.conv1 = nn.Sequential( 90 | nn.AdaptiveAvgPool2d((1, None)), 91 | Conv2DBNActiv(nin, nin, 1, 1, 0, activ=activ), 92 | ) 93 | self.conv2 = Conv2DBNActiv(nin, nin, 1, 1, 0, activ=activ) 94 | self.conv3 = SeperableConv2DBNActiv( 95 | nin, nin, 3, 1, dilations[0], dilations[0], activ=activ 96 | ) 97 | self.conv4 = SeperableConv2DBNActiv( 98 | nin, nin, 3, 1, dilations[1], dilations[1], activ=activ 99 | ) 100 | self.conv5 = SeperableConv2DBNActiv( 101 | nin, nin, 3, 1, dilations[2], dilations[2], activ=activ 102 | ) 103 | self.bottleneck = nn.Sequential( 104 | Conv2DBNActiv(nin * 5, nout, 1, 1, 0, activ=activ), nn.Dropout2d(0.1) 105 | ) 106 | 107 | def forward(self, x): 108 | _, _, h, w = x.size() 109 | feat1 = F.interpolate( 110 | self.conv1(x), size=(h, w), mode="bilinear", align_corners=True 111 | ) 112 | feat2 = self.conv2(x) 113 | feat3 = self.conv3(x) 114 | feat4 = self.conv4(x) 115 | feat5 = self.conv5(x) 116 | out = torch.cat((feat1, feat2, feat3, feat4, feat5), dim=1) 117 | bottle = self.bottleneck(out) 118 | return bottle 119 | -------------------------------------------------------------------------------- /infer/lib/train/mel_processing.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.utils.data 3 | from librosa.filters import mel as librosa_mel_fn 4 | import logging 5 | 6 | logger = logging.getLogger(__name__) 7 | 8 | MAX_WAV_VALUE = 32768.0 9 | 10 | 11 | def dynamic_range_compression_torch(x, C=1, clip_val=1e-5): 12 | """ 13 | PARAMS 14 | ------ 15 | C: compression factor 16 | """ 17 | return torch.log(torch.clamp(x, min=clip_val) * C) 18 | 19 | 20 | def dynamic_range_decompression_torch(x, C=1): 21 | """ 22 | PARAMS 23 | ------ 24 | C: compression factor used to compress 25 | """ 26 | return torch.exp(x) / C 27 | 28 | 29 | def spectral_normalize_torch(magnitudes): 30 | return dynamic_range_compression_torch(magnitudes) 31 | 32 | 33 | def spectral_de_normalize_torch(magnitudes): 34 | return dynamic_range_decompression_torch(magnitudes) 35 | 36 | 37 | # Reusable banks 38 | mel_basis = {} 39 | hann_window = {} 40 | 41 | 42 | def spectrogram_torch(y, n_fft, sampling_rate, hop_size, win_size, center=False): 43 | """Convert waveform into Linear-frequency Linear-amplitude spectrogram. 44 | 45 | Args: 46 | y :: (B, T) - Audio waveforms 47 | n_fft 48 | sampling_rate 49 | hop_size 50 | win_size 51 | center 52 | Returns: 53 | :: (B, Freq, Frame) - Linear-frequency Linear-amplitude spectrogram 54 | """ 55 | 56 | # Window - Cache if needed 57 | global hann_window 58 | dtype_device = str(y.dtype) + "_" + str(y.device) 59 | wnsize_dtype_device = str(win_size) + "_" + dtype_device 60 | if wnsize_dtype_device not in hann_window: 61 | hann_window[wnsize_dtype_device] = torch.hann_window(win_size).to( 62 | dtype=y.dtype, device=y.device 63 | ) 64 | 65 | # Padding 66 | y = torch.nn.functional.pad( 67 | y.unsqueeze(1), 68 | (int((n_fft - hop_size) / 2), int((n_fft - hop_size) / 2)), 69 | mode="reflect", 70 | ) 71 | y = y.squeeze(1) 72 | 73 | # Complex Spectrogram :: (B, T) -> (B, Freq, Frame, RealComplex=2) 74 | spec = torch.stft( 75 | y, 76 | n_fft, 77 | hop_length=hop_size, 78 | win_length=win_size, 79 | window=hann_window[wnsize_dtype_device], 80 | center=center, 81 | pad_mode="reflect", 82 | normalized=False, 83 | onesided=True, 84 | return_complex=True, 85 | ) 86 | 87 | # Linear-frequency Linear-amplitude spectrogram :: (B, Freq, Frame, RealComplex=2) -> (B, Freq, Frame) 88 | spec = torch.sqrt(spec.real.pow(2) + spec.imag.pow(2) + 1e-6) 89 | return spec 90 | 91 | 92 | def spec_to_mel_torch(spec, n_fft, num_mels, sampling_rate, fmin, fmax): 93 | # MelBasis - Cache if needed 94 | global mel_basis 95 | dtype_device = str(spec.dtype) + "_" + str(spec.device) 96 | fmax_dtype_device = str(fmax) + "_" + dtype_device 97 | if fmax_dtype_device not in mel_basis: 98 | mel = librosa_mel_fn( 99 | sr=sampling_rate, n_fft=n_fft, n_mels=num_mels, fmin=fmin, fmax=fmax 100 | ) 101 | mel_basis[fmax_dtype_device] = torch.from_numpy(mel).to( 102 | dtype=spec.dtype, device=spec.device 103 | ) 104 | 105 | # Mel-frequency Log-amplitude spectrogram :: (B, Freq=num_mels, Frame) 106 | melspec = torch.matmul(mel_basis[fmax_dtype_device], spec) 107 | melspec = spectral_normalize_torch(melspec) 108 | return melspec 109 | 110 | 111 | def mel_spectrogram_torch( 112 | y, n_fft, num_mels, sampling_rate, hop_size, win_size, fmin, fmax, center=False 113 | ): 114 | """Convert waveform into Mel-frequency Log-amplitude spectrogram. 115 | 116 | Args: 117 | y :: (B, T) - Waveforms 118 | Returns: 119 | melspec :: (B, Freq, Frame) - Mel-frequency Log-amplitude spectrogram 120 | """ 121 | # Linear-frequency Linear-amplitude spectrogram :: (B, T) -> (B, Freq, Frame) 122 | spec = spectrogram_torch(y, n_fft, sampling_rate, hop_size, win_size, center) 123 | 124 | # Mel-frequency Log-amplitude spectrogram :: (B, Freq, Frame) -> (B, Freq=num_mels, Frame) 125 | melspec = spec_to_mel_torch(spec, n_fft, num_mels, sampling_rate, fmin, fmax) 126 | 127 | return melspec 128 | -------------------------------------------------------------------------------- /infer/modules/uvr5/modules.py: -------------------------------------------------------------------------------- 1 | import os 2 | import traceback 3 | import logging 4 | 5 | logger = logging.getLogger(__name__) 6 | 7 | import ffmpeg 8 | import torch 9 | 10 | from configs.config import Config 11 | from infer.modules.uvr5.mdxnet import MDXNetDereverb 12 | from infer.modules.uvr5.vr import AudioPre, AudioPreDeEcho 13 | 14 | config = Config() 15 | 16 | 17 | def uvr(model_name, inp_root, save_root_vocal, paths, save_root_ins, agg, format0): 18 | infos = [] 19 | try: 20 | inp_root = inp_root.strip(" ").strip('"').strip("\n").strip('"').strip(" ") 21 | save_root_vocal = ( 22 | save_root_vocal.strip(" ").strip('"').strip("\n").strip('"').strip(" ") 23 | ) 24 | save_root_ins = ( 25 | save_root_ins.strip(" ").strip('"').strip("\n").strip('"').strip(" ") 26 | ) 27 | if model_name == "onnx_dereverb_By_FoxJoy": 28 | pre_fun = MDXNetDereverb(15, config.device) 29 | else: 30 | func = AudioPre if "DeEcho" not in model_name else AudioPreDeEcho 31 | pre_fun = func( 32 | agg=int(agg), 33 | model_path=os.path.join( 34 | os.getenv("weight_uvr5_root"), model_name + ".pth" 35 | ), 36 | device=config.device, 37 | is_half=config.is_half, 38 | ) 39 | is_hp3 = "HP3" in model_name 40 | if inp_root != "": 41 | paths = [os.path.join(inp_root, name) for name in os.listdir(inp_root)] 42 | else: 43 | paths = [path.name for path in paths] 44 | for path in paths: 45 | inp_path = os.path.join(inp_root, path) 46 | need_reformat = 1 47 | done = 0 48 | try: 49 | info = ffmpeg.probe(inp_path, cmd="ffprobe") 50 | if ( 51 | info["streams"][0]["channels"] == 2 52 | and info["streams"][0]["sample_rate"] == "44100" 53 | ): 54 | need_reformat = 0 55 | pre_fun._path_audio_( 56 | inp_path, save_root_ins, save_root_vocal, format0, is_hp3=is_hp3 57 | ) 58 | done = 1 59 | except: 60 | need_reformat = 1 61 | traceback.print_exc() 62 | if need_reformat == 1: 63 | tmp_path = "%s/%s.reformatted.wav" % ( 64 | os.path.join(os.environ["TEMP"]), 65 | os.path.basename(inp_path), 66 | ) 67 | os.system( 68 | "ffmpeg -i %s -vn -acodec pcm_s16le -ac 2 -ar 44100 %s -y" 69 | % (inp_path, tmp_path) 70 | ) 71 | inp_path = tmp_path 72 | try: 73 | if done == 0: 74 | pre_fun._path_audio_( 75 | inp_path, save_root_ins, save_root_vocal, format0 76 | ) 77 | infos.append("%s->Success" % (os.path.basename(inp_path))) 78 | yield "\n".join(infos) 79 | except: 80 | try: 81 | if done == 0: 82 | pre_fun._path_audio_( 83 | inp_path, save_root_ins, save_root_vocal, format0 84 | ) 85 | infos.append("%s->Success" % (os.path.basename(inp_path))) 86 | yield "\n".join(infos) 87 | except: 88 | infos.append( 89 | "%s->%s" % (os.path.basename(inp_path), traceback.format_exc()) 90 | ) 91 | yield "\n".join(infos) 92 | except: 93 | infos.append(traceback.format_exc()) 94 | yield "\n".join(infos) 95 | finally: 96 | try: 97 | if model_name == "onnx_dereverb_By_FoxJoy": 98 | del pre_fun.pred.model 99 | del pre_fun.pred.model_ 100 | else: 101 | del pre_fun.model 102 | del pre_fun 103 | except: 104 | traceback.print_exc() 105 | if torch.cuda.is_available(): 106 | torch.cuda.empty_cache() 107 | logger.info("Executed torch.cuda.empty_cache()") 108 | yield "\n".join(infos) 109 | -------------------------------------------------------------------------------- /infer/lib/uvr5_pack/lib_v5/nets.py: -------------------------------------------------------------------------------- 1 | import layers 2 | import torch 3 | import torch.nn.functional as F 4 | from torch import nn 5 | 6 | from . import spec_utils 7 | 8 | 9 | class BaseASPPNet(nn.Module): 10 | def __init__(self, nin, ch, dilations=(4, 8, 16)): 11 | super(BaseASPPNet, self).__init__() 12 | self.enc1 = layers.Encoder(nin, ch, 3, 2, 1) 13 | self.enc2 = layers.Encoder(ch, ch * 2, 3, 2, 1) 14 | self.enc3 = layers.Encoder(ch * 2, ch * 4, 3, 2, 1) 15 | self.enc4 = layers.Encoder(ch * 4, ch * 8, 3, 2, 1) 16 | 17 | self.aspp = layers.ASPPModule(ch * 8, ch * 16, dilations) 18 | 19 | self.dec4 = layers.Decoder(ch * (8 + 16), ch * 8, 3, 1, 1) 20 | self.dec3 = layers.Decoder(ch * (4 + 8), ch * 4, 3, 1, 1) 21 | self.dec2 = layers.Decoder(ch * (2 + 4), ch * 2, 3, 1, 1) 22 | self.dec1 = layers.Decoder(ch * (1 + 2), ch, 3, 1, 1) 23 | 24 | def __call__(self, x): 25 | h, e1 = self.enc1(x) 26 | h, e2 = self.enc2(h) 27 | h, e3 = self.enc3(h) 28 | h, e4 = self.enc4(h) 29 | 30 | h = self.aspp(h) 31 | 32 | h = self.dec4(h, e4) 33 | h = self.dec3(h, e3) 34 | h = self.dec2(h, e2) 35 | h = self.dec1(h, e1) 36 | 37 | return h 38 | 39 | 40 | class CascadedASPPNet(nn.Module): 41 | def __init__(self, n_fft): 42 | super(CascadedASPPNet, self).__init__() 43 | self.stg1_low_band_net = BaseASPPNet(2, 16) 44 | self.stg1_high_band_net = BaseASPPNet(2, 16) 45 | 46 | self.stg2_bridge = layers.Conv2DBNActiv(18, 8, 1, 1, 0) 47 | self.stg2_full_band_net = BaseASPPNet(8, 16) 48 | 49 | self.stg3_bridge = layers.Conv2DBNActiv(34, 16, 1, 1, 0) 50 | self.stg3_full_band_net = BaseASPPNet(16, 32) 51 | 52 | self.out = nn.Conv2d(32, 2, 1, bias=False) 53 | self.aux1_out = nn.Conv2d(16, 2, 1, bias=False) 54 | self.aux2_out = nn.Conv2d(16, 2, 1, bias=False) 55 | 56 | self.max_bin = n_fft // 2 57 | self.output_bin = n_fft // 2 + 1 58 | 59 | self.offset = 128 60 | 61 | def forward(self, x, aggressiveness=None): 62 | mix = x.detach() 63 | x = x.clone() 64 | 65 | x = x[:, :, : self.max_bin] 66 | 67 | bandw = x.size()[2] // 2 68 | aux1 = torch.cat( 69 | [ 70 | self.stg1_low_band_net(x[:, :, :bandw]), 71 | self.stg1_high_band_net(x[:, :, bandw:]), 72 | ], 73 | dim=2, 74 | ) 75 | 76 | h = torch.cat([x, aux1], dim=1) 77 | aux2 = self.stg2_full_band_net(self.stg2_bridge(h)) 78 | 79 | h = torch.cat([x, aux1, aux2], dim=1) 80 | h = self.stg3_full_band_net(self.stg3_bridge(h)) 81 | 82 | mask = torch.sigmoid(self.out(h)) 83 | mask = F.pad( 84 | input=mask, 85 | pad=(0, 0, 0, self.output_bin - mask.size()[2]), 86 | mode="replicate", 87 | ) 88 | 89 | if self.training: 90 | aux1 = torch.sigmoid(self.aux1_out(aux1)) 91 | aux1 = F.pad( 92 | input=aux1, 93 | pad=(0, 0, 0, self.output_bin - aux1.size()[2]), 94 | mode="replicate", 95 | ) 96 | aux2 = torch.sigmoid(self.aux2_out(aux2)) 97 | aux2 = F.pad( 98 | input=aux2, 99 | pad=(0, 0, 0, self.output_bin - aux2.size()[2]), 100 | mode="replicate", 101 | ) 102 | return mask * mix, aux1 * mix, aux2 * mix 103 | else: 104 | if aggressiveness: 105 | mask[:, :, : aggressiveness["split_bin"]] = torch.pow( 106 | mask[:, :, : aggressiveness["split_bin"]], 107 | 1 + aggressiveness["value"] / 3, 108 | ) 109 | mask[:, :, aggressiveness["split_bin"] :] = torch.pow( 110 | mask[:, :, aggressiveness["split_bin"] :], 111 | 1 + aggressiveness["value"], 112 | ) 113 | 114 | return mask * mix 115 | 116 | def predict(self, x_mag, aggressiveness=None): 117 | h = self.forward(x_mag, aggressiveness) 118 | 119 | if self.offset > 0: 120 | h = h[:, :, :, self.offset : -self.offset] 121 | assert h.size()[3] > 0 122 | 123 | return h 124 | -------------------------------------------------------------------------------- /infer/lib/uvr5_pack/lib_v5/nets_123812KB.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn.functional as F 3 | from torch import nn 4 | 5 | from . import layers_123821KB as layers 6 | 7 | 8 | class BaseASPPNet(nn.Module): 9 | def __init__(self, nin, ch, dilations=(4, 8, 16)): 10 | super(BaseASPPNet, self).__init__() 11 | self.enc1 = layers.Encoder(nin, ch, 3, 2, 1) 12 | self.enc2 = layers.Encoder(ch, ch * 2, 3, 2, 1) 13 | self.enc3 = layers.Encoder(ch * 2, ch * 4, 3, 2, 1) 14 | self.enc4 = layers.Encoder(ch * 4, ch * 8, 3, 2, 1) 15 | 16 | self.aspp = layers.ASPPModule(ch * 8, ch * 16, dilations) 17 | 18 | self.dec4 = layers.Decoder(ch * (8 + 16), ch * 8, 3, 1, 1) 19 | self.dec3 = layers.Decoder(ch * (4 + 8), ch * 4, 3, 1, 1) 20 | self.dec2 = layers.Decoder(ch * (2 + 4), ch * 2, 3, 1, 1) 21 | self.dec1 = layers.Decoder(ch * (1 + 2), ch, 3, 1, 1) 22 | 23 | def __call__(self, x): 24 | h, e1 = self.enc1(x) 25 | h, e2 = self.enc2(h) 26 | h, e3 = self.enc3(h) 27 | h, e4 = self.enc4(h) 28 | 29 | h = self.aspp(h) 30 | 31 | h = self.dec4(h, e4) 32 | h = self.dec3(h, e3) 33 | h = self.dec2(h, e2) 34 | h = self.dec1(h, e1) 35 | 36 | return h 37 | 38 | 39 | class CascadedASPPNet(nn.Module): 40 | def __init__(self, n_fft): 41 | super(CascadedASPPNet, self).__init__() 42 | self.stg1_low_band_net = BaseASPPNet(2, 32) 43 | self.stg1_high_band_net = BaseASPPNet(2, 32) 44 | 45 | self.stg2_bridge = layers.Conv2DBNActiv(34, 16, 1, 1, 0) 46 | self.stg2_full_band_net = BaseASPPNet(16, 32) 47 | 48 | self.stg3_bridge = layers.Conv2DBNActiv(66, 32, 1, 1, 0) 49 | self.stg3_full_band_net = BaseASPPNet(32, 64) 50 | 51 | self.out = nn.Conv2d(64, 2, 1, bias=False) 52 | self.aux1_out = nn.Conv2d(32, 2, 1, bias=False) 53 | self.aux2_out = nn.Conv2d(32, 2, 1, bias=False) 54 | 55 | self.max_bin = n_fft // 2 56 | self.output_bin = n_fft // 2 + 1 57 | 58 | self.offset = 128 59 | 60 | def forward(self, x, aggressiveness=None): 61 | mix = x.detach() 62 | x = x.clone() 63 | 64 | x = x[:, :, : self.max_bin] 65 | 66 | bandw = x.size()[2] // 2 67 | aux1 = torch.cat( 68 | [ 69 | self.stg1_low_band_net(x[:, :, :bandw]), 70 | self.stg1_high_band_net(x[:, :, bandw:]), 71 | ], 72 | dim=2, 73 | ) 74 | 75 | h = torch.cat([x, aux1], dim=1) 76 | aux2 = self.stg2_full_band_net(self.stg2_bridge(h)) 77 | 78 | h = torch.cat([x, aux1, aux2], dim=1) 79 | h = self.stg3_full_band_net(self.stg3_bridge(h)) 80 | 81 | mask = torch.sigmoid(self.out(h)) 82 | mask = F.pad( 83 | input=mask, 84 | pad=(0, 0, 0, self.output_bin - mask.size()[2]), 85 | mode="replicate", 86 | ) 87 | 88 | if self.training: 89 | aux1 = torch.sigmoid(self.aux1_out(aux1)) 90 | aux1 = F.pad( 91 | input=aux1, 92 | pad=(0, 0, 0, self.output_bin - aux1.size()[2]), 93 | mode="replicate", 94 | ) 95 | aux2 = torch.sigmoid(self.aux2_out(aux2)) 96 | aux2 = F.pad( 97 | input=aux2, 98 | pad=(0, 0, 0, self.output_bin - aux2.size()[2]), 99 | mode="replicate", 100 | ) 101 | return mask * mix, aux1 * mix, aux2 * mix 102 | else: 103 | if aggressiveness: 104 | mask[:, :, : aggressiveness["split_bin"]] = torch.pow( 105 | mask[:, :, : aggressiveness["split_bin"]], 106 | 1 + aggressiveness["value"] / 3, 107 | ) 108 | mask[:, :, aggressiveness["split_bin"] :] = torch.pow( 109 | mask[:, :, aggressiveness["split_bin"] :], 110 | 1 + aggressiveness["value"], 111 | ) 112 | 113 | return mask * mix 114 | 115 | def predict(self, x_mag, aggressiveness=None): 116 | h = self.forward(x_mag, aggressiveness) 117 | 118 | if self.offset > 0: 119 | h = h[:, :, :, self.offset : -self.offset] 120 | assert h.size()[3] > 0 121 | 122 | return h 123 | -------------------------------------------------------------------------------- /infer/lib/uvr5_pack/lib_v5/nets_123821KB.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn.functional as F 3 | from torch import nn 4 | 5 | from . import layers_123821KB as layers 6 | 7 | 8 | class BaseASPPNet(nn.Module): 9 | def __init__(self, nin, ch, dilations=(4, 8, 16)): 10 | super(BaseASPPNet, self).__init__() 11 | self.enc1 = layers.Encoder(nin, ch, 3, 2, 1) 12 | self.enc2 = layers.Encoder(ch, ch * 2, 3, 2, 1) 13 | self.enc3 = layers.Encoder(ch * 2, ch * 4, 3, 2, 1) 14 | self.enc4 = layers.Encoder(ch * 4, ch * 8, 3, 2, 1) 15 | 16 | self.aspp = layers.ASPPModule(ch * 8, ch * 16, dilations) 17 | 18 | self.dec4 = layers.Decoder(ch * (8 + 16), ch * 8, 3, 1, 1) 19 | self.dec3 = layers.Decoder(ch * (4 + 8), ch * 4, 3, 1, 1) 20 | self.dec2 = layers.Decoder(ch * (2 + 4), ch * 2, 3, 1, 1) 21 | self.dec1 = layers.Decoder(ch * (1 + 2), ch, 3, 1, 1) 22 | 23 | def __call__(self, x): 24 | h, e1 = self.enc1(x) 25 | h, e2 = self.enc2(h) 26 | h, e3 = self.enc3(h) 27 | h, e4 = self.enc4(h) 28 | 29 | h = self.aspp(h) 30 | 31 | h = self.dec4(h, e4) 32 | h = self.dec3(h, e3) 33 | h = self.dec2(h, e2) 34 | h = self.dec1(h, e1) 35 | 36 | return h 37 | 38 | 39 | class CascadedASPPNet(nn.Module): 40 | def __init__(self, n_fft): 41 | super(CascadedASPPNet, self).__init__() 42 | self.stg1_low_band_net = BaseASPPNet(2, 32) 43 | self.stg1_high_band_net = BaseASPPNet(2, 32) 44 | 45 | self.stg2_bridge = layers.Conv2DBNActiv(34, 16, 1, 1, 0) 46 | self.stg2_full_band_net = BaseASPPNet(16, 32) 47 | 48 | self.stg3_bridge = layers.Conv2DBNActiv(66, 32, 1, 1, 0) 49 | self.stg3_full_band_net = BaseASPPNet(32, 64) 50 | 51 | self.out = nn.Conv2d(64, 2, 1, bias=False) 52 | self.aux1_out = nn.Conv2d(32, 2, 1, bias=False) 53 | self.aux2_out = nn.Conv2d(32, 2, 1, bias=False) 54 | 55 | self.max_bin = n_fft // 2 56 | self.output_bin = n_fft // 2 + 1 57 | 58 | self.offset = 128 59 | 60 | def forward(self, x, aggressiveness=None): 61 | mix = x.detach() 62 | x = x.clone() 63 | 64 | x = x[:, :, : self.max_bin] 65 | 66 | bandw = x.size()[2] // 2 67 | aux1 = torch.cat( 68 | [ 69 | self.stg1_low_band_net(x[:, :, :bandw]), 70 | self.stg1_high_band_net(x[:, :, bandw:]), 71 | ], 72 | dim=2, 73 | ) 74 | 75 | h = torch.cat([x, aux1], dim=1) 76 | aux2 = self.stg2_full_band_net(self.stg2_bridge(h)) 77 | 78 | h = torch.cat([x, aux1, aux2], dim=1) 79 | h = self.stg3_full_band_net(self.stg3_bridge(h)) 80 | 81 | mask = torch.sigmoid(self.out(h)) 82 | mask = F.pad( 83 | input=mask, 84 | pad=(0, 0, 0, self.output_bin - mask.size()[2]), 85 | mode="replicate", 86 | ) 87 | 88 | if self.training: 89 | aux1 = torch.sigmoid(self.aux1_out(aux1)) 90 | aux1 = F.pad( 91 | input=aux1, 92 | pad=(0, 0, 0, self.output_bin - aux1.size()[2]), 93 | mode="replicate", 94 | ) 95 | aux2 = torch.sigmoid(self.aux2_out(aux2)) 96 | aux2 = F.pad( 97 | input=aux2, 98 | pad=(0, 0, 0, self.output_bin - aux2.size()[2]), 99 | mode="replicate", 100 | ) 101 | return mask * mix, aux1 * mix, aux2 * mix 102 | else: 103 | if aggressiveness: 104 | mask[:, :, : aggressiveness["split_bin"]] = torch.pow( 105 | mask[:, :, : aggressiveness["split_bin"]], 106 | 1 + aggressiveness["value"] / 3, 107 | ) 108 | mask[:, :, aggressiveness["split_bin"] :] = torch.pow( 109 | mask[:, :, aggressiveness["split_bin"] :], 110 | 1 + aggressiveness["value"], 111 | ) 112 | 113 | return mask * mix 114 | 115 | def predict(self, x_mag, aggressiveness=None): 116 | h = self.forward(x_mag, aggressiveness) 117 | 118 | if self.offset > 0: 119 | h = h[:, :, :, self.offset : -self.offset] 120 | assert h.size()[3] > 0 121 | 122 | return h 123 | -------------------------------------------------------------------------------- /infer/lib/uvr5_pack/lib_v5/nets_33966KB.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn.functional as F 3 | from torch import nn 4 | 5 | from . import layers_33966KB as layers 6 | 7 | 8 | class BaseASPPNet(nn.Module): 9 | def __init__(self, nin, ch, dilations=(4, 8, 16, 32)): 10 | super(BaseASPPNet, self).__init__() 11 | self.enc1 = layers.Encoder(nin, ch, 3, 2, 1) 12 | self.enc2 = layers.Encoder(ch, ch * 2, 3, 2, 1) 13 | self.enc3 = layers.Encoder(ch * 2, ch * 4, 3, 2, 1) 14 | self.enc4 = layers.Encoder(ch * 4, ch * 8, 3, 2, 1) 15 | 16 | self.aspp = layers.ASPPModule(ch * 8, ch * 16, dilations) 17 | 18 | self.dec4 = layers.Decoder(ch * (8 + 16), ch * 8, 3, 1, 1) 19 | self.dec3 = layers.Decoder(ch * (4 + 8), ch * 4, 3, 1, 1) 20 | self.dec2 = layers.Decoder(ch * (2 + 4), ch * 2, 3, 1, 1) 21 | self.dec1 = layers.Decoder(ch * (1 + 2), ch, 3, 1, 1) 22 | 23 | def __call__(self, x): 24 | h, e1 = self.enc1(x) 25 | h, e2 = self.enc2(h) 26 | h, e3 = self.enc3(h) 27 | h, e4 = self.enc4(h) 28 | 29 | h = self.aspp(h) 30 | 31 | h = self.dec4(h, e4) 32 | h = self.dec3(h, e3) 33 | h = self.dec2(h, e2) 34 | h = self.dec1(h, e1) 35 | 36 | return h 37 | 38 | 39 | class CascadedASPPNet(nn.Module): 40 | def __init__(self, n_fft): 41 | super(CascadedASPPNet, self).__init__() 42 | self.stg1_low_band_net = BaseASPPNet(2, 16) 43 | self.stg1_high_band_net = BaseASPPNet(2, 16) 44 | 45 | self.stg2_bridge = layers.Conv2DBNActiv(18, 8, 1, 1, 0) 46 | self.stg2_full_band_net = BaseASPPNet(8, 16) 47 | 48 | self.stg3_bridge = layers.Conv2DBNActiv(34, 16, 1, 1, 0) 49 | self.stg3_full_band_net = BaseASPPNet(16, 32) 50 | 51 | self.out = nn.Conv2d(32, 2, 1, bias=False) 52 | self.aux1_out = nn.Conv2d(16, 2, 1, bias=False) 53 | self.aux2_out = nn.Conv2d(16, 2, 1, bias=False) 54 | 55 | self.max_bin = n_fft // 2 56 | self.output_bin = n_fft // 2 + 1 57 | 58 | self.offset = 128 59 | 60 | def forward(self, x, aggressiveness=None): 61 | mix = x.detach() 62 | x = x.clone() 63 | 64 | x = x[:, :, : self.max_bin] 65 | 66 | bandw = x.size()[2] // 2 67 | aux1 = torch.cat( 68 | [ 69 | self.stg1_low_band_net(x[:, :, :bandw]), 70 | self.stg1_high_band_net(x[:, :, bandw:]), 71 | ], 72 | dim=2, 73 | ) 74 | 75 | h = torch.cat([x, aux1], dim=1) 76 | aux2 = self.stg2_full_band_net(self.stg2_bridge(h)) 77 | 78 | h = torch.cat([x, aux1, aux2], dim=1) 79 | h = self.stg3_full_band_net(self.stg3_bridge(h)) 80 | 81 | mask = torch.sigmoid(self.out(h)) 82 | mask = F.pad( 83 | input=mask, 84 | pad=(0, 0, 0, self.output_bin - mask.size()[2]), 85 | mode="replicate", 86 | ) 87 | 88 | if self.training: 89 | aux1 = torch.sigmoid(self.aux1_out(aux1)) 90 | aux1 = F.pad( 91 | input=aux1, 92 | pad=(0, 0, 0, self.output_bin - aux1.size()[2]), 93 | mode="replicate", 94 | ) 95 | aux2 = torch.sigmoid(self.aux2_out(aux2)) 96 | aux2 = F.pad( 97 | input=aux2, 98 | pad=(0, 0, 0, self.output_bin - aux2.size()[2]), 99 | mode="replicate", 100 | ) 101 | return mask * mix, aux1 * mix, aux2 * mix 102 | else: 103 | if aggressiveness: 104 | mask[:, :, : aggressiveness["split_bin"]] = torch.pow( 105 | mask[:, :, : aggressiveness["split_bin"]], 106 | 1 + aggressiveness["value"] / 3, 107 | ) 108 | mask[:, :, aggressiveness["split_bin"] :] = torch.pow( 109 | mask[:, :, aggressiveness["split_bin"] :], 110 | 1 + aggressiveness["value"], 111 | ) 112 | 113 | return mask * mix 114 | 115 | def predict(self, x_mag, aggressiveness=None): 116 | h = self.forward(x_mag, aggressiveness) 117 | 118 | if self.offset > 0: 119 | h = h[:, :, :, self.offset : -self.offset] 120 | assert h.size()[3] > 0 121 | 122 | return h 123 | -------------------------------------------------------------------------------- /infer/lib/uvr5_pack/lib_v5/nets_61968KB.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn.functional as F 3 | from torch import nn 4 | 5 | from . import layers_123821KB as layers 6 | 7 | 8 | class BaseASPPNet(nn.Module): 9 | def __init__(self, nin, ch, dilations=(4, 8, 16)): 10 | super(BaseASPPNet, self).__init__() 11 | self.enc1 = layers.Encoder(nin, ch, 3, 2, 1) 12 | self.enc2 = layers.Encoder(ch, ch * 2, 3, 2, 1) 13 | self.enc3 = layers.Encoder(ch * 2, ch * 4, 3, 2, 1) 14 | self.enc4 = layers.Encoder(ch * 4, ch * 8, 3, 2, 1) 15 | 16 | self.aspp = layers.ASPPModule(ch * 8, ch * 16, dilations) 17 | 18 | self.dec4 = layers.Decoder(ch * (8 + 16), ch * 8, 3, 1, 1) 19 | self.dec3 = layers.Decoder(ch * (4 + 8), ch * 4, 3, 1, 1) 20 | self.dec2 = layers.Decoder(ch * (2 + 4), ch * 2, 3, 1, 1) 21 | self.dec1 = layers.Decoder(ch * (1 + 2), ch, 3, 1, 1) 22 | 23 | def __call__(self, x): 24 | h, e1 = self.enc1(x) 25 | h, e2 = self.enc2(h) 26 | h, e3 = self.enc3(h) 27 | h, e4 = self.enc4(h) 28 | 29 | h = self.aspp(h) 30 | 31 | h = self.dec4(h, e4) 32 | h = self.dec3(h, e3) 33 | h = self.dec2(h, e2) 34 | h = self.dec1(h, e1) 35 | 36 | return h 37 | 38 | 39 | class CascadedASPPNet(nn.Module): 40 | def __init__(self, n_fft): 41 | super(CascadedASPPNet, self).__init__() 42 | self.stg1_low_band_net = BaseASPPNet(2, 32) 43 | self.stg1_high_band_net = BaseASPPNet(2, 32) 44 | 45 | self.stg2_bridge = layers.Conv2DBNActiv(34, 16, 1, 1, 0) 46 | self.stg2_full_band_net = BaseASPPNet(16, 32) 47 | 48 | self.stg3_bridge = layers.Conv2DBNActiv(66, 32, 1, 1, 0) 49 | self.stg3_full_band_net = BaseASPPNet(32, 64) 50 | 51 | self.out = nn.Conv2d(64, 2, 1, bias=False) 52 | self.aux1_out = nn.Conv2d(32, 2, 1, bias=False) 53 | self.aux2_out = nn.Conv2d(32, 2, 1, bias=False) 54 | 55 | self.max_bin = n_fft // 2 56 | self.output_bin = n_fft // 2 + 1 57 | 58 | self.offset = 128 59 | 60 | def forward(self, x, aggressiveness=None): 61 | mix = x.detach() 62 | x = x.clone() 63 | 64 | x = x[:, :, : self.max_bin] 65 | 66 | bandw = x.size()[2] // 2 67 | aux1 = torch.cat( 68 | [ 69 | self.stg1_low_band_net(x[:, :, :bandw]), 70 | self.stg1_high_band_net(x[:, :, bandw:]), 71 | ], 72 | dim=2, 73 | ) 74 | 75 | h = torch.cat([x, aux1], dim=1) 76 | aux2 = self.stg2_full_band_net(self.stg2_bridge(h)) 77 | 78 | h = torch.cat([x, aux1, aux2], dim=1) 79 | h = self.stg3_full_band_net(self.stg3_bridge(h)) 80 | 81 | mask = torch.sigmoid(self.out(h)) 82 | mask = F.pad( 83 | input=mask, 84 | pad=(0, 0, 0, self.output_bin - mask.size()[2]), 85 | mode="replicate", 86 | ) 87 | 88 | if self.training: 89 | aux1 = torch.sigmoid(self.aux1_out(aux1)) 90 | aux1 = F.pad( 91 | input=aux1, 92 | pad=(0, 0, 0, self.output_bin - aux1.size()[2]), 93 | mode="replicate", 94 | ) 95 | aux2 = torch.sigmoid(self.aux2_out(aux2)) 96 | aux2 = F.pad( 97 | input=aux2, 98 | pad=(0, 0, 0, self.output_bin - aux2.size()[2]), 99 | mode="replicate", 100 | ) 101 | return mask * mix, aux1 * mix, aux2 * mix 102 | else: 103 | if aggressiveness: 104 | mask[:, :, : aggressiveness["split_bin"]] = torch.pow( 105 | mask[:, :, : aggressiveness["split_bin"]], 106 | 1 + aggressiveness["value"] / 3, 107 | ) 108 | mask[:, :, aggressiveness["split_bin"] :] = torch.pow( 109 | mask[:, :, aggressiveness["split_bin"] :], 110 | 1 + aggressiveness["value"], 111 | ) 112 | 113 | return mask * mix 114 | 115 | def predict(self, x_mag, aggressiveness=None): 116 | h = self.forward(x_mag, aggressiveness) 117 | 118 | if self.offset > 0: 119 | h = h[:, :, :, self.offset : -self.offset] 120 | assert h.size()[3] > 0 121 | 122 | return h 123 | -------------------------------------------------------------------------------- /docs/cn/faq.md: -------------------------------------------------------------------------------- 1 | ## Q1:ffmpeg error/utf8 error. 2 | 3 | 大概率不是ffmpeg问题，而是音频路径问题；
4 | ffmpeg读取路径带空格、()等特殊符号，可能出现ffmpeg error；训练集音频带中文路径，在写入filelist.txt的时候可能出现utf8 error；
5 | 6 | ## Q2:一键训练结束没有索引 7 | 8 | 显示"Training is done. The program is closed."则模型训练成功，后续紧邻的报错是假的；
9 | 10 | 一键训练结束完成没有added开头的索引文件，可能是因为训练集太大卡住了添加索引的步骤；已通过批处理add索引解决内存add索引对内存需求过大的问题。临时可尝试再次点击"训练索引"按钮。
11 | 12 | ## Q3:训练结束推理没看到训练集的音色 13 | 点刷新音色再看看，如果还没有看看训练有没有报错，控制台和webui的截图，logs/实验名下的log，都可以发给开发者看看。
14 | 15 | ## Q4:如何分享模型 16 |   rvc_root/logs/实验名 下面存储的pth不是用来分享模型用来推理的，而是为了存储实验状态供复现，以及继续训练用的。用来分享的模型应该是weights文件夹下大小为60+MB的pth文件；
17 |   后续将把weights/exp_name.pth和logs/exp_name/added_xxx.index合并打包成weights/exp_name.zip省去填写index的步骤，那么zip文件用来分享，不要分享pth文件，除非是想换机器继续训练；
18 |   如果你把logs文件夹下的几百MB的pth文件复制/分享到weights文件夹下强行用于推理，可能会出现f0，tgt_sr等各种key不存在的报错。你需要用ckpt选项卡最下面，手工或自动（本地logs下如果能找到相关信息则会自动）选择是否携带音高、目标音频采样率的选项后进行ckpt小模型提取（输入路径填G开头的那个），提取完在weights文件夹下会出现60+MB的pth文件，刷新音色后可以选择使用。
19 | 20 | ## Q5:Connection Error. 21 | 也许你关闭了控制台（黑色窗口）。
22 | 23 | ## Q6:WebUI弹出Expecting value: line 1 column 1 (char 0). 24 | 请关闭系统局域网代理/全局代理。
25 | 26 | 这个不仅是客户端的代理，也包括服务端的代理（例如你使用autodl设置了http_proxy和https_proxy学术加速，使用时也需要unset关掉）
27 | 28 | ## Q7:不用WebUI如何通过命令训练推理 29 | 训练脚本：
30 | 可先跑通WebUI，消息窗内会显示数据集处理和训练用命令行；
31 | 32 | 推理脚本：
33 | https://huggingface.co/lj1995/VoiceConversionWebUI/blob/main/myinfer.py
34 | 35 | 例子：
36 | 37 | runtime\python.exe myinfer.py 0 "E:\codes\py39\RVC-beta\todo-songs\1111.wav" "E:\codes\py39\logs\mi-test\added_IVF677_Flat_nprobe_7.index" harvest "test.wav" "weights/mi-test.pth" 0.6 cuda:0 True
38 | 39 | f0up_key=sys.argv[1]
40 | input_path=sys.argv[2]
41 | index_path=sys.argv[3]
42 | f0method=sys.argv[4]#harvest or pm
43 | opt_path=sys.argv[5]
44 | model_path=sys.argv[6]
45 | index_rate=float(sys.argv[7])
46 | device=sys.argv[8]
47 | is_half=bool(sys.argv[9])
48 | 49 | ## Q8:Cuda error/Cuda out of memory. 50 | 小概率是cuda配置问题、设备不支持；大概率是显存不够（out of memory）；
51 | 52 | 训练的话缩小batch size（如果缩小到1还不够只能更换显卡训练），推理的话酌情缩小config.py结尾的x_pad，x_query，x_center，x_max。4G以下显存（例如1060（3G）和各种2G显卡）可以直接放弃，4G显存显卡还有救。
53 | 54 | ## Q9:total_epoch调多少比较好 55 | 56 | 如果训练集音质差底噪大，20~30足够了，调太高，底模音质无法带高你的低音质训练集
57 | 如果训练集音质高底噪低时长多，可以调高，200是ok的（训练速度很快，既然你有条件准备高音质训练集，显卡想必条件也不错，肯定不在乎多一些训练时间）
58 | 59 | ## Q10:需要多少训练集时长 60 |   推荐10min至50min
61 |   保证音质高底噪低的情况下，如果有个人特色的音色统一，则多多益善
62 |   高水平的训练集（精简+音色有特色），5min至10min也是ok的，仓库作者本人就经常这么玩
63 |   也有人拿1min至2min的数据来训练并且训练成功的，但是成功经验是其他人不可复现的，不太具备参考价值。这要求训练集音色特色非常明显（比如说高频气声较明显的萝莉少女音），且音质高；
64 |   1min以下时长数据目前没见有人尝试（成功）过。不建议进行这种鬼畜行为。
65 | 66 | ## Q11:index rate干嘛用的，怎么调（科普） 67 |   如果底模和推理源的音质高于训练集的音质，他们可以带高推理结果的音质，但代价可能是音色往底模/推理源的音色靠，这种现象叫做"音色泄露"；
68 |   index rate用来削减/解决音色泄露问题。调到1，则理论上不存在推理源的音色泄露问题，但音质更倾向于训练集。如果训练集音质比推理源低，则index rate调高可能降低音质。调到0，则不具备利用检索混合来保护训练集音色的效果；
69 |   如果训练集优质时长多，可调高total_epoch，此时模型本身不太会引用推理源和底模的音色，很少存在"音色泄露"问题，此时index_rate不重要，你甚至可以不建立/分享index索引文件。
70 | 71 | ## Q11:推理怎么选gpu 72 | config.py文件里device cuda:后面选择卡号；
73 | 卡号和显卡的映射关系，在训练选项卡的显卡信息栏里能看到。
74 | 75 | ## Q12:如何推理训练中间保存的pth 76 | 通过ckpt选项卡最下面提取小模型。
77 | 78 | 79 | ## Q13:如何中断和继续训练 80 | 现阶段只能关闭WebUI控制台双击go-web.bat重启程序。网页参数也要刷新重新填写；
81 | 继续训练：相同网页参数点训练模型，就会接着上次的checkpoint继续训练。
82 | 83 | ## Q14:训练时出现文件页面/内存error 84 | 进程开太多了，内存炸了。你可能可以通过如下方式解决
85 | 1、"提取音高和处理数据使用的CPU进程数" 酌情拉低；
86 | 2、训练集音频手工切一下，不要太长。
87 | 88 | 89 | ## Q15:如何中途加数据训练 90 | 1、所有数据新建一个实验名；
91 | 2、拷贝上一次的最新的那个G和D文件（或者你想基于哪个中间ckpt训练，也可以拷贝中间的）到新实验名；下
92 | 3、一键训练新实验名，他会继续上一次的最新进度训练。
93 | 94 | ## Q16: error about llvmlite.dll 95 | 96 | OSError: Could not load shared object file: llvmlite.dll 97 | 98 | FileNotFoundError: Could not find module lib\site-packages\llvmlite\binding\llvmlite.dll (or one of its dependencies). Try using the full path with constructor syntax. 99 | 100 | win平台会报这个错，装上https://aka.ms/vs/17/release/vc_redist.x64.exe这个再重启WebUI就好了。 101 | 102 | ## Q17: RuntimeError: The expanded size of the tensor (17280) must match the existing size (0) at non-singleton dimension 1. Target sizes: [1, 17280]. Tensor sizes: [0] 103 | 104 | wavs16k文件夹下，找到文件大小显著比其他都小的一些音频文件，删掉，点击训练模型，就不会报错了，不过由于一键流程中断了你训练完模型还要点训练索引。 105 | 106 | ## Q18: RuntimeError: The size of tensor a (24) must match the size of tensor b (16) at non-singleton dimension 2 107 | 108 | 不要中途变更采样率继续训练。如果一定要变更，应更换实验名从头训练。当然你也可以把上次提取的音高和特征（0/1/2/2b folders）拷贝过去加速训练流程。 109 | -------------------------------------------------------------------------------- /infer/lib/uvr5_pack/lib_v5/nets_537227KB.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | import torch.nn.functional as F 4 | from torch import nn 5 | 6 | from . import layers_537238KB as layers 7 | 8 | 9 | class BaseASPPNet(nn.Module): 10 | def __init__(self, nin, ch, dilations=(4, 8, 16)): 11 | super(BaseASPPNet, self).__init__() 12 | self.enc1 = layers.Encoder(nin, ch, 3, 2, 1) 13 | self.enc2 = layers.Encoder(ch, ch * 2, 3, 2, 1) 14 | self.enc3 = layers.Encoder(ch * 2, ch * 4, 3, 2, 1) 15 | self.enc4 = layers.Encoder(ch * 4, ch * 8, 3, 2, 1) 16 | 17 | self.aspp = layers.ASPPModule(ch * 8, ch * 16, dilations) 18 | 19 | self.dec4 = layers.Decoder(ch * (8 + 16), ch * 8, 3, 1, 1) 20 | self.dec3 = layers.Decoder(ch * (4 + 8), ch * 4, 3, 1, 1) 21 | self.dec2 = layers.Decoder(ch * (2 + 4), ch * 2, 3, 1, 1) 22 | self.dec1 = layers.Decoder(ch * (1 + 2), ch, 3, 1, 1) 23 | 24 | def __call__(self, x): 25 | h, e1 = self.enc1(x) 26 | h, e2 = self.enc2(h) 27 | h, e3 = self.enc3(h) 28 | h, e4 = self.enc4(h) 29 | 30 | h = self.aspp(h) 31 | 32 | h = self.dec4(h, e4) 33 | h = self.dec3(h, e3) 34 | h = self.dec2(h, e2) 35 | h = self.dec1(h, e1) 36 | 37 | return h 38 | 39 | 40 | class CascadedASPPNet(nn.Module): 41 | def __init__(self, n_fft): 42 | super(CascadedASPPNet, self).__init__() 43 | self.stg1_low_band_net = BaseASPPNet(2, 64) 44 | self.stg1_high_band_net = BaseASPPNet(2, 64) 45 | 46 | self.stg2_bridge = layers.Conv2DBNActiv(66, 32, 1, 1, 0) 47 | self.stg2_full_band_net = BaseASPPNet(32, 64) 48 | 49 | self.stg3_bridge = layers.Conv2DBNActiv(130, 64, 1, 1, 0) 50 | self.stg3_full_band_net = BaseASPPNet(64, 128) 51 | 52 | self.out = nn.Conv2d(128, 2, 1, bias=False) 53 | self.aux1_out = nn.Conv2d(64, 2, 1, bias=False) 54 | self.aux2_out = nn.Conv2d(64, 2, 1, bias=False) 55 | 56 | self.max_bin = n_fft // 2 57 | self.output_bin = n_fft // 2 + 1 58 | 59 | self.offset = 128 60 | 61 | def forward(self, x, aggressiveness=None): 62 | mix = x.detach() 63 | x = x.clone() 64 | 65 | x = x[:, :, : self.max_bin] 66 | 67 | bandw = x.size()[2] // 2 68 | aux1 = torch.cat( 69 | [ 70 | self.stg1_low_band_net(x[:, :, :bandw]), 71 | self.stg1_high_band_net(x[:, :, bandw:]), 72 | ], 73 | dim=2, 74 | ) 75 | 76 | h = torch.cat([x, aux1], dim=1) 77 | aux2 = self.stg2_full_band_net(self.stg2_bridge(h)) 78 | 79 | h = torch.cat([x, aux1, aux2], dim=1) 80 | h = self.stg3_full_band_net(self.stg3_bridge(h)) 81 | 82 | mask = torch.sigmoid(self.out(h)) 83 | mask = F.pad( 84 | input=mask, 85 | pad=(0, 0, 0, self.output_bin - mask.size()[2]), 86 | mode="replicate", 87 | ) 88 | 89 | if self.training: 90 | aux1 = torch.sigmoid(self.aux1_out(aux1)) 91 | aux1 = F.pad( 92 | input=aux1, 93 | pad=(0, 0, 0, self.output_bin - aux1.size()[2]), 94 | mode="replicate", 95 | ) 96 | aux2 = torch.sigmoid(self.aux2_out(aux2)) 97 | aux2 = F.pad( 98 | input=aux2, 99 | pad=(0, 0, 0, self.output_bin - aux2.size()[2]), 100 | mode="replicate", 101 | ) 102 | return mask * mix, aux1 * mix, aux2 * mix 103 | else: 104 | if aggressiveness: 105 | mask[:, :, : aggressiveness["split_bin"]] = torch.pow( 106 | mask[:, :, : aggressiveness["split_bin"]], 107 | 1 + aggressiveness["value"] / 3, 108 | ) 109 | mask[:, :, aggressiveness["split_bin"] :] = torch.pow( 110 | mask[:, :, aggressiveness["split_bin"] :], 111 | 1 + aggressiveness["value"], 112 | ) 113 | 114 | return mask * mix 115 | 116 | def predict(self, x_mag, aggressiveness=None): 117 | h = self.forward(x_mag, aggressiveness) 118 | 119 | if self.offset > 0: 120 | h = h[:, :, :, self.offset : -self.offset] 121 | assert h.size()[3] > 0 122 | 123 | return h 124 | -------------------------------------------------------------------------------- /infer/lib/uvr5_pack/lib_v5/nets_537238KB.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import torch 3 | import torch.nn.functional as F 4 | from torch import nn 5 | 6 | from . import layers_537238KB as layers 7 | 8 | 9 | class BaseASPPNet(nn.Module): 10 | def __init__(self, nin, ch, dilations=(4, 8, 16)): 11 | super(BaseASPPNet, self).__init__() 12 | self.enc1 = layers.Encoder(nin, ch, 3, 2, 1) 13 | self.enc2 = layers.Encoder(ch, ch * 2, 3, 2, 1) 14 | self.enc3 = layers.Encoder(ch * 2, ch * 4, 3, 2, 1) 15 | self.enc4 = layers.Encoder(ch * 4, ch * 8, 3, 2, 1) 16 | 17 | self.aspp = layers.ASPPModule(ch * 8, ch * 16, dilations) 18 | 19 | self.dec4 = layers.Decoder(ch * (8 + 16), ch * 8, 3, 1, 1) 20 | self.dec3 = layers.Decoder(ch * (4 + 8), ch * 4, 3, 1, 1) 21 | self.dec2 = layers.Decoder(ch * (2 + 4), ch * 2, 3, 1, 1) 22 | self.dec1 = layers.Decoder(ch * (1 + 2), ch, 3, 1, 1) 23 | 24 | def __call__(self, x): 25 | h, e1 = self.enc1(x) 26 | h, e2 = self.enc2(h) 27 | h, e3 = self.enc3(h) 28 | h, e4 = self.enc4(h) 29 | 30 | h = self.aspp(h) 31 | 32 | h = self.dec4(h, e4) 33 | h = self.dec3(h, e3) 34 | h = self.dec2(h, e2) 35 | h = self.dec1(h, e1) 36 | 37 | return h 38 | 39 | 40 | class CascadedASPPNet(nn.Module): 41 | def __init__(self, n_fft): 42 | super(CascadedASPPNet, self).__init__() 43 | self.stg1_low_band_net = BaseASPPNet(2, 64) 44 | self.stg1_high_band_net = BaseASPPNet(2, 64) 45 | 46 | self.stg2_bridge = layers.Conv2DBNActiv(66, 32, 1, 1, 0) 47 | self.stg2_full_band_net = BaseASPPNet(32, 64) 48 | 49 | self.stg3_bridge = layers.Conv2DBNActiv(130, 64, 1, 1, 0) 50 | self.stg3_full_band_net = BaseASPPNet(64, 128) 51 | 52 | self.out = nn.Conv2d(128, 2, 1, bias=False) 53 | self.aux1_out = nn.Conv2d(64, 2, 1, bias=False) 54 | self.aux2_out = nn.Conv2d(64, 2, 1, bias=False) 55 | 56 | self.max_bin = n_fft // 2 57 | self.output_bin = n_fft // 2 + 1 58 | 59 | self.offset = 128 60 | 61 | def forward(self, x, aggressiveness=None): 62 | mix = x.detach() 63 | x = x.clone() 64 | 65 | x = x[:, :, : self.max_bin] 66 | 67 | bandw = x.size()[2] // 2 68 | aux1 = torch.cat( 69 | [ 70 | self.stg1_low_band_net(x[:, :, :bandw]), 71 | self.stg1_high_band_net(x[:, :, bandw:]), 72 | ], 73 | dim=2, 74 | ) 75 | 76 | h = torch.cat([x, aux1], dim=1) 77 | aux2 = self.stg2_full_band_net(self.stg2_bridge(h)) 78 | 79 | h = torch.cat([x, aux1, aux2], dim=1) 80 | h = self.stg3_full_band_net(self.stg3_bridge(h)) 81 | 82 | mask = torch.sigmoid(self.out(h)) 83 | mask = F.pad( 84 | input=mask, 85 | pad=(0, 0, 0, self.output_bin - mask.size()[2]), 86 | mode="replicate", 87 | ) 88 | 89 | if self.training: 90 | aux1 = torch.sigmoid(self.aux1_out(aux1)) 91 | aux1 = F.pad( 92 | input=aux1, 93 | pad=(0, 0, 0, self.output_bin - aux1.size()[2]), 94 | mode="replicate", 95 | ) 96 | aux2 = torch.sigmoid(self.aux2_out(aux2)) 97 | aux2 = F.pad( 98 | input=aux2, 99 | pad=(0, 0, 0, self.output_bin - aux2.size()[2]), 100 | mode="replicate", 101 | ) 102 | return mask * mix, aux1 * mix, aux2 * mix 103 | else: 104 | if aggressiveness: 105 | mask[:, :, : aggressiveness["split_bin"]] = torch.pow( 106 | mask[:, :, : aggressiveness["split_bin"]], 107 | 1 + aggressiveness["value"] / 3, 108 | ) 109 | mask[:, :, aggressiveness["split_bin"] :] = torch.pow( 110 | mask[:, :, aggressiveness["split_bin"] :], 111 | 1 + aggressiveness["value"], 112 | ) 113 | 114 | return mask * mix 115 | 116 | def predict(self, x_mag, aggressiveness=None): 117 | h = self.forward(x_mag, aggressiveness) 118 | 119 | if self.offset > 0: 120 | h = h[:, :, :, self.offset : -self.offset] 121 | assert h.size()[3] > 0 122 | 123 | return h 124 | -------------------------------------------------------------------------------- /infer/lib/uvr5_pack/lib_v5/layers_new.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn.functional as F 3 | from torch import nn 4 | 5 | from . import spec_utils 6 | 7 | 8 | class Conv2DBNActiv(nn.Module): 9 | def __init__(self, nin, nout, ksize=3, stride=1, pad=1, dilation=1, activ=nn.ReLU): 10 | super(Conv2DBNActiv, self).__init__() 11 | self.conv = nn.Sequential( 12 | nn.Conv2d( 13 | nin, 14 | nout, 15 | kernel_size=ksize, 16 | stride=stride, 17 | padding=pad, 18 | dilation=dilation, 19 | bias=False, 20 | ), 21 | nn.BatchNorm2d(nout), 22 | activ(), 23 | ) 24 | 25 | def __call__(self, x): 26 | return self.conv(x) 27 | 28 | 29 | class Encoder(nn.Module): 30 | def __init__(self, nin, nout, ksize=3, stride=1, pad=1, activ=nn.LeakyReLU): 31 | super(Encoder, self).__init__() 32 | self.conv1 = Conv2DBNActiv(nin, nout, ksize, stride, pad, activ=activ) 33 | self.conv2 = Conv2DBNActiv(nout, nout, ksize, 1, pad, activ=activ) 34 | 35 | def __call__(self, x): 36 | h = self.conv1(x) 37 | h = self.conv2(h) 38 | 39 | return h 40 | 41 | 42 | class Decoder(nn.Module): 43 | def __init__( 44 | self, nin, nout, ksize=3, stride=1, pad=1, activ=nn.ReLU, dropout=False 45 | ): 46 | super(Decoder, self).__init__() 47 | self.conv1 = Conv2DBNActiv(nin, nout, ksize, 1, pad, activ=activ) 48 | # self.conv2 = Conv2DBNActiv(nout, nout, ksize, 1, pad, activ=activ) 49 | self.dropout = nn.Dropout2d(0.1) if dropout else None 50 | 51 | def __call__(self, x, skip=None): 52 | x = F.interpolate(x, scale_factor=2, mode="bilinear", align_corners=True) 53 | 54 | if skip is not None: 55 | skip = spec_utils.crop_center(skip, x) 56 | x = torch.cat([x, skip], dim=1) 57 | 58 | h = self.conv1(x) 59 | # h = self.conv2(h) 60 | 61 | if self.dropout is not None: 62 | h = self.dropout(h) 63 | 64 | return h 65 | 66 | 67 | class ASPPModule(nn.Module): 68 | def __init__(self, nin, nout, dilations=(4, 8, 12), activ=nn.ReLU, dropout=False): 69 | super(ASPPModule, self).__init__() 70 | self.conv1 = nn.Sequential( 71 | nn.AdaptiveAvgPool2d((1, None)), 72 | Conv2DBNActiv(nin, nout, 1, 1, 0, activ=activ), 73 | ) 74 | self.conv2 = Conv2DBNActiv(nin, nout, 1, 1, 0, activ=activ) 75 | self.conv3 = Conv2DBNActiv( 76 | nin, nout, 3, 1, dilations[0], dilations[0], activ=activ 77 | ) 78 | self.conv4 = Conv2DBNActiv( 79 | nin, nout, 3, 1, dilations[1], dilations[1], activ=activ 80 | ) 81 | self.conv5 = Conv2DBNActiv( 82 | nin, nout, 3, 1, dilations[2], dilations[2], activ=activ 83 | ) 84 | self.bottleneck = Conv2DBNActiv(nout * 5, nout, 1, 1, 0, activ=activ) 85 | self.dropout = nn.Dropout2d(0.1) if dropout else None 86 | 87 | def forward(self, x): 88 | _, _, h, w = x.size() 89 | feat1 = F.interpolate( 90 | self.conv1(x), size=(h, w), mode="bilinear", align_corners=True 91 | ) 92 | feat2 = self.conv2(x) 93 | feat3 = self.conv3(x) 94 | feat4 = self.conv4(x) 95 | feat5 = self.conv5(x) 96 | out = torch.cat((feat1, feat2, feat3, feat4, feat5), dim=1) 97 | out = self.bottleneck(out) 98 | 99 | if self.dropout is not None: 100 | out = self.dropout(out) 101 | 102 | return out 103 | 104 | 105 | class LSTMModule(nn.Module): 106 | def __init__(self, nin_conv, nin_lstm, nout_lstm): 107 | super(LSTMModule, self).__init__() 108 | self.conv = Conv2DBNActiv(nin_conv, 1, 1, 1, 0) 109 | self.lstm = nn.LSTM( 110 | input_size=nin_lstm, hidden_size=nout_lstm // 2, bidirectional=True 111 | ) 112 | self.dense = nn.Sequential( 113 | nn.Linear(nout_lstm, nin_lstm), nn.BatchNorm1d(nin_lstm), nn.ReLU() 114 | ) 115 | 116 | def forward(self, x): 117 | N, _, nbins, nframes = x.size() 118 | h = self.conv(x)[:, 0] # N, nbins, nframes 119 | h = h.permute(2, 0, 1) # nframes, N, nbins 120 | h, _ = self.lstm(h) 121 | h = self.dense(h.reshape(-1, h.size()[-1])) # nframes * N, nbins 122 | h = h.reshape(nframes, N, 1, nbins) 123 | h = h.permute(1, 2, 3, 0) 124 | 125 | return h 126 | -------------------------------------------------------------------------------- /infer/lib/uvr5_pack/lib_v5/layers_33966KB.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn.functional as F 3 | from torch import nn 4 | 5 | from . import spec_utils 6 | 7 | 8 | class Conv2DBNActiv(nn.Module): 9 | def __init__(self, nin, nout, ksize=3, stride=1, pad=1, dilation=1, activ=nn.ReLU): 10 | super(Conv2DBNActiv, self).__init__() 11 | self.conv = nn.Sequential( 12 | nn.Conv2d( 13 | nin, 14 | nout, 15 | kernel_size=ksize, 16 | stride=stride, 17 | padding=pad, 18 | dilation=dilation, 19 | bias=False, 20 | ), 21 | nn.BatchNorm2d(nout), 22 | activ(), 23 | ) 24 | 25 | def __call__(self, x): 26 | return self.conv(x) 27 | 28 | 29 | class SeperableConv2DBNActiv(nn.Module): 30 | def __init__(self, nin, nout, ksize=3, stride=1, pad=1, dilation=1, activ=nn.ReLU): 31 | super(SeperableConv2DBNActiv, self).__init__() 32 | self.conv = nn.Sequential( 33 | nn.Conv2d( 34 | nin, 35 | nin, 36 | kernel_size=ksize, 37 | stride=stride, 38 | padding=pad, 39 | dilation=dilation, 40 | groups=nin, 41 | bias=False, 42 | ), 43 | nn.Conv2d(nin, nout, kernel_size=1, bias=False), 44 | nn.BatchNorm2d(nout), 45 | activ(), 46 | ) 47 | 48 | def __call__(self, x): 49 | return self.conv(x) 50 | 51 | 52 | class Encoder(nn.Module): 53 | def __init__(self, nin, nout, ksize=3, stride=1, pad=1, activ=nn.LeakyReLU): 54 | super(Encoder, self).__init__() 55 | self.conv1 = Conv2DBNActiv(nin, nout, ksize, 1, pad, activ=activ) 56 | self.conv2 = Conv2DBNActiv(nout, nout, ksize, stride, pad, activ=activ) 57 | 58 | def __call__(self, x): 59 | skip = self.conv1(x) 60 | h = self.conv2(skip) 61 | 62 | return h, skip 63 | 64 | 65 | class Decoder(nn.Module): 66 | def __init__( 67 | self, nin, nout, ksize=3, stride=1, pad=1, activ=nn.ReLU, dropout=False 68 | ): 69 | super(Decoder, self).__init__() 70 | self.conv = Conv2DBNActiv(nin, nout, ksize, 1, pad, activ=activ) 71 | self.dropout = nn.Dropout2d(0.1) if dropout else None 72 | 73 | def __call__(self, x, skip=None): 74 | x = F.interpolate(x, scale_factor=2, mode="bilinear", align_corners=True) 75 | if skip is not None: 76 | skip = spec_utils.crop_center(skip, x) 77 | x = torch.cat([x, skip], dim=1) 78 | h = self.conv(x) 79 | 80 | if self.dropout is not None: 81 | h = self.dropout(h) 82 | 83 | return h 84 | 85 | 86 | class ASPPModule(nn.Module): 87 | def __init__(self, nin, nout, dilations=(4, 8, 16, 32, 64), activ=nn.ReLU): 88 | super(ASPPModule, self).__init__() 89 | self.conv1 = nn.Sequential( 90 | nn.AdaptiveAvgPool2d((1, None)), 91 | Conv2DBNActiv(nin, nin, 1, 1, 0, activ=activ), 92 | ) 93 | self.conv2 = Conv2DBNActiv(nin, nin, 1, 1, 0, activ=activ) 94 | self.conv3 = SeperableConv2DBNActiv( 95 | nin, nin, 3, 1, dilations[0], dilations[0], activ=activ 96 | ) 97 | self.conv4 = SeperableConv2DBNActiv( 98 | nin, nin, 3, 1, dilations[1], dilations[1], activ=activ 99 | ) 100 | self.conv5 = SeperableConv2DBNActiv( 101 | nin, nin, 3, 1, dilations[2], dilations[2], activ=activ 102 | ) 103 | self.conv6 = SeperableConv2DBNActiv( 104 | nin, nin, 3, 1, dilations[2], dilations[2], activ=activ 105 | ) 106 | self.conv7 = SeperableConv2DBNActiv( 107 | nin, nin, 3, 1, dilations[2], dilations[2], activ=activ 108 | ) 109 | self.bottleneck = nn.Sequential( 110 | Conv2DBNActiv(nin * 7, nout, 1, 1, 0, activ=activ), nn.Dropout2d(0.1) 111 | ) 112 | 113 | def forward(self, x): 114 | _, _, h, w = x.size() 115 | feat1 = F.interpolate( 116 | self.conv1(x), size=(h, w), mode="bilinear", align_corners=True 117 | ) 118 | feat2 = self.conv2(x) 119 | feat3 = self.conv3(x) 120 | feat4 = self.conv4(x) 121 | feat5 = self.conv5(x) 122 | feat6 = self.conv6(x) 123 | feat7 = self.conv7(x) 124 | out = torch.cat((feat1, feat2, feat3, feat4, feat5, feat6, feat7), dim=1) 125 | bottle = self.bottleneck(out) 126 | return bottle 127 | -------------------------------------------------------------------------------- /infer/lib/uvr5_pack/lib_v5/layers_537227KB.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn.functional as F 3 | from torch import nn 4 | 5 | from . import spec_utils 6 | 7 | 8 | class Conv2DBNActiv(nn.Module): 9 | def __init__(self, nin, nout, ksize=3, stride=1, pad=1, dilation=1, activ=nn.ReLU): 10 | super(Conv2DBNActiv, self).__init__() 11 | self.conv = nn.Sequential( 12 | nn.Conv2d( 13 | nin, 14 | nout, 15 | kernel_size=ksize, 16 | stride=stride, 17 | padding=pad, 18 | dilation=dilation, 19 | bias=False, 20 | ), 21 | nn.BatchNorm2d(nout), 22 | activ(), 23 | ) 24 | 25 | def __call__(self, x): 26 | return self.conv(x) 27 | 28 | 29 | class SeperableConv2DBNActiv(nn.Module): 30 | def __init__(self, nin, nout, ksize=3, stride=1, pad=1, dilation=1, activ=nn.ReLU): 31 | super(SeperableConv2DBNActiv, self).__init__() 32 | self.conv = nn.Sequential( 33 | nn.Conv2d( 34 | nin, 35 | nin, 36 | kernel_size=ksize, 37 | stride=stride, 38 | padding=pad, 39 | dilation=dilation, 40 | groups=nin, 41 | bias=False, 42 | ), 43 | nn.Conv2d(nin, nout, kernel_size=1, bias=False), 44 | nn.BatchNorm2d(nout), 45 | activ(), 46 | ) 47 | 48 | def __call__(self, x): 49 | return self.conv(x) 50 | 51 | 52 | class Encoder(nn.Module): 53 | def __init__(self, nin, nout, ksize=3, stride=1, pad=1, activ=nn.LeakyReLU): 54 | super(Encoder, self).__init__() 55 | self.conv1 = Conv2DBNActiv(nin, nout, ksize, 1, pad, activ=activ) 56 | self.conv2 = Conv2DBNActiv(nout, nout, ksize, stride, pad, activ=activ) 57 | 58 | def __call__(self, x): 59 | skip = self.conv1(x) 60 | h = self.conv2(skip) 61 | 62 | return h, skip 63 | 64 | 65 | class Decoder(nn.Module): 66 | def __init__( 67 | self, nin, nout, ksize=3, stride=1, pad=1, activ=nn.ReLU, dropout=False 68 | ): 69 | super(Decoder, self).__init__() 70 | self.conv = Conv2DBNActiv(nin, nout, ksize, 1, pad, activ=activ) 71 | self.dropout = nn.Dropout2d(0.1) if dropout else None 72 | 73 | def __call__(self, x, skip=None): 74 | x = F.interpolate(x, scale_factor=2, mode="bilinear", align_corners=True) 75 | if skip is not None: 76 | skip = spec_utils.crop_center(skip, x) 77 | x = torch.cat([x, skip], dim=1) 78 | h = self.conv(x) 79 | 80 | if self.dropout is not None: 81 | h = self.dropout(h) 82 | 83 | return h 84 | 85 | 86 | class ASPPModule(nn.Module): 87 | def __init__(self, nin, nout, dilations=(4, 8, 16, 32, 64), activ=nn.ReLU): 88 | super(ASPPModule, self).__init__() 89 | self.conv1 = nn.Sequential( 90 | nn.AdaptiveAvgPool2d((1, None)), 91 | Conv2DBNActiv(nin, nin, 1, 1, 0, activ=activ), 92 | ) 93 | self.conv2 = Conv2DBNActiv(nin, nin, 1, 1, 0, activ=activ) 94 | self.conv3 = SeperableConv2DBNActiv( 95 | nin, nin, 3, 1, dilations[0], dilations[0], activ=activ 96 | ) 97 | self.conv4 = SeperableConv2DBNActiv( 98 | nin, nin, 3, 1, dilations[1], dilations[1], activ=activ 99 | ) 100 | self.conv5 = SeperableConv2DBNActiv( 101 | nin, nin, 3, 1, dilations[2], dilations[2], activ=activ 102 | ) 103 | self.conv6 = SeperableConv2DBNActiv( 104 | nin, nin, 3, 1, dilations[2], dilations[2], activ=activ 105 | ) 106 | self.conv7 = SeperableConv2DBNActiv( 107 | nin, nin, 3, 1, dilations[2], dilations[2], activ=activ 108 | ) 109 | self.bottleneck = nn.Sequential( 110 | Conv2DBNActiv(nin * 7, nout, 1, 1, 0, activ=activ), nn.Dropout2d(0.1) 111 | ) 112 | 113 | def forward(self, x): 114 | _, _, h, w = x.size() 115 | feat1 = F.interpolate( 116 | self.conv1(x), size=(h, w), mode="bilinear", align_corners=True 117 | ) 118 | feat2 = self.conv2(x) 119 | feat3 = self.conv3(x) 120 | feat4 = self.conv4(x) 121 | feat5 = self.conv5(x) 122 | feat6 = self.conv6(x) 123 | feat7 = self.conv7(x) 124 | out = torch.cat((feat1, feat2, feat3, feat4, feat5, feat6, feat7), dim=1) 125 | bottle = self.bottleneck(out) 126 | return bottle 127 | -------------------------------------------------------------------------------- /infer/lib/uvr5_pack/lib_v5/layers_537238KB.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn.functional as F 3 | from torch import nn 4 | 5 | from . import spec_utils 6 | 7 | 8 | class Conv2DBNActiv(nn.Module): 9 | def __init__(self, nin, nout, ksize=3, stride=1, pad=1, dilation=1, activ=nn.ReLU): 10 | super(Conv2DBNActiv, self).__init__() 11 | self.conv = nn.Sequential( 12 | nn.Conv2d( 13 | nin, 14 | nout, 15 | kernel_size=ksize, 16 | stride=stride, 17 | padding=pad, 18 | dilation=dilation, 19 | bias=False, 20 | ), 21 | nn.BatchNorm2d(nout), 22 | activ(), 23 | ) 24 | 25 | def __call__(self, x): 26 | return self.conv(x) 27 | 28 | 29 | class SeperableConv2DBNActiv(nn.Module): 30 | def __init__(self, nin, nout, ksize=3, stride=1, pad=1, dilation=1, activ=nn.ReLU): 31 | super(SeperableConv2DBNActiv, self).__init__() 32 | self.conv = nn.Sequential( 33 | nn.Conv2d( 34 | nin, 35 | nin, 36 | kernel_size=ksize, 37 | stride=stride, 38 | padding=pad, 39 | dilation=dilation, 40 | groups=nin, 41 | bias=False, 42 | ), 43 | nn.Conv2d(nin, nout, kernel_size=1, bias=False), 44 | nn.BatchNorm2d(nout), 45 | activ(), 46 | ) 47 | 48 | def __call__(self, x): 49 | return self.conv(x) 50 | 51 | 52 | class Encoder(nn.Module): 53 | def __init__(self, nin, nout, ksize=3, stride=1, pad=1, activ=nn.LeakyReLU): 54 | super(Encoder, self).__init__() 55 | self.conv1 = Conv2DBNActiv(nin, nout, ksize, 1, pad, activ=activ) 56 | self.conv2 = Conv2DBNActiv(nout, nout, ksize, stride, pad, activ=activ) 57 | 58 | def __call__(self, x): 59 | skip = self.conv1(x) 60 | h = self.conv2(skip) 61 | 62 | return h, skip 63 | 64 | 65 | class Decoder(nn.Module): 66 | def __init__( 67 | self, nin, nout, ksize=3, stride=1, pad=1, activ=nn.ReLU, dropout=False 68 | ): 69 | super(Decoder, self).__init__() 70 | self.conv = Conv2DBNActiv(nin, nout, ksize, 1, pad, activ=activ) 71 | self.dropout = nn.Dropout2d(0.1) if dropout else None 72 | 73 | def __call__(self, x, skip=None): 74 | x = F.interpolate(x, scale_factor=2, mode="bilinear", align_corners=True) 75 | if skip is not None: 76 | skip = spec_utils.crop_center(skip, x) 77 | x = torch.cat([x, skip], dim=1) 78 | h = self.conv(x) 79 | 80 | if self.dropout is not None: 81 | h = self.dropout(h) 82 | 83 | return h 84 | 85 | 86 | class ASPPModule(nn.Module): 87 | def __init__(self, nin, nout, dilations=(4, 8, 16, 32, 64), activ=nn.ReLU): 88 | super(ASPPModule, self).__init__() 89 | self.conv1 = nn.Sequential( 90 | nn.AdaptiveAvgPool2d((1, None)), 91 | Conv2DBNActiv(nin, nin, 1, 1, 0, activ=activ), 92 | ) 93 | self.conv2 = Conv2DBNActiv(nin, nin, 1, 1, 0, activ=activ) 94 | self.conv3 = SeperableConv2DBNActiv( 95 | nin, nin, 3, 1, dilations[0], dilations[0], activ=activ 96 | ) 97 | self.conv4 = SeperableConv2DBNActiv( 98 | nin, nin, 3, 1, dilations[1], dilations[1], activ=activ 99 | ) 100 | self.conv5 = SeperableConv2DBNActiv( 101 | nin, nin, 3, 1, dilations[2], dilations[2], activ=activ 102 | ) 103 | self.conv6 = SeperableConv2DBNActiv( 104 | nin, nin, 3, 1, dilations[2], dilations[2], activ=activ 105 | ) 106 | self.conv7 = SeperableConv2DBNActiv( 107 | nin, nin, 3, 1, dilations[2], dilations[2], activ=activ 108 | ) 109 | self.bottleneck = nn.Sequential( 110 | Conv2DBNActiv(nin * 7, nout, 1, 1, 0, activ=activ), nn.Dropout2d(0.1) 111 | ) 112 | 113 | def forward(self, x): 114 | _, _, h, w = x.size() 115 | feat1 = F.interpolate( 116 | self.conv1(x), size=(h, w), mode="bilinear", align_corners=True 117 | ) 118 | feat2 = self.conv2(x) 119 | feat3 = self.conv3(x) 120 | feat4 = self.conv4(x) 121 | feat5 = self.conv5(x) 122 | feat6 = self.conv6(x) 123 | feat7 = self.conv7(x) 124 | out = torch.cat((feat1, feat2, feat3, feat4, feat5, feat6, feat7), dim=1) 125 | bottle = self.bottleneck(out) 126 | return bottle 127 | -------------------------------------------------------------------------------- /docs/kr/README.ko.han.md: -------------------------------------------------------------------------------- 1 |
2 | 3 |

Retrieval-based-Voice-Conversion-WebUI

4 | VITS基盤의 簡單하고使用하기 쉬운音聲變換틀

5 | 6 | [![madewithlove](https://img.shields.io/badge/made_with-%E2%9D%A4-red?style=for-the-badge&labelColor=orange 7 | )](https://github.com/RVC-Project/Retrieval-based-Voice-Conversion-WebUI) 8 | 9 |
10 | 11 | [![RVC v1](https://img.shields.io/badge/RVCv1-F9AB00?style=for-the-badge&logo=googlecolab&color=525252)](https://colab.research.google.com/github/RVC-Project/Retrieval-based-Voice-Conversion-WebUI/blob/main/tools/ipynb/v1.ipynb) 12 | [![RVC v2](https://img.shields.io/badge/RVCv2-F9AB00?style=for-the-badge&logo=googlecolab&color=525252)](https://colab.research.google.com/github/RVC-Project/Retrieval-based-Voice-Conversion-WebUI/blob/main/tools/ipynb/v2.ipynb) 13 | [![Licence](https://img.shields.io/github/license/RVC-Project/Retrieval-based-Voice-Conversion-WebUI?style=for-the-badge)](https://github.com/RVC-Project/Retrieval-based-Voice-Conversion-WebUI/blob/main/LICENSE) 14 | [![Huggingface](https://img.shields.io/badge/🤗%20-Spaces-yellow.svg?style=for-the-badge)](https://huggingface.co/lj1995/VoiceConversionWebUI/tree/main/) 15 | 16 | [![Discord](https://img.shields.io/badge/RVC%20Developers-Discord-7289DA?style=for-the-badge&logo=discord&logoColor=white)](https://discord.gg/HcsmBBGyVk) 17 | 18 |
19 | 20 | ------ 21 | [**更新日誌**](https://github.com/RVC-Project/Retrieval-based-Voice-Conversion-WebUI/blob/main/docs/Changelog_KO.md) 22 | 23 | [**English**](../en/README.en.md) | [**中文简体**](../../README.md) | [**日本語**](../jp/README.ja.md) | [**한국어**](../kr/README.ko.md) ([**韓國語**](../kr/README.ko.han.md)) | [**Français**](../fr/README.fr.md) | [**Türkçe**](../tr/README.tr.md) | [**Português**](../pt/README.pt.md) 24 | 25 | > [示範映像](https://www.bilibili.com/video/BV1pm4y1z7Gm/)을 確認해 보세요! 26 | 27 | > RVC를活用한實時間音聲變換: [w-okada/voice-changer](https://github.com/w-okada/voice-changer) 28 | 29 | > 基本모델은 50時間假量의 高品質 오픈 소스 VCTK 데이터셋을 使用하였으므로, 著作權上의 念慮가 없으니 安心하고 使用하시기 바랍니다. 30 | 31 | > 著作權問題가 없는 高品質의 노래를 以後에도 繼續해서 訓練할 豫定입니다. 32 | 33 | ## 紹介 34 | 本Repo는 다음과 같은 特徵을 가지고 있습니다: 35 | + top1檢索을利用하여 入力音色特徵을 訓練세트音色特徵으로 代替하여 音色의漏出을 防止; 36 | + 相對的으로 낮은性能의 GPU에서도 빠른訓練可能; 37 | + 적은量의 데이터로 訓練해도 좋은 結果를 얻을 수 있음 (最小10分以上의 低雜음音聲데이터를 使用하는 것을 勸獎); 38 | + 모델融合을通한 音色의 變調可能 (ckpt處理탭->ckpt混合選擇); 39 | + 使用하기 쉬운 WebUI (웹 使用者인터페이스); 40 | + UVR5 모델을 利用하여 목소리와 背景音樂의 빠른 分離; 41 | 42 | ## 環境의準備 43 | poetry를通해 依存를設置하는 것을 勸獎합니다. 44 | 45 | 다음命令은 Python 버전3.8以上의環境에서 實行되어야 합니다: 46 | ```bash 47 | # PyTorch 關聯主要依存設置, 이미設置되어 있는 境遇 건너뛰기 可能 48 | # 參照: https://pytorch.org/get-started/locally/ 49 | pip install torch torchvision torchaudio 50 | 51 | # Windows + Nvidia Ampere Architecture(RTX30xx)를 使用하고 있다面, #21 에서 명시된 것과 같이 PyTorch에 맞는 CUDA 버전을 指定해야 합니다. 52 | #pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu117 53 | 54 | # Poetry 設置, 이미設置되어 있는 境遇 건너뛰기 可能 55 | # Reference: https://python-poetry.org/docs/#installation 56 | curl -sSL https://install.python-poetry.org | python3 - 57 | 58 | # 依存設置 59 | poetry install 60 | ``` 61 | pip를 活用하여依存를 設置하여도 無妨합니다. 62 | 63 | ```bash 64 | pip install -r requirements.txt 65 | ``` 66 | 67 | ## 其他預備모델準備 68 | RVC 모델은 推論과訓練을 依하여 다른 預備모델이 必要합니다. 69 | 70 | [Huggingface space](https://huggingface.co/lj1995/VoiceConversionWebUI/tree/main/)를 通해서 다운로드 할 수 있습니다. 71 | 72 | 다음은 RVC에 必要한 預備모델 및 其他 파일 目錄입니다: 73 | ```bash 74 | ./assets/hubert/hubert_base.pt 75 | 76 | ./assets/pretrained 77 | 78 | ./assets/uvr5_weights 79 | 80 | V2 버전 모델을 테스트하려면 추가 다운로드가 필요합니다. 81 | 82 | ./assets/pretrained_v2 83 | 84 | # Windows를 使用하는境遇 이 사전도 必要할 수 있습니다. FFmpeg가 設置되어 있으면 건너뛰어도 됩니다. 85 | ffmpeg.exe 86 | ``` 87 | 그後 以下의 命令을 使用하여 WebUI를 始作할 수 있습니다: 88 | ```bash 89 | python infer-web.py 90 | ``` 91 | Windows를 使用하는境遇 `RVC-beta.7z`를 다운로드 및 壓縮解除하여 RVC를 直接使用하거나 `go-web.bat`을 使用하여 WebUi를 直接할 수 있습니다. 92 | 93 | ## 參考 94 | + [ContentVec](https://github.com/auspicious3000/contentvec/) 95 | + [VITS](https://github.com/jaywalnut310/vits) 96 | + [HIFIGAN](https://github.com/jik876/hifi-gan) 97 | + [Gradio](https://github.com/gradio-app/gradio) 98 | + [FFmpeg](https://github.com/FFmpeg/FFmpeg) 99 | + [Ultimate Vocal Remover](https://github.com/Anjok07/ultimatevocalremovergui) 100 | + [audio-slicer](https://github.com/openvpi/audio-slicer) 101 | ## 모든寄與者분들의勞力에感謝드립니다 102 | 103 | 104 | 105 | 106 | 107 | -------------------------------------------------------------------------------- /docs/en/training_tips_en.md: -------------------------------------------------------------------------------- 1 | Instructions and tips for RVC training 2 | ====================================== 3 | This TIPS explains how data training is done. 4 | 5 | # Training flow 6 | I will explain along the steps in the training tab of the GUI. 7 | 8 | ## step1 9 | Set the experiment name here. 10 | 11 | You can also set here whether the model should take pitch into account. 12 | If the model doesn't consider pitch, the model will be lighter, but not suitable for singing. 13 | 14 | Data for each experiment is placed in `/logs/your-experiment-name/`. 15 | 16 | ## step2a 17 | Loads and preprocesses audio. 18 | 19 | ### load audio 20 | If you specify a folder with audio, the audio files in that folder will be read automatically. 21 | For example, if you specify `C:Users\hoge\voices`, `C:Users\hoge\voices\voice.mp3` will be loaded, but `C:Users\hoge\voices\dir\voice.mp3` will Not loaded. 22 | 23 | Since ffmpeg is used internally for reading audio, if the extension is supported by ffmpeg, it will be read automatically. 24 | After converting to int16 with ffmpeg, convert to float32 and normalize between -1 to 1. 25 | 26 | ### denoising 27 | The audio is smoothed by scipy's filtfilt. 28 | 29 | ### Audio Split 30 | First, the input audio is divided by detecting parts of silence that last longer than a certain period (max_sil_kept=5 seconds?). After splitting the audio on silence, split the audio every 4 seconds with an overlap of 0.3 seconds. For audio separated within 4 seconds, after normalizing the volume, convert the wav file to `/logs/your-experiment-name/0_gt_wavs` and then convert it to 16k sampling rate to `/logs/your-experiment-name/1_16k_wavs ` as a wav file. 31 | 32 | ## step2b 33 | ### Extract pitch 34 | Extract pitch information from wav files. Extract the pitch information (=f0) using the method built into parselmouth or pyworld and save it in `/logs/your-experiment-name/2a_f0`. Then logarithmically convert the pitch information to an integer between 1 and 255 and save it in `/logs/your-experiment-name/2b-f0nsf`. 35 | 36 | ### Extract feature_print 37 | Convert the wav file to embedding in advance using HuBERT. Read the wav file saved in `/logs/your-experiment-name/1_16k_wavs`, convert the wav file to 256-dimensional features with HuBERT, and save in npy format in `/logs/your-experiment-name/3_feature256`. 38 | 39 | ## step3 40 | train the model. 41 | ### Glossary for Beginners 42 | In deep learning, the data set is divided and the learning proceeds little by little. In one model update (step), batch_size data are retrieved and predictions and error corrections are performed. Doing this once for a dataset counts as one epoch. 43 | 44 | Therefore, the learning time is the learning time per step x (the number of data in the dataset / batch size) x the number of epochs. In general, the larger the batch size, the more stable the learning becomes (learning time per step ÷ batch size) becomes smaller, but it uses more GPU memory. GPU RAM can be checked with the nvidia-smi command. Learning can be done in a short time by increasing the batch size as much as possible according to the machine of the execution environment. 45 | 46 | ### Specify pretrained model 47 | RVC starts training the model from pretrained weights instead of from 0, so it can be trained with a small dataset. 48 | 49 | By default 50 | 51 | - If you consider pitch, it loads `rvc-location/pretrained/f0G40k.pth` and `rvc-location/pretrained/f0D40k.pth`. 52 | - If you don't consider pitch, it loads `rvc-location/pretrained/G40k.pth` and `rvc-location/pretrained/D40k.pth`. 53 | 54 | When learning, model parameters are saved in `logs/your-experiment-name/G_{}.pth` and `logs/your-experiment-name/D_{}.pth` for each save_every_epoch, but by specifying this path, you can start learning. You can restart or start training from model weights learned in a different experiment. 55 | 56 | ### learning index 57 | RVC saves the HuBERT feature values used during training, and during inference, searches for feature values that are similar to the feature values used during learning to perform inference. In order to perform this search at high speed, the index is learned in advance. 58 | For index learning, we use the approximate neighborhood search library faiss. Read the feature value of `logs/your-experiment-name/3_feature256` and use it to learn the index, and save it as `logs/your-experiment-name/add_XXX.index`. 59 | 60 | (From the 20230428update version, it is read from the index, and saving / specifying is no longer necessary.) 61 | 62 | ### Button description 63 | - Train model: After executing step2b, press this button to train the model. 64 | - Train feature index: After training the model, perform index learning. 65 | - One-click training: step2b, model training and feature index training all at once. -------------------------------------------------------------------------------- /docs/jp/Changelog_JA.md: -------------------------------------------------------------------------------- 1 | ### 2023 年 10 月 6 日更新 2 | 3 | リアルタイム声変換のためのインターフェース go-realtime-gui.bat/gui_v1.py を作成しました（実際には既に存在していました）。今回のアップデートでは、リアルタイム声変換のパフォーマンスを重点的に最適化しました。0813 版との比較： 4 | 5 | - 1. インターフェース操作の最適化：パラメータのホット更新（パラメータ調整時に中断して再起動する必要がない）、レイジーロードモデル（既にロードされたモデルは再ロードする必要がない）、音量因子パラメータ追加（音量を入力オーディオに近づける） 6 | - 2. 内蔵ノイズリダクション効果と速度の最適化 7 | - 3. 推論速度の大幅な最適化 8 | 9 | 入出力デバイスは同じタイプを選択する必要があります。例えば、両方とも MME タイプを選択します。 10 | 11 | 1006 バージョンの全体的な更新は： 12 | 13 | - 1. rmvpe 音声ピッチ抽出アルゴリズムの効果をさらに向上、特に男性の低音部分で大きな改善 14 | - 2. 推論インターフェースレイアウトの最適化 15 | 16 | ### 2023 年 8 月 13 日更新 17 | 18 | 1-通常のバグ修正 19 | 20 | - 保存頻度と総ラウンド数の最小値を 1 に変更。総ラウンド数の最小値を 2 に変更 21 | - pretrain モデルなしでのトレーニングエラーを修正 22 | - 伴奏とボーカルの分離完了後の VRAM クリア 23 | - faiss 保存パスを絶対パスから相対パスに変更 24 | - パスに空白が含まれる場合のサポート（トレーニングセットのパス+実験名がサポートされ、エラーにならない） 25 | - filelist の強制的な utf8 エンコーディングをキャンセル 26 | - リアルタイム声変換中にインデックスを有効にすることによる CPU の大幅な使用問題を解決 27 | 28 | 2-重要なアップデート 29 | 30 | - 現在最も強力なオープンソースの人間の声のピッチ抽出モデル RMVPE をトレーニングし、RVC のトレーニング、オフライン/リアルタイム推論に使用。pytorch/onnx/DirectML をサポート 31 | - pytorch-dml を通じて A カードと I カードのサポート 32 | （1）リアルタイム声変換（2）推論（3）ボーカルと伴奏の分離（4）トレーニングはまだサポートされておらず、CPU でのトレーニングに切り替わります。onnx_dml を通じて rmvpe_gpu の推論をサポート 33 | 34 | ### 2023 年 6 月 18 日更新 35 | 36 | - v2 に 32k と 48k の 2 つの新しい事前トレーニングモデルを追加 37 | - 非 f0 モデルの推論エラーを修正 38 | - 1 時間を超えるトレーニングセットのインデックス構築フェーズでは、自動的に kmeans で特徴を縮小し、インデックスのトレーニングを加速し、検索に追加 39 | - 人間の声をギターに変換するおもちゃのリポジトリを添付 40 | - データ処理で異常値スライスを除外 41 | - onnx エクスポートオプションタブ 42 | 43 | 失敗した実験： 44 | 45 | - ~~特徴検索に時間次元を追加：ダメ、効果がない~~ 46 | - ~~特徴検索に PCAR 次元削減オプションを追加：ダメ、大きなデータは kmeans でデータ量を減らし、小さいデータは次元削減の時間が節約するマッチングの時間よりも長い~~ 47 | - ~~onnx 推論のサポート（推論のみの小さな圧縮パッケージ付き）：ダメ、nsf の生成には pytorch が必要~~ 48 | - ~~トレーニング中に音声、ジェンダー、eq、ノイズなどで入力をランダムに増強：ダメ、効果がない~~ 49 | - ~~小型声码器の接続調査：ダメ、効果が悪化~~ 50 | 51 | todolist： 52 | 53 | - ~~トレーニングセットの音声ピッチ認識に crepe をサポート：既に RMVPE に置き換えられているため不要~~ 54 | - ~~多プロセス harvest 推論：既に RMVPE に置き換えられているため不要~~ 55 | - ~~crepe の精度サポートと RVC-config の同期：既に RMVPE に置き換えられているため不要。これをサポートするには torchcrepe ライブラリも同期する必要があり、面倒~~ 56 | - F0 エディタとの連携 57 | 58 | ### 2023 年 5 月 28 日更新 59 | 60 | - v2 の jupyter notebook を追加、韓国語の changelog を追加、いくつかの環境依存関係を追加 61 | - 呼吸、清辅音、歯音の保護モードを追加 62 | - crepe-full 推論をサポート 63 | - UVR5 人間の声と伴奏の分離に 3 つの遅延除去モデルと MDX-Net の混响除去モデルを追加、HP3 人声抽出モデルを追加 64 | - インデックス名にバージョンと実験名を追加 65 | - 人間の声と伴奏の分離、推論のバッチエクスポートにオーディオエクスポートフォーマットオプションを追加 66 | - 32k モデルのトレーニングを廃止 67 | 68 | ### 2023 年 5 月 13 日更新 69 | 70 | - ワンクリックパッケージ内の古いバージョンの runtime 内の lib.infer_pack と uvr5_pack の残骸をクリア 71 | - トレーニングセットの事前処理の擬似マルチプロセスバグを修正 72 | - harvest による音声ピッチ認識で無声音現象を弱めるために中間値フィルターを追加、中間値フィルターの半径を調整可能 73 | - 音声エクスポートにポストプロセスリサンプリングを追加 74 | - トレーニング時の n_cpu プロセス数を「F0 抽出のみ調整」から「データ事前処理と F0 抽出の調整」に変更 75 | - logs フォルダ下の index パスを自動検出し、ドロップダウンリスト機能を提供 76 | - タブページに「よくある質問」を追加（または github-rvc-wiki を参照） 77 | - 同じパスの入力音声推論に音声ピッチキャッシュを追加（用途：harvest 音声ピッチ抽出を使用すると、全体のパイプラインが長く繰り返される音声ピッチ抽出プロセスを経験し、キャッシュを使用しない場合、異なる音色、インデックス、音声ピッチ中間値フィルター半径パラメーターをテストするユーザーは、最初のテスト後の待機結果が非常に苦痛になります） 78 | 79 | ### 2023 年 5 月 14 日更新 80 | 81 | - 音量エンベロープのアライメント入力ミックス（「入力が無音で出力がわずかなノイズ」の問題を緩和することができます。入力音声の背景ノイズが大きい場合は、オンにしないことをお勧めします。デフォルトではオフ（1 として扱われる）） 82 | - 指定された頻度で抽出された小型モデルを保存する機能をサポート（異なるエポックでの推論効果を試したいが、すべての大きなチェックポイントを保存して手動で小型モデルを抽出するのが面倒な場合、この機能は非常に便利です） 83 | - システム全体のプロキシが開かれている場合にブラウザの接続エラーが発生する問題を環境変数の設定で解決 84 | - v2 事前訓練モデルをサポート（現在、テストのために 40k バージョンのみが公開されており、他の 2 つのサンプリングレートはまだ完全に訓練されていません） 85 | - 推論前に 1 を超える過大な音量を制限 86 | - データ事前処理パラメーターを微調整 87 | 88 | ### 2023 年 4 月 9 日更新 89 | 90 | - トレーニングパラメーターを修正し、GPU の平均利用率を向上させる。A100 は最高 25％から約 90％に、V100 は 50％から約 90％に、2060S は 60％から約 85％に、P40 は 25％から約 95％に向上し、トレーニング速度が大幅に向上 91 | - パラメーターを修正：全体の batch_size を各カードの batch_size に変更 92 | - total_epoch を修正：最大制限 100 から 1000 に解除; デフォルト 10 からデフォルト 20 に引き上げ 93 | - ckpt 抽出時に音声ピッチの有無を誤って認識し、推論が異常になる問題を修正 94 | - 分散トレーニングで各ランクが ckpt を 1 回ずつ保存する問題を修正 95 | - 特徴抽出で nan 特徴をフィルタリング 96 | - 入力が無音で出力がランダムな子音またはノイズになる問題を修正（旧バージョンのモデルはトレーニングセットを作り直して再トレーニングする必要があります） 97 | 98 | ### 2023 年 4 月 16 日更新 99 | 100 | - ローカルリアルタイム音声変換ミニ GUI を新設、go-realtime-gui.bat をダブルクリックで起動 101 | - トレーニングと推論で 50Hz 以下の周波数帯をフィルタリング 102 | - トレーニングと推論の音声ピッチ抽出 pyworld の最低音声ピッチをデフォルトの 80 から 50 に下げ、50-80hz の男性低音声が無声にならないように 103 | - WebUI がシステムの地域に基づいて言語を変更する機能をサポート（現在サポートされているのは en_US、ja_JP、zh_CN、zh_HK、zh_SG、zh_TW、サポートされていない場合はデフォルトで en_US になります） 104 | - 一部のグラフィックカードの認識を修正（例えば V100-16G の認識失敗、P4 の認識失敗） 105 | 106 | ### 2023 年 4 月 28 日更新 107 | 108 | - faiss インデックス設定をアップグレードし、速度が速く、品質が高くなりました 109 | - total_npy 依存をキャンセルし、今後のモデル共有では total_npy の記入は不要 110 | - 16 シリーズの制限を解除。4G メモリ GPU に 4G の推論設定を提供 111 | - 一部のオーディオ形式で UVR5 の人声伴奏分離のバグを修正 112 | - リアルタイム音声変換ミニ gui に 40k 以外のモデルと妥協のない音声ピッチモデルのサポートを追加 113 | 114 | ### 今後の計画： 115 | 116 | 機能： 117 | 118 | - 複数人のトレーニングタブのサポート（最大 4 人） 119 | 120 | 底層モデル： 121 | 122 | - 呼吸 wav をトレーニングセットに追加し、呼吸が音声変換の電子音の問題を修正 123 | - 歌声トレーニングセットを追加した底層モデルをトレーニングしており、将来的には公開する予定です 124 | -------------------------------------------------------------------------------- /infer/lib/uvr5_pack/lib_v5/nets_new.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn.functional as F 3 | from torch import nn 4 | 5 | from . import layers_new 6 | 7 | 8 | class BaseNet(nn.Module): 9 | def __init__( 10 | self, nin, nout, nin_lstm, nout_lstm, dilations=((4, 2), (8, 4), (12, 6)) 11 | ): 12 | super(BaseNet, self).__init__() 13 | self.enc1 = layers_new.Conv2DBNActiv(nin, nout, 3, 1, 1) 14 | self.enc2 = layers_new.Encoder(nout, nout * 2, 3, 2, 1) 15 | self.enc3 = layers_new.Encoder(nout * 2, nout * 4, 3, 2, 1) 16 | self.enc4 = layers_new.Encoder(nout * 4, nout * 6, 3, 2, 1) 17 | self.enc5 = layers_new.Encoder(nout * 6, nout * 8, 3, 2, 1) 18 | 19 | self.aspp = layers_new.ASPPModule(nout * 8, nout * 8, dilations, dropout=True) 20 | 21 | self.dec4 = layers_new.Decoder(nout * (6 + 8), nout * 6, 3, 1, 1) 22 | self.dec3 = layers_new.Decoder(nout * (4 + 6), nout * 4, 3, 1, 1) 23 | self.dec2 = layers_new.Decoder(nout * (2 + 4), nout * 2, 3, 1, 1) 24 | self.lstm_dec2 = layers_new.LSTMModule(nout * 2, nin_lstm, nout_lstm) 25 | self.dec1 = layers_new.Decoder(nout * (1 + 2) + 1, nout * 1, 3, 1, 1) 26 | 27 | def __call__(self, x): 28 | e1 = self.enc1(x) 29 | e2 = self.enc2(e1) 30 | e3 = self.enc3(e2) 31 | e4 = self.enc4(e3) 32 | e5 = self.enc5(e4) 33 | 34 | h = self.aspp(e5) 35 | 36 | h = self.dec4(h, e4) 37 | h = self.dec3(h, e3) 38 | h = self.dec2(h, e2) 39 | h = torch.cat([h, self.lstm_dec2(h)], dim=1) 40 | h = self.dec1(h, e1) 41 | 42 | return h 43 | 44 | 45 | class CascadedNet(nn.Module): 46 | def __init__(self, n_fft, nout=32, nout_lstm=128): 47 | super(CascadedNet, self).__init__() 48 | 49 | self.max_bin = n_fft // 2 50 | self.output_bin = n_fft // 2 + 1 51 | self.nin_lstm = self.max_bin // 2 52 | self.offset = 64 53 | 54 | self.stg1_low_band_net = nn.Sequential( 55 | BaseNet(2, nout // 2, self.nin_lstm // 2, nout_lstm), 56 | layers_new.Conv2DBNActiv(nout // 2, nout // 4, 1, 1, 0), 57 | ) 58 | 59 | self.stg1_high_band_net = BaseNet( 60 | 2, nout // 4, self.nin_lstm // 2, nout_lstm // 2 61 | ) 62 | 63 | self.stg2_low_band_net = nn.Sequential( 64 | BaseNet(nout // 4 + 2, nout, self.nin_lstm // 2, nout_lstm), 65 | layers_new.Conv2DBNActiv(nout, nout // 2, 1, 1, 0), 66 | ) 67 | self.stg2_high_band_net = BaseNet( 68 | nout // 4 + 2, nout // 2, self.nin_lstm // 2, nout_lstm // 2 69 | ) 70 | 71 | self.stg3_full_band_net = BaseNet( 72 | 3 * nout // 4 + 2, nout, self.nin_lstm, nout_lstm 73 | ) 74 | 75 | self.out = nn.Conv2d(nout, 2, 1, bias=False) 76 | self.aux_out = nn.Conv2d(3 * nout // 4, 2, 1, bias=False) 77 | 78 | def forward(self, x): 79 | x = x[:, :, : self.max_bin] 80 | 81 | bandw = x.size()[2] // 2 82 | l1_in = x[:, :, :bandw] 83 | h1_in = x[:, :, bandw:] 84 | l1 = self.stg1_low_band_net(l1_in) 85 | h1 = self.stg1_high_band_net(h1_in) 86 | aux1 = torch.cat([l1, h1], dim=2) 87 | 88 | l2_in = torch.cat([l1_in, l1], dim=1) 89 | h2_in = torch.cat([h1_in, h1], dim=1) 90 | l2 = self.stg2_low_band_net(l2_in) 91 | h2 = self.stg2_high_band_net(h2_in) 92 | aux2 = torch.cat([l2, h2], dim=2) 93 | 94 | f3_in = torch.cat([x, aux1, aux2], dim=1) 95 | f3 = self.stg3_full_band_net(f3_in) 96 | 97 | mask = torch.sigmoid(self.out(f3)) 98 | mask = F.pad( 99 | input=mask, 100 | pad=(0, 0, 0, self.output_bin - mask.size()[2]), 101 | mode="replicate", 102 | ) 103 | 104 | if self.training: 105 | aux = torch.cat([aux1, aux2], dim=1) 106 | aux = torch.sigmoid(self.aux_out(aux)) 107 | aux = F.pad( 108 | input=aux, 109 | pad=(0, 0, 0, self.output_bin - aux.size()[2]), 110 | mode="replicate", 111 | ) 112 | return mask, aux 113 | else: 114 | return mask 115 | 116 | def predict_mask(self, x): 117 | mask = self.forward(x) 118 | 119 | if self.offset > 0: 120 | mask = mask[:, :, :, self.offset : -self.offset] 121 | assert mask.size()[3] > 0 122 | 123 | return mask 124 | 125 | def predict(self, x, aggressiveness=None): 126 | mask = self.forward(x) 127 | pred_mag = x * mask 128 | 129 | if self.offset > 0: 130 | pred_mag = pred_mag[:, :, :, self.offset : -self.offset] 131 | assert pred_mag.size()[3] > 0 132 | 133 | return pred_mag 134 | -------------------------------------------------------------------------------- /docs/tr/training_tips_tr.md: -------------------------------------------------------------------------------- 1 | ## RVC Eğitimi için Talimatlar ve İpuçları 2 | ====================================== 3 | Bu TALİMAT, veri eğitiminin nasıl yapıldığını açıklamaktadır. 4 | 5 | # Eğitim Akışı 6 | Eğitim sekmesindeki adımları takip ederek açıklayacağım. 7 | 8 | ## Adım 1 9 | Deney adını burada belirleyin. 10 | 11 | Ayrıca burada modelin pitch'i dikkate alıp almayacağını da belirleyebilirsiniz. 12 | Eğer model pitch'i dikkate almazsa, model daha hafif olacak, ancak şarkı söyleme için uygun olmayacaktır. 13 | 14 | Her deney için veriler `/logs/your-experiment-name/` dizinine yerleştirilir. 15 | 16 | ## Adım 2a 17 | Ses yüklenir ve ön işleme yapılır. 18 | 19 | ### Ses Yükleme 20 | Ses içeren bir klasör belirtirseniz, bu klasördeki ses dosyaları otomatik olarak okunur. 21 | Örneğin, `C:Users\hoge\voices` belirtirseniz, `C:Users\hoge\voices\voice.mp3` yüklenecek, ancak `C:Users\hoge\voices\dir\voice.mp3` yüklenmeyecektir. 22 | 23 | Ses okumak için dahili olarak ffmpeg kullanıldığından, uzantı ffmpeg tarafından destekleniyorsa otomatik olarak okunacaktır. 24 | ffmpeg ile int16'ya dönüştürüldükten sonra float32'ye dönüştürülüp -1 ile 1 arasında normalize edilir. 25 | 26 | ### Gürültü Temizleme 27 | Ses scipy'nin filtfilt işlevi ile yumuşatılır. 28 | 29 | ### Ses Ayırma 30 | İlk olarak, giriş sesi belirli bir süreden (max_sil_kept=5 saniye?) daha uzun süren sessiz kısımları tespit ederek böler. Sessizlik üzerinde ses bölündükten sonra sesi 4 saniyede bir 0.3 saniyelik bir örtüşme ile böler. 4 saniye içinde ayrılan sesler için ses normalleştirildikten sonra wav dosyası olarak `/logs/your-experiment-name/0_gt_wavs`'a, ardından 16 kHz örnekleme hızına dönüştürülerek `/logs/your-experiment-name/1_16k_wavs` olarak kaydedilir. 31 | 32 | ## Adım 2b 33 | ### Pitch Çıkarımı 34 | Wav dosyalarından pitch bilgisi çıkarılır. ParSelMouth veya PyWorld'e dahili olarak yerleştirilmiş yöntemi kullanarak pitch bilgisi (=f0) çıkarılır ve `/logs/your-experiment-name/2a_f0` dizinine kaydedilir. Ardından pitch bilgisi logaritmik olarak 1 ile 255 arasında bir tamsayıya dönüştürülüp `/logs/your-experiment-name/2b-f0nsf` dizinine kaydedilir. 35 | 36 | ### Özellik Çıkarımı 37 | HuBERT'i kullanarak önceden gömme olarak wav dosyasını çıkarır. `/logs/your-experiment-name/1_16k_wavs`'a kaydedilen wav dosyasını okuyarak, wav dosyasını 256 boyutlu HuBERT özelliklerine dönüştürür ve npy formatında `/logs/your-experiment-name/3_feature256` dizinine kaydeder. 38 | 39 | ## Adım 3 40 | Modeli eğit. 41 | ### Başlangıç Seviyesi Sözlüğü 42 | Derin öğrenmede, veri kümesi bölmeye ve öğrenmeye adım adım devam eder. Bir model güncellemesinde (adım), batch_size veri alınır ve tahminler ve hata düzeltmeleri yapılır. Bunun bir defa bir veri kümesi için yapılması bir dönem olarak sayılır. 43 | 44 | Bu nedenle, öğrenme zamanı adım başına öğrenme zamanı x (veri kümesindeki veri sayısı / batch boyutu) x dönem sayısıdır. Genel olarak, batch boyutu ne kadar büyükse, öğrenme daha istikrarlı hale gelir (adım başına öğrenme süresi ÷ batch boyutu) küçülür, ancak daha fazla GPU belleği kullanır. GPU RAM'ı nvidia-smi komutu ile kontrol edilebilir. Çalışma ortamının makinesine göre batch boyutunu mümkün olduğunca artırarak öğrenme süresini kısa sürede yapabilirsiniz. 45 | 46 | ### Önceden Eğitilmiş Modeli Belirtme 47 | RVC, modeli 0'dan değil önceden eğitilmiş ağırlıklardan başlatarak eğitir, bu nedenle küçük bir veri kümesi ile eğitilebilir. 48 | 49 | Varsayılan olarak 50 | 51 | - Eğer pitch'i dikkate alıyorsanız, `rvc-location/pretrained/f0G40k.pth` ve `rvc-location/pretrained/f0D40k.pth` yüklenir. 52 | - Eğer pitch'i dikkate almıyorsanız, yine `rvc-location/pretrained/f0G40k.pth` ve `rvc-location/pretrained/f0D40k.pth` yüklenir. 53 | 54 | Öğrenirken model parametreleri her save_every_epoch için `logs/your-experiment-name/G_{}.pth` ve `logs/your-experiment-name/D_{}.pth` olarak kaydedilir, ancak bu yolu belirterek öğrenmeye başlayabilirsiniz. Farklı bir deneyde öğrenilen model ağırlıklarından öğrenmeye yeniden başlayabilir veya eğitimi başlatabilirsiniz. 55 | 56 | ### Öğrenme İndeksi 57 | RVC, eğitim sırasında kullanılan HuBERT özellik değerlerini kaydeder ve çıkarım sırasında, öğrenme sırasında kullanılan özellik değerlerine benzer özellik değerlerini arayarak çıkarım yapar. Bu aramayı yüksek hızda gerçekleştirebilmek için indeks öğrenilir. 58 | İndeks öğrenimi için yaklaş 59 | 60 | ık komşuluk arama kütüphanesi faiss kullanılır. `/logs/your-experiment-name/3_feature256`'daki özellik değerini okur ve indeksi öğrenmek için kullanır, `logs/your-experiment-name/add_XXX.index` olarak kaydedilir. 61 | 62 | (20230428 güncelleme sürümünden itibaren indeks okunur ve kaydetmek/belirtmek artık gerekli değildir.) 63 | 64 | ### Düğme Açıklaması 65 | - Modeli Eğit: Adım 2b'yi çalıştırdıktan sonra, modeli eğitmek için bu düğmeye basın. 66 | - Özellik İndeksini Eğit: Modeli eğittikten sonra, indeks öğrenme işlemi yapın. 67 | - Tek Tıklamayla Eğitim: Adım 2b, model eğitimi ve özellik indeks eğitimini bir arada yapar. 68 | -------------------------------------------------------------------------------- /docs/kr/Changelog_KO.md: -------------------------------------------------------------------------------- 1 | ### 2023년 10월 6일 업데이트 2 | 3 | 실시간 음성 변환을 위한 인터페이스인 go-realtime-gui.bat/gui_v1.py를 제작했습니다(사실 이는 이미 존재했었습니다). 이번 업데이트는 주로 실시간 음성 변환 성능을 최적화하는 데 중점을 두었습니다. 0813 버전과 비교하여: 4 | 5 | - 1. 인터페이스 조작 최적화: 매개변수 핫 업데이트(매개변수 조정 시 중단 후 재시작 필요 없음), 모델 지연 로딩(이미 로드된 모델은 재로드 필요 없음), 음량 인자 매개변수 추가(음량을 입력 오디오에 가깝게 조정) 6 | - 2. 내장된 노이즈 감소 효과 및 속도 최적화 7 | - 3. 추론 속도 크게 향상 8 | 9 | 입력 및 출력 장치는 동일한 유형을 선택해야 합니다. 예를 들어, 모두 MME 유형을 선택해야 합니다. 10 | 11 | 1006 버전의 전체 업데이트는 다음과 같습니다: 12 | 13 | - 1. rmvpe 음성 피치 추출 알고리즘의 효과를 계속해서 향상, 특히 남성 저음역에 대한 개선이 큼 14 | - 2. 추론 인터페이스 레이아웃 최적화 15 | 16 | ### 2023년 08월 13일 업데이트 17 | 18 | 1-정기적인 버그 수정 19 | 20 | - 최소 총 에포크 수를 1로 변경하고, 최소 총 에포크 수를 2로 변경합니다. 21 | - 사전 훈련(pre-train) 모델을 사용하지 않는 훈련 오류 수정 22 | - 반주 보컬 분리 후 그래픽 메모리 지우기 23 | - 페이즈 저장 경로 절대 경로를 상대 경로로 변경 24 | - 공백이 포함된 경로 지원(훈련 세트 경로와 실험 이름 모두 지원되며 더 이상 오류가 보고되지 않음) 25 | - 파일 목록에서 필수 utf8 인코딩 취소 26 | - 실시간 음성 변경 중 faiss 검색으로 인한 CPU 소모 문제 해결 27 | 28 | 2-키 업데이트 29 | 30 | - 현재 가장 강력한 오픈 소스 보컬 피치 추출 모델 RMVPE를 훈련하고, 이를 RVC 훈련, 오프라인/실시간 추론에 사용하며, PyTorch/Onx/DirectML을 지원합니다. 31 | - 파이토치\_DML을 통한 AMD 및 인텔 그래픽 카드 지원 32 | (1) 실시간 음성 변화 (2) 추론 (3) 보컬 반주 분리 (4) 현재 지원되지 않는 훈련은 CPU 훈련으로 전환, Onnx_Dml을 통한 gpu의 RMVPE 추론 지원 33 | 34 | ### 2023년 6월 18일 업데이트 35 | 36 | - v2 버전에서 새로운 32k와 48k 사전 학습 모델을 추가. 37 | - non-f0 모델들의 추론 오류 수정. 38 | - 학습 세트가 1시간을 넘어가는 경우, 인덱스 생성 단계에서 minibatch-kmeans을 사용해, 학습속도 가속화. 39 | - [huggingface](https://huggingface.co/spaces/lj1995/vocal2guitar)에서 vocal2guitar 제공. 40 | - 데이터 처리 단계에서 이상 값 자동으로 제거. 41 | - ONNX로 내보내는(export) 옵션 탭 추가. 42 | 43 | 업데이트에 적용되지 않았지만 시도한 것들 : 44 | 45 | - ~~시계열 차원을 추가하여 특징 검색을 진행했지만, 유의미한 효과는 없었습니다.~~ 46 | - ~~PCA 차원 축소를 추가하여 특징 검색을 진행했지만, 유의미한 효과는 없었습니다.~~ 47 | - ~~ONNX 추론을 지원하는 것에 실패했습니다. nsf 생성시, Pytorch가 필요하기 때문입니다.~~ 48 | - ~~훈련 중에 입력에 대한 음고, 성별, 이퀄라이저, 노이즈 등 무작위로 강화하는 것에, 유의미한 효과는 없었습니다.~~ 49 | 50 | 추후 업데이트 목록: 51 | 52 | - ~~Vocos-RVC (소형 보코더) 통합 예정.~~ 53 | - ~~학습 단계에 음고 인식을 위한 Crepe 지원 예정.~~ 54 | - ~~Crepe의 정밀도를 REC-config와 동기화하여 지원 예정.~~ 55 | - FO 에디터 지원 예정. 56 | 57 | ### 2023년 5월 28일 업데이트 58 | 59 | - v2 jupyter notebook 추가, 한국어 업데이트 로그 추가, 의존성 모듈 일부 수정. 60 | - 무성음 및 숨소리 보호 모드 추가. 61 | - crepe-full pitch 감지 지원. 62 | - UVR5 보컬 분리: 디버브 및 디-에코 모델 지원. 63 | - index 이름에 experiment 이름과 버전 추가. 64 | - 배치 음성 변환 처리 및 UVR5 보컬 분리 시, 사용자가 수동으로 출력 오디오의 내보내기(export) 형식을 선택할 수 있도록 지원. 65 | - 32k 훈련 모델 지원 종료. 66 | 67 | ### 2023년 5월 13일 업데이트 68 | 69 | - 원클릭 패키지의 이전 버전 런타임 내, 불필요한 코드(lib.infer_pack 및 uvr5_pack) 제거. 70 | - 훈련 세트 전처리의 유사 다중 처리 버그 수정. 71 | - Harvest 피치 인식 알고리즘에 대한 중위수 필터링 반경 조정 추가. 72 | - 오디오 내보낼 때, 후처리 리샘플링 지원. 73 | - 훈련에 대한 다중 처리 "n_cpu" 설정이 "f0 추출"에서 "데이터 전처리 및 f0 추출"로 변경. 74 | - logs 폴더 하의 인덱스 경로를 자동으로 감지 및 드롭다운 목록 기능 제공. 75 | - 탭 페이지에 "자주 묻는 질문과 답변" 추가. (github RVC wiki 참조 가능) 76 | - 동일한 입력 오디오 경로를 사용할 때 추론, Harvest 피치를 캐시. 77 | (주의: Harvest 피치 추출을 사용하면 전체 파이프라인은 길고 반복적인 피치 추출 과정을 거치게됩니다. 캐싱을 하지 않는다면, 첫 inference 이후의 단계에서 timbre, 인덱스, 피치 중위수 필터링 반경 설정 등 대기시간이 엄청나게 길어집니다!) 78 | 79 | ### 2023년 5월 14일 업데이트 80 | 81 | - 입력의 볼륨 캡슐을 사용하여 출력의 볼륨 캡슐을 혼합하거나 대체. (입력이 무음이거나 출력의 노이즈 문제를 최소화 할 수 있습니다. 입력 오디오의 배경 노이즈(소음)가 큰 경우 해당 기능을 사용하지 않는 것이 좋습니다. 기본적으로 비활성화 되어있는 옵션입니다. (1: 비활성화 상태)) 82 | - 추출된 소형 모델을 지정된 빈도로 저장하는 기능을 지원. (다양한 에폭 하에서의 성능을 보려고 하지만 모든 대형 체크포인트를 저장하고 매번 ckpt 처리를 통해 소형 모델을 수동으로 추출하고 싶지 않은 경우 이 기능은 매우 유용합니다) 83 | - 환경 변수를 설정하여 서버의 전역 프록시로 인한 "연결 오류" 문제 해결. 84 | - 사전 훈련된 v2 모델 지원. (현재 40k 버전만 테스트를 위해 공개적으로 사용 가능하며, 다른 두 개의 샘플링 비율은 아직 완전히 훈련되지 않아 보류되었습니다.) 85 | - 추론 전, 1을 초과하는 과도한 볼륨 제한. 86 | - 데이터 전처리 매개변수 미세 조정. 87 | 88 | ### 2023년 4월 9일 업데이트 89 | 90 | - GPU 이용률 향상을 위해 훈련 파라미터 수정: A100은 25%에서 약 90%로 증가, V100: 50%에서 약 90%로 증가, 2060S: 60%에서 약 85%로 증가, P40: 25%에서 약 95%로 증가. 91 | 훈련 속도가 크게 향상. 92 | - 매개변수 기준 변경: total batch_size는 GPU당 batch_size를 의미. 93 | - total_epoch 변경: 최대 한도가 100에서 1000으로 증가. 기본값이 10에서 20으로 증가. 94 | - ckpt 추출이 피치를 잘못 인식하여 비정상적인 추론을 유발하는 문제 수정. 95 | - 분산 훈련 과정에서 각 랭크마다 ckpt를 저장하는 문제 수정. 96 | - 특성 추출 과정에 나노 특성 필터링 적용. 97 | - 무음 입력/출력이 랜덤하게 소음을 생성하는 문제 수정. (이전 모델은 새 데이터셋으로 다시 훈련해야 합니다) 98 | 99 | ### 2023년 4월 16일 업데이트 100 | 101 | - 로컬 실시간 음성 변경 미니-GUI 추가, go-realtime-gui.bat를 더블 클릭하여 시작. 102 | - 훈련 및 추론 중 50Hz 이하의 주파수 대역에 대해 필터링 적용. 103 | - 훈련 및 추론의 pyworld 최소 피치 추출을 기본 80에서 50으로 낮춤. 이로 인해, 50-80Hz 사이의 남성 저음이 무음화되지 않습니다. 104 | - 시스템 지역에 따른 WebUI 언어 변경 지원. (현재 en_US, ja_JP, zh_CN, zh_HK, zh_SG, zh_TW를 지원하며, 지원되지 않는 경우 기본값은 en_US) 105 | - 일부 GPU의 인식 수정. (예: V100-16G 인식 실패, P4 인식 실패) 106 | 107 | ### 2023년 4월 28일 업데이트 108 | 109 | - Faiss 인덱스 설정 업그레이드로 속도가 더 빨라지고 품질이 향상. 110 | - total_npy에 대한 의존성 제거. 추후의 모델 공유는 total_npy 입력을 필요로 하지 않습니다. 111 | - 16 시리즈 GPU에 대한 제한 해제, 4GB VRAM GPU에 대한 4GB 추론 설정 제공. 112 | - 일부 오디오 형식에 대한 UVR5 보컬 동반 분리에서의 버그 수정. 113 | - 실시간 음성 변경 미니-GUI는 이제 non-40k 및 non-lazy 피치 모델을 지원합니다. 114 | 115 | ### 추후 계획 116 | 117 | Features: 118 | 119 | - 다중 사용자 훈련 탭 지원.(최대 4명) 120 | 121 | Base model: 122 | 123 | - 훈련 데이터셋에 숨소리 wav 파일을 추가하여, 보컬의 호흡이 노이즈로 변환되는 문제 수정. 124 | - 보컬 훈련 세트의 기본 모델을 추가하기 위한 작업을 진행중이며, 이는 향후에 발표될 예정. 125 | --------------------------------------------------------------------------------