├── __init__.py ├── lib ├── __init__.py ├── infer_pack │ ├── __init__.py │ └── modules │ │ └── F0Predictor │ │ ├── __init__.py │ │ ├── F0Predictor.py │ │ ├── HarvestF0Predictor.py │ │ ├── DioF0Predictor.py │ │ └── PMF0Predictor.py ├── uvr5_pack │ ├── __init__.py │ ├── lib_v5 │ │ ├── __init__.py │ │ ├── modelparams │ │ │ ├── __init__.py │ │ │ ├── 1band_sr44100_hl256.json │ │ │ ├── 1band_sr16000_hl512.json │ │ │ ├── 1band_sr32000_hl512.json │ │ │ ├── 1band_sr33075_hl384.json │ │ │ ├── 1band_sr44100_hl1024.json │ │ │ ├── 1band_sr44100_hl512.json │ │ │ ├── 1band_sr44100_hl512_cut.json │ │ │ ├── 2band_48000.json │ │ │ ├── 2band_32000.json │ │ │ ├── 2band_44100_lofi.json │ │ │ ├── 3band_44100.json │ │ │ ├── ensemble.json │ │ │ ├── 3band_44100_mid.json │ │ │ ├── 3band_44100_msb2.json │ │ │ ├── 4band_v2.json │ │ │ ├── 4band_44100.json │ │ │ ├── 4band_44100_sw.json │ │ │ ├── 4band_44100_msb.json │ │ │ ├── 4band_44100_msb2.json │ │ │ ├── 4band_44100_reverse.json │ │ │ ├── 4band_v2_sn.json │ │ │ ├── 4band_44100_mid.json │ │ │ └── 4band_v3.json │ │ ├── model_param_init.py │ │ ├── layers.py │ │ ├── layers_123812KB .py │ │ ├── layers_123821KB.py │ │ ├── nets.py │ │ ├── nets_123812KB.py │ │ ├── nets_123821KB.py │ │ ├── nets_33966KB.py │ │ ├── nets_61968KB.py │ │ ├── nets_537227KB.py │ │ ├── nets_537238KB.py │ │ ├── layers_new.py │ │ ├── layers_33966KB.py │ │ ├── layers_537227KB.py │ │ ├── layers_537238KB.py │ │ └── nets_new.py │ └── utils.py ├── train │ ├── cmd.txt │ ├── losses.py │ └── mel_processing.py ├── audio.py └── i18n │ └── locale_diff.py ├── tools ├── __init__.py ├── infer │ ├── __init__.py │ ├── trans_weights.py │ ├── train-index.py │ └── train-index-v2.py ├── onnx_inference_demo.py ├── export_onnx.py └── calc_rvc_model_similarity.py ├── venv.sh ├── pretrained └── .gitignore ├── weights └── .gitignore ├── pretrained_v2 └── .gitignore ├── uvr5_weights └── .gitignore ├── go-realtime-gui.bat ├── MANIFEST.in ├── docs ├── 小白简易教程.doc ├── Changelog_CN.md ├── training_tips_ja.md ├── training_tips_ko.md ├── faq.md ├── faiss_tips_ja.md ├── Changelog_KO.md ├── README.ko.han.md ├── README.ja.md ├── README.ko.md ├── training_tips_en.md └── Changelog_EN.md ├── go-web.bat ├── .gitignore ├── logs └── mute │ ├── 1_16k_wavs │ └── mute.wav │ ├── 2a_f0 │ └── mute.wav.npy │ ├── 0_gt_wavs │ ├── mute32k.wav │ ├── mute40k.wav │ └── mute48k.wav │ ├── 2b-f0nsf │ └── mute.wav.npy │ ├── 3_feature256 │ └── mute.npy │ └── 3_feature768 │ └── mute.npy ├── Dockerfile ├── setup.py ├── requirements-win-for-realtime_vc_gui.txt ├── requirements.txt ├── i18n.py ├── extract_locale.py ├── .github └── workflows │ ├── genlocale.yml │ ├── pull_format.yml │ ├── unitest.yml │ ├── push_format.yml │ └── docker.yml ├── LICENSE ├── configs ├── 40k.json ├── 32k.json ├── 32k_v2.json ├── 48k.json └── 48k_v2.json ├── MIT协议暨相关引用库协议 ├── run.sh ├── infer_uvr.py ├── extract_feature_print.py ├── extract_f0_rmvpe.py ├── trainset_preprocess_pipeline_print.py ├── config.py └── README.md /__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lib/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tools/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tools/infer/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lib/infer_pack/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lib/uvr5_pack/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lib/uvr5_pack/lib_v5/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /venv.sh: -------------------------------------------------------------------------------- 1 | python3 -m venv .venv 2 | -------------------------------------------------------------------------------- /pretrained/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | !.gitignore 3 | -------------------------------------------------------------------------------- /weights/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | !.gitignore 3 | -------------------------------------------------------------------------------- /lib/infer_pack/modules/F0Predictor/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lib/uvr5_pack/lib_v5/modelparams/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pretrained_v2/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | !.gitignore 3 | -------------------------------------------------------------------------------- /uvr5_weights/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | !.gitignore 3 | -------------------------------------------------------------------------------- /go-realtime-gui.bat: -------------------------------------------------------------------------------- 1 | runtime\python.exe gui_v1.py 2 | pause 3 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | recursive-include lib/uvr5_pack/lib_v5/modelparams *.json 2 | -------------------------------------------------------------------------------- /docs/小白简易教程.doc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JarodMica/rvc/HEAD/docs/小白简易教程.doc -------------------------------------------------------------------------------- /go-web.bat: -------------------------------------------------------------------------------- 1 | runtime\python.exe infer-web.py --pycmd runtime\python.exe --port 7897 2 | pause 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | __pycache__ 3 | /TEMP 4 | *.pyd 5 | hubert_base.pt 6 | /logs 7 | .venv 8 | -------------------------------------------------------------------------------- /logs/mute/1_16k_wavs/mute.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JarodMica/rvc/HEAD/logs/mute/1_16k_wavs/mute.wav -------------------------------------------------------------------------------- /logs/mute/2a_f0/mute.wav.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JarodMica/rvc/HEAD/logs/mute/2a_f0/mute.wav.npy -------------------------------------------------------------------------------- /lib/train/cmd.txt: -------------------------------------------------------------------------------- 1 | python train_nsf_sim_cache_sid.py -c configs/mi_mix40k_nsf_co256_cs1sid_ms2048.json -m ft-mi -------------------------------------------------------------------------------- /logs/mute/0_gt_wavs/mute32k.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JarodMica/rvc/HEAD/logs/mute/0_gt_wavs/mute32k.wav -------------------------------------------------------------------------------- /logs/mute/0_gt_wavs/mute40k.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JarodMica/rvc/HEAD/logs/mute/0_gt_wavs/mute40k.wav -------------------------------------------------------------------------------- /logs/mute/0_gt_wavs/mute48k.wav: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JarodMica/rvc/HEAD/logs/mute/0_gt_wavs/mute48k.wav -------------------------------------------------------------------------------- /logs/mute/2b-f0nsf/mute.wav.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JarodMica/rvc/HEAD/logs/mute/2b-f0nsf/mute.wav.npy -------------------------------------------------------------------------------- /logs/mute/3_feature256/mute.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JarodMica/rvc/HEAD/logs/mute/3_feature256/mute.npy -------------------------------------------------------------------------------- /logs/mute/3_feature768/mute.npy: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JarodMica/rvc/HEAD/logs/mute/3_feature768/mute.npy -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # syntax=docker/dockerfile:1 2 | 3 | FROM python:3.10-bullseye 4 | 5 | EXPOSE 7865 6 | 7 | WORKDIR /app 8 | 9 | COPY . . 10 | 11 | RUN pip3 install -r requirements.txt 12 | 13 | CMD ["python3", "infer-web.py"] -------------------------------------------------------------------------------- /lib/uvr5_pack/lib_v5/modelparams/1band_sr44100_hl256.json: -------------------------------------------------------------------------------- 1 | { 2 | "bins": 256, 3 | "unstable_bins": 0, 4 | "reduction_bins": 0, 5 | "band": { 6 | "1": { 7 | "sr": 44100, 8 | "hl": 256, 9 | "n_fft": 512, 10 | "crop_start": 0, 11 | "crop_stop": 256, 12 | "hpf_start": -1, 13 | "res_type": "sinc_best" 14 | } 15 | }, 16 | "sr": 44100, 17 | "pre_filter_start": 256, 18 | "pre_filter_stop": 256 19 | } -------------------------------------------------------------------------------- /lib/uvr5_pack/lib_v5/modelparams/1band_sr16000_hl512.json: -------------------------------------------------------------------------------- 1 | { 2 | "bins": 1024, 3 | "unstable_bins": 0, 4 | "reduction_bins": 0, 5 | "band": { 6 | "1": { 7 | "sr": 16000, 8 | "hl": 512, 9 | "n_fft": 2048, 10 | "crop_start": 0, 11 | "crop_stop": 1024, 12 | "hpf_start": -1, 13 | "res_type": "sinc_best" 14 | } 15 | }, 16 | "sr": 16000, 17 | "pre_filter_start": 1023, 18 | "pre_filter_stop": 1024 19 | } -------------------------------------------------------------------------------- /lib/uvr5_pack/lib_v5/modelparams/1band_sr32000_hl512.json: -------------------------------------------------------------------------------- 1 | { 2 | "bins": 1024, 3 | "unstable_bins": 0, 4 | "reduction_bins": 0, 5 | "band": { 6 | "1": { 7 | "sr": 32000, 8 | "hl": 512, 9 | "n_fft": 2048, 10 | "crop_start": 0, 11 | "crop_stop": 1024, 12 | "hpf_start": -1, 13 | "res_type": "kaiser_fast" 14 | } 15 | }, 16 | "sr": 32000, 17 | "pre_filter_start": 1000, 18 | "pre_filter_stop": 1021 19 | } -------------------------------------------------------------------------------- /lib/uvr5_pack/lib_v5/modelparams/1band_sr33075_hl384.json: -------------------------------------------------------------------------------- 1 | { 2 | "bins": 1024, 3 | "unstable_bins": 0, 4 | "reduction_bins": 0, 5 | "band": { 6 | "1": { 7 | "sr": 33075, 8 | "hl": 384, 9 | "n_fft": 2048, 10 | "crop_start": 0, 11 | "crop_stop": 1024, 12 | "hpf_start": -1, 13 | "res_type": "sinc_best" 14 | } 15 | }, 16 | "sr": 33075, 17 | "pre_filter_start": 1000, 18 | "pre_filter_stop": 1021 19 | } -------------------------------------------------------------------------------- /lib/uvr5_pack/lib_v5/modelparams/1band_sr44100_hl1024.json: -------------------------------------------------------------------------------- 1 | { 2 | "bins": 1024, 3 | "unstable_bins": 0, 4 | "reduction_bins": 0, 5 | "band": { 6 | "1": { 7 | "sr": 44100, 8 | "hl": 1024, 9 | "n_fft": 2048, 10 | "crop_start": 0, 11 | "crop_stop": 1024, 12 | "hpf_start": -1, 13 | "res_type": "sinc_best" 14 | } 15 | }, 16 | "sr": 44100, 17 | "pre_filter_start": 1023, 18 | "pre_filter_stop": 1024 19 | } -------------------------------------------------------------------------------- /lib/uvr5_pack/lib_v5/modelparams/1band_sr44100_hl512.json: -------------------------------------------------------------------------------- 1 | { 2 | "bins": 1024, 3 | "unstable_bins": 0, 4 | "reduction_bins": 0, 5 | "band": { 6 | "1": { 7 | "sr": 44100, 8 | "hl": 512, 9 | "n_fft": 2048, 10 | "crop_start": 0, 11 | "crop_stop": 1024, 12 | "hpf_start": -1, 13 | "res_type": "sinc_best" 14 | } 15 | }, 16 | "sr": 44100, 17 | "pre_filter_start": 1023, 18 | "pre_filter_stop": 1024 19 | } -------------------------------------------------------------------------------- /lib/uvr5_pack/lib_v5/modelparams/1band_sr44100_hl512_cut.json: -------------------------------------------------------------------------------- 1 | { 2 | "bins": 1024, 3 | "unstable_bins": 0, 4 | "reduction_bins": 0, 5 | "band": { 6 | "1": { 7 | "sr": 44100, 8 | "hl": 512, 9 | "n_fft": 2048, 10 | "crop_start": 0, 11 | "crop_stop": 700, 12 | "hpf_start": -1, 13 | "res_type": "sinc_best" 14 | } 15 | }, 16 | "sr": 44100, 17 | "pre_filter_start": 1023, 18 | "pre_filter_stop": 700 19 | } -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | 3 | # Read the requirements.txt file 4 | with open('requirements.txt', 'r') as f: 5 | requirements = f.read().splitlines() 6 | 7 | setup( 8 | name='rvc', 9 | version='0.1', 10 | description='rvc package', 11 | author='RVC people', 12 | packages=find_packages(), 13 | install_requires=requirements, 14 | package_data={ 15 | 'lib.uvr5_pack.lib_v5.modelparams': ['*.json'] 16 | } 17 | 18 | ) 19 | -------------------------------------------------------------------------------- /lib/infer_pack/modules/F0Predictor/F0Predictor.py: -------------------------------------------------------------------------------- 1 | class F0Predictor(object): 2 | def compute_f0(self, wav, p_len): 3 | """ 4 | input: wav:[signal_length] 5 | p_len:int 6 | output: f0:[signal_length//hop_length] 7 | """ 8 | pass 9 | 10 | def compute_f0_uv(self, wav, p_len): 11 | """ 12 | input: wav:[signal_length] 13 | p_len:int 14 | output: f0:[signal_length//hop_length],uv:[signal_length//hop_length] 15 | """ 16 | pass 17 | -------------------------------------------------------------------------------- /lib/uvr5_pack/lib_v5/modelparams/2band_48000.json: -------------------------------------------------------------------------------- 1 | { 2 | "bins": 768, 3 | "unstable_bins": 7, 4 | "reduction_bins": 705, 5 | "band": { 6 | "1": { 7 | "sr": 6000, 8 | "hl": 66, 9 | "n_fft": 512, 10 | "crop_start": 0, 11 | "crop_stop": 240, 12 | "lpf_start": 60, 13 | "lpf_stop": 240, 14 | "res_type": "sinc_fastest" 15 | }, 16 | "2": { 17 | "sr": 48000, 18 | "hl": 528, 19 | "n_fft": 1536, 20 | "crop_start": 22, 21 | "crop_stop": 505, 22 | "hpf_start": 82, 23 | "hpf_stop": 22, 24 | "res_type": "sinc_medium" 25 | } 26 | }, 27 | "sr": 48000, 28 | "pre_filter_start": 710, 29 | "pre_filter_stop": 731 30 | } -------------------------------------------------------------------------------- /lib/uvr5_pack/lib_v5/modelparams/2band_32000.json: -------------------------------------------------------------------------------- 1 | { 2 | "bins": 768, 3 | "unstable_bins": 7, 4 | "reduction_bins": 705, 5 | "band": { 6 | "1": { 7 | "sr": 6000, 8 | "hl": 66, 9 | "n_fft": 512, 10 | "crop_start": 0, 11 | "crop_stop": 240, 12 | "lpf_start": 60, 13 | "lpf_stop": 118, 14 | "res_type": "sinc_fastest" 15 | }, 16 | "2": { 17 | "sr": 32000, 18 | "hl": 352, 19 | "n_fft": 1024, 20 | "crop_start": 22, 21 | "crop_stop": 505, 22 | "hpf_start": 44, 23 | "hpf_stop": 23, 24 | "res_type": "sinc_medium" 25 | } 26 | }, 27 | "sr": 32000, 28 | "pre_filter_start": 710, 29 | "pre_filter_stop": 731 30 | } 31 | -------------------------------------------------------------------------------- /lib/uvr5_pack/lib_v5/modelparams/2band_44100_lofi.json: -------------------------------------------------------------------------------- 1 | { 2 | "bins": 512, 3 | "unstable_bins": 7, 4 | "reduction_bins": 510, 5 | "band": { 6 | "1": { 7 | "sr": 11025, 8 | "hl": 160, 9 | "n_fft": 768, 10 | "crop_start": 0, 11 | "crop_stop": 192, 12 | "lpf_start": 41, 13 | "lpf_stop": 139, 14 | "res_type": "sinc_fastest" 15 | }, 16 | "2": { 17 | "sr": 44100, 18 | "hl": 640, 19 | "n_fft": 1024, 20 | "crop_start": 10, 21 | "crop_stop": 320, 22 | "hpf_start": 47, 23 | "hpf_stop": 15, 24 | "res_type": "sinc_medium" 25 | } 26 | }, 27 | "sr": 44100, 28 | "pre_filter_start": 510, 29 | "pre_filter_stop": 512 30 | } 31 | -------------------------------------------------------------------------------- /requirements-win-for-realtime_vc_gui.txt: -------------------------------------------------------------------------------- 1 | #1.Install torch from pytorch.org: 2 | #torch 2.0 with cuda 11.8 3 | #pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118 4 | #torch 1.11.0 with cuda 11.3 5 | #pip install torch==1.11.0+cu113 torchvision==0.12.0+cu113 torchaudio==0.11.0 --extra-index-url https://download.pytorch.org/whl/cu113 6 | einops 7 | fairseq 8 | flask 9 | flask_cors 10 | gin 11 | gin_config 12 | librosa 13 | local_attention 14 | matplotlib 15 | praat-parselmouth 16 | pyworld 17 | PyYAML 18 | resampy 19 | scikit_learn 20 | scipy 21 | SoundFile 22 | tensorboard 23 | tqdm 24 | wave 25 | PySimpleGUI 26 | sounddevice 27 | gradio 28 | noisereduce 29 | -------------------------------------------------------------------------------- /tools/onnx_inference_demo.py: -------------------------------------------------------------------------------- 1 | import soundfile 2 | from ..lib.infer_pack.onnx_inference import OnnxRVC 3 | 4 | hop_size = 512 5 | sampling_rate = 40000 # 采样率 6 | f0_up_key = 0 # 升降调 7 | sid = 0 # 角色ID 8 | f0_method = "dio" # F0提取算法 9 | model_path = "ShirohaRVC.onnx" # 模型的完整路径 10 | vec_name = "vec-256-layer-9" # 内部自动补齐为 f"pretrained/{vec_name}.onnx" 需要onnx的vec模型 11 | wav_path = "123.wav" # 输入路径或ByteIO实例 12 | out_path = "out.wav" # 输出路径或ByteIO实例 13 | 14 | model = OnnxRVC( 15 | model_path, vec_path=vec_name, sr=sampling_rate, hop_size=hop_size, device="cuda" 16 | ) 17 | 18 | audio = model.inference(wav_path, sid, f0_method=f0_method, f0_up_key=f0_up_key) 19 | 20 | soundfile.write(out_path, audio, sampling_rate) 21 | -------------------------------------------------------------------------------- /tools/infer/trans_weights.py: -------------------------------------------------------------------------------- 1 | import torch, pdb 2 | 3 | # a=torch.load(r"E:\codes\py39\vits_vc_gpu_train\logs\ft-mi-suc\G_1000.pth")["model"]#sim_nsf# 4 | # a=torch.load(r"E:\codes\py39\vits_vc_gpu_train\logs\ft-mi-freeze-vocoder-flow-enc_q\G_1000.pth")["model"]#sim_nsf# 5 | # a=torch.load(r"E:\codes\py39\vits_vc_gpu_train\logs\ft-mi-freeze-vocoder\G_1000.pth")["model"]#sim_nsf# 6 | # a=torch.load(r"E:\codes\py39\vits_vc_gpu_train\logs\ft-mi-test\G_1000.pth")["model"]#sim_nsf# 7 | a = torch.load( 8 | r"E:\codes\py39\vits_vc_gpu_train\logs\ft-mi-no_opt-no_dropout\G_1000.pth" 9 | )[ 10 | "model" 11 | ] # sim_nsf# 12 | for key in a.keys(): 13 | a[key] = a[key].half() 14 | # torch.save(a,"ft-mi-freeze-vocoder_true_1k.pt")# 15 | # torch.save(a,"ft-mi-sim1k.pt")# 16 | torch.save(a, "ft-mi-no_opt-no_dropout.pt") # 17 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | joblib>=1.1.0 2 | numba==0.56.4 3 | numpy==1.23.5 4 | scipy==1.9.3 5 | librosa==0.9.1 6 | llvmlite==0.39.0 7 | fairseq==0.12.2 8 | faiss-cpu==1.7.3 9 | gradio==3.14.0 10 | Cython 11 | pydub>=0.25.1 12 | soundfile>=0.12.1 13 | ffmpeg-python>=0.2.0 14 | tensorboardX 15 | Jinja2>=3.1.2 16 | json5 17 | Markdown 18 | matplotlib>=3.7.0 19 | matplotlib-inline>=0.1.3 20 | praat-parselmouth>=0.4.2 21 | Pillow>=9.1.1 22 | resampy>=0.4.2 23 | scikit-learn 24 | tensorboard 25 | tqdm>=4.63.1 26 | tornado>=6.1 27 | Werkzeug>=2.2.3 28 | uc-micro-py>=1.0.1 29 | sympy>=1.11.1 30 | tabulate>=0.8.10 31 | PyYAML>=6.0 32 | pyasn1>=0.4.8 33 | pyasn1-modules>=0.2.8 34 | fsspec>=2022.11.0 35 | absl-py>=1.2.0 36 | audioread 37 | uvicorn>=0.21.1 38 | colorama>=0.4.5 39 | pyworld==0.3.2 40 | httpx==0.23.0 41 | #onnxruntime-gpu 42 | torchcrepe==0.0.20 43 | fastapi==0.88 44 | ffmpy==0.3.1 45 | -------------------------------------------------------------------------------- /lib/audio.py: -------------------------------------------------------------------------------- 1 | import ffmpeg 2 | import numpy as np 3 | 4 | 5 | def load_audio(file, sr): 6 | try: 7 | # https://github.com/openai/whisper/blob/main/whisper/audio.py#L26 8 | # This launches a subprocess to decode audio while down-mixing and resampling as necessary. 9 | # Requires the ffmpeg CLI and `ffmpeg-python` package to be installed. 10 | file = ( 11 | file.strip(" ").strip('"').strip("\n").strip('"').strip(" ") 12 | ) # 防止小白拷路径头尾带了空格和"和回车 13 | out, _ = ( 14 | ffmpeg.input(file, threads=0) 15 | .output("-", format="f32le", acodec="pcm_f32le", ac=1, ar=sr) 16 | .run(cmd=["ffmpeg", "-nostdin"], capture_stdout=True, capture_stderr=True) 17 | ) 18 | except Exception as e: 19 | raise RuntimeError(f"Failed to load audio: {e}") 20 | 21 | return np.frombuffer(out, np.float32).flatten() 22 | -------------------------------------------------------------------------------- /lib/uvr5_pack/lib_v5/modelparams/3band_44100.json: -------------------------------------------------------------------------------- 1 | { 2 | "bins": 768, 3 | "unstable_bins": 5, 4 | "reduction_bins": 733, 5 | "band": { 6 | "1": { 7 | "sr": 11025, 8 | "hl": 128, 9 | "n_fft": 768, 10 | "crop_start": 0, 11 | "crop_stop": 278, 12 | "lpf_start": 28, 13 | "lpf_stop": 140, 14 | "res_type": "polyphase" 15 | }, 16 | "2": { 17 | "sr": 22050, 18 | "hl": 256, 19 | "n_fft": 768, 20 | "crop_start": 14, 21 | "crop_stop": 322, 22 | "hpf_start": 70, 23 | "hpf_stop": 14, 24 | "lpf_start": 283, 25 | "lpf_stop": 314, 26 | "res_type": "polyphase" 27 | }, 28 | "3": { 29 | "sr": 44100, 30 | "hl": 512, 31 | "n_fft": 768, 32 | "crop_start": 131, 33 | "crop_stop": 313, 34 | "hpf_start": 154, 35 | "hpf_stop": 141, 36 | "res_type": "sinc_medium" 37 | } 38 | }, 39 | "sr": 44100, 40 | "pre_filter_start": 757, 41 | "pre_filter_stop": 768 42 | } 43 | -------------------------------------------------------------------------------- /i18n.py: -------------------------------------------------------------------------------- 1 | import locale 2 | import json 3 | import os 4 | 5 | 6 | def load_language_list(language): 7 | with open(f"./lib/i18n/{language}.json", "r", encoding="utf-8") as f: 8 | language_list = json.load(f) 9 | return language_list 10 | 11 | 12 | class I18nAuto: 13 | def __init__(self, language=None): 14 | if language in ["Auto", None]: 15 | language = locale.getdefaultlocale()[ 16 | 0 17 | ] # getlocale can't identify the system's language ((None, None)) 18 | if not os.path.exists(f"./lib/i18n/{language}.json"): 19 | language = "en_US" 20 | self.language = language 21 | # print("Use Language:", language) 22 | self.language_map = load_language_list(language) 23 | 24 | def __call__(self, key): 25 | return self.language_map.get(key, key) 26 | 27 | def print(self): 28 | print("Use Language:", self.language) 29 | -------------------------------------------------------------------------------- /extract_locale.py: -------------------------------------------------------------------------------- 1 | import json 2 | import re 3 | 4 | # Define regular expression patterns 5 | pattern = r"""i18n\([\s\n\t]*(["'][^"']+["'])[\s\n\t]*\)""" 6 | 7 | # Initialize the dictionary to store key-value pairs 8 | data = {} 9 | 10 | 11 | def process(fn: str): 12 | global data 13 | with open(fn, "r", encoding="utf-8") as f: 14 | contents = f.read() 15 | matches = re.findall(pattern, contents) 16 | for key in matches: 17 | key = eval(key) 18 | print("extract:", key) 19 | data[key] = key 20 | 21 | 22 | print("processing infer-web.py") 23 | process("infer-web.py") 24 | 25 | print("processing gui_v0.py") 26 | process("gui_v0.py") 27 | 28 | print("processing gui_v1.py") 29 | process("gui_v1.py") 30 | 31 | # Save as a JSON file 32 | with open("./lib/i18n/zh_CN.json", "w", encoding="utf-8") as f: 33 | json.dump(data, f, ensure_ascii=False, indent=4) 34 | f.write("\n") 35 | -------------------------------------------------------------------------------- /lib/uvr5_pack/lib_v5/modelparams/ensemble.json: -------------------------------------------------------------------------------- 1 | { 2 | "mid_side_b2": true, 3 | "bins": 1280, 4 | "unstable_bins": 7, 5 | "reduction_bins": 565, 6 | "band": { 7 | "1": { 8 | "sr": 11025, 9 | "hl": 108, 10 | "n_fft": 2048, 11 | "crop_start": 0, 12 | "crop_stop": 374, 13 | "lpf_start": 92, 14 | "lpf_stop": 186, 15 | "res_type": "polyphase" 16 | }, 17 | "2": { 18 | "sr": 22050, 19 | "hl": 216, 20 | "n_fft": 1536, 21 | "crop_start": 0, 22 | "crop_stop": 424, 23 | "hpf_start": 68, 24 | "hpf_stop": 34, 25 | "lpf_start": 348, 26 | "lpf_stop": 418, 27 | "res_type": "polyphase" 28 | }, 29 | "3": { 30 | "sr": 44100, 31 | "hl": 432, 32 | "n_fft": 1280, 33 | "crop_start": 132, 34 | "crop_stop": 614, 35 | "hpf_start": 172, 36 | "hpf_stop": 144, 37 | "res_type": "polyphase" 38 | } 39 | }, 40 | "sr": 44100, 41 | "pre_filter_start": 1280, 42 | "pre_filter_stop": 1280 43 | } -------------------------------------------------------------------------------- /lib/uvr5_pack/lib_v5/modelparams/3band_44100_mid.json: -------------------------------------------------------------------------------- 1 | { 2 | "mid_side": true, 3 | "bins": 768, 4 | "unstable_bins": 5, 5 | "reduction_bins": 733, 6 | "band": { 7 | "1": { 8 | "sr": 11025, 9 | "hl": 128, 10 | "n_fft": 768, 11 | "crop_start": 0, 12 | "crop_stop": 278, 13 | "lpf_start": 28, 14 | "lpf_stop": 140, 15 | "res_type": "polyphase" 16 | }, 17 | "2": { 18 | "sr": 22050, 19 | "hl": 256, 20 | "n_fft": 768, 21 | "crop_start": 14, 22 | "crop_stop": 322, 23 | "hpf_start": 70, 24 | "hpf_stop": 14, 25 | "lpf_start": 283, 26 | "lpf_stop": 314, 27 | "res_type": "polyphase" 28 | }, 29 | "3": { 30 | "sr": 44100, 31 | "hl": 512, 32 | "n_fft": 768, 33 | "crop_start": 131, 34 | "crop_stop": 313, 35 | "hpf_start": 154, 36 | "hpf_stop": 141, 37 | "res_type": "sinc_medium" 38 | } 39 | }, 40 | "sr": 44100, 41 | "pre_filter_start": 757, 42 | "pre_filter_stop": 768 43 | } 44 | -------------------------------------------------------------------------------- /lib/uvr5_pack/lib_v5/modelparams/3band_44100_msb2.json: -------------------------------------------------------------------------------- 1 | { 2 | "mid_side_b2": true, 3 | "bins": 640, 4 | "unstable_bins": 7, 5 | "reduction_bins": 565, 6 | "band": { 7 | "1": { 8 | "sr": 11025, 9 | "hl": 108, 10 | "n_fft": 1024, 11 | "crop_start": 0, 12 | "crop_stop": 187, 13 | "lpf_start": 92, 14 | "lpf_stop": 186, 15 | "res_type": "polyphase" 16 | }, 17 | "2": { 18 | "sr": 22050, 19 | "hl": 216, 20 | "n_fft": 768, 21 | "crop_start": 0, 22 | "crop_stop": 212, 23 | "hpf_start": 68, 24 | "hpf_stop": 34, 25 | "lpf_start": 174, 26 | "lpf_stop": 209, 27 | "res_type": "polyphase" 28 | }, 29 | "3": { 30 | "sr": 44100, 31 | "hl": 432, 32 | "n_fft": 640, 33 | "crop_start": 66, 34 | "crop_stop": 307, 35 | "hpf_start": 86, 36 | "hpf_stop": 72, 37 | "res_type": "kaiser_fast" 38 | } 39 | }, 40 | "sr": 44100, 41 | "pre_filter_start": 639, 42 | "pre_filter_stop": 640 43 | } 44 | -------------------------------------------------------------------------------- /.github/workflows/genlocale.yml: -------------------------------------------------------------------------------- 1 | name: genlocale 2 | on: 3 | push: 4 | branches: 5 | - main 6 | jobs: 7 | genlocale: 8 | name: genlocale 9 | runs-on: ubuntu-latest 10 | steps: 11 | - name: Check out 12 | uses: actions/checkout@master 13 | 14 | - name: Run locale generation 15 | run: | 16 | python3 extract_locale.py 17 | cd lib/i18n && python3 locale_diff.py 18 | 19 | - name: Commit back 20 | if: ${{ !github.head_ref }} 21 | continue-on-error: true 22 | run: | 23 | git config --local user.name 'github-actions[bot]' 24 | git config --local user.email 'github-actions[bot]@users.noreply.github.com' 25 | git add --all 26 | git commit -m "🎨 同步 locale" 27 | 28 | - name: Create Pull Request 29 | if: ${{ !github.head_ref }} 30 | continue-on-error: true 31 | uses: peter-evans/create-pull-request@v4 32 | 33 | -------------------------------------------------------------------------------- /.github/workflows/pull_format.yml: -------------------------------------------------------------------------------- 1 | name: pull format 2 | 3 | on: [pull_request] 4 | 5 | permissions: 6 | contents: write 7 | 8 | jobs: 9 | pull_format: 10 | runs-on: ${{ matrix.os }} 11 | 12 | strategy: 13 | matrix: 14 | python-version: ["3.10"] 15 | os: [ubuntu-latest] 16 | fail-fast: false 17 | 18 | continue-on-error: true 19 | 20 | steps: 21 | - name: checkout 22 | continue-on-error: true 23 | uses: actions/checkout@v3 24 | with: 25 | ref: ${{ github.head_ref }} 26 | fetch-depth: 0 27 | 28 | - name: Set up Python ${{ matrix.python-version }} 29 | uses: actions/setup-python@v4 30 | with: 31 | python-version: ${{ matrix.python-version }} 32 | 33 | - name: Install Black 34 | run: pip install "black[jupyter]" 35 | 36 | - name: Run Black 37 | # run: black $(git ls-files '*.py') 38 | run: black . 39 | 40 | - name: Commit Back 41 | uses: stefanzweifel/git-auto-commit-action@v4 42 | with: 43 | commit_message: Apply Code Formatter Change 44 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 liujing04 4 | Copyright (c) 2023 源文雨 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | -------------------------------------------------------------------------------- /lib/uvr5_pack/lib_v5/modelparams/4band_v2.json: -------------------------------------------------------------------------------- 1 | { 2 | "bins": 672, 3 | "unstable_bins": 8, 4 | "reduction_bins": 637, 5 | "band": { 6 | "1": { 7 | "sr": 7350, 8 | "hl": 80, 9 | "n_fft": 640, 10 | "crop_start": 0, 11 | "crop_stop": 85, 12 | "lpf_start": 25, 13 | "lpf_stop": 53, 14 | "res_type": "polyphase" 15 | }, 16 | "2": { 17 | "sr": 7350, 18 | "hl": 80, 19 | "n_fft": 320, 20 | "crop_start": 4, 21 | "crop_stop": 87, 22 | "hpf_start": 25, 23 | "hpf_stop": 12, 24 | "lpf_start": 31, 25 | "lpf_stop": 62, 26 | "res_type": "polyphase" 27 | }, 28 | "3": { 29 | "sr": 14700, 30 | "hl": 160, 31 | "n_fft": 512, 32 | "crop_start": 17, 33 | "crop_stop": 216, 34 | "hpf_start": 48, 35 | "hpf_stop": 24, 36 | "lpf_start": 139, 37 | "lpf_stop": 210, 38 | "res_type": "polyphase" 39 | }, 40 | "4": { 41 | "sr": 44100, 42 | "hl": 480, 43 | "n_fft": 960, 44 | "crop_start": 78, 45 | "crop_stop": 383, 46 | "hpf_start": 130, 47 | "hpf_stop": 86, 48 | "res_type": "kaiser_fast" 49 | } 50 | }, 51 | "sr": 44100, 52 | "pre_filter_start": 668, 53 | "pre_filter_stop": 672 54 | } -------------------------------------------------------------------------------- /lib/uvr5_pack/lib_v5/modelparams/4band_44100.json: -------------------------------------------------------------------------------- 1 | { 2 | "bins": 768, 3 | "unstable_bins": 7, 4 | "reduction_bins": 668, 5 | "band": { 6 | "1": { 7 | "sr": 11025, 8 | "hl": 128, 9 | "n_fft": 1024, 10 | "crop_start": 0, 11 | "crop_stop": 186, 12 | "lpf_start": 37, 13 | "lpf_stop": 73, 14 | "res_type": "polyphase" 15 | }, 16 | "2": { 17 | "sr": 11025, 18 | "hl": 128, 19 | "n_fft": 512, 20 | "crop_start": 4, 21 | "crop_stop": 185, 22 | "hpf_start": 36, 23 | "hpf_stop": 18, 24 | "lpf_start": 93, 25 | "lpf_stop": 185, 26 | "res_type": "polyphase" 27 | }, 28 | "3": { 29 | "sr": 22050, 30 | "hl": 256, 31 | "n_fft": 512, 32 | "crop_start": 46, 33 | "crop_stop": 186, 34 | "hpf_start": 93, 35 | "hpf_stop": 46, 36 | "lpf_start": 164, 37 | "lpf_stop": 186, 38 | "res_type": "polyphase" 39 | }, 40 | "4": { 41 | "sr": 44100, 42 | "hl": 512, 43 | "n_fft": 768, 44 | "crop_start": 121, 45 | "crop_stop": 382, 46 | "hpf_start": 138, 47 | "hpf_stop": 123, 48 | "res_type": "sinc_medium" 49 | } 50 | }, 51 | "sr": 44100, 52 | "pre_filter_start": 740, 53 | "pre_filter_stop": 768 54 | } 55 | -------------------------------------------------------------------------------- /lib/uvr5_pack/lib_v5/modelparams/4band_44100_sw.json: -------------------------------------------------------------------------------- 1 | { 2 | "stereo_w": true, 3 | "bins": 768, 4 | "unstable_bins": 7, 5 | "reduction_bins": 668, 6 | "band": { 7 | "1": { 8 | "sr": 11025, 9 | "hl": 128, 10 | "n_fft": 1024, 11 | "crop_start": 0, 12 | "crop_stop": 186, 13 | "lpf_start": 37, 14 | "lpf_stop": 73, 15 | "res_type": "polyphase" 16 | }, 17 | "2": { 18 | "sr": 11025, 19 | "hl": 128, 20 | "n_fft": 512, 21 | "crop_start": 4, 22 | "crop_stop": 185, 23 | "hpf_start": 36, 24 | "hpf_stop": 18, 25 | "lpf_start": 93, 26 | "lpf_stop": 185, 27 | "res_type": "polyphase" 28 | }, 29 | "3": { 30 | "sr": 22050, 31 | "hl": 256, 32 | "n_fft": 512, 33 | "crop_start": 46, 34 | "crop_stop": 186, 35 | "hpf_start": 93, 36 | "hpf_stop": 46, 37 | "lpf_start": 164, 38 | "lpf_stop": 186, 39 | "res_type": "polyphase" 40 | }, 41 | "4": { 42 | "sr": 44100, 43 | "hl": 512, 44 | "n_fft": 768, 45 | "crop_start": 121, 46 | "crop_stop": 382, 47 | "hpf_start": 138, 48 | "hpf_stop": 123, 49 | "res_type": "sinc_medium" 50 | } 51 | }, 52 | "sr": 44100, 53 | "pre_filter_start": 740, 54 | "pre_filter_stop": 768 55 | } -------------------------------------------------------------------------------- /lib/uvr5_pack/lib_v5/modelparams/4band_44100_msb.json: -------------------------------------------------------------------------------- 1 | { 2 | "mid_side_b": true, 3 | "bins": 768, 4 | "unstable_bins": 7, 5 | "reduction_bins": 668, 6 | "band": { 7 | "1": { 8 | "sr": 11025, 9 | "hl": 128, 10 | "n_fft": 1024, 11 | "crop_start": 0, 12 | "crop_stop": 186, 13 | "lpf_start": 37, 14 | "lpf_stop": 73, 15 | "res_type": "polyphase" 16 | }, 17 | "2": { 18 | "sr": 11025, 19 | "hl": 128, 20 | "n_fft": 512, 21 | "crop_start": 4, 22 | "crop_stop": 185, 23 | "hpf_start": 36, 24 | "hpf_stop": 18, 25 | "lpf_start": 93, 26 | "lpf_stop": 185, 27 | "res_type": "polyphase" 28 | }, 29 | "3": { 30 | "sr": 22050, 31 | "hl": 256, 32 | "n_fft": 512, 33 | "crop_start": 46, 34 | "crop_stop": 186, 35 | "hpf_start": 93, 36 | "hpf_stop": 46, 37 | "lpf_start": 164, 38 | "lpf_stop": 186, 39 | "res_type": "polyphase" 40 | }, 41 | "4": { 42 | "sr": 44100, 43 | "hl": 512, 44 | "n_fft": 768, 45 | "crop_start": 121, 46 | "crop_stop": 382, 47 | "hpf_start": 138, 48 | "hpf_stop": 123, 49 | "res_type": "sinc_medium" 50 | } 51 | }, 52 | "sr": 44100, 53 | "pre_filter_start": 740, 54 | "pre_filter_stop": 768 55 | } -------------------------------------------------------------------------------- /lib/uvr5_pack/lib_v5/modelparams/4band_44100_msb2.json: -------------------------------------------------------------------------------- 1 | { 2 | "mid_side_b": true, 3 | "bins": 768, 4 | "unstable_bins": 7, 5 | "reduction_bins": 668, 6 | "band": { 7 | "1": { 8 | "sr": 11025, 9 | "hl": 128, 10 | "n_fft": 1024, 11 | "crop_start": 0, 12 | "crop_stop": 186, 13 | "lpf_start": 37, 14 | "lpf_stop": 73, 15 | "res_type": "polyphase" 16 | }, 17 | "2": { 18 | "sr": 11025, 19 | "hl": 128, 20 | "n_fft": 512, 21 | "crop_start": 4, 22 | "crop_stop": 185, 23 | "hpf_start": 36, 24 | "hpf_stop": 18, 25 | "lpf_start": 93, 26 | "lpf_stop": 185, 27 | "res_type": "polyphase" 28 | }, 29 | "3": { 30 | "sr": 22050, 31 | "hl": 256, 32 | "n_fft": 512, 33 | "crop_start": 46, 34 | "crop_stop": 186, 35 | "hpf_start": 93, 36 | "hpf_stop": 46, 37 | "lpf_start": 164, 38 | "lpf_stop": 186, 39 | "res_type": "polyphase" 40 | }, 41 | "4": { 42 | "sr": 44100, 43 | "hl": 512, 44 | "n_fft": 768, 45 | "crop_start": 121, 46 | "crop_stop": 382, 47 | "hpf_start": 138, 48 | "hpf_stop": 123, 49 | "res_type": "sinc_medium" 50 | } 51 | }, 52 | "sr": 44100, 53 | "pre_filter_start": 740, 54 | "pre_filter_stop": 768 55 | } -------------------------------------------------------------------------------- /lib/uvr5_pack/lib_v5/modelparams/4band_44100_reverse.json: -------------------------------------------------------------------------------- 1 | { 2 | "reverse": true, 3 | "bins": 768, 4 | "unstable_bins": 7, 5 | "reduction_bins": 668, 6 | "band": { 7 | "1": { 8 | "sr": 11025, 9 | "hl": 128, 10 | "n_fft": 1024, 11 | "crop_start": 0, 12 | "crop_stop": 186, 13 | "lpf_start": 37, 14 | "lpf_stop": 73, 15 | "res_type": "polyphase" 16 | }, 17 | "2": { 18 | "sr": 11025, 19 | "hl": 128, 20 | "n_fft": 512, 21 | "crop_start": 4, 22 | "crop_stop": 185, 23 | "hpf_start": 36, 24 | "hpf_stop": 18, 25 | "lpf_start": 93, 26 | "lpf_stop": 185, 27 | "res_type": "polyphase" 28 | }, 29 | "3": { 30 | "sr": 22050, 31 | "hl": 256, 32 | "n_fft": 512, 33 | "crop_start": 46, 34 | "crop_stop": 186, 35 | "hpf_start": 93, 36 | "hpf_stop": 46, 37 | "lpf_start": 164, 38 | "lpf_stop": 186, 39 | "res_type": "polyphase" 40 | }, 41 | "4": { 42 | "sr": 44100, 43 | "hl": 512, 44 | "n_fft": 768, 45 | "crop_start": 121, 46 | "crop_stop": 382, 47 | "hpf_start": 138, 48 | "hpf_stop": 123, 49 | "res_type": "sinc_medium" 50 | } 51 | }, 52 | "sr": 44100, 53 | "pre_filter_start": 740, 54 | "pre_filter_stop": 768 55 | } -------------------------------------------------------------------------------- /lib/uvr5_pack/lib_v5/modelparams/4band_v2_sn.json: -------------------------------------------------------------------------------- 1 | { 2 | "bins": 672, 3 | "unstable_bins": 8, 4 | "reduction_bins": 637, 5 | "band": { 6 | "1": { 7 | "sr": 7350, 8 | "hl": 80, 9 | "n_fft": 640, 10 | "crop_start": 0, 11 | "crop_stop": 85, 12 | "lpf_start": 25, 13 | "lpf_stop": 53, 14 | "res_type": "polyphase" 15 | }, 16 | "2": { 17 | "sr": 7350, 18 | "hl": 80, 19 | "n_fft": 320, 20 | "crop_start": 4, 21 | "crop_stop": 87, 22 | "hpf_start": 25, 23 | "hpf_stop": 12, 24 | "lpf_start": 31, 25 | "lpf_stop": 62, 26 | "res_type": "polyphase" 27 | }, 28 | "3": { 29 | "sr": 14700, 30 | "hl": 160, 31 | "n_fft": 512, 32 | "crop_start": 17, 33 | "crop_stop": 216, 34 | "hpf_start": 48, 35 | "hpf_stop": 24, 36 | "lpf_start": 139, 37 | "lpf_stop": 210, 38 | "res_type": "polyphase" 39 | }, 40 | "4": { 41 | "sr": 44100, 42 | "hl": 480, 43 | "n_fft": 960, 44 | "crop_start": 78, 45 | "crop_stop": 383, 46 | "hpf_start": 130, 47 | "hpf_stop": 86, 48 | "convert_channels": "stereo_n", 49 | "res_type": "kaiser_fast" 50 | } 51 | }, 52 | "sr": 44100, 53 | "pre_filter_start": 668, 54 | "pre_filter_stop": 672 55 | } -------------------------------------------------------------------------------- /lib/uvr5_pack/lib_v5/modelparams/4band_44100_mid.json: -------------------------------------------------------------------------------- 1 | { 2 | "bins": 768, 3 | "unstable_bins": 7, 4 | "mid_side": true, 5 | "reduction_bins": 668, 6 | "band": { 7 | "1": { 8 | "sr": 11025, 9 | "hl": 128, 10 | "n_fft": 1024, 11 | "crop_start": 0, 12 | "crop_stop": 186, 13 | "lpf_start": 37, 14 | "lpf_stop": 73, 15 | "res_type": "polyphase" 16 | }, 17 | "2": { 18 | "sr": 11025, 19 | "hl": 128, 20 | "n_fft": 512, 21 | "crop_start": 4, 22 | "crop_stop": 185, 23 | "hpf_start": 36, 24 | "hpf_stop": 18, 25 | "lpf_start": 93, 26 | "lpf_stop": 185, 27 | "res_type": "polyphase" 28 | }, 29 | "3": { 30 | "sr": 22050, 31 | "hl": 256, 32 | "n_fft": 512, 33 | "crop_start": 46, 34 | "crop_stop": 186, 35 | "hpf_start": 93, 36 | "hpf_stop": 46, 37 | "lpf_start": 164, 38 | "lpf_stop": 186, 39 | "res_type": "polyphase" 40 | }, 41 | "4": { 42 | "sr": 44100, 43 | "hl": 512, 44 | "n_fft": 768, 45 | "crop_start": 121, 46 | "crop_stop": 382, 47 | "hpf_start": 138, 48 | "hpf_stop": 123, 49 | "res_type": "sinc_medium" 50 | } 51 | }, 52 | "sr": 44100, 53 | "pre_filter_start": 740, 54 | "pre_filter_stop": 768 55 | } 56 | -------------------------------------------------------------------------------- /lib/uvr5_pack/lib_v5/modelparams/4band_v3.json: -------------------------------------------------------------------------------- 1 | { 2 | "bins": 672, 3 | "unstable_bins": 8, 4 | "reduction_bins": 530, 5 | "band": { 6 | "1": { 7 | "sr": 7350, 8 | "hl": 80, 9 | "n_fft": 640, 10 | "crop_start": 0, 11 | "crop_stop": 85, 12 | "lpf_start": 25, 13 | "lpf_stop": 53, 14 | "res_type": "polyphase" 15 | }, 16 | "2": { 17 | "sr": 7350, 18 | "hl": 80, 19 | "n_fft": 320, 20 | "crop_start": 4, 21 | "crop_stop": 87, 22 | "hpf_start": 25, 23 | "hpf_stop": 12, 24 | "lpf_start": 31, 25 | "lpf_stop": 62, 26 | "res_type": "polyphase" 27 | }, 28 | "3": { 29 | "sr": 14700, 30 | "hl": 160, 31 | "n_fft": 512, 32 | "crop_start": 17, 33 | "crop_stop": 216, 34 | "hpf_start": 48, 35 | "hpf_stop": 24, 36 | "lpf_start": 139, 37 | "lpf_stop": 210, 38 | "res_type": "polyphase" 39 | }, 40 | "4": { 41 | "sr": 44100, 42 | "hl": 480, 43 | "n_fft": 960, 44 | "crop_start": 78, 45 | "crop_stop": 383, 46 | "hpf_start": 130, 47 | "hpf_stop": 86, 48 | "res_type": "kaiser_fast" 49 | } 50 | }, 51 | "sr": 44100, 52 | "pre_filter_start": 668, 53 | "pre_filter_stop": 672 54 | } -------------------------------------------------------------------------------- /configs/40k.json: -------------------------------------------------------------------------------- 1 | { 2 | "train": { 3 | "log_interval": 200, 4 | "seed": 1234, 5 | "epochs": 20000, 6 | "learning_rate": 1e-4, 7 | "betas": [0.8, 0.99], 8 | "eps": 1e-9, 9 | "batch_size": 4, 10 | "fp16_run": true, 11 | "lr_decay": 0.999875, 12 | "segment_size": 12800, 13 | "init_lr_ratio": 1, 14 | "warmup_epochs": 0, 15 | "c_mel": 45, 16 | "c_kl": 1.0 17 | }, 18 | "data": { 19 | "max_wav_value": 32768.0, 20 | "sampling_rate": 40000, 21 | "filter_length": 2048, 22 | "hop_length": 400, 23 | "win_length": 2048, 24 | "n_mel_channels": 125, 25 | "mel_fmin": 0.0, 26 | "mel_fmax": null 27 | }, 28 | "model": { 29 | "inter_channels": 192, 30 | "hidden_channels": 192, 31 | "filter_channels": 768, 32 | "n_heads": 2, 33 | "n_layers": 6, 34 | "kernel_size": 3, 35 | "p_dropout": 0, 36 | "resblock": "1", 37 | "resblock_kernel_sizes": [3,7,11], 38 | "resblock_dilation_sizes": [[1,3,5], [1,3,5], [1,3,5]], 39 | "upsample_rates": [10,10,2,2], 40 | "upsample_initial_channel": 512, 41 | "upsample_kernel_sizes": [16,16,4,4], 42 | "use_spectral_norm": false, 43 | "gin_channels": 256, 44 | "spk_embed_dim": 109 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /configs/32k.json: -------------------------------------------------------------------------------- 1 | { 2 | "train": { 3 | "log_interval": 200, 4 | "seed": 1234, 5 | "epochs": 20000, 6 | "learning_rate": 1e-4, 7 | "betas": [0.8, 0.99], 8 | "eps": 1e-9, 9 | "batch_size": 4, 10 | "fp16_run": true, 11 | "lr_decay": 0.999875, 12 | "segment_size": 12800, 13 | "init_lr_ratio": 1, 14 | "warmup_epochs": 0, 15 | "c_mel": 45, 16 | "c_kl": 1.0 17 | }, 18 | "data": { 19 | "max_wav_value": 32768.0, 20 | "sampling_rate": 32000, 21 | "filter_length": 1024, 22 | "hop_length": 320, 23 | "win_length": 1024, 24 | "n_mel_channels": 80, 25 | "mel_fmin": 0.0, 26 | "mel_fmax": null 27 | }, 28 | "model": { 29 | "inter_channels": 192, 30 | "hidden_channels": 192, 31 | "filter_channels": 768, 32 | "n_heads": 2, 33 | "n_layers": 6, 34 | "kernel_size": 3, 35 | "p_dropout": 0, 36 | "resblock": "1", 37 | "resblock_kernel_sizes": [3,7,11], 38 | "resblock_dilation_sizes": [[1,3,5], [1,3,5], [1,3,5]], 39 | "upsample_rates": [10,4,2,2,2], 40 | "upsample_initial_channel": 512, 41 | "upsample_kernel_sizes": [16,16,4,4,4], 42 | "use_spectral_norm": false, 43 | "gin_channels": 256, 44 | "spk_embed_dim": 109 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /configs/32k_v2.json: -------------------------------------------------------------------------------- 1 | { 2 | "train": { 3 | "log_interval": 200, 4 | "seed": 1234, 5 | "epochs": 20000, 6 | "learning_rate": 1e-4, 7 | "betas": [0.8, 0.99], 8 | "eps": 1e-9, 9 | "batch_size": 4, 10 | "fp16_run": true, 11 | "lr_decay": 0.999875, 12 | "segment_size": 12800, 13 | "init_lr_ratio": 1, 14 | "warmup_epochs": 0, 15 | "c_mel": 45, 16 | "c_kl": 1.0 17 | }, 18 | "data": { 19 | "max_wav_value": 32768.0, 20 | "sampling_rate": 32000, 21 | "filter_length": 1024, 22 | "hop_length": 320, 23 | "win_length": 1024, 24 | "n_mel_channels": 80, 25 | "mel_fmin": 0.0, 26 | "mel_fmax": null 27 | }, 28 | "model": { 29 | "inter_channels": 192, 30 | "hidden_channels": 192, 31 | "filter_channels": 768, 32 | "n_heads": 2, 33 | "n_layers": 6, 34 | "kernel_size": 3, 35 | "p_dropout": 0, 36 | "resblock": "1", 37 | "resblock_kernel_sizes": [3,7,11], 38 | "resblock_dilation_sizes": [[1,3,5], [1,3,5], [1,3,5]], 39 | "upsample_rates": [10,8,2,2], 40 | "upsample_initial_channel": 512, 41 | "upsample_kernel_sizes": [20,16,4,4], 42 | "use_spectral_norm": false, 43 | "gin_channels": 256, 44 | "spk_embed_dim": 109 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /configs/48k.json: -------------------------------------------------------------------------------- 1 | { 2 | "train": { 3 | "log_interval": 200, 4 | "seed": 1234, 5 | "epochs": 20000, 6 | "learning_rate": 1e-4, 7 | "betas": [0.8, 0.99], 8 | "eps": 1e-9, 9 | "batch_size": 4, 10 | "fp16_run": true, 11 | "lr_decay": 0.999875, 12 | "segment_size": 11520, 13 | "init_lr_ratio": 1, 14 | "warmup_epochs": 0, 15 | "c_mel": 45, 16 | "c_kl": 1.0 17 | }, 18 | "data": { 19 | "max_wav_value": 32768.0, 20 | "sampling_rate": 48000, 21 | "filter_length": 2048, 22 | "hop_length": 480, 23 | "win_length": 2048, 24 | "n_mel_channels": 128, 25 | "mel_fmin": 0.0, 26 | "mel_fmax": null 27 | }, 28 | "model": { 29 | "inter_channels": 192, 30 | "hidden_channels": 192, 31 | "filter_channels": 768, 32 | "n_heads": 2, 33 | "n_layers": 6, 34 | "kernel_size": 3, 35 | "p_dropout": 0, 36 | "resblock": "1", 37 | "resblock_kernel_sizes": [3,7,11], 38 | "resblock_dilation_sizes": [[1,3,5], [1,3,5], [1,3,5]], 39 | "upsample_rates": [10,6,2,2,2], 40 | "upsample_initial_channel": 512, 41 | "upsample_kernel_sizes": [16,16,4,4,4], 42 | "use_spectral_norm": false, 43 | "gin_channels": 256, 44 | "spk_embed_dim": 109 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /configs/48k_v2.json: -------------------------------------------------------------------------------- 1 | { 2 | "train": { 3 | "log_interval": 200, 4 | "seed": 1234, 5 | "epochs": 20000, 6 | "learning_rate": 1e-4, 7 | "betas": [0.8, 0.99], 8 | "eps": 1e-9, 9 | "batch_size": 4, 10 | "fp16_run": true, 11 | "lr_decay": 0.999875, 12 | "segment_size": 17280, 13 | "init_lr_ratio": 1, 14 | "warmup_epochs": 0, 15 | "c_mel": 45, 16 | "c_kl": 1.0 17 | }, 18 | "data": { 19 | "max_wav_value": 32768.0, 20 | "sampling_rate": 48000, 21 | "filter_length": 2048, 22 | "hop_length": 480, 23 | "win_length": 2048, 24 | "n_mel_channels": 128, 25 | "mel_fmin": 0.0, 26 | "mel_fmax": null 27 | }, 28 | "model": { 29 | "inter_channels": 192, 30 | "hidden_channels": 192, 31 | "filter_channels": 768, 32 | "n_heads": 2, 33 | "n_layers": 6, 34 | "kernel_size": 3, 35 | "p_dropout": 0, 36 | "resblock": "1", 37 | "resblock_kernel_sizes": [3,7,11], 38 | "resblock_dilation_sizes": [[1,3,5], [1,3,5], [1,3,5]], 39 | "upsample_rates": [12,10,2,2], 40 | "upsample_initial_channel": 512, 41 | "upsample_kernel_sizes": [24,20,4,4], 42 | "use_spectral_norm": false, 43 | "gin_channels": 256, 44 | "spk_embed_dim": 109 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /tools/infer/train-index.py: -------------------------------------------------------------------------------- 1 | """ 2 | 格式：直接cid为自带的index位；aid放不下了，通过字典来查，反正就5w个 3 | """ 4 | import faiss, numpy as np, os 5 | 6 | # ###########如果是原始特征要先写save 7 | inp_root = r"E:\codes\py39\dataset\mi\2-co256" 8 | npys = [] 9 | for name in sorted(list(os.listdir(inp_root))): 10 | phone = np.load("%s/%s" % (inp_root, name)) 11 | npys.append(phone) 12 | big_npy = np.concatenate(npys, 0) 13 | print(big_npy.shape) # (6196072, 192)#fp32#4.43G 14 | np.save("infer/big_src_feature_mi.npy", big_npy) 15 | 16 | ##################train+add 17 | # big_npy=np.load("/bili-coeus/jupyter/jupyterhub-liujing04/vits_ch/inference_f0/big_src_feature_mi.npy") 18 | print(big_npy.shape) 19 | index = faiss.index_factory(256, "IVF512,Flat") # mi 20 | print("training") 21 | index_ivf = faiss.extract_index_ivf(index) # 22 | index_ivf.nprobe = 9 23 | index.train(big_npy) 24 | faiss.write_index(index, "infer/trained_IVF512_Flat_mi_baseline_src_feat.index") 25 | print("adding") 26 | index.add(big_npy) 27 | faiss.write_index(index, "infer/added_IVF512_Flat_mi_baseline_src_feat.index") 28 | """ 29 | 大小（都是FP32） 30 | big_src_feature 2.95G 31 | (3098036, 256) 32 | big_emb 4.43G 33 | (6196072, 192) 34 | big_emb双倍是因为求特征要repeat后再加pitch 35 | 36 | """ 37 | -------------------------------------------------------------------------------- /MIT协议暨相关引用库协议: -------------------------------------------------------------------------------- 1 | 本软件及其相关代码以MIT协议开源，作者不对软件具备任何控制力，使用软件者、传播软件导出的声音者自负全责。 2 | 如不认可该条款，则不能使用或引用软件包内任何代码和文件。 3 | 4 | 特此授予任何获得本软件和相关文档文件（以下简称“软件”）副本的人免费使用、复制、修改、合并、出版、分发、再授权和/或销售本软件的权利，以及授予本软件所提供的人使用本软件的权利，但须符合以下条件： 5 | 上述版权声明和本许可声明应包含在软件的所有副本或实质部分中。 6 | 软件是“按原样”提供的，没有任何明示或暗示的保证，包括但不限于适销性、适用于特定目的和不侵权的保证。在任何情况下，作者或版权持有人均不承担因软件或软件的使用或其他交易而产生、产生或与之相关的任何索赔、损害赔偿或其他责任，无论是在合同诉讼、侵权诉讼还是其他诉讼中。 7 | 8 | 9 | The LICENCEs for related libraries are as follows. 10 | 相关引用库协议如下： 11 | 12 | ContentVec 13 | https://github.com/auspicious3000/contentvec/blob/main/LICENSE 14 | MIT License 15 | 16 | VITS 17 | https://github.com/jaywalnut310/vits/blob/main/LICENSE 18 | MIT License 19 | 20 | HIFIGAN 21 | https://github.com/jik876/hifi-gan/blob/master/LICENSE 22 | MIT License 23 | 24 | gradio 25 | https://github.com/gradio-app/gradio/blob/main/LICENSE 26 | Apache License 2.0 27 | 28 | ffmpeg 29 | https://github.com/FFmpeg/FFmpeg/blob/master/COPYING.LGPLv3 30 | https://github.com/BtbN/FFmpeg-Builds/releases/download/autobuild-2021-02-28-12-32/ffmpeg-n4.3.2-160-gfbb9368226-win64-lgpl-4.3.zip 31 | LPGLv3 License 32 | MIT License 33 | 34 | ultimatevocalremovergui 35 | https://github.com/Anjok07/ultimatevocalremovergui/blob/master/LICENSE 36 | https://github.com/yang123qwe/vocal_separation_by_uvr5 37 | MIT License 38 | 39 | audio-slicer 40 | https://github.com/openvpi/audio-slicer/blob/main/LICENSE 41 | MIT License 42 | 43 | PySimpleGUI 44 | https://github.com/PySimpleGUI/PySimpleGUI/blob/master/license.txt 45 | LPGLv3 License 46 | -------------------------------------------------------------------------------- /.github/workflows/unitest.yml: -------------------------------------------------------------------------------- 1 | name: unitest 2 | on: [ push, pull_request ] 3 | jobs: 4 | build: 5 | runs-on: ${{ matrix.os }} 6 | strategy: 7 | matrix: 8 | python-version: ["3.8", "3.9", "3.10"] 9 | os: [ubuntu-latest] 10 | fail-fast: false 11 | 12 | steps: 13 | - uses: actions/checkout@master 14 | - name: Set up Python ${{ matrix.python-version }} 15 | uses: actions/setup-python@v4 16 | with: 17 | python-version: ${{ matrix.python-version }} 18 | - name: Install dependencies 19 | run: | 20 | sudo apt update 21 | sudo apt -y install ffmpeg 22 | sudo apt -y install -qq aria2 23 | aria2c --console-log-level=error -c -x 16 -s 16 -k 1M https://huggingface.co/lj1995/VoiceConversionWebUI/resolve/main/hubert_base.pt -d ./ -o hubert_base.pt 24 | python -m pip install --upgrade pip 25 | python -m pip install --upgrade setuptools 26 | python -m pip install --upgrade wheel 27 | pip install torch torchvision torchaudio 28 | pip install -r requirements.txt 29 | - name: Test step 1 & 2 30 | run: | 31 | mkdir -p logs/mi-test 32 | touch logs/mi-test/preprocess.log 33 | python trainset_preprocess_pipeline_print.py logs/mute/0_gt_wavs 48000 8 logs/mi-test True 34 | touch logs/mi-test/extract_f0_feature.log 35 | python extract_f0_print.py logs/mi-test $(nproc) pm 36 | python extract_feature_print.py cpu 1 0 0 logs/mi-test v1 37 | -------------------------------------------------------------------------------- /lib/i18n/locale_diff.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | from collections import OrderedDict 4 | 5 | # Define the standard file name 6 | standard_file = "zh_CN.json" 7 | 8 | # Find all JSON files in the directory 9 | dir_path = "./" 10 | languages = [ 11 | f for f in os.listdir(dir_path) if f.endswith(".json") and f != standard_file 12 | ] 13 | 14 | # Load the standard file 15 | with open(standard_file, "r", encoding="utf-8") as f: 16 | standard_data = json.load(f, object_pairs_hook=OrderedDict) 17 | 18 | # Loop through each language file 19 | for lang_file in languages: 20 | # Load the language file 21 | with open(lang_file, "r", encoding="utf-8") as f: 22 | lang_data = json.load(f, object_pairs_hook=OrderedDict) 23 | 24 | # Find the difference between the language file and the standard file 25 | diff = set(standard_data.keys()) - set(lang_data.keys()) 26 | 27 | miss = set(lang_data.keys()) - set(standard_data.keys()) 28 | 29 | # Add any missing keys to the language file 30 | for key in diff: 31 | lang_data[key] = key 32 | 33 | # Del any extra keys to the language file 34 | for key in miss: 35 | del lang_data[key] 36 | 37 | # Sort the keys of the language file to match the order of the standard file 38 | lang_data = OrderedDict( 39 | sorted(lang_data.items(), key=lambda x: list(standard_data.keys()).index(x[0])) 40 | ) 41 | 42 | # Save the updated language file 43 | with open(lang_file, "w", encoding="utf-8") as f: 44 | json.dump(lang_data, f, ensure_ascii=False, indent=4) 45 | f.write("\n") 46 | -------------------------------------------------------------------------------- /.github/workflows/push_format.yml: -------------------------------------------------------------------------------- 1 | name: push format 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | 8 | permissions: 9 | contents: write 10 | pull-requests: write 11 | 12 | jobs: 13 | push_format: 14 | runs-on: ${{ matrix.os }} 15 | 16 | strategy: 17 | matrix: 18 | python-version: ["3.10"] 19 | os: [ubuntu-latest] 20 | fail-fast: false 21 | 22 | steps: 23 | - uses: actions/checkout@v3 24 | with: 25 | ref: ${{github.ref_name}} 26 | 27 | - name: Set up Python ${{ matrix.python-version }} 28 | uses: actions/setup-python@v4 29 | with: 30 | python-version: ${{ matrix.python-version }} 31 | 32 | - name: Install Black 33 | run: pip install "black[jupyter]" 34 | 35 | - name: Run Black 36 | # run: black $(git ls-files '*.py') 37 | run: black . 38 | 39 | - name: Commit Back 40 | continue-on-error: true 41 | id: commitback 42 | run: | 43 | git config --local user.email "github-actions[bot]@users.noreply.github.com" 44 | git config --local user.name "github-actions[bot]" 45 | git add --all 46 | git commit -m "Format code" 47 | 48 | - name: Create Pull Request 49 | if: steps.commitback.outcome == 'success' 50 | continue-on-error: true 51 | uses: peter-evans/create-pull-request@v5 52 | with: 53 | delete-branch: true 54 | body: Apply Code Formatter Change 55 | title: Apply Code Formatter Change 56 | commit-message: Automatic code format 57 | -------------------------------------------------------------------------------- /run.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [[ "$(uname)" == "Darwin" ]]; then 4 | # macOS specific env: 5 | export PYTORCH_ENABLE_MPS_FALLBACK=1 6 | export PYTORCH_MPS_HIGH_WATERMARK_RATIO=0.0 7 | elif [[ "$(uname)" != "Linux" ]]; then 8 | echo "Unsupported operating system." 9 | exit 1 10 | fi 11 | 12 | requirements_file="requirements.txt" 13 | 14 | # Check if Python 3.8 is installed 15 | if ! command -v python3.8 &> /dev/null; then 16 | echo "Python 3.8 not found. Attempting to install..." 17 | if [[ "$(uname)" == "Darwin" ]] && command -v brew &> /dev/null; then 18 | brew install python@3.8 19 | elif [[ "$(uname)" == "Linux" ]] && command -v apt-get &> /dev/null; then 20 | sudo apt-get update 21 | sudo apt-get install python3.8 22 | else 23 | echo "Please install Python 3.8 manually." 24 | exit 1 25 | fi 26 | fi 27 | 28 | # Check if required packages are installed and install them if not 29 | if [ -f "${requirements_file}" ]; then 30 | installed_packages=$(python3.8 -m pip freeze) 31 | while IFS= read -r package; do 32 | [[ "${package}" =~ ^#.* ]] && continue 33 | package_name=$(echo "${package}" | sed 's/[<>=!].*//') 34 | if ! echo "${installed_packages}" | grep -q "${package_name}"; then 35 | echo "${package_name} not found. Attempting to install..." 36 | python3.8 -m pip install --upgrade "${package}" 37 | fi 38 | done < "${requirements_file}" 39 | else 40 | echo "${requirements_file} not found. Please ensure the requirements file with required packages exists." 41 | exit 1 42 | fi 43 | 44 | # Run the main script 45 | python3.8 infer-web.py --pycmd python3.8 -------------------------------------------------------------------------------- /lib/train/losses.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | 4 | def feature_loss(fmap_r, fmap_g): 5 | loss = 0 6 | for dr, dg in zip(fmap_r, fmap_g): 7 | for rl, gl in zip(dr, dg): 8 | rl = rl.float().detach() 9 | gl = gl.float() 10 | loss += torch.mean(torch.abs(rl - gl)) 11 | 12 | return loss * 2 13 | 14 | 15 | def discriminator_loss(disc_real_outputs, disc_generated_outputs): 16 | loss = 0 17 | r_losses = [] 18 | g_losses = [] 19 | for dr, dg in zip(disc_real_outputs, disc_generated_outputs): 20 | dr = dr.float() 21 | dg = dg.float() 22 | r_loss = torch.mean((1 - dr) ** 2) 23 | g_loss = torch.mean(dg**2) 24 | loss += r_loss + g_loss 25 | r_losses.append(r_loss.item()) 26 | g_losses.append(g_loss.item()) 27 | 28 | return loss, r_losses, g_losses 29 | 30 | 31 | def generator_loss(disc_outputs): 32 | loss = 0 33 | gen_losses = [] 34 | for dg in disc_outputs: 35 | dg = dg.float() 36 | l = torch.mean((1 - dg) ** 2) 37 | gen_losses.append(l) 38 | loss += l 39 | 40 | return loss, gen_losses 41 | 42 | 43 | def kl_loss(z_p, logs_q, m_p, logs_p, z_mask): 44 | """ 45 | z_p, logs_q: [b, h, t_t] 46 | m_p, logs_p: [b, h, t_t] 47 | """ 48 | z_p = z_p.float() 49 | logs_q = logs_q.float() 50 | m_p = m_p.float() 51 | logs_p = logs_p.float() 52 | z_mask = z_mask.float() 53 | 54 | kl = logs_p - logs_q - 0.5 55 | kl += 0.5 * ((z_p - m_p) ** 2) * torch.exp(-2.0 * logs_p) 56 | kl = torch.sum(kl * z_mask) 57 | l = kl / torch.sum(z_mask) 58 | return l 59 | -------------------------------------------------------------------------------- /lib/uvr5_pack/lib_v5/model_param_init.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import pathlib 4 | 5 | default_param = {} 6 | default_param["bins"] = 768 7 | default_param["unstable_bins"] = 9 # training only 8 | default_param["reduction_bins"] = 762 # training only 9 | default_param["sr"] = 44100 10 | default_param["pre_filter_start"] = 757 11 | default_param["pre_filter_stop"] = 768 12 | default_param["band"] = {} 13 | 14 | 15 | default_param["band"][1] = { 16 | "sr": 11025, 17 | "hl": 128, 18 | "n_fft": 960, 19 | "crop_start": 0, 20 | "crop_stop": 245, 21 | "lpf_start": 61, # inference only 22 | "res_type": "polyphase", 23 | } 24 | 25 | default_param["band"][2] = { 26 | "sr": 44100, 27 | "hl": 512, 28 | "n_fft": 1536, 29 | "crop_start": 24, 30 | "crop_stop": 547, 31 | "hpf_start": 81, # inference only 32 | "res_type": "sinc_best", 33 | } 34 | 35 | 36 | def int_keys(d): 37 | r = {} 38 | for k, v in d: 39 | if k.isdigit(): 40 | k = int(k) 41 | r[k] = v 42 | return r 43 | 44 | 45 | class ModelParameters(object): 46 | def __init__(self, config_path=""): 47 | if ".pth" == pathlib.Path(config_path).suffix: 48 | import zipfile 49 | 50 | with zipfile.ZipFile(config_path, "r") as zip: 51 | self.param = json.loads( 52 | zip.read("param.json"), object_pairs_hook=int_keys 53 | ) 54 | elif ".json" == pathlib.Path(config_path).suffix: 55 | with open(config_path, "r") as f: 56 | self.param = json.loads(f.read(), object_pairs_hook=int_keys) 57 | else: 58 | self.param = default_param 59 | 60 | for k in [ 61 | "mid_side", 62 | "mid_side_b", 63 | "mid_side_b2", 64 | "stereo_w", 65 | "stereo_n", 66 | "reverse", 67 | ]: 68 | if not k in self.param: 69 | self.param[k] = False 70 | -------------------------------------------------------------------------------- /tools/export_onnx.py: -------------------------------------------------------------------------------- 1 | from lib.infer_pack.models_onnx import SynthesizerTrnMsNSFsidM 2 | import torch 3 | 4 | if __name__ == "__main__": 5 | MoeVS = True # 模型是否为MoeVoiceStudio（原MoeSS）使用 6 | 7 | ModelPath = "Shiroha/shiroha.pth" # 模型路径 8 | ExportedPath = "model.onnx" # 输出路径 9 | hidden_channels = 256 # hidden_channels，为768Vec做准备 10 | cpt = torch.load(ModelPath, map_location="cpu") 11 | cpt["config"][-3] = cpt["weight"]["emb_g.weight"].shape[0] # n_spk 12 | print(*cpt["config"]) 13 | 14 | test_phone = torch.rand(1, 200, hidden_channels) # hidden unit 15 | test_phone_lengths = torch.tensor([200]).long() # hidden unit 长度（貌似没啥用） 16 | test_pitch = torch.randint(size=(1, 200), low=5, high=255) # 基频（单位赫兹） 17 | test_pitchf = torch.rand(1, 200) # nsf基频 18 | test_ds = torch.LongTensor([0]) # 说话人ID 19 | test_rnd = torch.rand(1, 192, 200) # 噪声（加入随机因子） 20 | 21 | device = "cpu" # 导出时设备（不影响使用模型） 22 | 23 | net_g = SynthesizerTrnMsNSFsidM( 24 | *cpt["config"], is_half=False 25 | ) # fp32导出（C++要支持fp16必须手动将内存重新排列所以暂时不用fp16） 26 | net_g.load_state_dict(cpt["weight"], strict=False) 27 | input_names = ["phone", "phone_lengths", "pitch", "pitchf", "ds", "rnd"] 28 | output_names = [ 29 | "audio", 30 | ] 31 | # net_g.construct_spkmixmap(n_speaker) 多角色混合轨道导出 32 | torch.onnx.export( 33 | net_g, 34 | ( 35 | test_phone.to(device), 36 | test_phone_lengths.to(device), 37 | test_pitch.to(device), 38 | test_pitchf.to(device), 39 | test_ds.to(device), 40 | test_rnd.to(device), 41 | ), 42 | ExportedPath, 43 | dynamic_axes={ 44 | "phone": [1], 45 | "pitch": [1], 46 | "pitchf": [1], 47 | "rnd": [2], 48 | }, 49 | do_constant_folding=False, 50 | opset_version=16, 51 | verbose=False, 52 | input_names=input_names, 53 | output_names=output_names, 54 | ) 55 | -------------------------------------------------------------------------------- /docs/Changelog_CN.md: -------------------------------------------------------------------------------- 1 | ### 20230618更新 2 | - v2增加32k和48k两个新预训练模型 3 | - 修复非f0模型推理报错 4 | - 对于超过一小时的训练集的索引建立环节，自动kmeans缩小特征处理以加速索引训练、加入和查询 5 | - 附送一个人声转吉他玩具仓库 6 | - 数据处理剔除异常值切片 7 | - onnx导出选项卡 8 | 9 | 失败的实验： 10 | - ~~特征检索增加时序维度：寄，没啥效果~~ 11 | - ~~特征检索增加PCAR降维可选项：寄，数据大用kmeans缩小数据量，数据小降维操作耗时比省下的匹配耗时还多~~ 12 | - ~~支持onnx推理（附带仅推理的小压缩包）：寄，生成nsf还是需要pytorch~~ 13 | - ~~训练时在音高、gender、eq、噪声等方面对输入进行随机增强：寄，没啥效果~~ 14 | 15 | todolist： 16 | - 接入小型声码器调研 17 | - 训练集音高识别支持crepe 18 | - crepe的精度支持和RVC-config同步 19 | - 对接F0编辑器 20 | 21 | 22 | ### 20230528更新 23 | - 增加v2的jupyter notebook，韩文changelog，增加一些环境依赖 24 | - 增加呼吸、清辅音、齿音保护模式 25 | - 支持crepe-full推理 26 | - UVR5人声伴奏分离加上3个去延迟模型和MDX-Net去混响模型，增加HP3人声提取模型 27 | - 索引名称增加版本和实验名称 28 | - 人声伴奏分离、推理批量导出增加音频导出格式选项 29 | - 废弃32k模型的训练 30 | 31 | ### 20230513更新 32 | - 清除一键包内部老版本runtime内残留的lib.infer_pack和uvr5_pack 33 | - 修复训练集预处理伪多进程的bug 34 | - 增加harvest识别音高可选通过中值滤波削弱哑音现象，可调整中值滤波半径 35 | - 导出音频增加后处理重采样 36 | - 训练n_cpu进程数从"仅调整f0提取"改为"调整数据预处理和f0提取" 37 | - 自动检测logs文件夹下的index路径，提供下拉列表功能 38 | - tab页增加"常见问题解答"（也可参考github-rvc-wiki） 39 | - 相同路径的输入音频推理增加了音高缓存（用途：使用harvest音高提取，整个pipeline会经历漫长且重复的音高提取过程，如果不使用缓存，实验不同音色、索引、音高中值滤波半径参数的用户在第一次测试后的等待结果会非常痛苦） 40 | 41 | ### 20230514更新 42 | - 音量包络对齐输入混合（可以缓解“输入静音输出小幅度噪声”的问题。如果输入音频背景底噪大则不建议开启，默认不开启（值为1可视为不开启）） 43 | - 支持按照指定频率保存提取的小模型（假如你想尝试不同epoch下的推理效果，但是不想保存所有大checkpoint并且每次都要ckpt手工处理提取小模型，这项功能会非常实用） 44 | - 通过设置环境变量解决服务端开了系统全局代理导致浏览器连接错误的问题 45 | - 支持v2预训练模型（目前只公开了40k版本进行测试，另外2个采样率还没有训练完全） 46 | - 推理前限制超过1的过大音量 47 | - 微调数据预处理参数 48 | 49 | 50 | ### 20230409更新 51 | - 修正训练参数，提升显卡平均利用率，A100最高从25%提升至90%左右，V100:50%->90%左右，2060S:60%->85%左右，P40:25%->95%左右，训练速度显著提升 52 | - 修正参数：总batch_size改为每张卡的batch_size 53 | - 修正total_epoch：最大限制100解锁至1000；默认10提升至默认20 54 | - 修复ckpt提取识别是否带音高错误导致推理异常的问题 55 | - 修复分布式训练每个rank都保存一次ckpt的问题 56 | - 特征提取进行nan特征过滤 57 | - 修复静音输入输出随机辅音or噪声的问题（老版模型需要重做训练集重训） 58 | 59 | ### 20230416更新 60 | - 新增本地实时变声迷你GUI，双击go-realtime-gui.bat启动 61 | - 训练推理均对<50Hz的频段进行滤波过滤 62 | - 训练推理音高提取pyworld最低音高从默认80下降至50,50-80hz间的男声低音不会哑 63 | - WebUI支持根据系统区域变更语言（现支持en_US，ja_JP，zh_CN，zh_HK，zh_SG，zh_TW，不支持的默认en_US） 64 | - 修正部分显卡识别（例如V100-16G识别失败，P4识别失败） 65 | 66 | ### 20230428更新 67 | - 升级faiss索引设置，速度更快，质量更高 68 | - 取消total_npy依赖，后续分享模型不再需要填写total_npy 69 | - 解锁16系限制。4G显存GPU给到4G的推理设置。 70 | - 修复部分音频格式下UVR5人声伴奏分离的bug 71 | - 实时变声迷你gui增加对非40k与不懈怠音高模型的支持 72 | 73 | ### 后续计划： 74 | 功能： 75 | - 支持多人训练选项卡（至多4人） 76 | 77 | 底模： 78 | - 收集呼吸wav加入训练集修正呼吸变声电音的问题 79 | - 我们正在训练增加了歌声训练集的底模，未来会公开 80 | 81 | -------------------------------------------------------------------------------- /.github/workflows/docker.yml: -------------------------------------------------------------------------------- 1 | name: Build And Push Docker Image 2 | 3 | on: 4 | workflow_dispatch: 5 | push: 6 | # Sequence of patterns matched against refs/tags 7 | tags: 8 | - 'v*' # Push events to matching v*, i.e. v1.0, v20.15.10 9 | 10 | jobs: 11 | build: 12 | runs-on: ubuntu-latest 13 | permissions: 14 | packages: write 15 | contents: read 16 | steps: 17 | - uses: actions/checkout@v3 18 | - name: Set time zone 19 | uses: szenius/set-timezone@v1.0 20 | with: 21 | timezoneLinux: "Asia/Shanghai" 22 | timezoneMacos: "Asia/Shanghai" 23 | timezoneWindows: "China Standard Time" 24 | 25 | # # 如果有 dockerhub 账户，可以在github的secrets中配置下面两个，然后取消下面注释的这几行，并在meta步骤的images增加一行 ${{ github.repository }} 26 | # - name: Login to DockerHub 27 | # uses: docker/login-action@v1 28 | # with: 29 | # username: ${{ secrets.DOCKERHUB_USERNAME }} 30 | # password: ${{ secrets.DOCKERHUB_TOKEN }} 31 | 32 | - name: Login to GHCR 33 | uses: docker/login-action@v2 34 | with: 35 | registry: ghcr.io 36 | username: ${{ github.repository_owner }} 37 | password: ${{ secrets.GITHUB_TOKEN }} 38 | 39 | - name: Extract metadata (tags, labels) for Docker 40 | id: meta 41 | uses: docker/metadata-action@v4 42 | with: 43 | images: | 44 | ghcr.io/${{ github.repository }} 45 | # generate Docker tags based on the following events/attributes 46 | # nightly, master, pr-2, 1.2.3, 1.2, 1 47 | tags: | 48 | type=schedule,pattern=nightly 49 | type=edge 50 | type=ref,event=branch 51 | type=ref,event=pr 52 | type=semver,pattern={{version}} 53 | type=semver,pattern={{major}}.{{minor}} 54 | type=semver,pattern={{major}} 55 | 56 | - name: Set up QEMU 57 | uses: docker/setup-qemu-action@v2 58 | 59 | - name: Set up Docker Buildx 60 | uses: docker/setup-buildx-action@v2 61 | 62 | - name: Build and push 63 | id: docker_build 64 | uses: docker/build-push-action@v4 65 | with: 66 | context: . 67 | platforms: linux/amd64,linux/arm64 68 | push: true 69 | tags: ${{ steps.meta.outputs.tags }} 70 | labels: ${{ steps.meta.outputs.labels }} 71 | -------------------------------------------------------------------------------- /tools/infer/train-index-v2.py: -------------------------------------------------------------------------------- 1 | """ 2 | 格式：直接cid为自带的index位；aid放不下了，通过字典来查，反正就5w个 3 | """ 4 | import faiss, numpy as np, os 5 | from sklearn.cluster import MiniBatchKMeans 6 | import traceback 7 | from multiprocessing import cpu_count 8 | 9 | # ###########如果是原始特征要先写save 10 | n_cpu = 0 11 | if n_cpu == 0: 12 | n_cpu = cpu_count() 13 | inp_root = r"./logs/anz/3_feature768" 14 | npys = [] 15 | listdir_res = list(os.listdir(inp_root)) 16 | for name in sorted(listdir_res): 17 | phone = np.load("%s/%s" % (inp_root, name)) 18 | npys.append(phone) 19 | big_npy = np.concatenate(npys, 0) 20 | big_npy_idx = np.arange(big_npy.shape[0]) 21 | np.random.shuffle(big_npy_idx) 22 | big_npy = big_npy[big_npy_idx] 23 | print(big_npy.shape) # (6196072, 192)#fp32#4.43G 24 | if big_npy.shape[0] > 2e5: 25 | # if(1): 26 | info = "Trying doing kmeans %s shape to 10k centers." % big_npy.shape[0] 27 | print(info) 28 | try: 29 | big_npy = ( 30 | MiniBatchKMeans( 31 | n_clusters=10000, 32 | verbose=True, 33 | batch_size=256 * n_cpu, 34 | compute_labels=False, 35 | init="random", 36 | ) 37 | .fit(big_npy) 38 | .cluster_centers_ 39 | ) 40 | except: 41 | info = traceback.format_exc() 42 | print(info) 43 | 44 | np.save("tools/infer/big_src_feature_mi.npy", big_npy) 45 | 46 | ##################train+add 47 | # big_npy=np.load("/bili-coeus/jupyter/jupyterhub-liujing04/vits_ch/inference_f0/big_src_feature_mi.npy") 48 | n_ivf = min(int(16 * np.sqrt(big_npy.shape[0])), big_npy.shape[0] // 39) 49 | index = faiss.index_factory(768, "IVF%s,Flat" % n_ivf) # mi 50 | print("training") 51 | index_ivf = faiss.extract_index_ivf(index) # 52 | index_ivf.nprobe = 1 53 | index.train(big_npy) 54 | faiss.write_index( 55 | index, "tools/infer/trained_IVF%s_Flat_baseline_src_feat_v2.index" % (n_ivf) 56 | ) 57 | print("adding") 58 | batch_size_add = 8192 59 | for i in range(0, big_npy.shape[0], batch_size_add): 60 | index.add(big_npy[i : i + batch_size_add]) 61 | faiss.write_index( 62 | index, "tools/infer/added_IVF%s_Flat_mi_baseline_src_feat.index" % (n_ivf) 63 | ) 64 | """ 65 | 大小（都是FP32） 66 | big_src_feature 2.95G 67 | (3098036, 256) 68 | big_emb 4.43G 69 | (6196072, 192) 70 | big_emb双倍是因为求特征要repeat后再加pitch 71 | 72 | """ 73 | -------------------------------------------------------------------------------- /docs/training_tips_ja.md: -------------------------------------------------------------------------------- 1 | RVCの訓練における説明、およびTIPS 2 | =============================== 3 | 本TIPSではどのようにデータの訓練が行われているかを説明します。 4 | 5 | # 訓練の流れ 6 | GUIの訓練タブのstepに沿って説明します。 7 | 8 | ## step1 9 | 実験名の設定を行います。 10 | 11 | また、モデルに音高ガイド(ピッチ)を考慮させるかもここで設定できます。考慮させない場合はモデルは軽量になりますが、歌唱には向かなくなります。 12 | 13 | 各実験のデータは`/logs/実験名/`に配置されます。 14 | 15 | ## step2a 16 | 音声の読み込みと前処理を行います。 17 | 18 | ### load audio 19 | 音声のあるフォルダを指定すると、そのフォルダ内にある音声ファイルを自動で読み込みます。 20 | 例えば`C:Users\hoge\voices`を指定した場合、`C:Users\hoge\voices\voice.mp3`は読み込まれますが、`C:Users\hoge\voices\dir\voice.mp3`は読み込まれません。 21 | 22 | 音声の読み込みには内部でffmpegを利用しているので、ffmpegで対応している拡張子であれば自動的に読み込まれます。 23 | ffmpegでint16に変換した後、float32に変換し、-1 ~ 1の間に正規化されます。 24 | 25 | ### denoising 26 | 音声についてscipyのfiltfiltによる平滑化を行います。 27 | 28 | ### 音声の分割 29 | 入力した音声はまず、一定期間(max_sil_kept=5秒?)より長く無音が続く部分を検知して音声を分割します。無音で音声を分割した後は、0.3秒のoverlapを含む4秒ごとに音声を分割します。4秒以内に区切られた音声は、音量の正規化を行った後wavファイルを`/logs/実験名/0_gt_wavs`に、そこから16kのサンプリングレートに変換して`/logs/実験名/1_16k_wavs`にwavファイルで保存します。 30 | 31 | ## step2b 32 | ### ピッチの抽出 33 | wavファイルからピッチ(音の高低)の情報を抽出します。parselmouthやpyworldに内蔵されている手法でピッチ情報(=f0)を抽出し、`/logs/実験名/2a_f0`に保存します。その後、ピッチ情報を対数で変換して1~255の整数に変換し、`/logs/実験名/2b-f0nsf`に保存します。 34 | 35 | ### feature_printの抽出 36 | HuBERTを用いてwavファイルを事前にembeddingに変換します。`/logs/実験名/1_16k_wavs`に保存したwavファイルを読み込み、HuBERTでwavファイルを256次元の特徴量に変換し、npy形式で`/logs/実験名/3_feature256`に保存します。 37 | 38 | ## step3 39 | モデルのトレーニングを行います。 40 | ### 初心者向け用語解説 41 | 深層学習ではデータセットを分割し、少しずつ学習を進めていきます。一回のモデルの更新(step)では、batch_size個のデータを取り出し予測と誤差の修正を行います。これをデータセットに対して一通り行うと一epochと数えます。 42 | 43 | そのため、学習時間は 1step当たりの学習時間 x (データセット内のデータ数 ÷ バッチサイズ) x epoch数 かかります。一般にバッチサイズを大きくするほど学習は安定し、(1step当たりの学習時間÷バッチサイズ)は小さくなりますが、その分GPUのメモリを多く使用します。GPUのRAMはnvidia-smiコマンド等で確認できます。実行環境のマシンに合わせてバッチサイズをできるだけ大きくするとより短時間で学習が可能です。 44 | 45 | ### pretrained modelの指定 46 | RVCではモデルの訓練を0からではなく、事前学習済みの重みから開始するため、少ないデータセットで学習を行えます。 47 | 48 | デフォルトでは 49 | 50 | - 音高ガイドを考慮する場合、`RVCのある場所/pretrained/f0G40k.pth`と`RVCのある場所/pretrained/f0D40k.pth`を読み込みます。 51 | - 音高ガイドを考慮しない場合、`RVCのある場所/pretrained/G40k.pth`と`RVCのある場所/pretrained/D40k.pth`を読み込みます。 52 | 53 | 学習時はsave_every_epochごとにモデルのパラメータが`logs/実験名/G_{}.pth`と`logs/実験名/D_{}.pth`に保存されますが、このパスを指定することで学習を再開したり、もしくは違う実験で学習したモデルの重みから学習を開始できます。 54 | 55 | ### indexの学習 56 | RVCでは学習時に使われたHuBERTの特徴量を保存し、推論時は学習時の特徴量から近い特徴量を探してきて推論を行います。この検索を高速に行うために事前にindexの学習を行います。 57 | indexの学習には近似近傍探索ライブラリのfaissを用います。`/logs/実験名/3_feature256`の特徴量を読み込み、それを用いて学習したindexを`/logs/実験名/add_XXX.index`として保存します。 58 | (20230428updateよりtotal_fea.npyはindexから読み込むので不要になりました。) 59 | 60 | ### ボタンの説明 61 | - モデルのトレーニング: step2bまでを実行した後、このボタンを押すとモデルの学習を行います。 62 | - 特徴インデックスのトレーニング: モデルのトレーニング後、indexの学習を行います。 63 | - ワンクリックトレーニング: step2bまでとモデルのトレーニング、特徴インデックスのトレーニングを一括で行います。 64 | 65 | -------------------------------------------------------------------------------- /docs/training_tips_ko.md: -------------------------------------------------------------------------------- 1 | RVC 훈련에 대한 설명과 팁들 2 | ====================================== 3 | 본 팁에서는 어떻게 데이터 훈련이 이루어지고 있는지 설명합니다. 4 | 5 | # 훈련의 흐름 6 | GUI의 훈련 탭의 단계를 따라 설명합니다. 7 | 8 | ## step1 9 | 실험 이름을 지정합니다. 또한, 모델이 피치(소리의 높낮이)를 고려해야 하는지 여부를 여기에서 설정할 수도 있습니다.. 10 | 각 실험을 위한 데이터는 `/logs/experiment name/`에 배치됩니다.. 11 | 12 | ## step2a 13 | 음성 파일을 불러오고 전처리합니다. 14 | 15 | ### 음성 파일 불러오기 16 | 음성 파일이 있는 폴더를 지정하면 해당 폴더에 있는 음성 파일이 자동으로 가져와집니다. 17 | 예를 들어 `C:Users\hoge\voices`를 지정하면 `C:Users\hoge\voices\voice.mp3`가 읽히지만 `C:Users\hoge\voices\dir\voice.mp3`는 읽히지 않습니다. 18 | 19 | 음성 로드에는 내부적으로 ffmpeg를 이용하고 있으므로, ffmpeg로 대응하고 있는 확장자라면 자동적으로 읽힙니다. 20 | ffmpeg에서 int16으로 변환한 후 float32로 변환하고 -1과 1 사이에 정규화됩니다. 21 | 22 | ### 잡음 제거 23 | 음성 파일에 대해 scipy의 filtfilt를 이용하여 잡음을 처리합니다. 24 | 25 | ### 음성 분할 26 | 입력한 음성 파일은 먼저 일정 기간(max_sil_kept=5초?)보다 길게 무음이 지속되는 부분을 감지하여 음성을 분할합니다.무음으로 음성을 분할한 후에는 0.3초의 overlap을 포함하여 4초마다 음성을 분할합니다.4초 이내에 구분된 음성은 음량의 정규화를 실시한 후 wav 파일을 `/logs/실험명/0_gt_wavs`로, 거기에서 16k의 샘플링 레이트로 변환해 `/logs/실험명/1_16k_wavs`에 wav 파일로 저장합니다. 27 | 28 | ## step2b 29 | ### 피치 추출 30 | wav 파일에서 피치(소리의 높낮이) 정보를 추출합니다. parselmouth나 pyworld에 내장되어 있는 메서드으로 피치 정보(=f0)를 추출해, `/logs/실험명/2a_f0`에 저장합니다. 그 후 피치 정보를 로그로 변환하여 1~255 정수로 변환하고 `/logs/실험명/2b-f0nsf`에 저장합니다. 31 | 32 | ### feature_print 추출 33 | HuBERT를 이용하여 wav 파일을 미리 embedding으로 변환합니다. `/logs/실험명/1_16k_wavs`에 저장한 wav 파일을 읽고 HuBERT에서 wav 파일을 256차원 feature들로 변환한 후 npy 형식으로 `/logs/실험명/3_feature256`에 저장합니다. 34 | 35 | ## step3 36 | 모델의 훈련을 진행합니다. 37 | 38 | ### 초보자용 용어 해설 39 | 심층학습(딥러닝)에서는 데이터셋을 분할하여 조금씩 학습을 진행합니다.한 번의 모델 업데이트(step) 단계 당 batch_size개의 데이터를 탐색하여 예측과 오차를 수정합니다. 데이터셋 전부에 대해 이 작업을 한 번 수행하는 이를 하나의 epoch라고 계산합니다. 40 | 41 | 따라서 학습 시간은 단계당 학습 시간 x (데이터셋 내 데이터의 수 / batch size) x epoch 수가 소요됩니다. 일반적으로 batch size가 클수록 학습이 안정적이게 됩니다. (step당 학습 시간 ÷ batch size)는 작아지지만 GPU 메모리를 더 많이 사용합니다. GPU RAM은 nvidia-smi 명령어를 통해 확인할 수 있습니다. 실행 환경에 따라 배치 크기를 최대한 늘리면 짧은 시간 내에 학습이 가능합니다. 42 | 43 | ### 사전 학습된 모델 지정 44 | RVC는 적은 데이터셋으로도 훈련이 가능하도록 사전 훈련된 가중치에서 모델 훈련을 시작합니다. 기본적으로 `rvc-location/pretrained/f0G40k.pth` 및 `rvc-location/pretrained/f0D40k.pth`를 불러옵니다. 학습을 할 시에, 모델 파라미터는 각 save_every_epoch별로 `logs/experiment name/G_{}.pth` 와 `logs/experiment name/D_{}.pth`로 저장이 되는데, 이 경로를 지정함으로써 학습을 재개하거나, 다른 실험에서 학습한 모델의 가중치에서 학습을 시작할 수 있습니다. 45 | 46 | ### index의 학습 47 | RVC에서는 학습시에 사용된 HuBERT의 feature값을 저장하고, 추론 시에는 학습 시 사용한 feature값과 유사한 feature 값을 탐색해 추론을 진행합니다. 이 탐색을 고속으로 수행하기 위해 사전에 index을 학습하게 됩니다. 48 | Index 학습에는 근사 근접 탐색법 라이브러리인 Faiss를 사용하게 됩니다. `/logs/실험명/3_feature256`의 feature값을 불러와, 이를 모두 결합시킨 feature값을 `/logs/실험명/total_fea.npy`로서 저장, 그것을 사용해 학습한 index를`/logs/실험명/add_XXX.index`로 저장합니다. 49 | 50 | ### 버튼 설명 51 | - モデルのトレーニング (모델 학습): step2b까지 실행한 후, 이 버튼을 눌러 모델을 학습합니다. 52 | - 特徴インデックスのトレーニング (특징 지수 훈련): 모델의 훈련 후, index를 학습합니다. 53 | - ワンクリックトレーニング (원클릭 트레이닝): step2b까지의 모델 훈련, feature index 훈련을 일괄로 실시합니다. -------------------------------------------------------------------------------- /tools/calc_rvc_model_similarity.py: -------------------------------------------------------------------------------- 1 | # This code references https://huggingface.co/JosephusCheung/ASimilarityCalculatior/blob/main/qwerty.py 2 | # Fill in the path of the model to be queried and the root directory of the reference models, and this script will return the similarity between the model to be queried and all reference models. 3 | import sys, os 4 | import torch 5 | import torch.nn as nn 6 | import torch.nn.functional as F 7 | 8 | 9 | def cal_cross_attn(to_q, to_k, to_v, rand_input): 10 | hidden_dim, embed_dim = to_q.shape 11 | attn_to_q = nn.Linear(hidden_dim, embed_dim, bias=False) 12 | attn_to_k = nn.Linear(hidden_dim, embed_dim, bias=False) 13 | attn_to_v = nn.Linear(hidden_dim, embed_dim, bias=False) 14 | attn_to_q.load_state_dict({"weight": to_q}) 15 | attn_to_k.load_state_dict({"weight": to_k}) 16 | attn_to_v.load_state_dict({"weight": to_v}) 17 | 18 | return torch.einsum( 19 | "ik, jk -> ik", 20 | F.softmax( 21 | torch.einsum("ij, kj -> ik", attn_to_q(rand_input), attn_to_k(rand_input)), 22 | dim=-1, 23 | ), 24 | attn_to_v(rand_input), 25 | ) 26 | 27 | 28 | def model_hash(filename): 29 | try: 30 | with open(filename, "rb") as file: 31 | import hashlib 32 | 33 | m = hashlib.sha256() 34 | 35 | file.seek(0x100000) 36 | m.update(file.read(0x10000)) 37 | return m.hexdigest()[0:8] 38 | except FileNotFoundError: 39 | return "NOFILE" 40 | 41 | 42 | def eval(model, n, input): 43 | qk = f"enc_p.encoder.attn_layers.{n}.conv_q.weight" 44 | uk = f"enc_p.encoder.attn_layers.{n}.conv_k.weight" 45 | vk = f"enc_p.encoder.attn_layers.{n}.conv_v.weight" 46 | atoq, atok, atov = model[qk][:, :, 0], model[uk][:, :, 0], model[vk][:, :, 0] 47 | 48 | attn = cal_cross_attn(atoq, atok, atov, input) 49 | return attn 50 | 51 | 52 | def main(path, root): 53 | torch.manual_seed(114514) 54 | model_a = torch.load(path, map_location="cpu")["weight"] 55 | 56 | print("query:\t\t%s\t%s" % (path, model_hash(path))) 57 | 58 | map_attn_a = {} 59 | map_rand_input = {} 60 | for n in range(6): 61 | hidden_dim, embed_dim, _ = model_a[ 62 | f"enc_p.encoder.attn_layers.{n}.conv_v.weight" 63 | ].shape 64 | rand_input = torch.randn([embed_dim, hidden_dim]) 65 | 66 | map_attn_a[n] = eval(model_a, n, rand_input) 67 | map_rand_input[n] = rand_input 68 | 69 | del model_a 70 | 71 | for name in sorted(list(os.listdir(root))): 72 | path = "%s/%s" % (root, name) 73 | model_b = torch.load(path, map_location="cpu")["weight"] 74 | 75 | sims = [] 76 | for n in range(6): 77 | attn_a = map_attn_a[n] 78 | attn_b = eval(model_b, n, map_rand_input[n]) 79 | 80 | sim = torch.mean(torch.cosine_similarity(attn_a, attn_b)) 81 | sims.append(sim) 82 | 83 | print( 84 | "reference:\t%s\t%s\t%s" 85 | % (path, model_hash(path), f"{torch.mean(torch.stack(sims)) * 1e2:.2f}%") 86 | ) 87 | 88 | 89 | if __name__ == "__main__": 90 | query_path = r"weights\mi v3.pth" 91 | reference_root = r"weights" 92 | main(query_path, reference_root) 93 | -------------------------------------------------------------------------------- /lib/infer_pack/modules/F0Predictor/HarvestF0Predictor.py: -------------------------------------------------------------------------------- 1 | from lib.infer_pack.modules.F0Predictor.F0Predictor import F0Predictor 2 | import pyworld 3 | import numpy as np 4 | 5 | 6 | class HarvestF0Predictor(F0Predictor): 7 | def __init__(self, hop_length=512, f0_min=50, f0_max=1100, sampling_rate=44100): 8 | self.hop_length = hop_length 9 | self.f0_min = f0_min 10 | self.f0_max = f0_max 11 | self.sampling_rate = sampling_rate 12 | 13 | def interpolate_f0(self, f0): 14 | """ 15 | 对F0进行插值处理 16 | """ 17 | 18 | data = np.reshape(f0, (f0.size, 1)) 19 | 20 | vuv_vector = np.zeros((data.size, 1), dtype=np.float32) 21 | vuv_vector[data > 0.0] = 1.0 22 | vuv_vector[data <= 0.0] = 0.0 23 | 24 | ip_data = data 25 | 26 | frame_number = data.size 27 | last_value = 0.0 28 | for i in range(frame_number): 29 | if data[i] <= 0.0: 30 | j = i + 1 31 | for j in range(i + 1, frame_number): 32 | if data[j] > 0.0: 33 | break 34 | if j < frame_number - 1: 35 | if last_value > 0.0: 36 | step = (data[j] - data[i - 1]) / float(j - i) 37 | for k in range(i, j): 38 | ip_data[k] = data[i - 1] + step * (k - i + 1) 39 | else: 40 | for k in range(i, j): 41 | ip_data[k] = data[j] 42 | else: 43 | for k in range(i, frame_number): 44 | ip_data[k] = last_value 45 | else: 46 | ip_data[i] = data[i] # 这里可能存在一个没有必要的拷贝 47 | last_value = data[i] 48 | 49 | return ip_data[:, 0], vuv_vector[:, 0] 50 | 51 | def resize_f0(self, x, target_len): 52 | source = np.array(x) 53 | source[source < 0.001] = np.nan 54 | target = np.interp( 55 | np.arange(0, len(source) * target_len, len(source)) / target_len, 56 | np.arange(0, len(source)), 57 | source, 58 | ) 59 | res = np.nan_to_num(target) 60 | return res 61 | 62 | def compute_f0(self, wav, p_len=None): 63 | if p_len is None: 64 | p_len = wav.shape[0] // self.hop_length 65 | f0, t = pyworld.harvest( 66 | wav.astype(np.double), 67 | fs=self.hop_length, 68 | f0_ceil=self.f0_max, 69 | f0_floor=self.f0_min, 70 | frame_period=1000 * self.hop_length / self.sampling_rate, 71 | ) 72 | f0 = pyworld.stonemask(wav.astype(np.double), f0, t, self.fs) 73 | return self.interpolate_f0(self.resize_f0(f0, p_len))[0] 74 | 75 | def compute_f0_uv(self, wav, p_len=None): 76 | if p_len is None: 77 | p_len = wav.shape[0] // self.hop_length 78 | f0, t = pyworld.harvest( 79 | wav.astype(np.double), 80 | fs=self.sampling_rate, 81 | f0_floor=self.f0_min, 82 | f0_ceil=self.f0_max, 83 | frame_period=1000 * self.hop_length / self.sampling_rate, 84 | ) 85 | f0 = pyworld.stonemask(wav.astype(np.double), f0, t, self.sampling_rate) 86 | return self.interpolate_f0(self.resize_f0(f0, p_len)) 87 | -------------------------------------------------------------------------------- /docs/faq.md: -------------------------------------------------------------------------------- 1 | ## Q1:ffmpeg error/utf8 error. 2 | 3 | 大概率不是ffmpeg问题，而是音频路径问题；
4 | ffmpeg读取路径带空格、()等特殊符号，可能出现ffmpeg error；训练集音频带中文路径，在写入filelist.txt的时候可能出现utf8 error；
5 | 6 | ## Q2:一键训练结束没有索引 7 | 8 | 显示"Training is done. The program is closed."则模型训练成功，后续紧邻的报错是假的；
9 | 10 | 一键训练结束完成没有added开头的索引文件，可能是因为训练集太大卡住了添加索引的步骤；已通过批处理add索引解决内存add索引对内存需求过大的问题。临时可尝试再次点击"训练索引"按钮。
11 | 12 | ## Q3:训练结束推理没看到训练集的音色 13 | 点刷新音色再看看，如果还没有看看训练有没有报错，控制台和webui的截图，logs/实验名下的log，都可以发给开发者看看。
14 | 15 | ## Q4:如何分享模型 16 |   rvc_root/logs/实验名 下面存储的pth不是用来分享模型用来推理的，而是为了存储实验状态供复现，以及继续训练用的。用来分享的模型应该是weights文件夹下大小为60+MB的pth文件；
17 |   后续将把weights/exp_name.pth和logs/exp_name/added_xxx.index合并打包成weights/exp_name.zip省去填写index的步骤，那么zip文件用来分享，不要分享pth文件，除非是想换机器继续训练；
18 |   如果你把logs文件夹下的几百MB的pth文件复制/分享到weights文件夹下强行用于推理，可能会出现f0，tgt_sr等各种key不存在的报错。你需要用ckpt选项卡最下面，手工或自动（本地logs下如果能找到相关信息则会自动）选择是否携带音高、目标音频采样率的选项后进行ckpt小模型提取（输入路径填G开头的那个），提取完在weights文件夹下会出现60+MB的pth文件，刷新音色后可以选择使用。
19 | 20 | ## Q5:Connection Error. 21 | 也许你关闭了控制台（黑色窗口）。
22 | 23 | ## Q6:WebUI弹出Expecting value: line 1 column 1 (char 0). 24 | 请关闭系统局域网代理/全局代理。
25 | 26 | 这个不仅是客户端的代理，也包括服务端的代理（例如你使用autodl设置了http_proxy和https_proxy学术加速，使用时也需要unset关掉）
27 | 28 | ## Q7:不用WebUI如何通过命令训练推理 29 | 训练脚本：
30 | 可先跑通WebUI，消息窗内会显示数据集处理和训练用命令行；
31 | 32 | 推理脚本：
33 | https://huggingface.co/lj1995/VoiceConversionWebUI/blob/main/myinfer.py
34 | 35 | 例子：
36 | 37 | runtime\python.exe myinfer.py 0 "E:\codes\py39\RVC-beta\todo-songs\1111.wav" "E:\codes\py39\logs\mi-test\added_IVF677_Flat_nprobe_7.index" harvest "test.wav" "weights/mi-test.pth" 0.6 cuda:0 True
38 | 39 | f0up_key=sys.argv[1]
40 | input_path=sys.argv[2]
41 | index_path=sys.argv[3]
42 | f0method=sys.argv[4]#harvest or pm
43 | opt_path=sys.argv[5]
44 | model_path=sys.argv[6]
45 | index_rate=float(sys.argv[7])
46 | device=sys.argv[8]
47 | is_half=bool(sys.argv[9])
48 | 49 | ## Q8:Cuda error/Cuda out of memory. 50 | 小概率是cuda配置问题、设备不支持；大概率是显存不够（out of memory）；
51 | 52 | 训练的话缩小batch size（如果缩小到1还不够只能更换显卡训练），推理的话酌情缩小config.py结尾的x_pad，x_query，x_center，x_max。4G以下显存（例如1060（3G）和各种2G显卡）可以直接放弃，4G显存显卡还有救。
53 | 54 | ## Q9:total_epoch调多少比较好 55 | 56 | 如果训练集音质差底噪大，20~30足够了，调太高，底模音质无法带高你的低音质训练集
57 | 如果训练集音质高底噪低时长多，可以调高，200是ok的（训练速度很快，既然你有条件准备高音质训练集，显卡想必条件也不错，肯定不在乎多一些训练时间）
58 | 59 | ## Q10:需要多少训练集时长 60 |   推荐10min至50min
61 |   保证音质高底噪低的情况下，如果有个人特色的音色统一，则多多益善
62 |   高水平的训练集（精简+音色有特色），5min至10min也是ok的，仓库作者本人就经常这么玩
63 |   也有人拿1min至2min的数据来训练并且训练成功的，但是成功经验是其他人不可复现的，不太具备参考价值。这要求训练集音色特色非常明显（比如说高频气声较明显的萝莉少女音），且音质高；
64 |   1min以下时长数据目前没见有人尝试（成功）过。不建议进行这种鬼畜行为。
65 | 66 | ## Q11:index rate干嘛用的，怎么调（科普） 67 |   如果底模和推理源的音质高于训练集的音质，他们可以带高推理结果的音质，但代价可能是音色往底模/推理源的音色靠，这种现象叫做"音色泄露"；
68 |   index rate用来削减/解决音色泄露问题。调到1，则理论上不存在推理源的音色泄露问题，但音质更倾向于训练集。如果训练集音质比推理源低，则index rate调高可能降低音质。调到0，则不具备利用检索混合来保护训练集音色的效果；
69 |   如果训练集优质时长多，可调高total_epoch，此时模型本身不太会引用推理源和底模的音色，很少存在"音色泄露"问题，此时index_rate不重要，你甚至可以不建立/分享index索引文件。
70 | 71 | ## Q11:推理怎么选gpu 72 | config.py文件里device cuda:后面选择卡号；
73 | 卡号和显卡的映射关系，在训练选项卡的显卡信息栏里能看到。
74 | 75 | ## Q12:如何推理训练中间保存的pth 76 | 通过ckpt选项卡最下面提取小模型。
77 | 78 | 79 | ## Q13:如何中断和继续训练 80 | 现阶段只能关闭WebUI控制台双击go-web.bat重启程序。网页参数也要刷新重新填写；
81 | 继续训练：相同网页参数点训练模型，就会接着上次的checkpoint继续训练。
82 | 83 | ## Q14:训练时出现文件页面/内存error 84 | 进程开太多了，内存炸了。你可能可以通过如下方式解决
85 | 1、"提取音高和处理数据使用的CPU进程数" 酌情拉低；
86 | 2、训练集音频手工切一下，不要太长。
87 | 88 | 89 | ## Q15:如何中途加数据训练 90 | 1、所有数据新建一个实验名；
91 | 2、拷贝上一次的最新的那个G和D文件（或者你想基于哪个中间ckpt训练，也可以拷贝中间的）到新实验名；下
92 | 3、一键训练新实验名，他会继续上一次的最新进度训练。
93 | 94 | -------------------------------------------------------------------------------- /docs/faiss_tips_ja.md: -------------------------------------------------------------------------------- 1 | faiss tuning TIPS 2 | ================== 3 | # about faiss 4 | faissはfacebook researchの開発する、密なベクトルに対する近傍探索をまとめたライブラリで、多くの近似近傍探索の手法を効率的に実装しています。 5 | 近似近傍探索はある程度精度を犠牲にしながら高速に類似するベクトルを探します。 6 | 7 | ## faiss in RVC 8 | RVCではHuBERTで変換した特徴量のEmbeddingに対し、学習データから生成されたEmbeddingと類似するものを検索し、混ぜることでより元の音声に近い変換を実現しています。ただ、この検索は愚直に行うと時間がかかるため、近似近傍探索を用いることで高速な変換を実現しています。 9 | 10 | # 実装のoverview 11 | モデルが配置されている '/logs/your-experiment/3_feature256'には各音声データからHuBERTで抽出された特徴量が配置されています。 12 | ここからnpyファイルをファイル名でソートした順番で読み込み、ベクトルを連結してbig_npyを作成しfaissを学習させます。(このベクトルのshapeは[N, 256]です。) 13 | 14 | 本Tipsではまずこれらのパラメータの意味を解説します。 15 | 16 | # 手法の解説 17 | ## index factory 18 | index factoryは複数の近似近傍探索の手法を繋げるパイプラインをstringで表記するfaiss独自の記法です。 19 | これにより、index factoryの文字列を変更するだけで様々な近似近傍探索の手法を試せます。 20 | RVCでは以下のように使われています。 21 | 22 | ```python 23 | index = faiss.index_factory(256, "IVF%s,Flat" % n_ivf) 24 | ``` 25 | index_factoryの引数のうち、1つ目はベクトルの次元数、2つ目はindex factoryの文字列で、3つ目には用いる距離を指定することができます。 26 | 27 | より詳細な記法については 28 | https://github.com/facebookresearch/faiss/wiki/The-index-factory 29 | 30 | ## 距離指標 31 | embeddingの類似度として用いられる代表的な指標として以下の二つがあります。 32 | 33 | - ユークリッド距離(METRIC_L2) 34 | - 内積(METRIC_INNER_PRODUCT) 35 | 36 | ユークリッド距離では各次元において二乗の差をとり、全次元の差を足してから平方根をとります。これは日常的に用いる2次元、3次元での距離と同じです。 37 | 内積はこのままでは類似度の指標として用いず、一般的にはL2ノルムで正規化してから内積をとるコサイン類似度を用います。 38 | 39 | どちらがよいかは場合によりますが、word2vec等で得られるembeddingやArcFace等で学習した類似画像検索のモデルではコサイン類似度が用いられることが多いです。ベクトルXに対してl2正規化をnumpyで行う場合は、0 divisionを避けるために十分に小さな値をepsとして以下のコードで可能です。 40 | 41 | ```python 42 | X_normed = X / np.maximum(eps, np.linalg.norm(X, ord=2, axis=-1, keepdims=True)) 43 | ``` 44 | 45 | また、index factoryには第3引数に渡す値を選ぶことで計算に用いる距離指標を変更できます。 46 | 47 | ```python 48 | index = faiss.index_factory(dimention, text, faiss.METRIC_INNER_PRODUCT) 49 | ``` 50 | 51 | ## IVF 52 | IVF(Inverted file indexes)は全文検索における転置インデックスと似たようなアルゴリズムです。 53 | 学習時には検索対象に対してkmeansでクラスタリングを行い、クラスタ中心を用いてボロノイ分割を行います。各データ点には一つずつクラスタが割り当てられるので、クラスタからデータ点を逆引きする辞書を作成します。 54 | 55 | 例えば以下のようにクラスタが割り当てられた場合 56 | |index|クラスタ| 57 | |-----|-------| 58 | |1|A| 59 | |2|B| 60 | |3|A| 61 | |4|C| 62 | |5|B| 63 | 64 | 作成される転置インデックスは以下のようになります。 65 | 66 | |クラスタ|index| 67 | |-------|-----| 68 | |A|1, 3| 69 | |B|2, 5| 70 | |C|4| 71 | 72 | 検索時にはまずクラスタからn_probe個のクラスタを検索し、次にそれぞれのクラスタに属するデータ点について距離を計算します。 73 | 74 | # 推奨されるパラメータ 75 | indexの選び方については公式にガイドラインがあるので、それに準じて説明します。 76 | https://github.com/facebookresearch/faiss/wiki/Guidelines-to-choose-an-index 77 | 78 | 1M以下のデータセットにおいては4bit-PQが2023年4月時点ではfaissで利用できる最も効率的な手法です。 79 | これをIVFと組み合わせ、4bit-PQで候補を絞り、最後に正確な指標で距離を再計算するには以下のindex factoryを用いることで記載できます。 80 | 81 | ```python 82 | index = faiss.index_factory(256, "IVF1024,PQ128x4fs,RFlat") 83 | ``` 84 | 85 | ## IVFの推奨パラメータ 86 | IVFの数が多すぎる場合、たとえばデータ数の数だけIVFによる粗量子化を行うと、これは愚直な全探索と同じになり効率が悪いです。 87 | 1M以下の場合ではIVFの値はデータ点の数Nに対して4*sqrt(N) ~ 16*sqrt(N)に推奨しています。 88 | 89 | n_probeはn_probeの数に比例して計算時間が増えるので、精度と相談して適切に選んでください。個人的にはRVCにおいてそこまで精度は必要ないと思うのでn_probe = 1で良いと思います。 90 | 91 | ## FastScan 92 | FastScanは直積量子化で大まかに距離を近似するのを、レジスタ内で行うことにより高速に行うようにした手法です。 93 | 直積量子化は学習時にd次元ごと(通常はd=2)に独立してクラスタリングを行い、クラスタ同士の距離を事前計算してlookup tableを作成します。予測時はlookup tableを見ることで各次元の距離をO(1)で計算できます。 94 | そのため、PQの次に指定する数字は通常ベクトルの半分の次元を指定します。 95 | 96 | FastScanに関するより詳細な説明は公式のドキュメントを参照してください。 97 | https://github.com/facebookresearch/faiss/wiki/Fast-accumulation-of-PQ-and-AQ-codes-(FastScan) 98 | 99 | ## RFlat 100 | RFlatはFastScanで計算した大まかな距離を、index factoryの第三引数で指定した正確な距離で再計算する指示です。 101 | k個の近傍を取得する際は、k*k_factor個の点について再計算が行われます。 102 | -------------------------------------------------------------------------------- /lib/infer_pack/modules/F0Predictor/DioF0Predictor.py: -------------------------------------------------------------------------------- 1 | from lib.infer_pack.modules.F0Predictor.F0Predictor import F0Predictor 2 | import pyworld 3 | import numpy as np 4 | 5 | 6 | class DioF0Predictor(F0Predictor): 7 | def __init__(self, hop_length=512, f0_min=50, f0_max=1100, sampling_rate=44100): 8 | self.hop_length = hop_length 9 | self.f0_min = f0_min 10 | self.f0_max = f0_max 11 | self.sampling_rate = sampling_rate 12 | 13 | def interpolate_f0(self, f0): 14 | """ 15 | 对F0进行插值处理 16 | """ 17 | 18 | data = np.reshape(f0, (f0.size, 1)) 19 | 20 | vuv_vector = np.zeros((data.size, 1), dtype=np.float32) 21 | vuv_vector[data > 0.0] = 1.0 22 | vuv_vector[data <= 0.0] = 0.0 23 | 24 | ip_data = data 25 | 26 | frame_number = data.size 27 | last_value = 0.0 28 | for i in range(frame_number): 29 | if data[i] <= 0.0: 30 | j = i + 1 31 | for j in range(i + 1, frame_number): 32 | if data[j] > 0.0: 33 | break 34 | if j < frame_number - 1: 35 | if last_value > 0.0: 36 | step = (data[j] - data[i - 1]) / float(j - i) 37 | for k in range(i, j): 38 | ip_data[k] = data[i - 1] + step * (k - i + 1) 39 | else: 40 | for k in range(i, j): 41 | ip_data[k] = data[j] 42 | else: 43 | for k in range(i, frame_number): 44 | ip_data[k] = last_value 45 | else: 46 | ip_data[i] = data[i] # 这里可能存在一个没有必要的拷贝 47 | last_value = data[i] 48 | 49 | return ip_data[:, 0], vuv_vector[:, 0] 50 | 51 | def resize_f0(self, x, target_len): 52 | source = np.array(x) 53 | source[source < 0.001] = np.nan 54 | target = np.interp( 55 | np.arange(0, len(source) * target_len, len(source)) / target_len, 56 | np.arange(0, len(source)), 57 | source, 58 | ) 59 | res = np.nan_to_num(target) 60 | return res 61 | 62 | def compute_f0(self, wav, p_len=None): 63 | if p_len is None: 64 | p_len = wav.shape[0] // self.hop_length 65 | f0, t = pyworld.dio( 66 | wav.astype(np.double), 67 | fs=self.sampling_rate, 68 | f0_floor=self.f0_min, 69 | f0_ceil=self.f0_max, 70 | frame_period=1000 * self.hop_length / self.sampling_rate, 71 | ) 72 | f0 = pyworld.stonemask(wav.astype(np.double), f0, t, self.sampling_rate) 73 | for index, pitch in enumerate(f0): 74 | f0[index] = round(pitch, 1) 75 | return self.interpolate_f0(self.resize_f0(f0, p_len))[0] 76 | 77 | def compute_f0_uv(self, wav, p_len=None): 78 | if p_len is None: 79 | p_len = wav.shape[0] // self.hop_length 80 | f0, t = pyworld.dio( 81 | wav.astype(np.double), 82 | fs=self.sampling_rate, 83 | f0_floor=self.f0_min, 84 | f0_ceil=self.f0_max, 85 | frame_period=1000 * self.hop_length / self.sampling_rate, 86 | ) 87 | f0 = pyworld.stonemask(wav.astype(np.double), f0, t, self.sampling_rate) 88 | for index, pitch in enumerate(f0): 89 | f0[index] = round(pitch, 1) 90 | return self.interpolate_f0(self.resize_f0(f0, p_len)) 91 | -------------------------------------------------------------------------------- /docs/Changelog_KO.md: -------------------------------------------------------------------------------- 1 | ### 2023년 6월 18일 업데이트 2 | 3 | - v2 버전에서 새로운 32k와 48k 사전 학습 모델을 추가. 4 | - non-f0 모델들의 추론 오류 수정. 5 | - 학습 세트가 1시간을 넘어가는 경우, 인덱스 생성 단계에서 minibatch-kmeans을 사용해, 학습속도 가속화. 6 | - [huggingface](https://huggingface.co/spaces/lj1995/vocal2guitar)에서 vocal2guitar 제공. 7 | - 데이터 처리 단계에서 이상 값 자동으로 제거. 8 | - ONNX로 내보내는(export) 옵션 탭 추가. 9 | 10 | 업데이트에 적용되지 않았지만 시도한 것들 : 11 | 12 | - 시계열 차원을 추가하여 특징 검색을 진행했지만, 유의미한 효과는 없었습니다. 13 | - PCA 차원 축소를 추가하여 특징 검색을 진행했지만, 유의미한 효과는 없었습니다. 14 | - ONNX 추론을 지원하는 것에 실패했습니다. nsf 생성시, Pytorch가 필요하기 때문입니다. 15 | - 훈련 중에 입력에 대한 음고, 성별, 이퀄라이저, 노이즈 등 무작위로 강화하는 것에, 유의미한 효과는 없었습니다. 16 | 17 | 추후 업데이트 목록: 18 | 19 | - Vocos-RVC (소형 보코더) 통합 예정. 20 | - 학습 단계에 음고 인식을 위한 Crepe 지원 예정. 21 | - Crepe의 정밀도를 REC-config와 동기화하여 지원 예정. 22 | - FO 에디터 지원 예정. 23 | 24 | ### 2023년 5월 28일 업데이트 25 | 26 | - v2 jupyter notebook 추가, 한국어 업데이트 로그 추가, 의존성 모듈 일부 수정. 27 | - 무성음 및 숨소리 보호 모드 추가. 28 | - crepe-full pitch 감지 지원. 29 | - UVR5 보컬 분리: 디버브 및 디-에코 모델 지원. 30 | - index 이름에 experiment 이름과 버전 추가. 31 | - 배치 음성 변환 처리 및 UVR5 보컬 분리 시, 사용자가 수동으로 출력 오디오의 내보내기(export) 형식을 선택할 수 있도록 지원. 32 | - 32k 훈련 모델 지원 종료. 33 | 34 | ### 2023년 5월 13일 업데이트 35 | 36 | - 원클릭 패키지의 이전 버전 런타임 내, 불필요한 코드(lib.infer_pack 및 uvr5_pack) 제거. 37 | - 훈련 세트 전처리의 유사 다중 처리 버그 수정. 38 | - Harvest 피치 인식 알고리즘에 대한 중위수 필터링 반경 조정 추가. 39 | - 오디오 내보낼 때, 후처리 리샘플링 지원. 40 | - 훈련에 대한 다중 처리 "n_cpu" 설정이 "f0 추출"에서 "데이터 전처리 및 f0 추출"로 변경. 41 | - logs 폴더 하의 인덱스 경로를 자동으로 감지 및 드롭다운 목록 기능 제공. 42 | - 탭 페이지에 "자주 묻는 질문과 답변" 추가. (github RVC wiki 참조 가능) 43 | - 동일한 입력 오디오 경로를 사용할 때 추론, Harvest 피치를 캐시. 44 | (주의: Harvest 피치 추출을 사용하면 전체 파이프라인은 길고 반복적인 피치 추출 과정을 거치게됩니다. 캐싱을 하지 않는다면, 첫 inference 이후의 단계에서 timbre, 인덱스, 피치 중위수 필터링 반경 설정 등 대기시간이 엄청나게 길어집니다!) 45 | 46 | ### 2023년 5월 14일 업데이트 47 | 48 | - 입력의 볼륨 캡슐을 사용하여 출력의 볼륨 캡슐을 혼합하거나 대체. (입력이 무음이거나 출력의 노이즈 문제를 최소화 할 수 있습니다. 입력 오디오의 배경 노이즈(소음)가 큰 경우 해당 기능을 사용하지 않는 것이 좋습니다. 기본적으로 비활성화 되어있는 옵션입니다. (1: 비활성화 상태)) 49 | - 추출된 소형 모델을 지정된 빈도로 저장하는 기능을 지원. (다양한 에폭 하에서의 성능을 보려고 하지만 모든 대형 체크포인트를 저장하고 매번 ckpt 처리를 통해 소형 모델을 수동으로 추출하고 싶지 않은 경우 이 기능은 매우 유용합니다) 50 | - 환경 변수를 설정하여 서버의 전역 프록시로 인한 "연결 오류" 문제 해결. 51 | - 사전 훈련된 v2 모델 지원. (현재 40k 버전만 테스트를 위해 공개적으로 사용 가능하며, 다른 두 개의 샘플링 비율은 아직 완전히 훈련되지 않아 보류되었습니다.) 52 | - 추론 전, 1을 초과하는 과도한 볼륨 제한. 53 | - 데이터 전처리 매개변수 미세 조정. 54 | 55 | ### 2023년 4월 9일 업데이트 56 | 57 | - GPU 이용률 향상을 위해 훈련 파라미터 수정: A100은 25%에서 약 90%로 증가, V100: 50%에서 약 90%로 증가, 2060S: 60%에서 약 85%로 증가, P40: 25%에서 약 95%로 증가. 58 | 훈련 속도가 크게 향상. 59 | - 매개변수 기준 변경: total batch_size는 GPU당 batch_size를 의미. 60 | - total_epoch 변경: 최대 한도가 100에서 1000으로 증가. 기본값이 10에서 20으로 증가. 61 | - ckpt 추출이 피치를 잘못 인식하여 비정상적인 추론을 유발하는 문제 수정. 62 | - 분산 훈련 과정에서 각 랭크마다 ckpt를 저장하는 문제 수정. 63 | - 특성 추출 과정에 나노 특성 필터링 적용. 64 | - 무음 입력/출력이 랜덤하게 소음을 생성하는 문제 수정. (이전 모델은 새 데이터셋으로 다시 훈련해야 합니다) 65 | 66 | ### 2023년 4월 16일 업데이트 67 | 68 | - 로컬 실시간 음성 변경 미니-GUI 추가, go-realtime-gui.bat를 더블 클릭하여 시작. 69 | - 훈련 및 추론 중 50Hz 이하의 주파수 대역에 대해 필터링 적용. 70 | - 훈련 및 추론의 pyworld 최소 피치 추출을 기본 80에서 50으로 낮춤. 이로 인해, 50-80Hz 사이의 남성 저음이 무음화되지 않습니다. 71 | - 시스템 지역에 따른 WebUI 언어 변경 지원. (현재 en_US, ja_JP, zh_CN, zh_HK, zh_SG, zh_TW를 지원하며, 지원되지 않는 경우 기본값은 en_US) 72 | - 일부 GPU의 인식 수정. (예: V100-16G 인식 실패, P4 인식 실패) 73 | 74 | ### 2023년 4월 28일 업데이트 75 | 76 | - Faiss 인덱스 설정 업그레이드로 속도가 더 빨라지고 품질이 향상. 77 | - total_npy에 대한 의존성 제거. 추후의 모델 공유는 total_npy 입력을 필요로 하지 않습니다. 78 | - 16 시리즈 GPU에 대한 제한 해제, 4GB VRAM GPU에 대한 4GB 추론 설정 제공. 79 | - 일부 오디오 형식에 대한 UVR5 보컬 동반 분리에서의 버그 수정. 80 | - 실시간 음성 변경 미니-GUI는 이제 non-40k 및 non-lazy 피치 모델을 지원합니다. 81 | 82 | ### 추후 계획 83 | 84 | Features: 85 | 86 | - 다중 사용자 훈련 탭 지원.(최대 4명) 87 | 88 | Base model: 89 | 90 | - 훈련 데이터셋에 숨소리 wav 파일을 추가하여, 보컬의 호흡이 노이즈로 변환되는 문제 수정. 91 | - 보컬 훈련 세트의 기본 모델을 추가하기 위한 작업을 진행중이며, 이는 향후에 발표될 예정. 92 | -------------------------------------------------------------------------------- /lib/infer_pack/modules/F0Predictor/PMF0Predictor.py: -------------------------------------------------------------------------------- 1 | from lib.infer_pack.modules.F0Predictor.F0Predictor import F0Predictor 2 | import parselmouth 3 | import numpy as np 4 | 5 | 6 | class PMF0Predictor(F0Predictor): 7 | def __init__(self, hop_length=512, f0_min=50, f0_max=1100, sampling_rate=44100): 8 | self.hop_length = hop_length 9 | self.f0_min = f0_min 10 | self.f0_max = f0_max 11 | self.sampling_rate = sampling_rate 12 | 13 | def interpolate_f0(self, f0): 14 | """ 15 | 对F0进行插值处理 16 | """ 17 | 18 | data = np.reshape(f0, (f0.size, 1)) 19 | 20 | vuv_vector = np.zeros((data.size, 1), dtype=np.float32) 21 | vuv_vector[data > 0.0] = 1.0 22 | vuv_vector[data <= 0.0] = 0.0 23 | 24 | ip_data = data 25 | 26 | frame_number = data.size 27 | last_value = 0.0 28 | for i in range(frame_number): 29 | if data[i] <= 0.0: 30 | j = i + 1 31 | for j in range(i + 1, frame_number): 32 | if data[j] > 0.0: 33 | break 34 | if j < frame_number - 1: 35 | if last_value > 0.0: 36 | step = (data[j] - data[i - 1]) / float(j - i) 37 | for k in range(i, j): 38 | ip_data[k] = data[i - 1] + step * (k - i + 1) 39 | else: 40 | for k in range(i, j): 41 | ip_data[k] = data[j] 42 | else: 43 | for k in range(i, frame_number): 44 | ip_data[k] = last_value 45 | else: 46 | ip_data[i] = data[i] # 这里可能存在一个没有必要的拷贝 47 | last_value = data[i] 48 | 49 | return ip_data[:, 0], vuv_vector[:, 0] 50 | 51 | def compute_f0(self, wav, p_len=None): 52 | x = wav 53 | if p_len is None: 54 | p_len = x.shape[0] // self.hop_length 55 | else: 56 | assert abs(p_len - x.shape[0] // self.hop_length) < 4, "pad length error" 57 | time_step = self.hop_length / self.sampling_rate * 1000 58 | f0 = ( 59 | parselmouth.Sound(x, self.sampling_rate) 60 | .to_pitch_ac( 61 | time_step=time_step / 1000, 62 | voicing_threshold=0.6, 63 | pitch_floor=self.f0_min, 64 | pitch_ceiling=self.f0_max, 65 | ) 66 | .selected_array["frequency"] 67 | ) 68 | 69 | pad_size = (p_len - len(f0) + 1) // 2 70 | if pad_size > 0 or p_len - len(f0) - pad_size > 0: 71 | f0 = np.pad(f0, [[pad_size, p_len - len(f0) - pad_size]], mode="constant") 72 | f0, uv = self.interpolate_f0(f0) 73 | return f0 74 | 75 | def compute_f0_uv(self, wav, p_len=None): 76 | x = wav 77 | if p_len is None: 78 | p_len = x.shape[0] // self.hop_length 79 | else: 80 | assert abs(p_len - x.shape[0] // self.hop_length) < 4, "pad length error" 81 | time_step = self.hop_length / self.sampling_rate * 1000 82 | f0 = ( 83 | parselmouth.Sound(x, self.sampling_rate) 84 | .to_pitch_ac( 85 | time_step=time_step / 1000, 86 | voicing_threshold=0.6, 87 | pitch_floor=self.f0_min, 88 | pitch_ceiling=self.f0_max, 89 | ) 90 | .selected_array["frequency"] 91 | ) 92 | 93 | pad_size = (p_len - len(f0) + 1) // 2 94 | if pad_size > 0 or p_len - len(f0) - pad_size > 0: 95 | f0 = np.pad(f0, [[pad_size, p_len - len(f0) - pad_size]], mode="constant") 96 | f0, uv = self.interpolate_f0(f0) 97 | return f0, uv 98 | -------------------------------------------------------------------------------- /infer_uvr.py: -------------------------------------------------------------------------------- 1 | import os 2 | import shutil 3 | import sys 4 | import traceback 5 | import torch 6 | 7 | os.environ["OPENBLAS_NUM_THREADS"] = "1" 8 | os.environ["no_proxy"] = "localhost, 127.0.0.1, ::1" 9 | 10 | import ffmpeg 11 | from infer_uvr5 import _audio_pre_, _audio_pre_new 12 | 13 | now_dir = os.getcwd() 14 | sys.path.append(now_dir) 15 | tmp = os.path.join(now_dir, "TEMP") 16 | # shutil.rmtree(tmp, ignore_errors=True) 17 | shutil.rmtree("%s/runtime/Lib/site-packages/infer_pack" % (now_dir), ignore_errors=True) 18 | shutil.rmtree("%s/runtime/Lib/site-packages/uvr5_pack" % (now_dir), ignore_errors=True) 19 | os.makedirs(tmp, exist_ok=True) 20 | os.makedirs(os.path.join(now_dir, "logs"), exist_ok=True) 21 | os.makedirs(os.path.join(now_dir, "weights"), exist_ok=True) 22 | os.environ["TEMP"] = tmp 23 | torch.manual_seed(114514) 24 | 25 | weight_root = "weights" 26 | weight_uvr5_root = "uvr5_weights" 27 | index_root = "logs" 28 | 29 | def uvr(model_name, inp_root, save_root_vocal, paths, save_root_ins, agg, format0): 30 | infos = [] 31 | try: 32 | inp_root = inp_root.strip(" ").strip('"').strip("\n").strip('"').strip(" ") 33 | save_root_vocal = save_root_vocal.strip(" ").strip('"').strip("\n").strip('"').strip(" ") 34 | save_root_ins = save_root_ins.strip(" ").strip('"').strip("\n").strip('"').strip(" ") 35 | 36 | if model_name == "onnx_dereverb_By_FoxJoy": 37 | from MDXNet import MDXNetDereverb 38 | pre_fun = MDXNetDereverb(15) 39 | else: 40 | func = _audio_pre_ if "DeEcho" not in model_name else _audio_pre_new 41 | pre_fun = func( 42 | agg=int(agg), 43 | model_path=os.path.join(weight_uvr5_root, model_name + ".pth"), 44 | device="cuda", 45 | is_half=True 46 | ) 47 | 48 | need_reformat = 1 49 | done = 0 50 | try: 51 | info = ffmpeg.probe(inp_root, cmd="ffprobe") 52 | if (info["streams"][0]["channels"] == 2 and info["streams"][0]["sample_rate"] == "44100"): 53 | need_reformat = 0 54 | pre_fun._path_audio_(inp_root, save_root_ins, save_root_vocal, format0) 55 | done = 1 56 | except: 57 | need_reformat = 1 58 | infos.append(traceback.format_exc()) 59 | 60 | if need_reformat == 1: 61 | tmp_path = os.path.join(tmp, "{}.reformatted.wav".format(os.path.basename(inp_root))) 62 | 63 | # Wrap paths in double quotes to handle spaces 64 | os.system('ffmpeg -i "{}" -vn -acodec pcm_s16le -ac 2 -ar 44100 "{}" -y'.format(inp_root, tmp_path)) 65 | 66 | inp_root = tmp_path 67 | 68 | try: 69 | if done == 0: 70 | pre_fun._path_audio_(inp_root, save_root_ins, save_root_vocal, format0) 71 | infos.append("{}->Success".format(os.path.basename(inp_root))) 72 | except: 73 | infos.append("{}->{}".format(os.path.basename(inp_root), traceback.format_exc())) 74 | 75 | except Exception as e: 76 | infos.append(traceback.format_exc()) 77 | 78 | finally: 79 | try: 80 | if model_name == "onnx_dereverb_By_FoxJoy": 81 | del pre_fun.pred.model 82 | del pre_fun.pred.model_ 83 | else: 84 | del pre_fun.model 85 | del pre_fun 86 | except: 87 | infos.append(traceback.format_exc()) 88 | 89 | print("clean_empty_cache") 90 | if torch.cuda.is_available(): 91 | torch.cuda.empty_cache() 92 | 93 | for info in infos: 94 | print(info) 95 | 96 | if __name__ == "__main__": 97 | uvr(model_name="5_HP-Karaoke-UVR", 98 | inp_root="songs\heythere.mp4", 99 | save_root_vocal="opt", 100 | paths="", 101 | save_root_ins="opt", 102 | agg=10, 103 | format0="wav") -------------------------------------------------------------------------------- /docs/README.ko.han.md: -------------------------------------------------------------------------------- 1 |
2 | 3 |

Retrieval-based-Voice-Conversion-WebUI

4 | VITS基盤의 簡單하고使用하기 쉬운音聲變換틀

5 | 6 | [![madewithlove](https://forthebadge.com/images/badges/built-with-love.svg)](https://github.com/RVC-Project/Retrieval-based-Voice-Conversion-WebUI) 7 | 8 |
9 | 10 | [![Open In Colab](https://img.shields.io/badge/Colab-F9AB00?style=for-the-badge&logo=googlecolab&color=525252)](https://colab.research.google.com/github/RVC-Project/Retrieval-based-Voice-Conversion-WebUI/blob/main/Retrieval_based_Voice_Conversion_WebUI.ipynb) 11 | [![Licence](https://img.shields.io/github/license/RVC-Project/Retrieval-based-Voice-Conversion-WebUI?style=for-the-badge)](https://github.com/RVC-Project/Retrieval-based-Voice-Conversion-WebUI/blob/main/LICENSE) 12 | [![Huggingface](https://img.shields.io/badge/🤗%20-Spaces-yellow.svg?style=for-the-badge)](https://huggingface.co/lj1995/VoiceConversionWebUI/tree/main/) 13 | 14 | [![Discord](https://img.shields.io/badge/RVC%20Developers-Discord-7289DA?style=for-the-badge&logo=discord&logoColor=white)](https://discord.gg/HcsmBBGyVk) 15 | 16 |
17 | 18 | ------ 19 | [**更新日誌**](https://github.com/RVC-Project/Retrieval-based-Voice-Conversion-WebUI/blob/main/docs/Changelog_KO.md) 20 | 21 | [**English**](./README.en.md) | [**中文简体**](../README.md) | [**日本語**](./README.ja.md) | [**한국어**](./README.ko.md) ([**韓國語**](./README.ko.han.md)) 22 | 23 | > [示範映像](https://www.bilibili.com/video/BV1pm4y1z7Gm/)을 確認해 보세요! 24 | 25 | > RVC를活用한實時間音聲變換: [w-okada/voice-changer](https://github.com/w-okada/voice-changer) 26 | 27 | > 基本모델은 50時間假量의 高品質 오픈 소스 VCTK 데이터셋을 使用하였으므로, 著作權上의 念慮가 없으니 安心하고 使用하시기 바랍니다. 28 | 29 | > 著作權問題가 없는 高品質의 노래를 以後에도 繼續해서 訓練할 豫定입니다. 30 | 31 | ## 紹介 32 | 本Repo는 다음과 같은 特徵을 가지고 있습니다: 33 | + top1檢索을利用하여 入力音色特徵을 訓練세트音色特徵으로 代替하여 音色의漏出을 防止; 34 | + 相對的으로 낮은性能의 GPU에서도 빠른訓練可能; 35 | + 적은量의 데이터로 訓練해도 좋은 結果를 얻을 수 있음 (最小10分以上의 低雜음音聲데이터를 使用하는 것을 勸獎); 36 | + 모델融合을通한 音色의 變調可能 (ckpt處理탭->ckpt混合選擇); 37 | + 使用하기 쉬운 WebUI (웹 使用者인터페이스); 38 | + UVR5 모델을 利用하여 목소리와 背景音樂의 빠른 分離; 39 | 40 | ## 環境의準備 41 | poetry를通해 依存를設置하는 것을 勸獎합니다. 42 | 43 | 다음命令은 Python 버전3.8以上의環境에서 實行되어야 합니다: 44 | ```bash 45 | # PyTorch 關聯主要依存設置, 이미設置되어 있는 境遇 건너뛰기 可能 46 | # 參照: https://pytorch.org/get-started/locally/ 47 | pip install torch torchvision torchaudio 48 | 49 | # Windows + Nvidia Ampere Architecture(RTX30xx)를 使用하고 있다面, #21 에서 명시된 것과 같이 PyTorch에 맞는 CUDA 버전을 指定해야 합니다. 50 | #pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu117 51 | 52 | # Poetry 設置, 이미設置되어 있는 境遇 건너뛰기 可能 53 | # Reference: https://python-poetry.org/docs/#installation 54 | curl -sSL https://install.python-poetry.org | python3 - 55 | 56 | # 依存設置 57 | poetry install 58 | ``` 59 | pip를 活用하여依存를 設置하여도 無妨합니다. 60 | 61 | ```bash 62 | pip install -r requirements.txt 63 | ``` 64 | 65 | ## 其他預備모델準備 66 | RVC 모델은 推論과訓練을 依하여 다른 預備모델이 必要합니다. 67 | 68 | [Huggingface space](https://huggingface.co/lj1995/VoiceConversionWebUI/tree/main/)를 通해서 다운로드 할 수 있습니다. 69 | 70 | 다음은 RVC에 必要한 預備모델 및 其他 파일 目錄입니다: 71 | ```bash 72 | hubert_base.pt 73 | 74 | ./pretrained 75 | 76 | ./uvr5_weights 77 | 78 | # Windows를 使用하는境遇 이 사전도 必要할 수 있습니다. FFmpeg가 設置되어 있으면 건너뛰어도 됩니다. 79 | ffmpeg.exe 80 | ``` 81 | 그後 以下의 命令을 使用하여 WebUI를 始作할 수 있습니다: 82 | ```bash 83 | python infer-web.py 84 | ``` 85 | Windows를 使用하는境遇 `RVC-beta.7z`를 다운로드 및 壓縮解除하여 RVC를 直接使用하거나 `go-web.bat`을 使用하여 WebUi를 直接할 수 있습니다. 86 | 87 | ## 參考 88 | + [ContentVec](https://github.com/auspicious3000/contentvec/) 89 | + [VITS](https://github.com/jaywalnut310/vits) 90 | + [HIFIGAN](https://github.com/jik876/hifi-gan) 91 | + [Gradio](https://github.com/gradio-app/gradio) 92 | + [FFmpeg](https://github.com/FFmpeg/FFmpeg) 93 | + [Ultimate Vocal Remover](https://github.com/Anjok07/ultimatevocalremovergui) 94 | + [audio-slicer](https://github.com/openvpi/audio-slicer) 95 | ## 모든寄與者분들의勞力에感謝드립니다 96 | 97 | 98 | 99 | 100 | 101 | -------------------------------------------------------------------------------- /lib/uvr5_pack/utils.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import numpy as np 3 | from tqdm import tqdm 4 | import json 5 | 6 | 7 | def load_data(file_name: str = "./lib/uvr5_pack/name_params.json") -> dict: 8 | with open(file_name, "r") as f: 9 | data = json.load(f) 10 | 11 | return data 12 | 13 | 14 | def make_padding(width, cropsize, offset): 15 | left = offset 16 | roi_size = cropsize - left * 2 17 | if roi_size == 0: 18 | roi_size = cropsize 19 | right = roi_size - (width % roi_size) + left 20 | 21 | return left, right, roi_size 22 | 23 | 24 | def inference(X_spec, device, model, aggressiveness, data): 25 | """ 26 | data ： dic configs 27 | """ 28 | 29 | def _execute( 30 | X_mag_pad, roi_size, n_window, device, model, aggressiveness, is_half=True 31 | ): 32 | model.eval() 33 | with torch.no_grad(): 34 | preds = [] 35 | 36 | iterations = [n_window] 37 | 38 | total_iterations = sum(iterations) 39 | for i in tqdm(range(n_window)): 40 | start = i * roi_size 41 | X_mag_window = X_mag_pad[ 42 | None, :, :, start : start + data["window_size"] 43 | ] 44 | X_mag_window = torch.from_numpy(X_mag_window) 45 | if is_half: 46 | X_mag_window = X_mag_window.half() 47 | X_mag_window = X_mag_window.to(device) 48 | 49 | pred = model.predict(X_mag_window, aggressiveness) 50 | 51 | pred = pred.detach().cpu().numpy() 52 | preds.append(pred[0]) 53 | 54 | pred = np.concatenate(preds, axis=2) 55 | return pred 56 | 57 | def preprocess(X_spec): 58 | X_mag = np.abs(X_spec) 59 | X_phase = np.angle(X_spec) 60 | 61 | return X_mag, X_phase 62 | 63 | X_mag, X_phase = preprocess(X_spec) 64 | 65 | coef = X_mag.max() 66 | X_mag_pre = X_mag / coef 67 | 68 | n_frame = X_mag_pre.shape[2] 69 | pad_l, pad_r, roi_size = make_padding(n_frame, data["window_size"], model.offset) 70 | n_window = int(np.ceil(n_frame / roi_size)) 71 | 72 | X_mag_pad = np.pad(X_mag_pre, ((0, 0), (0, 0), (pad_l, pad_r)), mode="constant") 73 | 74 | if list(model.state_dict().values())[0].dtype == torch.float16: 75 | is_half = True 76 | else: 77 | is_half = False 78 | pred = _execute( 79 | X_mag_pad, roi_size, n_window, device, model, aggressiveness, is_half 80 | ) 81 | pred = pred[:, :, :n_frame] 82 | 83 | if data["tta"]: 84 | pad_l += roi_size // 2 85 | pad_r += roi_size // 2 86 | n_window += 1 87 | 88 | X_mag_pad = np.pad(X_mag_pre, ((0, 0), (0, 0), (pad_l, pad_r)), mode="constant") 89 | 90 | pred_tta = _execute( 91 | X_mag_pad, roi_size, n_window, device, model, aggressiveness, is_half 92 | ) 93 | pred_tta = pred_tta[:, :, roi_size // 2 :] 94 | pred_tta = pred_tta[:, :, :n_frame] 95 | 96 | return (pred + pred_tta) * 0.5 * coef, X_mag, np.exp(1.0j * X_phase) 97 | else: 98 | return pred * coef, X_mag, np.exp(1.0j * X_phase) 99 | 100 | 101 | def _get_name_params(model_path, model_hash): 102 | data = load_data() 103 | flag = False 104 | ModelName = model_path 105 | for type in list(data): 106 | for model in list(data[type][0]): 107 | for i in range(len(data[type][0][model])): 108 | if str(data[type][0][model][i]["hash_name"]) == model_hash: 109 | flag = True 110 | elif str(data[type][0][model][i]["hash_name"]) in ModelName: 111 | flag = True 112 | 113 | if flag: 114 | model_params_auto = data[type][0][model][i]["model_params"] 115 | param_name_auto = data[type][0][model][i]["param_name"] 116 | if type == "equivalent": 117 | return param_name_auto, model_params_auto 118 | else: 119 | flag = False 120 | return param_name_auto, model_params_auto 121 | -------------------------------------------------------------------------------- /docs/README.ja.md: -------------------------------------------------------------------------------- 1 |
2 | 3 |

Retrieval-based-Voice-Conversion-WebUI

4 | VITSに基づく使いやすい音声変換（voice changer）framework

5 | 6 | [![madewithlove](https://forthebadge.com/images/badges/built-with-love.svg)](https://github.com/RVC-Project/Retrieval-based-Voice-Conversion-WebUI) 7 | 8 |
9 | 10 | [![Open In Colab](https://img.shields.io/badge/Colab-F9AB00?style=for-the-badge&logo=googlecolab&color=525252)](https://colab.research.google.com/github/RVC-Project/Retrieval-based-Voice-Conversion-WebUI/blob/main/Retrieval_based_Voice_Conversion_WebUI.ipynb) 11 | [![Licence](https://img.shields.io/github/license/RVC-Project/Retrieval-based-Voice-Conversion-WebUI?style=for-the-badge)](https://github.com/RVC-Project/Retrieval-based-Voice-Conversion-WebUI/blob/main/LICENSE) 12 | [![Huggingface](https://img.shields.io/badge/🤗%20-Spaces-yellow.svg?style=for-the-badge)](https://huggingface.co/lj1995/VoiceConversionWebUI/tree/main/) 13 | 14 | [![Discord](https://img.shields.io/badge/RVC%20Developers-Discord-7289DA?style=for-the-badge&logo=discord&logoColor=white)](https://discord.gg/HcsmBBGyVk) 15 | 16 |
17 | 18 | ------ 19 | 20 | [**更新日誌**](https://github.com/RVC-Project/Retrieval-based-Voice-Conversion-WebUI/blob/main/docs/Changelog_CN.md) 21 | 22 | [**English**](./README.en.md) | [**中文简体**](../README.md) | [**日本語**](./README.ja.md) | [**한국어**](./README.ko.md) ([**韓國語**](./README.ko.han.md)) 23 | 24 | > デモ動画は[こちら](https://www.bilibili.com/video/BV1pm4y1z7Gm/)でご覧ください。 25 | 26 | > RVCによるリアルタイム音声変換: [w-okada/voice-changer](https://github.com/w-okada/voice-changer) 27 | 28 | > 著作権侵害を心配することなく使用できるように、基底モデルは約50時間の高品質なオープンソースデータセットで訓練されています。 29 | 30 | > 今後も、次々と使用許可のある高品質な歌声の資料集を追加し、基底モデルを訓練する予定です。 31 | 32 | ## はじめに 33 | 本リポジトリには下記の特徴があります。 34 | 35 | + Top1検索を用いることで、生の特徴量を訓練用データセット特徴量に変換し、トーンリーケージを削減します。 36 | + 比較的貧弱なGPUでも、高速かつ簡単に訓練できます。 37 | + 少量のデータセットからでも、比較的良い結果を得ることができます。（10分以上のノイズの少ない音声を推奨します。） 38 | + モデルを融合することで、音声を混ぜることができます。（ckpt processingタブの、ckpt mergeを使用します。） 39 | + 使いやすいWebUI。 40 | + UVR5 Modelも含んでいるため、人の声とBGMを素早く分離できます。 41 | 42 | ## 環境構築 43 | Poetryで依存関係をインストールすることをお勧めします。 44 | 45 | 下記のコマンドは、Python3.8以上の環境で実行する必要があります: 46 | ```bash 47 | # PyTorch関連の依存関係をインストール。インストール済の場合は省略。 48 | # 参照先: https://pytorch.org/get-started/locally/ 49 | pip install torch torchvision torchaudio 50 | 51 | #Windows＋ Nvidia Ampere Architecture(RTX30xx)の場合、 #21 に従い、pytorchに対応するcuda versionを指定する必要があります。 52 | #pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu117 53 | 54 | # PyTorch関連の依存関係をインストール。インストール済の場合は省略。 55 | # 参照先: https://python-poetry.org/docs/#installation 56 | curl -sSL https://install.python-poetry.org | python3 - 57 | 58 | # Poetry経由で依存関係をインストール 59 | poetry install 60 | ``` 61 | 62 | pipでも依存関係のインストールが可能です: 63 | 64 | ```bash 65 | pip install -r requirements.txt 66 | ``` 67 | 68 | ## 基底modelsを準備 69 | RVCは推論/訓練のために、様々な事前訓練を行った基底モデルを必要とします。 70 | 71 | modelsは[Hugging Face space](https://huggingface.co/lj1995/VoiceConversionWebUI/tree/main/)からダウンロードできます。 72 | 73 | 以下は、RVCに必要な基底モデルやその他のファイルの一覧です。 74 | ```bash 75 | hubert_base.pt 76 | 77 | ./pretrained 78 | 79 | ./uvr5_weights 80 | 81 | # ffmpegがすでにinstallされている場合は省略 82 | ./ffmpeg 83 | ``` 84 | その後、下記のコマンドでWebUIを起動します。 85 | ```bash 86 | python infer-web.py 87 | ``` 88 | Windowsをお使いの方は、直接`RVC-beta.7z`をダウンロード後に展開し、`go-web.bat`をクリックすることで、WebUIを起動することができます。(7zipが必要です。) 89 | 90 | また、リポジトリに[小白简易教程.doc](./小白简易教程.doc)がありますので、参考にしてください（中国語版のみ）。 91 | 92 | ## 参考プロジェクト 93 | + [ContentVec](https://github.com/auspicious3000/contentvec/) 94 | + [VITS](https://github.com/jaywalnut310/vits) 95 | + [HIFIGAN](https://github.com/jik876/hifi-gan) 96 | + [Gradio](https://github.com/gradio-app/gradio) 97 | + [FFmpeg](https://github.com/FFmpeg/FFmpeg) 98 | + [Ultimate Vocal Remover](https://github.com/Anjok07/ultimatevocalremovergui) 99 | + [audio-slicer](https://github.com/openvpi/audio-slicer) 100 | 101 | ## 貢献者(contributor)の皆様の尽力に感謝します 102 | 103 | 104 | 105 | -------------------------------------------------------------------------------- /lib/uvr5_pack/lib_v5/layers.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch import nn 3 | import torch.nn.functional as F 4 | 5 | from . import spec_utils 6 | 7 | 8 | class Conv2DBNActiv(nn.Module): 9 | def __init__(self, nin, nout, ksize=3, stride=1, pad=1, dilation=1, activ=nn.ReLU): 10 | super(Conv2DBNActiv, self).__init__() 11 | self.conv = nn.Sequential( 12 | nn.Conv2d( 13 | nin, 14 | nout, 15 | kernel_size=ksize, 16 | stride=stride, 17 | padding=pad, 18 | dilation=dilation, 19 | bias=False, 20 | ), 21 | nn.BatchNorm2d(nout), 22 | activ(), 23 | ) 24 | 25 | def __call__(self, x): 26 | return self.conv(x) 27 | 28 | 29 | class SeperableConv2DBNActiv(nn.Module): 30 | def __init__(self, nin, nout, ksize=3, stride=1, pad=1, dilation=1, activ=nn.ReLU): 31 | super(SeperableConv2DBNActiv, self).__init__() 32 | self.conv = nn.Sequential( 33 | nn.Conv2d( 34 | nin, 35 | nin, 36 | kernel_size=ksize, 37 | stride=stride, 38 | padding=pad, 39 | dilation=dilation, 40 | groups=nin, 41 | bias=False, 42 | ), 43 | nn.Conv2d(nin, nout, kernel_size=1, bias=False), 44 | nn.BatchNorm2d(nout), 45 | activ(), 46 | ) 47 | 48 | def __call__(self, x): 49 | return self.conv(x) 50 | 51 | 52 | class Encoder(nn.Module): 53 | def __init__(self, nin, nout, ksize=3, stride=1, pad=1, activ=nn.LeakyReLU): 54 | super(Encoder, self).__init__() 55 | self.conv1 = Conv2DBNActiv(nin, nout, ksize, 1, pad, activ=activ) 56 | self.conv2 = Conv2DBNActiv(nout, nout, ksize, stride, pad, activ=activ) 57 | 58 | def __call__(self, x): 59 | skip = self.conv1(x) 60 | h = self.conv2(skip) 61 | 62 | return h, skip 63 | 64 | 65 | class Decoder(nn.Module): 66 | def __init__( 67 | self, nin, nout, ksize=3, stride=1, pad=1, activ=nn.ReLU, dropout=False 68 | ): 69 | super(Decoder, self).__init__() 70 | self.conv = Conv2DBNActiv(nin, nout, ksize, 1, pad, activ=activ) 71 | self.dropout = nn.Dropout2d(0.1) if dropout else None 72 | 73 | def __call__(self, x, skip=None): 74 | x = F.interpolate(x, scale_factor=2, mode="bilinear", align_corners=True) 75 | if skip is not None: 76 | skip = spec_utils.crop_center(skip, x) 77 | x = torch.cat([x, skip], dim=1) 78 | h = self.conv(x) 79 | 80 | if self.dropout is not None: 81 | h = self.dropout(h) 82 | 83 | return h 84 | 85 | 86 | class ASPPModule(nn.Module): 87 | def __init__(self, nin, nout, dilations=(4, 8, 16), activ=nn.ReLU): 88 | super(ASPPModule, self).__init__() 89 | self.conv1 = nn.Sequential( 90 | nn.AdaptiveAvgPool2d((1, None)), 91 | Conv2DBNActiv(nin, nin, 1, 1, 0, activ=activ), 92 | ) 93 | self.conv2 = Conv2DBNActiv(nin, nin, 1, 1, 0, activ=activ) 94 | self.conv3 = SeperableConv2DBNActiv( 95 | nin, nin, 3, 1, dilations[0], dilations[0], activ=activ 96 | ) 97 | self.conv4 = SeperableConv2DBNActiv( 98 | nin, nin, 3, 1, dilations[1], dilations[1], activ=activ 99 | ) 100 | self.conv5 = SeperableConv2DBNActiv( 101 | nin, nin, 3, 1, dilations[2], dilations[2], activ=activ 102 | ) 103 | self.bottleneck = nn.Sequential( 104 | Conv2DBNActiv(nin * 5, nout, 1, 1, 0, activ=activ), nn.Dropout2d(0.1) 105 | ) 106 | 107 | def forward(self, x): 108 | _, _, h, w = x.size() 109 | feat1 = F.interpolate( 110 | self.conv1(x), size=(h, w), mode="bilinear", align_corners=True 111 | ) 112 | feat2 = self.conv2(x) 113 | feat3 = self.conv3(x) 114 | feat4 = self.conv4(x) 115 | feat5 = self.conv5(x) 116 | out = torch.cat((feat1, feat2, feat3, feat4, feat5), dim=1) 117 | bottle = self.bottleneck(out) 118 | return bottle 119 | -------------------------------------------------------------------------------- /lib/uvr5_pack/lib_v5/layers_123812KB .py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch import nn 3 | import torch.nn.functional as F 4 | 5 | from . import spec_utils 6 | 7 | 8 | class Conv2DBNActiv(nn.Module): 9 | def __init__(self, nin, nout, ksize=3, stride=1, pad=1, dilation=1, activ=nn.ReLU): 10 | super(Conv2DBNActiv, self).__init__() 11 | self.conv = nn.Sequential( 12 | nn.Conv2d( 13 | nin, 14 | nout, 15 | kernel_size=ksize, 16 | stride=stride, 17 | padding=pad, 18 | dilation=dilation, 19 | bias=False, 20 | ), 21 | nn.BatchNorm2d(nout), 22 | activ(), 23 | ) 24 | 25 | def __call__(self, x): 26 | return self.conv(x) 27 | 28 | 29 | class SeperableConv2DBNActiv(nn.Module): 30 | def __init__(self, nin, nout, ksize=3, stride=1, pad=1, dilation=1, activ=nn.ReLU): 31 | super(SeperableConv2DBNActiv, self).__init__() 32 | self.conv = nn.Sequential( 33 | nn.Conv2d( 34 | nin, 35 | nin, 36 | kernel_size=ksize, 37 | stride=stride, 38 | padding=pad, 39 | dilation=dilation, 40 | groups=nin, 41 | bias=False, 42 | ), 43 | nn.Conv2d(nin, nout, kernel_size=1, bias=False), 44 | nn.BatchNorm2d(nout), 45 | activ(), 46 | ) 47 | 48 | def __call__(self, x): 49 | return self.conv(x) 50 | 51 | 52 | class Encoder(nn.Module): 53 | def __init__(self, nin, nout, ksize=3, stride=1, pad=1, activ=nn.LeakyReLU): 54 | super(Encoder, self).__init__() 55 | self.conv1 = Conv2DBNActiv(nin, nout, ksize, 1, pad, activ=activ) 56 | self.conv2 = Conv2DBNActiv(nout, nout, ksize, stride, pad, activ=activ) 57 | 58 | def __call__(self, x): 59 | skip = self.conv1(x) 60 | h = self.conv2(skip) 61 | 62 | return h, skip 63 | 64 | 65 | class Decoder(nn.Module): 66 | def __init__( 67 | self, nin, nout, ksize=3, stride=1, pad=1, activ=nn.ReLU, dropout=False 68 | ): 69 | super(Decoder, self).__init__() 70 | self.conv = Conv2DBNActiv(nin, nout, ksize, 1, pad, activ=activ) 71 | self.dropout = nn.Dropout2d(0.1) if dropout else None 72 | 73 | def __call__(self, x, skip=None): 74 | x = F.interpolate(x, scale_factor=2, mode="bilinear", align_corners=True) 75 | if skip is not None: 76 | skip = spec_utils.crop_center(skip, x) 77 | x = torch.cat([x, skip], dim=1) 78 | h = self.conv(x) 79 | 80 | if self.dropout is not None: 81 | h = self.dropout(h) 82 | 83 | return h 84 | 85 | 86 | class ASPPModule(nn.Module): 87 | def __init__(self, nin, nout, dilations=(4, 8, 16), activ=nn.ReLU): 88 | super(ASPPModule, self).__init__() 89 | self.conv1 = nn.Sequential( 90 | nn.AdaptiveAvgPool2d((1, None)), 91 | Conv2DBNActiv(nin, nin, 1, 1, 0, activ=activ), 92 | ) 93 | self.conv2 = Conv2DBNActiv(nin, nin, 1, 1, 0, activ=activ) 94 | self.conv3 = SeperableConv2DBNActiv( 95 | nin, nin, 3, 1, dilations[0], dilations[0], activ=activ 96 | ) 97 | self.conv4 = SeperableConv2DBNActiv( 98 | nin, nin, 3, 1, dilations[1], dilations[1], activ=activ 99 | ) 100 | self.conv5 = SeperableConv2DBNActiv( 101 | nin, nin, 3, 1, dilations[2], dilations[2], activ=activ 102 | ) 103 | self.bottleneck = nn.Sequential( 104 | Conv2DBNActiv(nin * 5, nout, 1, 1, 0, activ=activ), nn.Dropout2d(0.1) 105 | ) 106 | 107 | def forward(self, x): 108 | _, _, h, w = x.size() 109 | feat1 = F.interpolate( 110 | self.conv1(x), size=(h, w), mode="bilinear", align_corners=True 111 | ) 112 | feat2 = self.conv2(x) 113 | feat3 = self.conv3(x) 114 | feat4 = self.conv4(x) 115 | feat5 = self.conv5(x) 116 | out = torch.cat((feat1, feat2, feat3, feat4, feat5), dim=1) 117 | bottle = self.bottleneck(out) 118 | return bottle 119 | -------------------------------------------------------------------------------- /lib/uvr5_pack/lib_v5/layers_123821KB.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch import nn 3 | import torch.nn.functional as F 4 | 5 | from . import spec_utils 6 | 7 | 8 | class Conv2DBNActiv(nn.Module): 9 | def __init__(self, nin, nout, ksize=3, stride=1, pad=1, dilation=1, activ=nn.ReLU): 10 | super(Conv2DBNActiv, self).__init__() 11 | self.conv = nn.Sequential( 12 | nn.Conv2d( 13 | nin, 14 | nout, 15 | kernel_size=ksize, 16 | stride=stride, 17 | padding=pad, 18 | dilation=dilation, 19 | bias=False, 20 | ), 21 | nn.BatchNorm2d(nout), 22 | activ(), 23 | ) 24 | 25 | def __call__(self, x): 26 | return self.conv(x) 27 | 28 | 29 | class SeperableConv2DBNActiv(nn.Module): 30 | def __init__(self, nin, nout, ksize=3, stride=1, pad=1, dilation=1, activ=nn.ReLU): 31 | super(SeperableConv2DBNActiv, self).__init__() 32 | self.conv = nn.Sequential( 33 | nn.Conv2d( 34 | nin, 35 | nin, 36 | kernel_size=ksize, 37 | stride=stride, 38 | padding=pad, 39 | dilation=dilation, 40 | groups=nin, 41 | bias=False, 42 | ), 43 | nn.Conv2d(nin, nout, kernel_size=1, bias=False), 44 | nn.BatchNorm2d(nout), 45 | activ(), 46 | ) 47 | 48 | def __call__(self, x): 49 | return self.conv(x) 50 | 51 | 52 | class Encoder(nn.Module): 53 | def __init__(self, nin, nout, ksize=3, stride=1, pad=1, activ=nn.LeakyReLU): 54 | super(Encoder, self).__init__() 55 | self.conv1 = Conv2DBNActiv(nin, nout, ksize, 1, pad, activ=activ) 56 | self.conv2 = Conv2DBNActiv(nout, nout, ksize, stride, pad, activ=activ) 57 | 58 | def __call__(self, x): 59 | skip = self.conv1(x) 60 | h = self.conv2(skip) 61 | 62 | return h, skip 63 | 64 | 65 | class Decoder(nn.Module): 66 | def __init__( 67 | self, nin, nout, ksize=3, stride=1, pad=1, activ=nn.ReLU, dropout=False 68 | ): 69 | super(Decoder, self).__init__() 70 | self.conv = Conv2DBNActiv(nin, nout, ksize, 1, pad, activ=activ) 71 | self.dropout = nn.Dropout2d(0.1) if dropout else None 72 | 73 | def __call__(self, x, skip=None): 74 | x = F.interpolate(x, scale_factor=2, mode="bilinear", align_corners=True) 75 | if skip is not None: 76 | skip = spec_utils.crop_center(skip, x) 77 | x = torch.cat([x, skip], dim=1) 78 | h = self.conv(x) 79 | 80 | if self.dropout is not None: 81 | h = self.dropout(h) 82 | 83 | return h 84 | 85 | 86 | class ASPPModule(nn.Module): 87 | def __init__(self, nin, nout, dilations=(4, 8, 16), activ=nn.ReLU): 88 | super(ASPPModule, self).__init__() 89 | self.conv1 = nn.Sequential( 90 | nn.AdaptiveAvgPool2d((1, None)), 91 | Conv2DBNActiv(nin, nin, 1, 1, 0, activ=activ), 92 | ) 93 | self.conv2 = Conv2DBNActiv(nin, nin, 1, 1, 0, activ=activ) 94 | self.conv3 = SeperableConv2DBNActiv( 95 | nin, nin, 3, 1, dilations[0], dilations[0], activ=activ 96 | ) 97 | self.conv4 = SeperableConv2DBNActiv( 98 | nin, nin, 3, 1, dilations[1], dilations[1], activ=activ 99 | ) 100 | self.conv5 = SeperableConv2DBNActiv( 101 | nin, nin, 3, 1, dilations[2], dilations[2], activ=activ 102 | ) 103 | self.bottleneck = nn.Sequential( 104 | Conv2DBNActiv(nin * 5, nout, 1, 1, 0, activ=activ), nn.Dropout2d(0.1) 105 | ) 106 | 107 | def forward(self, x): 108 | _, _, h, w = x.size() 109 | feat1 = F.interpolate( 110 | self.conv1(x), size=(h, w), mode="bilinear", align_corners=True 111 | ) 112 | feat2 = self.conv2(x) 113 | feat3 = self.conv3(x) 114 | feat4 = self.conv4(x) 115 | feat5 = self.conv5(x) 116 | out = torch.cat((feat1, feat2, feat3, feat4, feat5), dim=1) 117 | bottle = self.bottleneck(out) 118 | return bottle 119 | -------------------------------------------------------------------------------- /lib/uvr5_pack/lib_v5/nets.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch import nn 3 | import torch.nn.functional as F 4 | 5 | import layers 6 | from . import spec_utils 7 | 8 | 9 | class BaseASPPNet(nn.Module): 10 | def __init__(self, nin, ch, dilations=(4, 8, 16)): 11 | super(BaseASPPNet, self).__init__() 12 | self.enc1 = layers.Encoder(nin, ch, 3, 2, 1) 13 | self.enc2 = layers.Encoder(ch, ch * 2, 3, 2, 1) 14 | self.enc3 = layers.Encoder(ch * 2, ch * 4, 3, 2, 1) 15 | self.enc4 = layers.Encoder(ch * 4, ch * 8, 3, 2, 1) 16 | 17 | self.aspp = layers.ASPPModule(ch * 8, ch * 16, dilations) 18 | 19 | self.dec4 = layers.Decoder(ch * (8 + 16), ch * 8, 3, 1, 1) 20 | self.dec3 = layers.Decoder(ch * (4 + 8), ch * 4, 3, 1, 1) 21 | self.dec2 = layers.Decoder(ch * (2 + 4), ch * 2, 3, 1, 1) 22 | self.dec1 = layers.Decoder(ch * (1 + 2), ch, 3, 1, 1) 23 | 24 | def __call__(self, x): 25 | h, e1 = self.enc1(x) 26 | h, e2 = self.enc2(h) 27 | h, e3 = self.enc3(h) 28 | h, e4 = self.enc4(h) 29 | 30 | h = self.aspp(h) 31 | 32 | h = self.dec4(h, e4) 33 | h = self.dec3(h, e3) 34 | h = self.dec2(h, e2) 35 | h = self.dec1(h, e1) 36 | 37 | return h 38 | 39 | 40 | class CascadedASPPNet(nn.Module): 41 | def __init__(self, n_fft): 42 | super(CascadedASPPNet, self).__init__() 43 | self.stg1_low_band_net = BaseASPPNet(2, 16) 44 | self.stg1_high_band_net = BaseASPPNet(2, 16) 45 | 46 | self.stg2_bridge = layers.Conv2DBNActiv(18, 8, 1, 1, 0) 47 | self.stg2_full_band_net = BaseASPPNet(8, 16) 48 | 49 | self.stg3_bridge = layers.Conv2DBNActiv(34, 16, 1, 1, 0) 50 | self.stg3_full_band_net = BaseASPPNet(16, 32) 51 | 52 | self.out = nn.Conv2d(32, 2, 1, bias=False) 53 | self.aux1_out = nn.Conv2d(16, 2, 1, bias=False) 54 | self.aux2_out = nn.Conv2d(16, 2, 1, bias=False) 55 | 56 | self.max_bin = n_fft // 2 57 | self.output_bin = n_fft // 2 + 1 58 | 59 | self.offset = 128 60 | 61 | def forward(self, x, aggressiveness=None): 62 | mix = x.detach() 63 | x = x.clone() 64 | 65 | x = x[:, :, : self.max_bin] 66 | 67 | bandw = x.size()[2] // 2 68 | aux1 = torch.cat( 69 | [ 70 | self.stg1_low_band_net(x[:, :, :bandw]), 71 | self.stg1_high_band_net(x[:, :, bandw:]), 72 | ], 73 | dim=2, 74 | ) 75 | 76 | h = torch.cat([x, aux1], dim=1) 77 | aux2 = self.stg2_full_band_net(self.stg2_bridge(h)) 78 | 79 | h = torch.cat([x, aux1, aux2], dim=1) 80 | h = self.stg3_full_band_net(self.stg3_bridge(h)) 81 | 82 | mask = torch.sigmoid(self.out(h)) 83 | mask = F.pad( 84 | input=mask, 85 | pad=(0, 0, 0, self.output_bin - mask.size()[2]), 86 | mode="replicate", 87 | ) 88 | 89 | if self.training: 90 | aux1 = torch.sigmoid(self.aux1_out(aux1)) 91 | aux1 = F.pad( 92 | input=aux1, 93 | pad=(0, 0, 0, self.output_bin - aux1.size()[2]), 94 | mode="replicate", 95 | ) 96 | aux2 = torch.sigmoid(self.aux2_out(aux2)) 97 | aux2 = F.pad( 98 | input=aux2, 99 | pad=(0, 0, 0, self.output_bin - aux2.size()[2]), 100 | mode="replicate", 101 | ) 102 | return mask * mix, aux1 * mix, aux2 * mix 103 | else: 104 | if aggressiveness: 105 | mask[:, :, : aggressiveness["split_bin"]] = torch.pow( 106 | mask[:, :, : aggressiveness["split_bin"]], 107 | 1 + aggressiveness["value"] / 3, 108 | ) 109 | mask[:, :, aggressiveness["split_bin"] :] = torch.pow( 110 | mask[:, :, aggressiveness["split_bin"] :], 111 | 1 + aggressiveness["value"], 112 | ) 113 | 114 | return mask * mix 115 | 116 | def predict(self, x_mag, aggressiveness=None): 117 | h = self.forward(x_mag, aggressiveness) 118 | 119 | if self.offset > 0: 120 | h = h[:, :, :, self.offset : -self.offset] 121 | assert h.size()[3] > 0 122 | 123 | return h 124 | -------------------------------------------------------------------------------- /docs/README.ko.md: -------------------------------------------------------------------------------- 1 |
2 | 3 |

Retrieval-based-Voice-Conversion-WebUI

4 | VITS 기반의 간단하고 사용하기 쉬운 음성 변환 프레임워크.

5 | 6 | [![madewithlove](https://forthebadge.com/images/badges/built-with-love.svg)](https://github.com/RVC-Project/Retrieval-based-Voice-Conversion-WebUI) 7 | 8 |
9 | 10 | [![Open In Colab](https://img.shields.io/badge/Colab-F9AB00?style=for-the-badge&logo=googlecolab&color=525252)](https://colab.research.google.com/github/RVC-Project/Retrieval-based-Voice-Conversion-WebUI/blob/main/Retrieval_based_Voice_Conversion_WebUI.ipynb) 11 | [![Licence](https://img.shields.io/github/license/RVC-Project/Retrieval-based-Voice-Conversion-WebUI?style=for-the-badge)](https://github.com/RVC-Project/Retrieval-based-Voice-Conversion-WebUI/blob/main/LICENSE) 12 | [![Huggingface](https://img.shields.io/badge/🤗%20-Spaces-yellow.svg?style=for-the-badge)](https://huggingface.co/lj1995/VoiceConversionWebUI/tree/main/) 13 | 14 | [![Discord](https://img.shields.io/badge/RVC%20Developers-Discord-7289DA?style=for-the-badge&logo=discord&logoColor=white)](https://discord.gg/HcsmBBGyVk) 15 | 16 |
17 | 18 | --- 19 | 20 | [**업데이트 로그**](https://github.com/RVC-Project/Retrieval-based-Voice-Conversion-WebUI/blob/main/docs/Changelog_KO.md) 21 | 22 | [**English**](./README.en.md) | [**中文简体**](../README.md) | [**日本語**](./README.ja.md) | [**한국어**](./README.ko.md) ([**韓國語**](./README.ko.han.md)) 23 | 24 | > [데모 영상](https://www.bilibili.com/video/BV1pm4y1z7Gm/)을 확인해 보세요! 25 | 26 | > RVC를 활용한 실시간 음성변환: [w-okada/voice-changer](https://github.com/w-okada/voice-changer) 27 | 28 | > 기본 모델은 50시간 가량의 고퀄리티 오픈 소스 VCTK 데이터셋을 사용하였으므로, 저작권상의 염려가 없으니 안심하고 사용하시기 바랍니다. 29 | 30 | > 저작권 문제가 없는 고퀄리티의 노래를 이후에도 계속해서 훈련할 예정입니다. 31 | 32 | ## 소개 33 | 34 | 본 Repo는 다음과 같은 특징을 가지고 있습니다: 35 | 36 | - top1 검색을 이용하여 입력 음색 특징을 훈련 세트 음색 특징으로 대체하여 음색의 누출을 방지; 37 | - 상대적으로 낮은 성능의 GPU에서도 빠른 훈련 가능; 38 | - 적은 양의 데이터로 훈련해도 좋은 결과를 얻을 수 있음 (최소 10분 이상의 저잡음 음성 데이터를 사용하는 것을 권장); 39 | - 모델 융합을 통한 음색의 변조 가능 (ckpt 처리 탭->ckpt 병합 선택); 40 | - 사용하기 쉬운 WebUI (웹 인터페이스); 41 | - UVR5 모델을 이용하여 목소리와 배경음악의 빠른 분리; 42 | 43 | ## 환경의 준비 44 | 45 | poetry를 통해 dependecies를 설치하는 것을 권장합니다. 46 | 47 | 다음 명령은 Python 버전 3.8 이상의 환경에서 실행되어야 합니다: 48 | 49 | ```bash 50 | # PyTorch 관련 주요 dependencies 설치, 이미 설치되어 있는 경우 건너뛰기 가능 51 | # 참조: https://pytorch.org/get-started/locally/ 52 | pip install torch torchvision torchaudio 53 | 54 | # Windows + Nvidia Ampere Architecture(RTX30xx)를 사용하고 있다면, https://github.com/RVC-Project/Retrieval-based-Voice-Conversion-WebUI/issues/21 에서 명시된 것과 같이 PyTorch에 맞는 CUDA 버전을 지정해야 합니다. 55 | #pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu117 56 | 57 | # Poetry 설치, 이미 설치되어 있는 경우 건너뛰기 가능 58 | # Reference: https://python-poetry.org/docs/#installation 59 | curl -sSL https://install.python-poetry.org | python3 - 60 | 61 | # Dependecies 설치 62 | poetry install 63 | ``` 64 | 65 | pip를 활용하여 dependencies를 설치하여도 무방합니다. 66 | 67 | ```bash 68 | pip install -r requirements.txt 69 | ``` 70 | 71 | ## 기타 사전 모델 준비 72 | 73 | RVC 모델은 추론과 훈련을 위하여 다른 사전 모델이 필요합니다. 74 | 75 | [Huggingface space](https://huggingface.co/lj1995/VoiceConversionWebUI/tree/main/)를 통해서 다운로드 할 수 있습니다. 76 | 77 | 다음은 RVC에 필요한 사전 모델 및 기타 파일 목록입니다: 78 | 79 | ```bash 80 | hubert_base.pt 81 | 82 | ./pretrained 83 | 84 | ./uvr5_weights 85 | 86 | # Windows를 사용하는 경우 이 사전도 필요할 수 있습니다. FFmpeg가 설치되어 있으면 건너뛰어도 됩니다. 87 | ffmpeg.exe 88 | ``` 89 | 90 | 그 후 이하의 명령을 사용하여 WebUI를 시작할 수 있습니다: 91 | 92 | ```bash 93 | python infer-web.py 94 | ``` 95 | 96 | Windows를 사용하는 경우 `RVC-beta.7z`를 다운로드 및 압축 해제하여 RVC를 직접 사용하거나 `go-web.bat`을 사용하여 WebUi를 시작할 수 있습니다. 97 | 98 | ## 참고 99 | 100 | - [ContentVec](https://github.com/auspicious3000/contentvec/) 101 | - [VITS](https://github.com/jaywalnut310/vits) 102 | - [HIFIGAN](https://github.com/jik876/hifi-gan) 103 | - [Gradio](https://github.com/gradio-app/gradio) 104 | - [FFmpeg](https://github.com/FFmpeg/FFmpeg) 105 | - [Ultimate Vocal Remover](https://github.com/Anjok07/ultimatevocalremovergui) 106 | - [audio-slicer](https://github.com/openvpi/audio-slicer) 107 | 108 | ## 모든 기여자 분들의 노력에 감사드립니다. 109 | 110 | 111 | 112 | 113 | -------------------------------------------------------------------------------- /lib/uvr5_pack/lib_v5/nets_123812KB.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch import nn 3 | import torch.nn.functional as F 4 | 5 | from . import layers_123821KB as layers 6 | 7 | 8 | class BaseASPPNet(nn.Module): 9 | def __init__(self, nin, ch, dilations=(4, 8, 16)): 10 | super(BaseASPPNet, self).__init__() 11 | self.enc1 = layers.Encoder(nin, ch, 3, 2, 1) 12 | self.enc2 = layers.Encoder(ch, ch * 2, 3, 2, 1) 13 | self.enc3 = layers.Encoder(ch * 2, ch * 4, 3, 2, 1) 14 | self.enc4 = layers.Encoder(ch * 4, ch * 8, 3, 2, 1) 15 | 16 | self.aspp = layers.ASPPModule(ch * 8, ch * 16, dilations) 17 | 18 | self.dec4 = layers.Decoder(ch * (8 + 16), ch * 8, 3, 1, 1) 19 | self.dec3 = layers.Decoder(ch * (4 + 8), ch * 4, 3, 1, 1) 20 | self.dec2 = layers.Decoder(ch * (2 + 4), ch * 2, 3, 1, 1) 21 | self.dec1 = layers.Decoder(ch * (1 + 2), ch, 3, 1, 1) 22 | 23 | def __call__(self, x): 24 | h, e1 = self.enc1(x) 25 | h, e2 = self.enc2(h) 26 | h, e3 = self.enc3(h) 27 | h, e4 = self.enc4(h) 28 | 29 | h = self.aspp(h) 30 | 31 | h = self.dec4(h, e4) 32 | h = self.dec3(h, e3) 33 | h = self.dec2(h, e2) 34 | h = self.dec1(h, e1) 35 | 36 | return h 37 | 38 | 39 | class CascadedASPPNet(nn.Module): 40 | def __init__(self, n_fft): 41 | super(CascadedASPPNet, self).__init__() 42 | self.stg1_low_band_net = BaseASPPNet(2, 32) 43 | self.stg1_high_band_net = BaseASPPNet(2, 32) 44 | 45 | self.stg2_bridge = layers.Conv2DBNActiv(34, 16, 1, 1, 0) 46 | self.stg2_full_band_net = BaseASPPNet(16, 32) 47 | 48 | self.stg3_bridge = layers.Conv2DBNActiv(66, 32, 1, 1, 0) 49 | self.stg3_full_band_net = BaseASPPNet(32, 64) 50 | 51 | self.out = nn.Conv2d(64, 2, 1, bias=False) 52 | self.aux1_out = nn.Conv2d(32, 2, 1, bias=False) 53 | self.aux2_out = nn.Conv2d(32, 2, 1, bias=False) 54 | 55 | self.max_bin = n_fft // 2 56 | self.output_bin = n_fft // 2 + 1 57 | 58 | self.offset = 128 59 | 60 | def forward(self, x, aggressiveness=None): 61 | mix = x.detach() 62 | x = x.clone() 63 | 64 | x = x[:, :, : self.max_bin] 65 | 66 | bandw = x.size()[2] // 2 67 | aux1 = torch.cat( 68 | [ 69 | self.stg1_low_band_net(x[:, :, :bandw]), 70 | self.stg1_high_band_net(x[:, :, bandw:]), 71 | ], 72 | dim=2, 73 | ) 74 | 75 | h = torch.cat([x, aux1], dim=1) 76 | aux2 = self.stg2_full_band_net(self.stg2_bridge(h)) 77 | 78 | h = torch.cat([x, aux1, aux2], dim=1) 79 | h = self.stg3_full_band_net(self.stg3_bridge(h)) 80 | 81 | mask = torch.sigmoid(self.out(h)) 82 | mask = F.pad( 83 | input=mask, 84 | pad=(0, 0, 0, self.output_bin - mask.size()[2]), 85 | mode="replicate", 86 | ) 87 | 88 | if self.training: 89 | aux1 = torch.sigmoid(self.aux1_out(aux1)) 90 | aux1 = F.pad( 91 | input=aux1, 92 | pad=(0, 0, 0, self.output_bin - aux1.size()[2]), 93 | mode="replicate", 94 | ) 95 | aux2 = torch.sigmoid(self.aux2_out(aux2)) 96 | aux2 = F.pad( 97 | input=aux2, 98 | pad=(0, 0, 0, self.output_bin - aux2.size()[2]), 99 | mode="replicate", 100 | ) 101 | return mask * mix, aux1 * mix, aux2 * mix 102 | else: 103 | if aggressiveness: 104 | mask[:, :, : aggressiveness["split_bin"]] = torch.pow( 105 | mask[:, :, : aggressiveness["split_bin"]], 106 | 1 + aggressiveness["value"] / 3, 107 | ) 108 | mask[:, :, aggressiveness["split_bin"] :] = torch.pow( 109 | mask[:, :, aggressiveness["split_bin"] :], 110 | 1 + aggressiveness["value"], 111 | ) 112 | 113 | return mask * mix 114 | 115 | def predict(self, x_mag, aggressiveness=None): 116 | h = self.forward(x_mag, aggressiveness) 117 | 118 | if self.offset > 0: 119 | h = h[:, :, :, self.offset : -self.offset] 120 | assert h.size()[3] > 0 121 | 122 | return h 123 | -------------------------------------------------------------------------------- /lib/uvr5_pack/lib_v5/nets_123821KB.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch import nn 3 | import torch.nn.functional as F 4 | 5 | from . import layers_123821KB as layers 6 | 7 | 8 | class BaseASPPNet(nn.Module): 9 | def __init__(self, nin, ch, dilations=(4, 8, 16)): 10 | super(BaseASPPNet, self).__init__() 11 | self.enc1 = layers.Encoder(nin, ch, 3, 2, 1) 12 | self.enc2 = layers.Encoder(ch, ch * 2, 3, 2, 1) 13 | self.enc3 = layers.Encoder(ch * 2, ch * 4, 3, 2, 1) 14 | self.enc4 = layers.Encoder(ch * 4, ch * 8, 3, 2, 1) 15 | 16 | self.aspp = layers.ASPPModule(ch * 8, ch * 16, dilations) 17 | 18 | self.dec4 = layers.Decoder(ch * (8 + 16), ch * 8, 3, 1, 1) 19 | self.dec3 = layers.Decoder(ch * (4 + 8), ch * 4, 3, 1, 1) 20 | self.dec2 = layers.Decoder(ch * (2 + 4), ch * 2, 3, 1, 1) 21 | self.dec1 = layers.Decoder(ch * (1 + 2), ch, 3, 1, 1) 22 | 23 | def __call__(self, x): 24 | h, e1 = self.enc1(x) 25 | h, e2 = self.enc2(h) 26 | h, e3 = self.enc3(h) 27 | h, e4 = self.enc4(h) 28 | 29 | h = self.aspp(h) 30 | 31 | h = self.dec4(h, e4) 32 | h = self.dec3(h, e3) 33 | h = self.dec2(h, e2) 34 | h = self.dec1(h, e1) 35 | 36 | return h 37 | 38 | 39 | class CascadedASPPNet(nn.Module): 40 | def __init__(self, n_fft): 41 | super(CascadedASPPNet, self).__init__() 42 | self.stg1_low_band_net = BaseASPPNet(2, 32) 43 | self.stg1_high_band_net = BaseASPPNet(2, 32) 44 | 45 | self.stg2_bridge = layers.Conv2DBNActiv(34, 16, 1, 1, 0) 46 | self.stg2_full_band_net = BaseASPPNet(16, 32) 47 | 48 | self.stg3_bridge = layers.Conv2DBNActiv(66, 32, 1, 1, 0) 49 | self.stg3_full_band_net = BaseASPPNet(32, 64) 50 | 51 | self.out = nn.Conv2d(64, 2, 1, bias=False) 52 | self.aux1_out = nn.Conv2d(32, 2, 1, bias=False) 53 | self.aux2_out = nn.Conv2d(32, 2, 1, bias=False) 54 | 55 | self.max_bin = n_fft // 2 56 | self.output_bin = n_fft // 2 + 1 57 | 58 | self.offset = 128 59 | 60 | def forward(self, x, aggressiveness=None): 61 | mix = x.detach() 62 | x = x.clone() 63 | 64 | x = x[:, :, : self.max_bin] 65 | 66 | bandw = x.size()[2] // 2 67 | aux1 = torch.cat( 68 | [ 69 | self.stg1_low_band_net(x[:, :, :bandw]), 70 | self.stg1_high_band_net(x[:, :, bandw:]), 71 | ], 72 | dim=2, 73 | ) 74 | 75 | h = torch.cat([x, aux1], dim=1) 76 | aux2 = self.stg2_full_band_net(self.stg2_bridge(h)) 77 | 78 | h = torch.cat([x, aux1, aux2], dim=1) 79 | h = self.stg3_full_band_net(self.stg3_bridge(h)) 80 | 81 | mask = torch.sigmoid(self.out(h)) 82 | mask = F.pad( 83 | input=mask, 84 | pad=(0, 0, 0, self.output_bin - mask.size()[2]), 85 | mode="replicate", 86 | ) 87 | 88 | if self.training: 89 | aux1 = torch.sigmoid(self.aux1_out(aux1)) 90 | aux1 = F.pad( 91 | input=aux1, 92 | pad=(0, 0, 0, self.output_bin - aux1.size()[2]), 93 | mode="replicate", 94 | ) 95 | aux2 = torch.sigmoid(self.aux2_out(aux2)) 96 | aux2 = F.pad( 97 | input=aux2, 98 | pad=(0, 0, 0, self.output_bin - aux2.size()[2]), 99 | mode="replicate", 100 | ) 101 | return mask * mix, aux1 * mix, aux2 * mix 102 | else: 103 | if aggressiveness: 104 | mask[:, :, : aggressiveness["split_bin"]] = torch.pow( 105 | mask[:, :, : aggressiveness["split_bin"]], 106 | 1 + aggressiveness["value"] / 3, 107 | ) 108 | mask[:, :, aggressiveness["split_bin"] :] = torch.pow( 109 | mask[:, :, aggressiveness["split_bin"] :], 110 | 1 + aggressiveness["value"], 111 | ) 112 | 113 | return mask * mix 114 | 115 | def predict(self, x_mag, aggressiveness=None): 116 | h = self.forward(x_mag, aggressiveness) 117 | 118 | if self.offset > 0: 119 | h = h[:, :, :, self.offset : -self.offset] 120 | assert h.size()[3] > 0 121 | 122 | return h 123 | -------------------------------------------------------------------------------- /lib/uvr5_pack/lib_v5/nets_33966KB.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch import nn 3 | import torch.nn.functional as F 4 | 5 | from . import layers_33966KB as layers 6 | 7 | 8 | class BaseASPPNet(nn.Module): 9 | def __init__(self, nin, ch, dilations=(4, 8, 16, 32)): 10 | super(BaseASPPNet, self).__init__() 11 | self.enc1 = layers.Encoder(nin, ch, 3, 2, 1) 12 | self.enc2 = layers.Encoder(ch, ch * 2, 3, 2, 1) 13 | self.enc3 = layers.Encoder(ch * 2, ch * 4, 3, 2, 1) 14 | self.enc4 = layers.Encoder(ch * 4, ch * 8, 3, 2, 1) 15 | 16 | self.aspp = layers.ASPPModule(ch * 8, ch * 16, dilations) 17 | 18 | self.dec4 = layers.Decoder(ch * (8 + 16), ch * 8, 3, 1, 1) 19 | self.dec3 = layers.Decoder(ch * (4 + 8), ch * 4, 3, 1, 1) 20 | self.dec2 = layers.Decoder(ch * (2 + 4), ch * 2, 3, 1, 1) 21 | self.dec1 = layers.Decoder(ch * (1 + 2), ch, 3, 1, 1) 22 | 23 | def __call__(self, x): 24 | h, e1 = self.enc1(x) 25 | h, e2 = self.enc2(h) 26 | h, e3 = self.enc3(h) 27 | h, e4 = self.enc4(h) 28 | 29 | h = self.aspp(h) 30 | 31 | h = self.dec4(h, e4) 32 | h = self.dec3(h, e3) 33 | h = self.dec2(h, e2) 34 | h = self.dec1(h, e1) 35 | 36 | return h 37 | 38 | 39 | class CascadedASPPNet(nn.Module): 40 | def __init__(self, n_fft): 41 | super(CascadedASPPNet, self).__init__() 42 | self.stg1_low_band_net = BaseASPPNet(2, 16) 43 | self.stg1_high_band_net = BaseASPPNet(2, 16) 44 | 45 | self.stg2_bridge = layers.Conv2DBNActiv(18, 8, 1, 1, 0) 46 | self.stg2_full_band_net = BaseASPPNet(8, 16) 47 | 48 | self.stg3_bridge = layers.Conv2DBNActiv(34, 16, 1, 1, 0) 49 | self.stg3_full_band_net = BaseASPPNet(16, 32) 50 | 51 | self.out = nn.Conv2d(32, 2, 1, bias=False) 52 | self.aux1_out = nn.Conv2d(16, 2, 1, bias=False) 53 | self.aux2_out = nn.Conv2d(16, 2, 1, bias=False) 54 | 55 | self.max_bin = n_fft // 2 56 | self.output_bin = n_fft // 2 + 1 57 | 58 | self.offset = 128 59 | 60 | def forward(self, x, aggressiveness=None): 61 | mix = x.detach() 62 | x = x.clone() 63 | 64 | x = x[:, :, : self.max_bin] 65 | 66 | bandw = x.size()[2] // 2 67 | aux1 = torch.cat( 68 | [ 69 | self.stg1_low_band_net(x[:, :, :bandw]), 70 | self.stg1_high_band_net(x[:, :, bandw:]), 71 | ], 72 | dim=2, 73 | ) 74 | 75 | h = torch.cat([x, aux1], dim=1) 76 | aux2 = self.stg2_full_band_net(self.stg2_bridge(h)) 77 | 78 | h = torch.cat([x, aux1, aux2], dim=1) 79 | h = self.stg3_full_band_net(self.stg3_bridge(h)) 80 | 81 | mask = torch.sigmoid(self.out(h)) 82 | mask = F.pad( 83 | input=mask, 84 | pad=(0, 0, 0, self.output_bin - mask.size()[2]), 85 | mode="replicate", 86 | ) 87 | 88 | if self.training: 89 | aux1 = torch.sigmoid(self.aux1_out(aux1)) 90 | aux1 = F.pad( 91 | input=aux1, 92 | pad=(0, 0, 0, self.output_bin - aux1.size()[2]), 93 | mode="replicate", 94 | ) 95 | aux2 = torch.sigmoid(self.aux2_out(aux2)) 96 | aux2 = F.pad( 97 | input=aux2, 98 | pad=(0, 0, 0, self.output_bin - aux2.size()[2]), 99 | mode="replicate", 100 | ) 101 | return mask * mix, aux1 * mix, aux2 * mix 102 | else: 103 | if aggressiveness: 104 | mask[:, :, : aggressiveness["split_bin"]] = torch.pow( 105 | mask[:, :, : aggressiveness["split_bin"]], 106 | 1 + aggressiveness["value"] / 3, 107 | ) 108 | mask[:, :, aggressiveness["split_bin"] :] = torch.pow( 109 | mask[:, :, aggressiveness["split_bin"] :], 110 | 1 + aggressiveness["value"], 111 | ) 112 | 113 | return mask * mix 114 | 115 | def predict(self, x_mag, aggressiveness=None): 116 | h = self.forward(x_mag, aggressiveness) 117 | 118 | if self.offset > 0: 119 | h = h[:, :, :, self.offset : -self.offset] 120 | assert h.size()[3] > 0 121 | 122 | return h 123 | -------------------------------------------------------------------------------- /lib/uvr5_pack/lib_v5/nets_61968KB.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch import nn 3 | import torch.nn.functional as F 4 | 5 | from . import layers_123821KB as layers 6 | 7 | 8 | class BaseASPPNet(nn.Module): 9 | def __init__(self, nin, ch, dilations=(4, 8, 16)): 10 | super(BaseASPPNet, self).__init__() 11 | self.enc1 = layers.Encoder(nin, ch, 3, 2, 1) 12 | self.enc2 = layers.Encoder(ch, ch * 2, 3, 2, 1) 13 | self.enc3 = layers.Encoder(ch * 2, ch * 4, 3, 2, 1) 14 | self.enc4 = layers.Encoder(ch * 4, ch * 8, 3, 2, 1) 15 | 16 | self.aspp = layers.ASPPModule(ch * 8, ch * 16, dilations) 17 | 18 | self.dec4 = layers.Decoder(ch * (8 + 16), ch * 8, 3, 1, 1) 19 | self.dec3 = layers.Decoder(ch * (4 + 8), ch * 4, 3, 1, 1) 20 | self.dec2 = layers.Decoder(ch * (2 + 4), ch * 2, 3, 1, 1) 21 | self.dec1 = layers.Decoder(ch * (1 + 2), ch, 3, 1, 1) 22 | 23 | def __call__(self, x): 24 | h, e1 = self.enc1(x) 25 | h, e2 = self.enc2(h) 26 | h, e3 = self.enc3(h) 27 | h, e4 = self.enc4(h) 28 | 29 | h = self.aspp(h) 30 | 31 | h = self.dec4(h, e4) 32 | h = self.dec3(h, e3) 33 | h = self.dec2(h, e2) 34 | h = self.dec1(h, e1) 35 | 36 | return h 37 | 38 | 39 | class CascadedASPPNet(nn.Module): 40 | def __init__(self, n_fft): 41 | super(CascadedASPPNet, self).__init__() 42 | self.stg1_low_band_net = BaseASPPNet(2, 32) 43 | self.stg1_high_band_net = BaseASPPNet(2, 32) 44 | 45 | self.stg2_bridge = layers.Conv2DBNActiv(34, 16, 1, 1, 0) 46 | self.stg2_full_band_net = BaseASPPNet(16, 32) 47 | 48 | self.stg3_bridge = layers.Conv2DBNActiv(66, 32, 1, 1, 0) 49 | self.stg3_full_band_net = BaseASPPNet(32, 64) 50 | 51 | self.out = nn.Conv2d(64, 2, 1, bias=False) 52 | self.aux1_out = nn.Conv2d(32, 2, 1, bias=False) 53 | self.aux2_out = nn.Conv2d(32, 2, 1, bias=False) 54 | 55 | self.max_bin = n_fft // 2 56 | self.output_bin = n_fft // 2 + 1 57 | 58 | self.offset = 128 59 | 60 | def forward(self, x, aggressiveness=None): 61 | mix = x.detach() 62 | x = x.clone() 63 | 64 | x = x[:, :, : self.max_bin] 65 | 66 | bandw = x.size()[2] // 2 67 | aux1 = torch.cat( 68 | [ 69 | self.stg1_low_band_net(x[:, :, :bandw]), 70 | self.stg1_high_band_net(x[:, :, bandw:]), 71 | ], 72 | dim=2, 73 | ) 74 | 75 | h = torch.cat([x, aux1], dim=1) 76 | aux2 = self.stg2_full_band_net(self.stg2_bridge(h)) 77 | 78 | h = torch.cat([x, aux1, aux2], dim=1) 79 | h = self.stg3_full_band_net(self.stg3_bridge(h)) 80 | 81 | mask = torch.sigmoid(self.out(h)) 82 | mask = F.pad( 83 | input=mask, 84 | pad=(0, 0, 0, self.output_bin - mask.size()[2]), 85 | mode="replicate", 86 | ) 87 | 88 | if self.training: 89 | aux1 = torch.sigmoid(self.aux1_out(aux1)) 90 | aux1 = F.pad( 91 | input=aux1, 92 | pad=(0, 0, 0, self.output_bin - aux1.size()[2]), 93 | mode="replicate", 94 | ) 95 | aux2 = torch.sigmoid(self.aux2_out(aux2)) 96 | aux2 = F.pad( 97 | input=aux2, 98 | pad=(0, 0, 0, self.output_bin - aux2.size()[2]), 99 | mode="replicate", 100 | ) 101 | return mask * mix, aux1 * mix, aux2 * mix 102 | else: 103 | if aggressiveness: 104 | mask[:, :, : aggressiveness["split_bin"]] = torch.pow( 105 | mask[:, :, : aggressiveness["split_bin"]], 106 | 1 + aggressiveness["value"] / 3, 107 | ) 108 | mask[:, :, aggressiveness["split_bin"] :] = torch.pow( 109 | mask[:, :, aggressiveness["split_bin"] :], 110 | 1 + aggressiveness["value"], 111 | ) 112 | 113 | return mask * mix 114 | 115 | def predict(self, x_mag, aggressiveness=None): 116 | h = self.forward(x_mag, aggressiveness) 117 | 118 | if self.offset > 0: 119 | h = h[:, :, :, self.offset : -self.offset] 120 | assert h.size()[3] > 0 121 | 122 | return h 123 | -------------------------------------------------------------------------------- /extract_feature_print.py: -------------------------------------------------------------------------------- 1 | import os, sys, traceback 2 | 3 | os.environ["PYTORCH_ENABLE_MPS_FALLBACK"] = "1" 4 | os.environ["PYTORCH_MPS_HIGH_WATERMARK_RATIO"] = "0.0" 5 | 6 | # device=sys.argv[1] 7 | n_part = int(sys.argv[2]) 8 | i_part = int(sys.argv[3]) 9 | if len(sys.argv) == 6: 10 | exp_dir = sys.argv[4] 11 | version = sys.argv[5] 12 | else: 13 | i_gpu = sys.argv[4] 14 | exp_dir = sys.argv[5] 15 | os.environ["CUDA_VISIBLE_DEVICES"] = str(i_gpu) 16 | version = sys.argv[6] 17 | import torch 18 | import torch.nn.functional as F 19 | import soundfile as sf 20 | import numpy as np 21 | from fairseq import checkpoint_utils 22 | 23 | device = "cpu" 24 | if torch.cuda.is_available(): 25 | device = "cuda" 26 | elif torch.backends.mps.is_available(): 27 | device = "mps" 28 | 29 | f = open("%s/extract_f0_feature.log" % exp_dir, "a+") 30 | 31 | 32 | def printt(strr): 33 | print(strr) 34 | f.write("%s\n" % strr) 35 | f.flush() 36 | 37 | 38 | printt(sys.argv) 39 | model_path = "hubert_base.pt" 40 | 41 | printt(exp_dir) 42 | wavPath = "%s/1_16k_wavs" % exp_dir 43 | outPath = ( 44 | "%s/3_feature256" % exp_dir if version == "v1" else "%s/3_feature768" % exp_dir 45 | ) 46 | os.makedirs(outPath, exist_ok=True) 47 | 48 | 49 | # wave must be 16k, hop_size=320 50 | def readwave(wav_path, normalize=False): 51 | wav, sr = sf.read(wav_path) 52 | assert sr == 16000 53 | feats = torch.from_numpy(wav).float() 54 | if feats.dim() == 2: # double channels 55 | feats = feats.mean(-1) 56 | assert feats.dim() == 1, feats.dim() 57 | if normalize: 58 | with torch.no_grad(): 59 | feats = F.layer_norm(feats, feats.shape) 60 | feats = feats.view(1, -1) 61 | return feats 62 | 63 | 64 | # HuBERT model 65 | printt("load model(s) from {}".format(model_path)) 66 | # if hubert model is exist 67 | if os.access(model_path, os.F_OK) == False: 68 | printt( 69 | "Error: Extracting is shut down because %s does not exist, you may download it from https://huggingface.co/lj1995/VoiceConversionWebUI/tree/main" 70 | % model_path 71 | ) 72 | exit(0) 73 | models, saved_cfg, task = checkpoint_utils.load_model_ensemble_and_task( 74 | [model_path], 75 | suffix="", 76 | ) 77 | model = models[0] 78 | model = model.to(device) 79 | printt("move model to %s" % device) 80 | if device not in ["mps", "cpu"]: 81 | model = model.half() 82 | model.eval() 83 | 84 | todo = sorted(list(os.listdir(wavPath)))[i_part::n_part] 85 | n = max(1, len(todo) // 10) # 最多打印十条 86 | if len(todo) == 0: 87 | printt("no-feature-todo") 88 | else: 89 | printt("all-feature-%s" % len(todo)) 90 | for idx, file in enumerate(todo): 91 | try: 92 | if file.endswith(".wav"): 93 | wav_path = "%s/%s" % (wavPath, file) 94 | out_path = "%s/%s" % (outPath, file.replace("wav", "npy")) 95 | 96 | if os.path.exists(out_path): 97 | continue 98 | 99 | feats = readwave(wav_path, normalize=saved_cfg.task.normalize) 100 | padding_mask = torch.BoolTensor(feats.shape).fill_(False) 101 | inputs = { 102 | "source": feats.half().to(device) 103 | if device not in ["mps", "cpu"] 104 | else feats.to(device), 105 | "padding_mask": padding_mask.to(device), 106 | "output_layer": 9 if version == "v1" else 12, # layer 9 107 | } 108 | with torch.no_grad(): 109 | logits = model.extract_features(**inputs) 110 | feats = ( 111 | model.final_proj(logits[0]) if version == "v1" else logits[0] 112 | ) 113 | 114 | feats = feats.squeeze(0).float().cpu().numpy() 115 | if np.isnan(feats).sum() == 0: 116 | np.save(out_path, feats, allow_pickle=False) 117 | else: 118 | printt("%s-contains nan" % file) 119 | if idx % n == 0: 120 | printt("now-%s,all-%s,%s,%s" % (len(todo), idx, file, feats.shape)) 121 | except: 122 | printt(traceback.format_exc()) 123 | printt("all-feature-done") 124 | -------------------------------------------------------------------------------- /lib/uvr5_pack/lib_v5/nets_537227KB.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import numpy as np 3 | from torch import nn 4 | import torch.nn.functional as F 5 | 6 | from . import layers_537238KB as layers 7 | 8 | 9 | class BaseASPPNet(nn.Module): 10 | def __init__(self, nin, ch, dilations=(4, 8, 16)): 11 | super(BaseASPPNet, self).__init__() 12 | self.enc1 = layers.Encoder(nin, ch, 3, 2, 1) 13 | self.enc2 = layers.Encoder(ch, ch * 2, 3, 2, 1) 14 | self.enc3 = layers.Encoder(ch * 2, ch * 4, 3, 2, 1) 15 | self.enc4 = layers.Encoder(ch * 4, ch * 8, 3, 2, 1) 16 | 17 | self.aspp = layers.ASPPModule(ch * 8, ch * 16, dilations) 18 | 19 | self.dec4 = layers.Decoder(ch * (8 + 16), ch * 8, 3, 1, 1) 20 | self.dec3 = layers.Decoder(ch * (4 + 8), ch * 4, 3, 1, 1) 21 | self.dec2 = layers.Decoder(ch * (2 + 4), ch * 2, 3, 1, 1) 22 | self.dec1 = layers.Decoder(ch * (1 + 2), ch, 3, 1, 1) 23 | 24 | def __call__(self, x): 25 | h, e1 = self.enc1(x) 26 | h, e2 = self.enc2(h) 27 | h, e3 = self.enc3(h) 28 | h, e4 = self.enc4(h) 29 | 30 | h = self.aspp(h) 31 | 32 | h = self.dec4(h, e4) 33 | h = self.dec3(h, e3) 34 | h = self.dec2(h, e2) 35 | h = self.dec1(h, e1) 36 | 37 | return h 38 | 39 | 40 | class CascadedASPPNet(nn.Module): 41 | def __init__(self, n_fft): 42 | super(CascadedASPPNet, self).__init__() 43 | self.stg1_low_band_net = BaseASPPNet(2, 64) 44 | self.stg1_high_band_net = BaseASPPNet(2, 64) 45 | 46 | self.stg2_bridge = layers.Conv2DBNActiv(66, 32, 1, 1, 0) 47 | self.stg2_full_band_net = BaseASPPNet(32, 64) 48 | 49 | self.stg3_bridge = layers.Conv2DBNActiv(130, 64, 1, 1, 0) 50 | self.stg3_full_band_net = BaseASPPNet(64, 128) 51 | 52 | self.out = nn.Conv2d(128, 2, 1, bias=False) 53 | self.aux1_out = nn.Conv2d(64, 2, 1, bias=False) 54 | self.aux2_out = nn.Conv2d(64, 2, 1, bias=False) 55 | 56 | self.max_bin = n_fft // 2 57 | self.output_bin = n_fft // 2 + 1 58 | 59 | self.offset = 128 60 | 61 | def forward(self, x, aggressiveness=None): 62 | mix = x.detach() 63 | x = x.clone() 64 | 65 | x = x[:, :, : self.max_bin] 66 | 67 | bandw = x.size()[2] // 2 68 | aux1 = torch.cat( 69 | [ 70 | self.stg1_low_band_net(x[:, :, :bandw]), 71 | self.stg1_high_band_net(x[:, :, bandw:]), 72 | ], 73 | dim=2, 74 | ) 75 | 76 | h = torch.cat([x, aux1], dim=1) 77 | aux2 = self.stg2_full_band_net(self.stg2_bridge(h)) 78 | 79 | h = torch.cat([x, aux1, aux2], dim=1) 80 | h = self.stg3_full_band_net(self.stg3_bridge(h)) 81 | 82 | mask = torch.sigmoid(self.out(h)) 83 | mask = F.pad( 84 | input=mask, 85 | pad=(0, 0, 0, self.output_bin - mask.size()[2]), 86 | mode="replicate", 87 | ) 88 | 89 | if self.training: 90 | aux1 = torch.sigmoid(self.aux1_out(aux1)) 91 | aux1 = F.pad( 92 | input=aux1, 93 | pad=(0, 0, 0, self.output_bin - aux1.size()[2]), 94 | mode="replicate", 95 | ) 96 | aux2 = torch.sigmoid(self.aux2_out(aux2)) 97 | aux2 = F.pad( 98 | input=aux2, 99 | pad=(0, 0, 0, self.output_bin - aux2.size()[2]), 100 | mode="replicate", 101 | ) 102 | return mask * mix, aux1 * mix, aux2 * mix 103 | else: 104 | if aggressiveness: 105 | mask[:, :, : aggressiveness["split_bin"]] = torch.pow( 106 | mask[:, :, : aggressiveness["split_bin"]], 107 | 1 + aggressiveness["value"] / 3, 108 | ) 109 | mask[:, :, aggressiveness["split_bin"] :] = torch.pow( 110 | mask[:, :, aggressiveness["split_bin"] :], 111 | 1 + aggressiveness["value"], 112 | ) 113 | 114 | return mask * mix 115 | 116 | def predict(self, x_mag, aggressiveness=None): 117 | h = self.forward(x_mag, aggressiveness) 118 | 119 | if self.offset > 0: 120 | h = h[:, :, :, self.offset : -self.offset] 121 | assert h.size()[3] > 0 122 | 123 | return h 124 | -------------------------------------------------------------------------------- /lib/uvr5_pack/lib_v5/nets_537238KB.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import numpy as np 3 | from torch import nn 4 | import torch.nn.functional as F 5 | 6 | from . import layers_537238KB as layers 7 | 8 | 9 | class BaseASPPNet(nn.Module): 10 | def __init__(self, nin, ch, dilations=(4, 8, 16)): 11 | super(BaseASPPNet, self).__init__() 12 | self.enc1 = layers.Encoder(nin, ch, 3, 2, 1) 13 | self.enc2 = layers.Encoder(ch, ch * 2, 3, 2, 1) 14 | self.enc3 = layers.Encoder(ch * 2, ch * 4, 3, 2, 1) 15 | self.enc4 = layers.Encoder(ch * 4, ch * 8, 3, 2, 1) 16 | 17 | self.aspp = layers.ASPPModule(ch * 8, ch * 16, dilations) 18 | 19 | self.dec4 = layers.Decoder(ch * (8 + 16), ch * 8, 3, 1, 1) 20 | self.dec3 = layers.Decoder(ch * (4 + 8), ch * 4, 3, 1, 1) 21 | self.dec2 = layers.Decoder(ch * (2 + 4), ch * 2, 3, 1, 1) 22 | self.dec1 = layers.Decoder(ch * (1 + 2), ch, 3, 1, 1) 23 | 24 | def __call__(self, x): 25 | h, e1 = self.enc1(x) 26 | h, e2 = self.enc2(h) 27 | h, e3 = self.enc3(h) 28 | h, e4 = self.enc4(h) 29 | 30 | h = self.aspp(h) 31 | 32 | h = self.dec4(h, e4) 33 | h = self.dec3(h, e3) 34 | h = self.dec2(h, e2) 35 | h = self.dec1(h, e1) 36 | 37 | return h 38 | 39 | 40 | class CascadedASPPNet(nn.Module): 41 | def __init__(self, n_fft): 42 | super(CascadedASPPNet, self).__init__() 43 | self.stg1_low_band_net = BaseASPPNet(2, 64) 44 | self.stg1_high_band_net = BaseASPPNet(2, 64) 45 | 46 | self.stg2_bridge = layers.Conv2DBNActiv(66, 32, 1, 1, 0) 47 | self.stg2_full_band_net = BaseASPPNet(32, 64) 48 | 49 | self.stg3_bridge = layers.Conv2DBNActiv(130, 64, 1, 1, 0) 50 | self.stg3_full_band_net = BaseASPPNet(64, 128) 51 | 52 | self.out = nn.Conv2d(128, 2, 1, bias=False) 53 | self.aux1_out = nn.Conv2d(64, 2, 1, bias=False) 54 | self.aux2_out = nn.Conv2d(64, 2, 1, bias=False) 55 | 56 | self.max_bin = n_fft // 2 57 | self.output_bin = n_fft // 2 + 1 58 | 59 | self.offset = 128 60 | 61 | def forward(self, x, aggressiveness=None): 62 | mix = x.detach() 63 | x = x.clone() 64 | 65 | x = x[:, :, : self.max_bin] 66 | 67 | bandw = x.size()[2] // 2 68 | aux1 = torch.cat( 69 | [ 70 | self.stg1_low_band_net(x[:, :, :bandw]), 71 | self.stg1_high_band_net(x[:, :, bandw:]), 72 | ], 73 | dim=2, 74 | ) 75 | 76 | h = torch.cat([x, aux1], dim=1) 77 | aux2 = self.stg2_full_band_net(self.stg2_bridge(h)) 78 | 79 | h = torch.cat([x, aux1, aux2], dim=1) 80 | h = self.stg3_full_band_net(self.stg3_bridge(h)) 81 | 82 | mask = torch.sigmoid(self.out(h)) 83 | mask = F.pad( 84 | input=mask, 85 | pad=(0, 0, 0, self.output_bin - mask.size()[2]), 86 | mode="replicate", 87 | ) 88 | 89 | if self.training: 90 | aux1 = torch.sigmoid(self.aux1_out(aux1)) 91 | aux1 = F.pad( 92 | input=aux1, 93 | pad=(0, 0, 0, self.output_bin - aux1.size()[2]), 94 | mode="replicate", 95 | ) 96 | aux2 = torch.sigmoid(self.aux2_out(aux2)) 97 | aux2 = F.pad( 98 | input=aux2, 99 | pad=(0, 0, 0, self.output_bin - aux2.size()[2]), 100 | mode="replicate", 101 | ) 102 | return mask * mix, aux1 * mix, aux2 * mix 103 | else: 104 | if aggressiveness: 105 | mask[:, :, : aggressiveness["split_bin"]] = torch.pow( 106 | mask[:, :, : aggressiveness["split_bin"]], 107 | 1 + aggressiveness["value"] / 3, 108 | ) 109 | mask[:, :, aggressiveness["split_bin"] :] = torch.pow( 110 | mask[:, :, aggressiveness["split_bin"] :], 111 | 1 + aggressiveness["value"], 112 | ) 113 | 114 | return mask * mix 115 | 116 | def predict(self, x_mag, aggressiveness=None): 117 | h = self.forward(x_mag, aggressiveness) 118 | 119 | if self.offset > 0: 120 | h = h[:, :, :, self.offset : -self.offset] 121 | assert h.size()[3] > 0 122 | 123 | return h 124 | -------------------------------------------------------------------------------- /lib/train/mel_processing.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.utils.data 3 | from librosa.filters import mel as librosa_mel_fn 4 | 5 | 6 | MAX_WAV_VALUE = 32768.0 7 | 8 | 9 | def dynamic_range_compression_torch(x, C=1, clip_val=1e-5): 10 | """ 11 | PARAMS 12 | ------ 13 | C: compression factor 14 | """ 15 | return torch.log(torch.clamp(x, min=clip_val) * C) 16 | 17 | 18 | def dynamic_range_decompression_torch(x, C=1): 19 | """ 20 | PARAMS 21 | ------ 22 | C: compression factor used to compress 23 | """ 24 | return torch.exp(x) / C 25 | 26 | 27 | def spectral_normalize_torch(magnitudes): 28 | return dynamic_range_compression_torch(magnitudes) 29 | 30 | 31 | def spectral_de_normalize_torch(magnitudes): 32 | return dynamic_range_decompression_torch(magnitudes) 33 | 34 | 35 | # Reusable banks 36 | mel_basis = {} 37 | hann_window = {} 38 | 39 | 40 | def spectrogram_torch(y, n_fft, sampling_rate, hop_size, win_size, center=False): 41 | """Convert waveform into Linear-frequency Linear-amplitude spectrogram. 42 | 43 | Args: 44 | y :: (B, T) - Audio waveforms 45 | n_fft 46 | sampling_rate 47 | hop_size 48 | win_size 49 | center 50 | Returns: 51 | :: (B, Freq, Frame) - Linear-frequency Linear-amplitude spectrogram 52 | """ 53 | # Validation 54 | if torch.min(y) < -1.07: 55 | print("min value is ", torch.min(y)) 56 | if torch.max(y) > 1.07: 57 | print("max value is ", torch.max(y)) 58 | 59 | # Window - Cache if needed 60 | global hann_window 61 | dtype_device = str(y.dtype) + "_" + str(y.device) 62 | wnsize_dtype_device = str(win_size) + "_" + dtype_device 63 | if wnsize_dtype_device not in hann_window: 64 | hann_window[wnsize_dtype_device] = torch.hann_window(win_size).to( 65 | dtype=y.dtype, device=y.device 66 | ) 67 | 68 | # Padding 69 | y = torch.nn.functional.pad( 70 | y.unsqueeze(1), 71 | (int((n_fft - hop_size) / 2), int((n_fft - hop_size) / 2)), 72 | mode="reflect", 73 | ) 74 | y = y.squeeze(1) 75 | 76 | # Complex Spectrogram :: (B, T) -> (B, Freq, Frame, RealComplex=2) 77 | spec = torch.stft( 78 | y, 79 | n_fft, 80 | hop_length=hop_size, 81 | win_length=win_size, 82 | window=hann_window[wnsize_dtype_device], 83 | center=center, 84 | pad_mode="reflect", 85 | normalized=False, 86 | onesided=True, 87 | return_complex=False, 88 | ) 89 | 90 | # Linear-frequency Linear-amplitude spectrogram :: (B, Freq, Frame, RealComplex=2) -> (B, Freq, Frame) 91 | spec = torch.sqrt(spec.pow(2).sum(-1) + 1e-6) 92 | return spec 93 | 94 | 95 | def spec_to_mel_torch(spec, n_fft, num_mels, sampling_rate, fmin, fmax): 96 | # MelBasis - Cache if needed 97 | global mel_basis 98 | dtype_device = str(spec.dtype) + "_" + str(spec.device) 99 | fmax_dtype_device = str(fmax) + "_" + dtype_device 100 | if fmax_dtype_device not in mel_basis: 101 | mel = librosa_mel_fn( 102 | sr=sampling_rate, n_fft=n_fft, n_mels=num_mels, fmin=fmin, fmax=fmax 103 | ) 104 | mel_basis[fmax_dtype_device] = torch.from_numpy(mel).to( 105 | dtype=spec.dtype, device=spec.device 106 | ) 107 | 108 | # Mel-frequency Log-amplitude spectrogram :: (B, Freq=num_mels, Frame) 109 | melspec = torch.matmul(mel_basis[fmax_dtype_device], spec) 110 | melspec = spectral_normalize_torch(melspec) 111 | return melspec 112 | 113 | 114 | def mel_spectrogram_torch( 115 | y, n_fft, num_mels, sampling_rate, hop_size, win_size, fmin, fmax, center=False 116 | ): 117 | """Convert waveform into Mel-frequency Log-amplitude spectrogram. 118 | 119 | Args: 120 | y :: (B, T) - Waveforms 121 | Returns: 122 | melspec :: (B, Freq, Frame) - Mel-frequency Log-amplitude spectrogram 123 | """ 124 | # Linear-frequency Linear-amplitude spectrogram :: (B, T) -> (B, Freq, Frame) 125 | spec = spectrogram_torch(y, n_fft, sampling_rate, hop_size, win_size, center) 126 | 127 | # Mel-frequency Log-amplitude spectrogram :: (B, Freq, Frame) -> (B, Freq=num_mels, Frame) 128 | melspec = spec_to_mel_torch(spec, n_fft, num_mels, sampling_rate, fmin, fmax) 129 | 130 | return melspec 131 | -------------------------------------------------------------------------------- /lib/uvr5_pack/lib_v5/layers_new.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch import nn 3 | import torch.nn.functional as F 4 | 5 | from . import spec_utils 6 | 7 | 8 | class Conv2DBNActiv(nn.Module): 9 | def __init__(self, nin, nout, ksize=3, stride=1, pad=1, dilation=1, activ=nn.ReLU): 10 | super(Conv2DBNActiv, self).__init__() 11 | self.conv = nn.Sequential( 12 | nn.Conv2d( 13 | nin, 14 | nout, 15 | kernel_size=ksize, 16 | stride=stride, 17 | padding=pad, 18 | dilation=dilation, 19 | bias=False, 20 | ), 21 | nn.BatchNorm2d(nout), 22 | activ(), 23 | ) 24 | 25 | def __call__(self, x): 26 | return self.conv(x) 27 | 28 | 29 | class Encoder(nn.Module): 30 | def __init__(self, nin, nout, ksize=3, stride=1, pad=1, activ=nn.LeakyReLU): 31 | super(Encoder, self).__init__() 32 | self.conv1 = Conv2DBNActiv(nin, nout, ksize, stride, pad, activ=activ) 33 | self.conv2 = Conv2DBNActiv(nout, nout, ksize, 1, pad, activ=activ) 34 | 35 | def __call__(self, x): 36 | h = self.conv1(x) 37 | h = self.conv2(h) 38 | 39 | return h 40 | 41 | 42 | class Decoder(nn.Module): 43 | def __init__( 44 | self, nin, nout, ksize=3, stride=1, pad=1, activ=nn.ReLU, dropout=False 45 | ): 46 | super(Decoder, self).__init__() 47 | self.conv1 = Conv2DBNActiv(nin, nout, ksize, 1, pad, activ=activ) 48 | # self.conv2 = Conv2DBNActiv(nout, nout, ksize, 1, pad, activ=activ) 49 | self.dropout = nn.Dropout2d(0.1) if dropout else None 50 | 51 | def __call__(self, x, skip=None): 52 | x = F.interpolate(x, scale_factor=2, mode="bilinear", align_corners=True) 53 | 54 | if skip is not None: 55 | skip = spec_utils.crop_center(skip, x) 56 | x = torch.cat([x, skip], dim=1) 57 | 58 | h = self.conv1(x) 59 | # h = self.conv2(h) 60 | 61 | if self.dropout is not None: 62 | h = self.dropout(h) 63 | 64 | return h 65 | 66 | 67 | class ASPPModule(nn.Module): 68 | def __init__(self, nin, nout, dilations=(4, 8, 12), activ=nn.ReLU, dropout=False): 69 | super(ASPPModule, self).__init__() 70 | self.conv1 = nn.Sequential( 71 | nn.AdaptiveAvgPool2d((1, None)), 72 | Conv2DBNActiv(nin, nout, 1, 1, 0, activ=activ), 73 | ) 74 | self.conv2 = Conv2DBNActiv(nin, nout, 1, 1, 0, activ=activ) 75 | self.conv3 = Conv2DBNActiv( 76 | nin, nout, 3, 1, dilations[0], dilations[0], activ=activ 77 | ) 78 | self.conv4 = Conv2DBNActiv( 79 | nin, nout, 3, 1, dilations[1], dilations[1], activ=activ 80 | ) 81 | self.conv5 = Conv2DBNActiv( 82 | nin, nout, 3, 1, dilations[2], dilations[2], activ=activ 83 | ) 84 | self.bottleneck = Conv2DBNActiv(nout * 5, nout, 1, 1, 0, activ=activ) 85 | self.dropout = nn.Dropout2d(0.1) if dropout else None 86 | 87 | def forward(self, x): 88 | _, _, h, w = x.size() 89 | feat1 = F.interpolate( 90 | self.conv1(x), size=(h, w), mode="bilinear", align_corners=True 91 | ) 92 | feat2 = self.conv2(x) 93 | feat3 = self.conv3(x) 94 | feat4 = self.conv4(x) 95 | feat5 = self.conv5(x) 96 | out = torch.cat((feat1, feat2, feat3, feat4, feat5), dim=1) 97 | out = self.bottleneck(out) 98 | 99 | if self.dropout is not None: 100 | out = self.dropout(out) 101 | 102 | return out 103 | 104 | 105 | class LSTMModule(nn.Module): 106 | def __init__(self, nin_conv, nin_lstm, nout_lstm): 107 | super(LSTMModule, self).__init__() 108 | self.conv = Conv2DBNActiv(nin_conv, 1, 1, 1, 0) 109 | self.lstm = nn.LSTM( 110 | input_size=nin_lstm, hidden_size=nout_lstm // 2, bidirectional=True 111 | ) 112 | self.dense = nn.Sequential( 113 | nn.Linear(nout_lstm, nin_lstm), nn.BatchNorm1d(nin_lstm), nn.ReLU() 114 | ) 115 | 116 | def forward(self, x): 117 | N, _, nbins, nframes = x.size() 118 | h = self.conv(x)[:, 0] # N, nbins, nframes 119 | h = h.permute(2, 0, 1) # nframes, N, nbins 120 | h, _ = self.lstm(h) 121 | h = self.dense(h.reshape(-1, h.size()[-1])) # nframes * N, nbins 122 | h = h.reshape(nframes, N, 1, nbins) 123 | h = h.permute(1, 2, 3, 0) 124 | 125 | return h 126 | -------------------------------------------------------------------------------- /lib/uvr5_pack/lib_v5/layers_33966KB.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch import nn 3 | import torch.nn.functional as F 4 | 5 | from . import spec_utils 6 | 7 | 8 | class Conv2DBNActiv(nn.Module): 9 | def __init__(self, nin, nout, ksize=3, stride=1, pad=1, dilation=1, activ=nn.ReLU): 10 | super(Conv2DBNActiv, self).__init__() 11 | self.conv = nn.Sequential( 12 | nn.Conv2d( 13 | nin, 14 | nout, 15 | kernel_size=ksize, 16 | stride=stride, 17 | padding=pad, 18 | dilation=dilation, 19 | bias=False, 20 | ), 21 | nn.BatchNorm2d(nout), 22 | activ(), 23 | ) 24 | 25 | def __call__(self, x): 26 | return self.conv(x) 27 | 28 | 29 | class SeperableConv2DBNActiv(nn.Module): 30 | def __init__(self, nin, nout, ksize=3, stride=1, pad=1, dilation=1, activ=nn.ReLU): 31 | super(SeperableConv2DBNActiv, self).__init__() 32 | self.conv = nn.Sequential( 33 | nn.Conv2d( 34 | nin, 35 | nin, 36 | kernel_size=ksize, 37 | stride=stride, 38 | padding=pad, 39 | dilation=dilation, 40 | groups=nin, 41 | bias=False, 42 | ), 43 | nn.Conv2d(nin, nout, kernel_size=1, bias=False), 44 | nn.BatchNorm2d(nout), 45 | activ(), 46 | ) 47 | 48 | def __call__(self, x): 49 | return self.conv(x) 50 | 51 | 52 | class Encoder(nn.Module): 53 | def __init__(self, nin, nout, ksize=3, stride=1, pad=1, activ=nn.LeakyReLU): 54 | super(Encoder, self).__init__() 55 | self.conv1 = Conv2DBNActiv(nin, nout, ksize, 1, pad, activ=activ) 56 | self.conv2 = Conv2DBNActiv(nout, nout, ksize, stride, pad, activ=activ) 57 | 58 | def __call__(self, x): 59 | skip = self.conv1(x) 60 | h = self.conv2(skip) 61 | 62 | return h, skip 63 | 64 | 65 | class Decoder(nn.Module): 66 | def __init__( 67 | self, nin, nout, ksize=3, stride=1, pad=1, activ=nn.ReLU, dropout=False 68 | ): 69 | super(Decoder, self).__init__() 70 | self.conv = Conv2DBNActiv(nin, nout, ksize, 1, pad, activ=activ) 71 | self.dropout = nn.Dropout2d(0.1) if dropout else None 72 | 73 | def __call__(self, x, skip=None): 74 | x = F.interpolate(x, scale_factor=2, mode="bilinear", align_corners=True) 75 | if skip is not None: 76 | skip = spec_utils.crop_center(skip, x) 77 | x = torch.cat([x, skip], dim=1) 78 | h = self.conv(x) 79 | 80 | if self.dropout is not None: 81 | h = self.dropout(h) 82 | 83 | return h 84 | 85 | 86 | class ASPPModule(nn.Module): 87 | def __init__(self, nin, nout, dilations=(4, 8, 16, 32, 64), activ=nn.ReLU): 88 | super(ASPPModule, self).__init__() 89 | self.conv1 = nn.Sequential( 90 | nn.AdaptiveAvgPool2d((1, None)), 91 | Conv2DBNActiv(nin, nin, 1, 1, 0, activ=activ), 92 | ) 93 | self.conv2 = Conv2DBNActiv(nin, nin, 1, 1, 0, activ=activ) 94 | self.conv3 = SeperableConv2DBNActiv( 95 | nin, nin, 3, 1, dilations[0], dilations[0], activ=activ 96 | ) 97 | self.conv4 = SeperableConv2DBNActiv( 98 | nin, nin, 3, 1, dilations[1], dilations[1], activ=activ 99 | ) 100 | self.conv5 = SeperableConv2DBNActiv( 101 | nin, nin, 3, 1, dilations[2], dilations[2], activ=activ 102 | ) 103 | self.conv6 = SeperableConv2DBNActiv( 104 | nin, nin, 3, 1, dilations[2], dilations[2], activ=activ 105 | ) 106 | self.conv7 = SeperableConv2DBNActiv( 107 | nin, nin, 3, 1, dilations[2], dilations[2], activ=activ 108 | ) 109 | self.bottleneck = nn.Sequential( 110 | Conv2DBNActiv(nin * 7, nout, 1, 1, 0, activ=activ), nn.Dropout2d(0.1) 111 | ) 112 | 113 | def forward(self, x): 114 | _, _, h, w = x.size() 115 | feat1 = F.interpolate( 116 | self.conv1(x), size=(h, w), mode="bilinear", align_corners=True 117 | ) 118 | feat2 = self.conv2(x) 119 | feat3 = self.conv3(x) 120 | feat4 = self.conv4(x) 121 | feat5 = self.conv5(x) 122 | feat6 = self.conv6(x) 123 | feat7 = self.conv7(x) 124 | out = torch.cat((feat1, feat2, feat3, feat4, feat5, feat6, feat7), dim=1) 125 | bottle = self.bottleneck(out) 126 | return bottle 127 | -------------------------------------------------------------------------------- /lib/uvr5_pack/lib_v5/layers_537227KB.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch import nn 3 | import torch.nn.functional as F 4 | 5 | from . import spec_utils 6 | 7 | 8 | class Conv2DBNActiv(nn.Module): 9 | def __init__(self, nin, nout, ksize=3, stride=1, pad=1, dilation=1, activ=nn.ReLU): 10 | super(Conv2DBNActiv, self).__init__() 11 | self.conv = nn.Sequential( 12 | nn.Conv2d( 13 | nin, 14 | nout, 15 | kernel_size=ksize, 16 | stride=stride, 17 | padding=pad, 18 | dilation=dilation, 19 | bias=False, 20 | ), 21 | nn.BatchNorm2d(nout), 22 | activ(), 23 | ) 24 | 25 | def __call__(self, x): 26 | return self.conv(x) 27 | 28 | 29 | class SeperableConv2DBNActiv(nn.Module): 30 | def __init__(self, nin, nout, ksize=3, stride=1, pad=1, dilation=1, activ=nn.ReLU): 31 | super(SeperableConv2DBNActiv, self).__init__() 32 | self.conv = nn.Sequential( 33 | nn.Conv2d( 34 | nin, 35 | nin, 36 | kernel_size=ksize, 37 | stride=stride, 38 | padding=pad, 39 | dilation=dilation, 40 | groups=nin, 41 | bias=False, 42 | ), 43 | nn.Conv2d(nin, nout, kernel_size=1, bias=False), 44 | nn.BatchNorm2d(nout), 45 | activ(), 46 | ) 47 | 48 | def __call__(self, x): 49 | return self.conv(x) 50 | 51 | 52 | class Encoder(nn.Module): 53 | def __init__(self, nin, nout, ksize=3, stride=1, pad=1, activ=nn.LeakyReLU): 54 | super(Encoder, self).__init__() 55 | self.conv1 = Conv2DBNActiv(nin, nout, ksize, 1, pad, activ=activ) 56 | self.conv2 = Conv2DBNActiv(nout, nout, ksize, stride, pad, activ=activ) 57 | 58 | def __call__(self, x): 59 | skip = self.conv1(x) 60 | h = self.conv2(skip) 61 | 62 | return h, skip 63 | 64 | 65 | class Decoder(nn.Module): 66 | def __init__( 67 | self, nin, nout, ksize=3, stride=1, pad=1, activ=nn.ReLU, dropout=False 68 | ): 69 | super(Decoder, self).__init__() 70 | self.conv = Conv2DBNActiv(nin, nout, ksize, 1, pad, activ=activ) 71 | self.dropout = nn.Dropout2d(0.1) if dropout else None 72 | 73 | def __call__(self, x, skip=None): 74 | x = F.interpolate(x, scale_factor=2, mode="bilinear", align_corners=True) 75 | if skip is not None: 76 | skip = spec_utils.crop_center(skip, x) 77 | x = torch.cat([x, skip], dim=1) 78 | h = self.conv(x) 79 | 80 | if self.dropout is not None: 81 | h = self.dropout(h) 82 | 83 | return h 84 | 85 | 86 | class ASPPModule(nn.Module): 87 | def __init__(self, nin, nout, dilations=(4, 8, 16, 32, 64), activ=nn.ReLU): 88 | super(ASPPModule, self).__init__() 89 | self.conv1 = nn.Sequential( 90 | nn.AdaptiveAvgPool2d((1, None)), 91 | Conv2DBNActiv(nin, nin, 1, 1, 0, activ=activ), 92 | ) 93 | self.conv2 = Conv2DBNActiv(nin, nin, 1, 1, 0, activ=activ) 94 | self.conv3 = SeperableConv2DBNActiv( 95 | nin, nin, 3, 1, dilations[0], dilations[0], activ=activ 96 | ) 97 | self.conv4 = SeperableConv2DBNActiv( 98 | nin, nin, 3, 1, dilations[1], dilations[1], activ=activ 99 | ) 100 | self.conv5 = SeperableConv2DBNActiv( 101 | nin, nin, 3, 1, dilations[2], dilations[2], activ=activ 102 | ) 103 | self.conv6 = SeperableConv2DBNActiv( 104 | nin, nin, 3, 1, dilations[2], dilations[2], activ=activ 105 | ) 106 | self.conv7 = SeperableConv2DBNActiv( 107 | nin, nin, 3, 1, dilations[2], dilations[2], activ=activ 108 | ) 109 | self.bottleneck = nn.Sequential( 110 | Conv2DBNActiv(nin * 7, nout, 1, 1, 0, activ=activ), nn.Dropout2d(0.1) 111 | ) 112 | 113 | def forward(self, x): 114 | _, _, h, w = x.size() 115 | feat1 = F.interpolate( 116 | self.conv1(x), size=(h, w), mode="bilinear", align_corners=True 117 | ) 118 | feat2 = self.conv2(x) 119 | feat3 = self.conv3(x) 120 | feat4 = self.conv4(x) 121 | feat5 = self.conv5(x) 122 | feat6 = self.conv6(x) 123 | feat7 = self.conv7(x) 124 | out = torch.cat((feat1, feat2, feat3, feat4, feat5, feat6, feat7), dim=1) 125 | bottle = self.bottleneck(out) 126 | return bottle 127 | -------------------------------------------------------------------------------- /lib/uvr5_pack/lib_v5/layers_537238KB.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch import nn 3 | import torch.nn.functional as F 4 | 5 | from . import spec_utils 6 | 7 | 8 | class Conv2DBNActiv(nn.Module): 9 | def __init__(self, nin, nout, ksize=3, stride=1, pad=1, dilation=1, activ=nn.ReLU): 10 | super(Conv2DBNActiv, self).__init__() 11 | self.conv = nn.Sequential( 12 | nn.Conv2d( 13 | nin, 14 | nout, 15 | kernel_size=ksize, 16 | stride=stride, 17 | padding=pad, 18 | dilation=dilation, 19 | bias=False, 20 | ), 21 | nn.BatchNorm2d(nout), 22 | activ(), 23 | ) 24 | 25 | def __call__(self, x): 26 | return self.conv(x) 27 | 28 | 29 | class SeperableConv2DBNActiv(nn.Module): 30 | def __init__(self, nin, nout, ksize=3, stride=1, pad=1, dilation=1, activ=nn.ReLU): 31 | super(SeperableConv2DBNActiv, self).__init__() 32 | self.conv = nn.Sequential( 33 | nn.Conv2d( 34 | nin, 35 | nin, 36 | kernel_size=ksize, 37 | stride=stride, 38 | padding=pad, 39 | dilation=dilation, 40 | groups=nin, 41 | bias=False, 42 | ), 43 | nn.Conv2d(nin, nout, kernel_size=1, bias=False), 44 | nn.BatchNorm2d(nout), 45 | activ(), 46 | ) 47 | 48 | def __call__(self, x): 49 | return self.conv(x) 50 | 51 | 52 | class Encoder(nn.Module): 53 | def __init__(self, nin, nout, ksize=3, stride=1, pad=1, activ=nn.LeakyReLU): 54 | super(Encoder, self).__init__() 55 | self.conv1 = Conv2DBNActiv(nin, nout, ksize, 1, pad, activ=activ) 56 | self.conv2 = Conv2DBNActiv(nout, nout, ksize, stride, pad, activ=activ) 57 | 58 | def __call__(self, x): 59 | skip = self.conv1(x) 60 | h = self.conv2(skip) 61 | 62 | return h, skip 63 | 64 | 65 | class Decoder(nn.Module): 66 | def __init__( 67 | self, nin, nout, ksize=3, stride=1, pad=1, activ=nn.ReLU, dropout=False 68 | ): 69 | super(Decoder, self).__init__() 70 | self.conv = Conv2DBNActiv(nin, nout, ksize, 1, pad, activ=activ) 71 | self.dropout = nn.Dropout2d(0.1) if dropout else None 72 | 73 | def __call__(self, x, skip=None): 74 | x = F.interpolate(x, scale_factor=2, mode="bilinear", align_corners=True) 75 | if skip is not None: 76 | skip = spec_utils.crop_center(skip, x) 77 | x = torch.cat([x, skip], dim=1) 78 | h = self.conv(x) 79 | 80 | if self.dropout is not None: 81 | h = self.dropout(h) 82 | 83 | return h 84 | 85 | 86 | class ASPPModule(nn.Module): 87 | def __init__(self, nin, nout, dilations=(4, 8, 16, 32, 64), activ=nn.ReLU): 88 | super(ASPPModule, self).__init__() 89 | self.conv1 = nn.Sequential( 90 | nn.AdaptiveAvgPool2d((1, None)), 91 | Conv2DBNActiv(nin, nin, 1, 1, 0, activ=activ), 92 | ) 93 | self.conv2 = Conv2DBNActiv(nin, nin, 1, 1, 0, activ=activ) 94 | self.conv3 = SeperableConv2DBNActiv( 95 | nin, nin, 3, 1, dilations[0], dilations[0], activ=activ 96 | ) 97 | self.conv4 = SeperableConv2DBNActiv( 98 | nin, nin, 3, 1, dilations[1], dilations[1], activ=activ 99 | ) 100 | self.conv5 = SeperableConv2DBNActiv( 101 | nin, nin, 3, 1, dilations[2], dilations[2], activ=activ 102 | ) 103 | self.conv6 = SeperableConv2DBNActiv( 104 | nin, nin, 3, 1, dilations[2], dilations[2], activ=activ 105 | ) 106 | self.conv7 = SeperableConv2DBNActiv( 107 | nin, nin, 3, 1, dilations[2], dilations[2], activ=activ 108 | ) 109 | self.bottleneck = nn.Sequential( 110 | Conv2DBNActiv(nin * 7, nout, 1, 1, 0, activ=activ), nn.Dropout2d(0.1) 111 | ) 112 | 113 | def forward(self, x): 114 | _, _, h, w = x.size() 115 | feat1 = F.interpolate( 116 | self.conv1(x), size=(h, w), mode="bilinear", align_corners=True 117 | ) 118 | feat2 = self.conv2(x) 119 | feat3 = self.conv3(x) 120 | feat4 = self.conv4(x) 121 | feat5 = self.conv5(x) 122 | feat6 = self.conv6(x) 123 | feat7 = self.conv7(x) 124 | out = torch.cat((feat1, feat2, feat3, feat4, feat5, feat6, feat7), dim=1) 125 | bottle = self.bottleneck(out) 126 | return bottle 127 | -------------------------------------------------------------------------------- /docs/training_tips_en.md: -------------------------------------------------------------------------------- 1 | Instructions and tips for RVC training 2 | ====================================== 3 | This TIPS explains how data training is done. 4 | 5 | # Training flow 6 | I will explain along the steps in the training tab of the GUI. 7 | 8 | ## step1 9 | Set the experiment name here. 10 | 11 | You can also set here whether the model should take pitch into account. 12 | If the model doesn't consider pitch, the model will be lighter, but not suitable for singing. 13 | 14 | Data for each experiment is placed in `/logs/your-experiment-name/`. 15 | 16 | ## step2a 17 | Loads and preprocesses audio. 18 | 19 | ### load audio 20 | If you specify a folder with audio, the audio files in that folder will be read automatically. 21 | For example, if you specify `C:Users\hoge\voices`, `C:Users\hoge\voices\voice.mp3` will be loaded, but `C:Users\hoge\voices\dir\voice.mp3` will Not loaded. 22 | 23 | Since ffmpeg is used internally for reading audio, if the extension is supported by ffmpeg, it will be read automatically. 24 | After converting to int16 with ffmpeg, convert to float32 and normalize between -1 to 1. 25 | 26 | ### denoising 27 | The audio is smoothed by scipy's filtfilt. 28 | 29 | ### Audio Split 30 | First, the input audio is divided by detecting parts of silence that last longer than a certain period (max_sil_kept=5 seconds?). After splitting the audio on silence, split the audio every 4 seconds with an overlap of 0.3 seconds. For audio separated within 4 seconds, after normalizing the volume, convert the wav file to `/logs/your-experiment-name/0_gt_wavs` and then convert it to 16k sampling rate to `/logs/your-experiment-name/1_16k_wavs ` as a wav file. 31 | 32 | ## step2b 33 | ### Extract pitch 34 | Extract pitch information from wav files. Extract the pitch information (=f0) using the method built into parselmouth or pyworld and save it in `/logs/your-experiment-name/2a_f0`. Then logarithmically convert the pitch information to an integer between 1 and 255 and save it in `/logs/your-experiment-name/2b-f0nsf`. 35 | 36 | ### Extract feature_print 37 | Convert the wav file to embedding in advance using HuBERT. Read the wav file saved in `/logs/your-experiment-name/1_16k_wavs`, convert the wav file to 256-dimensional features with HuBERT, and save in npy format in `/logs/your-experiment-name/3_feature256`. 38 | 39 | ## step3 40 | train the model. 41 | ### Glossary for Beginners 42 | In deep learning, the data set is divided and the learning proceeds little by little. In one model update (step), batch_size data are retrieved and predictions and error corrections are performed. Doing this once for a dataset counts as one epoch. 43 | 44 | Therefore, the learning time is the learning time per step x (the number of data in the dataset / batch size) x the number of epochs. In general, the larger the batch size, the more stable the learning becomes (learning time per step ÷ batch size) becomes smaller, but it uses more GPU memory. GPU RAM can be checked with the nvidia-smi command. Learning can be done in a short time by increasing the batch size as much as possible according to the machine of the execution environment. 45 | 46 | ### Specify pretrained model 47 | RVC starts training the model from pretrained weights instead of from 0, so it can be trained with a small dataset. 48 | 49 | By default 50 | 51 | - If you consider pitch, it loads `rvc-location/pretrained/f0G40k.pth` and `rvc-location/pretrained/f0D40k.pth`. 52 | - If you don't consider pitch, it loads `rvc-location/pretrained/f0G40k.pth` and `rvc-location/pretrained/f0D40k.pth`. 53 | 54 | When learning, model parameters are saved in `logs/your-experiment-name/G_{}.pth` and `logs/your-experiment-name/D_{}.pth` for each save_every_epoch, but by specifying this path, you can start learning. You can restart or start training from model weights learned in a different experiment. 55 | 56 | ### learning index 57 | RVC saves the HuBERT feature values used during training, and during inference, searches for feature values that are similar to the feature values used during learning to perform inference. In order to perform this search at high speed, the index is learned in advance. 58 | For index learning, we use the approximate neighborhood search library faiss. Read the feature value of `logs/your-experiment-name/3_feature256` and use it to learn the index, and save it as `logs/your-experiment-name/add_XXX.index`. 59 | 60 | (From the 20230428update version, it is read from the index, and saving / specifying is no longer necessary.) 61 | 62 | ### Button description 63 | - Train model: After executing step2b, press this button to train the model. 64 | - Train feature index: After training the model, perform index learning. 65 | - One-click training: step2b, model training and feature index training all at once. -------------------------------------------------------------------------------- /lib/uvr5_pack/lib_v5/nets_new.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from torch import nn 3 | import torch.nn.functional as F 4 | from . import layers_new 5 | 6 | 7 | class BaseNet(nn.Module): 8 | def __init__( 9 | self, nin, nout, nin_lstm, nout_lstm, dilations=((4, 2), (8, 4), (12, 6)) 10 | ): 11 | super(BaseNet, self).__init__() 12 | self.enc1 = layers_new.Conv2DBNActiv(nin, nout, 3, 1, 1) 13 | self.enc2 = layers_new.Encoder(nout, nout * 2, 3, 2, 1) 14 | self.enc3 = layers_new.Encoder(nout * 2, nout * 4, 3, 2, 1) 15 | self.enc4 = layers_new.Encoder(nout * 4, nout * 6, 3, 2, 1) 16 | self.enc5 = layers_new.Encoder(nout * 6, nout * 8, 3, 2, 1) 17 | 18 | self.aspp = layers_new.ASPPModule(nout * 8, nout * 8, dilations, dropout=True) 19 | 20 | self.dec4 = layers_new.Decoder(nout * (6 + 8), nout * 6, 3, 1, 1) 21 | self.dec3 = layers_new.Decoder(nout * (4 + 6), nout * 4, 3, 1, 1) 22 | self.dec2 = layers_new.Decoder(nout * (2 + 4), nout * 2, 3, 1, 1) 23 | self.lstm_dec2 = layers_new.LSTMModule(nout * 2, nin_lstm, nout_lstm) 24 | self.dec1 = layers_new.Decoder(nout * (1 + 2) + 1, nout * 1, 3, 1, 1) 25 | 26 | def __call__(self, x): 27 | e1 = self.enc1(x) 28 | e2 = self.enc2(e1) 29 | e3 = self.enc3(e2) 30 | e4 = self.enc4(e3) 31 | e5 = self.enc5(e4) 32 | 33 | h = self.aspp(e5) 34 | 35 | h = self.dec4(h, e4) 36 | h = self.dec3(h, e3) 37 | h = self.dec2(h, e2) 38 | h = torch.cat([h, self.lstm_dec2(h)], dim=1) 39 | h = self.dec1(h, e1) 40 | 41 | return h 42 | 43 | 44 | class CascadedNet(nn.Module): 45 | def __init__(self, n_fft, nout=32, nout_lstm=128): 46 | super(CascadedNet, self).__init__() 47 | 48 | self.max_bin = n_fft // 2 49 | self.output_bin = n_fft // 2 + 1 50 | self.nin_lstm = self.max_bin // 2 51 | self.offset = 64 52 | 53 | self.stg1_low_band_net = nn.Sequential( 54 | BaseNet(2, nout // 2, self.nin_lstm // 2, nout_lstm), 55 | layers_new.Conv2DBNActiv(nout // 2, nout // 4, 1, 1, 0), 56 | ) 57 | 58 | self.stg1_high_band_net = BaseNet( 59 | 2, nout // 4, self.nin_lstm // 2, nout_lstm // 2 60 | ) 61 | 62 | self.stg2_low_band_net = nn.Sequential( 63 | BaseNet(nout // 4 + 2, nout, self.nin_lstm // 2, nout_lstm), 64 | layers_new.Conv2DBNActiv(nout, nout // 2, 1, 1, 0), 65 | ) 66 | self.stg2_high_band_net = BaseNet( 67 | nout // 4 + 2, nout // 2, self.nin_lstm // 2, nout_lstm // 2 68 | ) 69 | 70 | self.stg3_full_band_net = BaseNet( 71 | 3 * nout // 4 + 2, nout, self.nin_lstm, nout_lstm 72 | ) 73 | 74 | self.out = nn.Conv2d(nout, 2, 1, bias=False) 75 | self.aux_out = nn.Conv2d(3 * nout // 4, 2, 1, bias=False) 76 | 77 | def forward(self, x): 78 | x = x[:, :, : self.max_bin] 79 | 80 | bandw = x.size()[2] // 2 81 | l1_in = x[:, :, :bandw] 82 | h1_in = x[:, :, bandw:] 83 | l1 = self.stg1_low_band_net(l1_in) 84 | h1 = self.stg1_high_band_net(h1_in) 85 | aux1 = torch.cat([l1, h1], dim=2) 86 | 87 | l2_in = torch.cat([l1_in, l1], dim=1) 88 | h2_in = torch.cat([h1_in, h1], dim=1) 89 | l2 = self.stg2_low_band_net(l2_in) 90 | h2 = self.stg2_high_band_net(h2_in) 91 | aux2 = torch.cat([l2, h2], dim=2) 92 | 93 | f3_in = torch.cat([x, aux1, aux2], dim=1) 94 | f3 = self.stg3_full_band_net(f3_in) 95 | 96 | mask = torch.sigmoid(self.out(f3)) 97 | mask = F.pad( 98 | input=mask, 99 | pad=(0, 0, 0, self.output_bin - mask.size()[2]), 100 | mode="replicate", 101 | ) 102 | 103 | if self.training: 104 | aux = torch.cat([aux1, aux2], dim=1) 105 | aux = torch.sigmoid(self.aux_out(aux)) 106 | aux = F.pad( 107 | input=aux, 108 | pad=(0, 0, 0, self.output_bin - aux.size()[2]), 109 | mode="replicate", 110 | ) 111 | return mask, aux 112 | else: 113 | return mask 114 | 115 | def predict_mask(self, x): 116 | mask = self.forward(x) 117 | 118 | if self.offset > 0: 119 | mask = mask[:, :, :, self.offset : -self.offset] 120 | assert mask.size()[3] > 0 121 | 122 | return mask 123 | 124 | def predict(self, x, aggressiveness=None): 125 | mask = self.forward(x) 126 | pred_mag = x * mask 127 | 128 | if self.offset > 0: 129 | pred_mag = pred_mag[:, :, :, self.offset : -self.offset] 130 | assert pred_mag.size()[3] > 0 131 | 132 | return pred_mag 133 | -------------------------------------------------------------------------------- /extract_f0_rmvpe.py: -------------------------------------------------------------------------------- 1 | import os, traceback, sys, parselmouth 2 | 3 | now_dir = os.getcwd() 4 | sys.path.append(now_dir) 5 | from lib.audio import load_audio 6 | import pyworld 7 | import numpy as np, logging 8 | 9 | logging.getLogger("numba").setLevel(logging.WARNING) 10 | 11 | n_part = int(sys.argv[1]) 12 | i_part = int(sys.argv[2]) 13 | i_gpu = sys.argv[3] 14 | os.environ["CUDA_VISIBLE_DEVICES"] = str(i_gpu) 15 | exp_dir = sys.argv[4] 16 | is_half = sys.argv[5] 17 | f = open("%s/extract_f0_feature.log" % exp_dir, "a+") 18 | 19 | 20 | def printt(strr): 21 | print(strr) 22 | f.write("%s\n" % strr) 23 | f.flush() 24 | 25 | 26 | class FeatureInput(object): 27 | def __init__(self, samplerate=16000, hop_size=160): 28 | self.fs = samplerate 29 | self.hop = hop_size 30 | 31 | self.f0_bin = 256 32 | self.f0_max = 1100.0 33 | self.f0_min = 50.0 34 | self.f0_mel_min = 1127 * np.log(1 + self.f0_min / 700) 35 | self.f0_mel_max = 1127 * np.log(1 + self.f0_max / 700) 36 | 37 | def compute_f0(self, path, f0_method): 38 | x = load_audio(path, self.fs) 39 | p_len = x.shape[0] // self.hop 40 | if f0_method == "rmvpe": 41 | if hasattr(self, "model_rmvpe") == False: 42 | from lib.rmvpe import RMVPE 43 | 44 | print("loading rmvpe model") 45 | self.model_rmvpe = RMVPE("rmvpe.pt", is_half=True, device="cuda") 46 | f0 = self.model_rmvpe.infer_from_audio(x, thred=0.03) 47 | return f0 48 | 49 | def coarse_f0(self, f0): 50 | f0_mel = 1127 * np.log(1 + f0 / 700) 51 | f0_mel[f0_mel > 0] = (f0_mel[f0_mel > 0] - self.f0_mel_min) * ( 52 | self.f0_bin - 2 53 | ) / (self.f0_mel_max - self.f0_mel_min) + 1 54 | 55 | # use 0 or 1 56 | f0_mel[f0_mel <= 1] = 1 57 | f0_mel[f0_mel > self.f0_bin - 1] = self.f0_bin - 1 58 | f0_coarse = np.rint(f0_mel).astype(int) 59 | assert f0_coarse.max() <= 255 and f0_coarse.min() >= 1, ( 60 | f0_coarse.max(), 61 | f0_coarse.min(), 62 | ) 63 | return f0_coarse 64 | 65 | def go(self, paths, f0_method): 66 | if len(paths) == 0: 67 | printt("no-f0-todo") 68 | else: 69 | printt("todo-f0-%s" % len(paths)) 70 | n = max(len(paths) // 5, 1) # 每个进程最多打印5条 71 | for idx, (inp_path, opt_path1, opt_path2) in enumerate(paths): 72 | try: 73 | if idx % n == 0: 74 | printt("f0ing,now-%s,all-%s,-%s" % (idx, len(paths), inp_path)) 75 | if ( 76 | os.path.exists(opt_path1 + ".npy") == True 77 | and os.path.exists(opt_path2 + ".npy") == True 78 | ): 79 | continue 80 | featur_pit = self.compute_f0(inp_path, f0_method) 81 | np.save( 82 | opt_path2, 83 | featur_pit, 84 | allow_pickle=False, 85 | ) # nsf 86 | coarse_pit = self.coarse_f0(featur_pit) 87 | np.save( 88 | opt_path1, 89 | coarse_pit, 90 | allow_pickle=False, 91 | ) # ori 92 | except: 93 | printt("f0fail-%s-%s-%s" % (idx, inp_path, traceback.format_exc())) 94 | 95 | 96 | if __name__ == "__main__": 97 | # exp_dir=r"E:\codes\py39\dataset\mi-test" 98 | # n_p=16 99 | # f = open("%s/log_extract_f0.log"%exp_dir, "w") 100 | printt(sys.argv) 101 | featureInput = FeatureInput() 102 | paths = [] 103 | inp_root = "%s/1_16k_wavs" % (exp_dir) 104 | opt_root1 = "%s/2a_f0" % (exp_dir) 105 | opt_root2 = "%s/2b-f0nsf" % (exp_dir) 106 | 107 | os.makedirs(opt_root1, exist_ok=True) 108 | os.makedirs(opt_root2, exist_ok=True) 109 | for name in sorted(list(os.listdir(inp_root))): 110 | inp_path = "%s/%s" % (inp_root, name) 111 | if "spec" in inp_path: 112 | continue 113 | opt_path1 = "%s/%s" % (opt_root1, name) 114 | opt_path2 = "%s/%s" % (opt_root2, name) 115 | paths.append([inp_path, opt_path1, opt_path2]) 116 | try: 117 | featureInput.go(paths[i_part::n_part], "rmvpe") 118 | except: 119 | printt("f0_all_fail-%s" % (traceback.format_exc())) 120 | # ps = [] 121 | # for i in range(n_p): 122 | # p = Process( 123 | # target=featureInput.go, 124 | # args=( 125 | # paths[i::n_p], 126 | # f0method, 127 | # ), 128 | # ) 129 | # ps.append(p) 130 | # p.start() 131 | # for i in range(n_p): 132 | # ps[i].join() 133 | -------------------------------------------------------------------------------- /trainset_preprocess_pipeline_print.py: -------------------------------------------------------------------------------- 1 | import sys, os, multiprocessing 2 | from scipy import signal 3 | 4 | now_dir = os.getcwd() 5 | sys.path.append(now_dir) 6 | print(sys.argv) 7 | inp_root = sys.argv[1] 8 | sr = int(sys.argv[2]) 9 | n_p = int(sys.argv[3]) 10 | exp_dir = sys.argv[4] 11 | noparallel = sys.argv[5] == "True" 12 | import numpy as np, os, traceback 13 | from lib.slicer2 import Slicer 14 | import librosa, traceback 15 | from scipy.io import wavfile 16 | import multiprocessing 17 | from lib.audio import load_audio 18 | 19 | mutex = multiprocessing.Lock() 20 | f = open("%s/preprocess.log" % exp_dir, "a+") 21 | 22 | 23 | def println(strr): 24 | mutex.acquire() 25 | print(strr) 26 | f.write("%s\n" % strr) 27 | f.flush() 28 | mutex.release() 29 | 30 | 31 | class PreProcess: 32 | def __init__(self, sr, exp_dir): 33 | self.slicer = Slicer( 34 | sr=sr, 35 | threshold=-42, 36 | min_length=1500, 37 | min_interval=400, 38 | hop_size=15, 39 | max_sil_kept=500, 40 | ) 41 | self.sr = sr 42 | self.bh, self.ah = signal.butter(N=5, Wn=48, btype="high", fs=self.sr) 43 | self.per = 3.0 44 | self.overlap = 0.3 45 | self.tail = self.per + self.overlap 46 | self.max = 0.9 47 | self.alpha = 0.75 48 | self.exp_dir = exp_dir 49 | self.gt_wavs_dir = "%s/0_gt_wavs" % exp_dir 50 | self.wavs16k_dir = "%s/1_16k_wavs" % exp_dir 51 | os.makedirs(self.exp_dir, exist_ok=True) 52 | os.makedirs(self.gt_wavs_dir, exist_ok=True) 53 | os.makedirs(self.wavs16k_dir, exist_ok=True) 54 | 55 | def norm_write(self, tmp_audio, idx0, idx1): 56 | tmp_max = np.abs(tmp_audio).max() 57 | if tmp_max > 2.5: 58 | print("%s-%s-%s-filtered" % (idx0, idx1, tmp_max)) 59 | return 60 | tmp_audio = (tmp_audio / tmp_max * (self.max * self.alpha)) + ( 61 | 1 - self.alpha 62 | ) * tmp_audio 63 | wavfile.write( 64 | "%s/%s_%s.wav" % (self.gt_wavs_dir, idx0, idx1), 65 | self.sr, 66 | tmp_audio.astype(np.float32), 67 | ) 68 | tmp_audio = librosa.resample( 69 | tmp_audio, orig_sr=self.sr, target_sr=16000 70 | ) # , res_type="soxr_vhq" 71 | wavfile.write( 72 | "%s/%s_%s.wav" % (self.wavs16k_dir, idx0, idx1), 73 | 16000, 74 | tmp_audio.astype(np.float32), 75 | ) 76 | 77 | def pipeline(self, path, idx0): 78 | try: 79 | audio = load_audio(path, self.sr) 80 | # zero phased digital filter cause pre-ringing noise... 81 | # audio = signal.filtfilt(self.bh, self.ah, audio) 82 | audio = signal.lfilter(self.bh, self.ah, audio) 83 | 84 | idx1 = 0 85 | for audio in self.slicer.slice(audio): 86 | i = 0 87 | while 1: 88 | start = int(self.sr * (self.per - self.overlap) * i) 89 | i += 1 90 | if len(audio[start:]) > self.tail * self.sr: 91 | tmp_audio = audio[start : start + int(self.per * self.sr)] 92 | self.norm_write(tmp_audio, idx0, idx1) 93 | idx1 += 1 94 | else: 95 | tmp_audio = audio[start:] 96 | idx1 += 1 97 | break 98 | self.norm_write(tmp_audio, idx0, idx1) 99 | println("%s->Suc." % path) 100 | except: 101 | println("%s->%s" % (path, traceback.format_exc())) 102 | 103 | def pipeline_mp(self, infos): 104 | for path, idx0 in infos: 105 | self.pipeline(path, idx0) 106 | 107 | def pipeline_mp_inp_dir(self, inp_root, n_p): 108 | try: 109 | infos = [ 110 | ("%s/%s" % (inp_root, name), idx) 111 | for idx, name in enumerate(sorted(list(os.listdir(inp_root)))) 112 | ] 113 | if noparallel: 114 | for i in range(n_p): 115 | self.pipeline_mp(infos[i::n_p]) 116 | else: 117 | ps = [] 118 | for i in range(n_p): 119 | p = multiprocessing.Process( 120 | target=self.pipeline_mp, args=(infos[i::n_p],) 121 | ) 122 | ps.append(p) 123 | p.start() 124 | for i in range(n_p): 125 | ps[i].join() 126 | except: 127 | println("Fail. %s" % traceback.format_exc()) 128 | 129 | 130 | def preprocess_trainset(inp_root, sr, n_p, exp_dir): 131 | pp = PreProcess(sr, exp_dir) 132 | println("start preprocess") 133 | println(sys.argv) 134 | pp.pipeline_mp_inp_dir(inp_root, n_p) 135 | println("end preprocess") 136 | 137 | 138 | if __name__ == "__main__": 139 | preprocess_trainset(inp_root, sr, n_p, exp_dir) 140 | -------------------------------------------------------------------------------- /config.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import sys 3 | import torch 4 | from multiprocessing import cpu_count 5 | 6 | 7 | def use_fp32_config(): 8 | for config_file in [ 9 | "32k.json", 10 | "40k.json", 11 | "48k.json", 12 | "48k_v2.json", 13 | "32k_v2.json", 14 | ]: 15 | with open(f"configs/{config_file}", "r") as f: 16 | strr = f.read().replace("true", "false") 17 | with open(f"configs/{config_file}", "w") as f: 18 | f.write(strr) 19 | with open("trainset_preprocess_pipeline_print.py", "r") as f: 20 | strr = f.read().replace("3.7", "3.0") 21 | with open("trainset_preprocess_pipeline_print.py", "w") as f: 22 | f.write(strr) 23 | 24 | 25 | class Config: 26 | def __init__(self): 27 | self.device = "cuda:0" 28 | self.is_half = True 29 | self.n_cpu = 0 30 | self.gpu_name = None 31 | self.gpu_mem = None 32 | ( 33 | self.python_cmd, 34 | self.listen_port, 35 | self.iscolab, 36 | self.noparallel, 37 | self.noautoopen, 38 | ) = self.arg_parse() 39 | self.x_pad, self.x_query, self.x_center, self.x_max = self.device_config() 40 | 41 | @staticmethod 42 | def arg_parse() -> tuple: 43 | exe = sys.executable or "python" 44 | parser = argparse.ArgumentParser() 45 | parser.add_argument("--port", type=int, default=7865, help="Listen port") 46 | parser.add_argument("--pycmd", type=str, default=exe, help="Python command") 47 | parser.add_argument("--colab", action="store_true", help="Launch in colab") 48 | parser.add_argument( 49 | "--noparallel", action="store_true", help="Disable parallel processing" 50 | ) 51 | parser.add_argument( 52 | "--noautoopen", 53 | action="store_true", 54 | help="Do not open in browser automatically", 55 | ) 56 | cmd_opts = parser.parse_args() 57 | 58 | cmd_opts.port = cmd_opts.port if 0 <= cmd_opts.port <= 65535 else 7865 59 | 60 | return ( 61 | cmd_opts.pycmd, 62 | cmd_opts.port, 63 | cmd_opts.colab, 64 | cmd_opts.noparallel, 65 | cmd_opts.noautoopen, 66 | ) 67 | 68 | # has_mps is only available in nightly pytorch (for now) and MasOS 12.3+. 69 | # check `getattr` and try it for compatibility 70 | @staticmethod 71 | def has_mps() -> bool: 72 | if not torch.backends.mps.is_available(): 73 | return False 74 | try: 75 | torch.zeros(1).to(torch.device("mps")) 76 | return True 77 | except Exception: 78 | return False 79 | 80 | def device_config(self) -> tuple: 81 | if torch.cuda.is_available(): 82 | i_device = int(self.device.split(":")[-1]) 83 | self.gpu_name = torch.cuda.get_device_name(i_device) 84 | if ( 85 | ("16" in self.gpu_name and "V100" not in self.gpu_name.upper()) 86 | or "P40" in self.gpu_name.upper() 87 | or "1060" in self.gpu_name 88 | or "1070" in self.gpu_name 89 | or "1080" in self.gpu_name 90 | ): 91 | print("Found GPU", self.gpu_name, ", force to fp32") 92 | self.is_half = False 93 | use_fp32_config() 94 | else: 95 | print("Found GPU", self.gpu_name) 96 | self.gpu_mem = int( 97 | torch.cuda.get_device_properties(i_device).total_memory 98 | / 1024 99 | / 1024 100 | / 1024 101 | + 0.4 102 | ) 103 | if self.gpu_mem <= 4: 104 | with open("trainset_preprocess_pipeline_print.py", "r") as f: 105 | strr = f.read().replace("3.7", "3.0") 106 | with open("trainset_preprocess_pipeline_print.py", "w") as f: 107 | f.write(strr) 108 | elif self.has_mps(): 109 | print("No supported Nvidia GPU found, use MPS instead") 110 | self.device = "mps" 111 | self.is_half = False 112 | use_fp32_config() 113 | else: 114 | print("No supported Nvidia GPU found, use CPU instead") 115 | self.device = "cpu" 116 | self.is_half = False 117 | use_fp32_config() 118 | 119 | if self.n_cpu == 0: 120 | self.n_cpu = cpu_count() 121 | 122 | if self.is_half: 123 | # 6G显存配置 124 | x_pad = 3 125 | x_query = 10 126 | x_center = 60 127 | x_max = 65 128 | else: 129 | # 5G显存配置 130 | x_pad = 1 131 | x_query = 6 132 | x_center = 38 133 | x_max = 41 134 | 135 | if self.gpu_mem != None and self.gpu_mem <= 4: 136 | x_pad = 1 137 | x_query = 5 138 | x_center = 30 139 | x_max = 32 140 | 141 | return x_pad, x_query, x_center, x_max 142 | -------------------------------------------------------------------------------- /docs/Changelog_EN.md: -------------------------------------------------------------------------------- 1 | ### 2023-06-18 2 | - New pretrained v2 models: 32k and 48k 3 | - Fix non-f0 model inference errors 4 | - For training-set exceeding 1 hour, do automatic minibatch-kmeans to reduce feature shape, so that index training, adding, and searching will be much faster. 5 | - Provide a toy vocal2guitar huggingface space 6 | - Auto delete outlier short cut training-set audios 7 | - Onnx export tab 8 | 9 | Failed experiments: 10 | - ~~Feature retrieval: add temporal feature retrieval: not effective~~ 11 | - ~~Feature retrieval: add PCAR dimensionality reduction: searching is even slower~~ 12 | - ~~Random data augmentation when training: not effective~~ 13 | 14 | todolist： 15 | - Vocos-RVC (tiny vocoder) 16 | - Crepe support for training 17 | - Half precision crepe inference 18 | - F0 editor support 19 | 20 | ### 2023-05-28 21 | - Add v2 jupyter notebook, korean changelog, fix some environment requirments 22 | - Add voiceless consonant and breath protection mode 23 | - Support crepe-full pitch detect 24 | - UVR5 vocal separation: support dereverb models and de-echo models 25 | - Add experiment name and version on the name of index 26 | - Support users to manually select export format of output audios when batch voice conversion processing and UVR5 vocal separation 27 | - v1 32k model training is no more supported 28 | 29 | ### 2023-05-13 30 | - Clear the redundant codes in the old version of runtime in the one-click-package: lib.infer_pack and uvr5_pack 31 | - Fix pseudo multiprocessing bug in training set preprocessing 32 | - Adding median filtering radius adjustment for harvest pitch recognize algorithm 33 | - Support post processing resampling for exporting audio 34 | - Multi processing "n_cpu" setting for training is changed from "f0 extraction" to "data preprocessing and f0 extraction" 35 | - Automatically detect the index paths under the logs folder and provide a drop-down list function 36 | - Add "Frequently Asked Questions and Answers" on the tab page (you can also refer to github RVC wiki) 37 | - When inference, harvest pitch is cached when using same input audio path (purpose: using harvest pitch extraction, the entire pipeline will go through a long and repetitive pitch extraction process. If caching is not used, users who experiment with different timbre, index, and pitch median filtering radius settings will experience a very painful waiting process after the first inference) 38 | 39 | ### 2023-05-14 40 | - Use volume envelope of input to mix or replace the volume envelope of output (can alleviate the problem of "input muting and output small amplitude noise". If the input audio background noise is high, it is not recommended to turn it on, and it is not turned on by default (1 can be considered as not turned on) 41 | - Support saving extracted small models at a specified frequency (if you want to see the performance under different epochs, but do not want to save all large checkpoints and manually extract small models by ckpt-processing every time, this feature will be very practical) 42 | - Resolve the issue of "connection errors" caused by the server's global proxy by setting environment variables 43 | - Supports pre-trained v2 models (currently only 40k versions are publicly available for testing, and the other two sampling rates have not been fully trained yet) 44 | - Limit excessive volume exceeding 1 before inference 45 | - Slightly adjusted the settings of training-set preprocessing 46 | 47 | 48 | ####################### 49 | 50 | History changelogs: 51 | 52 | ### 2023-04-09 53 | - Fixed training parameters to improve GPU utilization rate: A100 increased from 25% to around 90%, V100: 50% to around 90%, 2060S: 60% to around 85%, P40: 25% to around 95%; significantly improved training speed 54 | - Changed parameter: total batch_size is now per GPU batch_size 55 | - Changed total_epoch: maximum limit increased from 100 to 1000; default increased from 10 to 20 56 | - Fixed issue of ckpt extraction recognizing pitch incorrectly, causing abnormal inference 57 | - Fixed issue of distributed training saving ckpt for each rank 58 | - Applied nan feature filtering for feature extraction 59 | - Fixed issue with silent input/output producing random consonants or noise (old models need to retrain with a new dataset) 60 | 61 | ### 2023-04-16 Update 62 | - Added local real-time voice changing mini-GUI, start by double-clicking go-realtime-gui.bat 63 | - Applied filtering for frequency bands below 50Hz during training and inference 64 | - Lowered the minimum pitch extraction of pyworld from the default 80 to 50 for training and inference, allowing male low-pitched voices between 50-80Hz not to be muted 65 | - WebUI supports changing languages according to system locale (currently supporting en_US, ja_JP, zh_CN, zh_HK, zh_SG, zh_TW; defaults to en_US if not supported) 66 | - Fixed recognition of some GPUs (e.g., V100-16G recognition failure, P4 recognition failure) 67 | 68 | ### 2023-04-28 Update 69 | - Upgraded faiss index settings for faster speed and higher quality 70 | - Removed dependency on total_npy; future model sharing will not require total_npy input 71 | - Unlocked restrictions for the 16-series GPUs, providing 4GB inference settings for 4GB VRAM GPUs 72 | - Fixed bug in UVR5 vocal accompaniment separation for certain audio formats 73 | - Real-time voice changing mini-GUI now supports non-40k and non-lazy pitch models 74 | 75 | ### Future Plans: 76 | Features: 77 | - Add option: extract small models for each epoch save 78 | - Add option: export additional mp3 to the specified path during inference 79 | - Support multi-person training tab (up to 4 people) 80 | 81 | Base model: 82 | - Collect breathing wav files to add to the training dataset to fix the issue of distorted breath sounds 83 | - We are currently training a base model with an extended singing dataset, which will be released in the future 84 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |
2 | 3 |

Retrieval-based-Voice-Conversion-WebUI

4 | 一个基于VITS的简单易用的语音转换（变声器）框架

5 | 6 | [![madewithlove](https://img.shields.io/badge/made_with-%E2%9D%A4-red?style=for-the-badge&labelColor=orange 7 | )](https://github.com/RVC-Project/Retrieval-based-Voice-Conversion-WebUI) 8 | 9 |
10 | 11 | [![Open In Colab](https://img.shields.io/badge/Colab-F9AB00?style=for-the-badge&logo=googlecolab&color=525252)](https://colab.research.google.com/github/RVC-Project/Retrieval-based-Voice-Conversion-WebUI/blob/main/Retrieval_based_Voice_Conversion_WebUI.ipynb) 12 | [![Licence](https://img.shields.io/badge/LICENSE-MIT-green.svg?style=for-the-badge)](https://github.com/RVC-Project/Retrieval-based-Voice-Conversion-WebUI/blob/main/LICENSE) 13 | [![Huggingface](https://img.shields.io/badge/🤗%20-Spaces-yellow.svg?style=for-the-badge)](https://huggingface.co/lj1995/VoiceConversionWebUI/tree/main/) 14 | 15 | [![Discord](https://img.shields.io/badge/RVC%20Developers-Discord-7289DA?style=for-the-badge&logo=discord&logoColor=white)](https://discord.gg/HcsmBBGyVk) 16 | 17 | [**更新日志**](https://github.com/RVC-Project/Retrieval-based-Voice-Conversion-WebUI/blob/main/docs/Changelog_CN.md) | [**常见问题解答**](https://github.com/RVC-Project/Retrieval-based-Voice-Conversion-WebUI/wiki/%E5%B8%B8%E8%A7%81%E9%97%AE%E9%A2%98%E8%A7%A3%E7%AD%94) | [**AutoDL·5毛钱训练AI歌手**](https://github.com/RVC-Project/Retrieval-based-Voice-Conversion-WebUI/wiki/Autodl%E8%AE%AD%E7%BB%83RVC%C2%B7AI%E6%AD%8C%E6%89%8B%E6%95%99%E7%A8%8B) | [**对照实验记录**](https://github.com/RVC-Project/Retrieval-based-Voice-Conversion-WebUI/wiki/Autodl%E8%AE%AD%E7%BB%83RVC%C2%B7AI%E6%AD%8C%E6%89%8B%E6%95%99%E7%A8%8B](https://github.com/RVC-Project/Retrieval-based-Voice-Conversion-WebUI/wiki/%E5%AF%B9%E7%85%A7%E5%AE%9E%E9%AA%8C%C2%B7%E5%AE%9E%E9%AA%8C%E8%AE%B0%E5%BD%95)) | [**在线演示**](https://huggingface.co/spaces/Ricecake123/RVC-demo) 18 | 19 |
20 | 21 | ------ 22 | 23 | [**English**](./docs/README.en.md) | [**中文简体**](./README.md) | [**日本語**](./docs/README.ja.md) | [**한국어**](./docs/README.ko.md) ([**韓國語**](./docs/README.ko.han.md)) 24 | 25 | 点此查看我们的[演示视频](https://www.bilibili.com/video/BV1pm4y1z7Gm/) ! 26 | 27 | > 使用了RVC的实时语音转换: [w-okada/voice-changer](https://github.com/w-okada/voice-changer) 28 | 29 | > 使用了RVC变声器训练的人声转木吉他模型在线demo ：https://huggingface.co/spaces/lj1995/vocal2guitar 30 | 31 | > RVC人声转吉他效果展示视频 ：https://www.bilibili.com/video/BV19W4y1D7tT/ 32 | 33 | > 底模使用接近50小时的开源高质量VCTK训练集训练，无版权方面的顾虑，请大家放心使用 34 | 35 | > 后续会陆续加入高质量有授权歌声训练集训练底模 36 | 37 | ## 简介 38 | 本仓库具有以下特点 39 | + 使用top1检索替换输入源特征为训练集特征来杜绝音色泄漏 40 | + 即便在相对较差的显卡上也能快速训练 41 | + 使用少量数据进行训练也能得到较好结果(推荐至少收集10分钟低底噪语音数据) 42 | + 可以通过模型融合来改变音色(借助ckpt处理选项卡中的ckpt-merge) 43 | + 简单易用的网页界面 44 | + 可调用UVR5模型来快速分离人声和伴奏 45 | + 使用最先进的[人声音高提取算法InterSpeech2023-RMVPE](#参考项目)根绝哑音问题。效果最好（显著地）但比crepe_full更快、资源占用更小 46 | 47 | ## 环境配置 48 | 以下指令需在 Python 版本大于3.8的环境中执行。 49 | 50 | (Windows/Linux) 51 | 首先通过 pip 安装主要依赖: 52 | ```bash 53 | # 安装Pytorch及其核心依赖，若已安装则跳过 54 | # 参考自: https://pytorch.org/get-started/locally/ 55 | pip install torch torchvision torchaudio 56 | 57 | #如果是win系统+Nvidia Ampere架构(RTX30xx)，根据 #21 的经验，需要指定pytorch对应的cuda版本 58 | #pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu117 59 | ``` 60 | 61 | 可以使用 poetry 来安装依赖： 62 | ```bash 63 | # 安装 Poetry 依赖管理工具, 若已安装则跳过 64 | # 参考自: https://python-poetry.org/docs/#installation 65 | curl -sSL https://install.python-poetry.org | python3 - 66 | 67 | # 通过poetry安装依赖 68 | poetry install 69 | ``` 70 | 71 | 你也可以通过 pip 来安装依赖： 72 | ```bash 73 | pip install -r requirements.txt 74 | ``` 75 | 76 | ------ 77 | Mac 用户可以通过 `run.sh` 来安装依赖： 78 | ```bash 79 | sh ./run.sh 80 | ``` 81 | 82 | ## 其他预模型准备 83 | RVC需要其他一些预模型来推理和训练。 84 | 85 | 你可以从我们的[Hugging Face space](https://huggingface.co/lj1995/VoiceConversionWebUI/tree/main/)下载到这些模型。 86 | 87 | 以下是一份清单，包括了所有RVC所需的预模型和其他文件的名称: 88 | ```bash 89 | hubert_base.pt 90 | 91 | ./pretrained 92 | 93 | ./uvr5_weights 94 | 95 | 想测试v2版本模型的话，需要额外下载 96 | 97 | ./pretrained_v2 98 | 99 | 如果你正在使用Windows，则你可能需要这个文件，若ffmpeg和ffprobe已安装则跳过; ubuntu/debian 用户可以通过apt install ffmpeg来安装这2个库, Mac 用户则可以通过brew install ffmpeg来安装 (需要预先安装brew) 100 | 101 | ./ffmpeg 102 | 103 | https://huggingface.co/lj1995/VoiceConversionWebUI/blob/main/ffmpeg.exe 104 | 105 | ./ffprobe 106 | 107 | https://huggingface.co/lj1995/VoiceConversionWebUI/blob/main/ffprobe.exe 108 | 109 | 如果你想使用最新的RMVPE人声音高提取算法，则你需要下载音高提取模型参数并放置于RVC根目录 110 | 111 | https://huggingface.co/lj1995/VoiceConversionWebUI/blob/main/rmvpe.pt 112 | 113 | ``` 114 | 之后使用以下指令来启动WebUI: 115 | ```bash 116 | python infer-web.py 117 | ``` 118 | 119 | 如果你正在使用Windows 或 macOS，你可以直接下载并解压`RVC-beta.7z`，前者可以运行`go-web.bat`以启动WebUI，后者则运行命令`sh ./run.sh`以启动WebUI。 120 | 121 | 仓库内还有一份`小白简易教程.doc`以供参考。 122 | 123 | ## 参考项目 124 | + [ContentVec](https://github.com/auspicious3000/contentvec/) 125 | + [VITS](https://github.com/jaywalnut310/vits) 126 | + [HIFIGAN](https://github.com/jik876/hifi-gan) 127 | + [Gradio](https://github.com/gradio-app/gradio) 128 | + [FFmpeg](https://github.com/FFmpeg/FFmpeg) 129 | + [Ultimate Vocal Remover](https://github.com/Anjok07/ultimatevocalremovergui) 130 | + [audio-slicer](https://github.com/openvpi/audio-slicer) 131 | + [Vocal pitch extraction:RMVPE](https://github.com/Dream-High/RMVPE) 132 | + The pretrained model is trained and tested by [yxlllc](https://github.com/yxlllc/RMVPE) and [RVC-Boss](https://github.com/RVC-Boss). 133 | 134 | ## 感谢所有贡献者作出的努力 135 | 136 | 137 | 138 | --------------------------------------------------------------------------------