├── __init__.py ├── index └── .gitignore ├── models └── .gitignore ├── .gitignore ├── changelog.md ├── setup.py ├── rvc copy.yaml.example ├── LICENSE ├── README.md └── rvc_infer.py /__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /index/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | !.gitignore -------------------------------------------------------------------------------- /models/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | !.gitignore -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /output 2 | /rvc 3 | /venv 4 | /.venv 5 | /rvc_voices 6 | 7 | *.wav 8 | *.whl 9 | *.7z 10 | 11 | hubert_base.pt 12 | rmvpe.pt 13 | rvc.yaml -------------------------------------------------------------------------------- /changelog.md: -------------------------------------------------------------------------------- 1 | # Changelogs 2 | 3 | ## 8/18/2023 4 | - Added a return path in rvc_convert 5 | - Added mps for Mac for device configuration (untested as I don't have a Mac) 6 | 7 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | 3 | setup( 4 | name='rvc_tts_pipe', 5 | version='0.1', 6 | description='tts-pipeline', 7 | author='Jarod Mica', 8 | packages=find_packages(), 9 | ) -------------------------------------------------------------------------------- /rvc copy.yaml.example: -------------------------------------------------------------------------------- 1 | # If in quotes, do NOT remove quotes as code is expecting strings 2 | transpose: 0 # change pitch of voice 3 | audio_file: "path to audio file" # audio file path 4 | output_dir: "" # If you wanna change the name of output dir 5 | model_path: "models\\enter_pth_name" # Pytorch model name 6 | device: "cuda:0" # Uses CUDA GPU 7 | is_half: "False" 8 | f0method: "rmvpe" # options are: dio, harvest, crepe (good), rmvpe(also good) 9 | file_index: "" # path to voice index file if using it, leave blank if not 10 | file_index2: "" 11 | index_rate: 1 # strength of the index from 0 to 1 12 | filter_radius: 3 13 | resample_sr: 0 14 | rms_mix_rate: 1.0 15 | protect: 0.33 -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Jarod Mica 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # RVC-TTS-Pipeline 2 | Pipeline for TTS to RVC. This seems to produce the best sounding TTS with the closest representation to the original speaker's voice that one may have trained on. This works by passing in an audio file generated from some type of TTS (tortoise, vits, etc.) and then converting it using the trained weights from an RVC model. 3 | 4 | To get this to work, pytorch must be installed first on the system to allow RVC to be installable. If it's not, I was running into issues of having to uninstall and reinstall torch (though probably I should just adjust the requirements inside of rvc). 5 | 6 | **It is still a work in progress, there will be bugs and issues.** 7 | 8 | ## Installation 9 | 10 | 1. Install pytorch first here: https://pytorch.org/get-started/locally/ 11 | 12 | 2. Then, to install rvc, run the following: 13 | 14 | ``` 15 | pip install -e git+https://github.com/JarodMica/rvc.git#egg=rvc 16 | ``` 17 | 18 | 3. Lastly, you need to get the ```hubert_base.pt``` and ```rmvpe.pt``` files from rvc and put them into the parent directory of whatever project you're working on (or the SAME location of whereever you're running the scripts) 19 | 20 | **If you want to install rvc-tts-pipeline as it's own package, run the following (recommended)** 21 | 22 | ``` 23 | pip install -e git+https://github.com/JarodMica/rvc-tts-pipeline.git#egg=rvc_tts_pipe 24 | ``` 25 | 26 | This will allow you to import ```rvc_infer``` so that you do not have to move this package around. 27 | 28 | ## Basic usage 29 | The only function that should be called is the ```rvc_convert``` function. The only required parameters that are absolutely needed are: 30 | 31 | ```model_path = path to the model``` 32 | 33 | ```input_path = path to the audio file to convert (or tts output audio file)``` 34 | 35 | Then, it can simply be called like the following: 36 | 37 | ``` 38 | from rvc_infer import rvc_convert 39 | 40 | rvc_convert(model_path="your_model_path_here", input_path="your_audio_path_here") 41 | ``` 42 | 43 | The docstrings of rvc_convert details out other values you may want to play around with, probably the most important being pitch and f0method. 44 | 45 | ## Notes 46 | Currently, the github packages ONLY work if you install them in editable mode. Why exactly, I am not too sure but may have to do with package structure etc. If a time comes to where "-e" is not suitable for my projects, I will look for a solution when that comes. 47 | 48 | ## Acknowledgements 49 | Huge thanks to the RVC creators as none of this would be possible without them. This takes and uses a lot of their code in order to make this possible. 50 | -------------------------------------------------------------------------------- /rvc_infer.py: -------------------------------------------------------------------------------- 1 | import os,sys,pdb,torch 2 | now_dir = os.getcwd() 3 | sys.path.append(now_dir) 4 | import argparse 5 | import glob 6 | import sys 7 | import torch 8 | import numpy as np 9 | import yaml 10 | import pkg_resources 11 | import logging 12 | 13 | from multiprocessing import cpu_count 14 | from vc_infer_pipeline import VC 15 | from lib.infer_pack.models import SynthesizerTrnMs256NSFsid, SynthesizerTrnMs256NSFsid_nono, SynthesizerTrnMs768NSFsid, SynthesizerTrnMs768NSFsid_nono 16 | from lib.audio import load_audio 17 | 18 | from fairseq import checkpoint_utils 19 | from scipy.io import wavfile 20 | 21 | 22 | class Config: 23 | def __init__(self,device,is_half): 24 | self.device = device 25 | self.is_half = is_half 26 | self.n_cpu = 0 27 | self.gpu_name = None 28 | self.gpu_mem = None 29 | self.x_pad, self.x_query, self.x_center, self.x_max = self.device_config() 30 | 31 | def device_config(self) -> tuple: 32 | if torch.cuda.is_available(): 33 | i_device = int(self.device.split(":")[-1]) 34 | self.gpu_name = torch.cuda.get_device_name(i_device) 35 | if ( 36 | ("16" in self.gpu_name and "V100" not in self.gpu_name.upper()) 37 | or "P40" in self.gpu_name.upper() 38 | or "1060" in self.gpu_name 39 | or "1070" in self.gpu_name 40 | or "1080" in self.gpu_name 41 | ): 42 | print("16系/10系显卡和P40强制单精度") 43 | self.is_half = False 44 | for config_file in ["32k.json", "40k.json", "48k.json"]: 45 | with open(f"configs/{config_file}", "r") as f: 46 | strr = f.read().replace("true", "false") 47 | with open(f"configs/{config_file}", "w") as f: 48 | f.write(strr) 49 | with open("trainset_preprocess_pipeline_print.py", "r") as f: 50 | strr = f.read().replace("3.7", "3.0") 51 | with open("trainset_preprocess_pipeline_print.py", "w") as f: 52 | f.write(strr) 53 | else: 54 | self.gpu_name = None 55 | self.gpu_mem = int( 56 | torch.cuda.get_device_properties(i_device).total_memory 57 | / 1024 58 | / 1024 59 | / 1024 60 | + 0.4 61 | ) 62 | if self.gpu_mem <= 4: 63 | with open("trainset_preprocess_pipeline_print.py", "r") as f: 64 | strr = f.read().replace("3.7", "3.0") 65 | with open("trainset_preprocess_pipeline_print.py", "w") as f: 66 | f.write(strr) 67 | elif torch.backends.mps.is_available(): 68 | print("没有发现支持的N卡, 使用MPS进行推理") 69 | self.device = "mps" 70 | else: 71 | print("没有发现支持的N卡, 使用CPU进行推理") 72 | self.device = "cpu" 73 | self.is_half = True 74 | 75 | if self.n_cpu == 0: 76 | self.n_cpu = cpu_count() 77 | 78 | if self.is_half: 79 | # 6G显存配置 80 | x_pad = 3 81 | x_query = 10 82 | x_center = 60 83 | x_max = 65 84 | else: 85 | # 5G显存配置 86 | x_pad = 1 87 | x_query = 6 88 | x_center = 38 89 | x_max = 41 90 | 91 | if self.gpu_mem != None and self.gpu_mem <= 4: 92 | x_pad = 1 93 | x_query = 5 94 | x_center = 30 95 | x_max = 32 96 | 97 | return x_pad, x_query, x_center, x_max 98 | 99 | 100 | def get_path(name): 101 | ''' 102 | Built to get the path of a file based on where the initial script is being run from. 103 | 104 | Args: 105 | - name(str) : name of the file/folder 106 | ''' 107 | return os.path.join(os.getcwd(), name) 108 | 109 | def create_directory(name): 110 | ''' 111 | Creates a directory based on the location from which the script is run. Relies on 112 | get_path() 113 | 114 | Args: 115 | - name(str) : name of the file/folder 116 | ''' 117 | dir_name = get_path(name) 118 | if not os.path.exists(dir_name): 119 | os.makedirs(dir_name) 120 | 121 | def load_hubert(file_path="hubert_base.pt"): 122 | ''' 123 | Args: 124 | file_fath (str) : Direct path location to the hubert_base. If not specified, defaults to top level directory. 125 | ''' 126 | global hubert_model 127 | file_path = file_path 128 | models, _, _ = checkpoint_utils.load_model_ensemble_and_task( 129 | [file_path], 130 | suffix="", 131 | ) 132 | hubert_model = models[0] 133 | hubert_model = hubert_model.to(config.device) 134 | if config.is_half: 135 | hubert_model = hubert_model.half() 136 | else: 137 | hubert_model = hubert_model.float() 138 | hubert_model.eval() 139 | 140 | def vc_single( 141 | sid, 142 | input_audio_path, 143 | f0_up_key, 144 | f0_file, 145 | f0_method, 146 | file_index, 147 | file_index2, 148 | # file_big_npy, 149 | index_rate, 150 | filter_radius, 151 | resample_sr, 152 | rms_mix_rate, 153 | protect, 154 | ): # spk_item, input_audio0, vc_transform0,f0_file,f0method0 155 | global tgt_sr, net_g, vc, hubert_model, version 156 | f0_file = None 157 | if input_audio_path is None: 158 | return "You need to upload an audio", None 159 | f0_up_key = int(f0_up_key) 160 | audio = load_audio(input_audio_path, 16000) 161 | audio_max = np.abs(audio).max() / 0.95 162 | if audio_max > 1: 163 | audio /= audio_max 164 | times = [0, 0, 0] 165 | if not hubert_model: 166 | load_hubert() 167 | if_f0 = cpt.get("f0", 1) 168 | file_index = ( 169 | ( 170 | file_index.strip(" ") 171 | .strip('"') 172 | .strip("\n") 173 | .strip('"') 174 | .strip(" ") 175 | .replace("trained", "added") 176 | ) 177 | if file_index != "" 178 | else file_index2 179 | ) # 防止小白写错，自动帮他替换掉 180 | # file_big_npy = ( 181 | # file_big_npy.strip(" ").strip('"').strip("\n").strip('"').strip(" ") 182 | # ) 183 | audio_opt = vc.pipeline( 184 | hubert_model, 185 | net_g, 186 | sid, 187 | audio, 188 | input_audio_path, 189 | times, 190 | f0_up_key, 191 | f0_method, 192 | file_index, 193 | # file_big_npy, 194 | index_rate, 195 | if_f0, 196 | filter_radius, 197 | tgt_sr, 198 | resample_sr, 199 | rms_mix_rate, 200 | version, 201 | protect, 202 | f0_file=f0_file, 203 | ) 204 | return audio_opt 205 | 206 | def get_vc(model_path): 207 | global n_spk,tgt_sr,net_g,vc,cpt,device,is_half, version 208 | print("loading pth %s"%model_path) 209 | cpt = torch.load(model_path, map_location="cpu") 210 | tgt_sr = cpt["config"][-1] 211 | cpt["config"][-3]=cpt["weight"]["emb_g.weight"].shape[0]#n_spk 212 | if_f0=cpt.get("f0",1) 213 | version = cpt.get("version", "v1") 214 | if version == "v1": 215 | if if_f0 == 1: 216 | net_g = SynthesizerTrnMs256NSFsid( 217 | *cpt["config"], is_half=config.is_half 218 | ) 219 | else: 220 | net_g = SynthesizerTrnMs256NSFsid_nono(*cpt["config"]) 221 | elif version == "v2": 222 | if if_f0 == 1: 223 | net_g = SynthesizerTrnMs768NSFsid( 224 | *cpt["config"], is_half=config.is_half 225 | ) 226 | else: 227 | net_g = SynthesizerTrnMs768NSFsid_nono(*cpt["config"]) 228 | del net_g.enc_q 229 | print(net_g.load_state_dict(cpt["weight"], strict=False)) 230 | net_g.eval().to(device) 231 | if (is_half):net_g = net_g.half() 232 | else:net_g = net_g.float() 233 | vc = VC(tgt_sr, config) 234 | n_spk=cpt["config"][-3] 235 | # return {"visible": True,"maximum": n_spk, "__type__": "update"} 236 | 237 | def load_config(): 238 | current_dir = os.path.dirname(os.path.abspath(__file__)) 239 | yaml_file = os.path.join(current_dir, "rvc.yaml") 240 | 241 | with open(yaml_file, "r") as file: 242 | rvc_conf = yaml.safe_load(file) 243 | 244 | return rvc_conf 245 | 246 | def rvc_convert(model_path, 247 | f0_up_key=0, 248 | input_path=None, 249 | output_dir_path=None, 250 | _is_half="False", 251 | f0method="rmvpe", 252 | file_index="", 253 | file_index2="", 254 | index_rate=1, 255 | filter_radius=3, 256 | resample_sr=0, 257 | rms_mix_rate=0.5, 258 | protect=0.33, 259 | verbose=False 260 | ): 261 | ''' 262 | Function to call for the rvc voice conversion. All parameters are the same present in that of the webui 263 | 264 | Args: 265 | model_path (str) : path to the rvc voice model you're using 266 | f0_up_key (int) : transpose of the audio file, changes pitch (positive makes voice higher pitch) 267 | input_path (str) : path to audio file (use wav file) 268 | output_dir_path (str) : path to output directory, defaults to parent directory in output folder 269 | _is_half (str) : Determines half-precision 270 | f0method (str) : picks which f0 method to use: dio, harvest, crepe, rmvpe (requires rmvpe.pt) 271 | file_index (str) : path to file_index, defaults to None 272 | file_index2 (str) : path to file_index2, defaults to None. #honestly don't know what this is for 273 | index_rate (int) : strength of the index file if provided 274 | filter_radius (int) : if >=3: apply median filtering to the harvested pitch results. The value represents the filter radius and can reduce breathiness. 275 | resample_sr (int) : quality at which to resample audio to, defaults to no resample 276 | rmx_mix_rate (int) : adjust the volume envelope scaling. Closer to 0, the more it mimicks the volume of the original vocals. Can help mask noise and make volume sound more natural when set relatively low. Closer to 1 will be more of a consistently loud volume 277 | protect (int) : protect voiceless consonants and breath sounds to prevent artifacts such as tearing in electronic music. Set to 0.5 to disable. Decrease the value to increase protection, but it may reduce indexing accuracy 278 | 279 | Returns: 280 | output_file_path (str) : file path and name of tshe output wav file 281 | 282 | ''' 283 | global config, now_dir, hubert_model, tgt_sr, net_g, vc, cpt, device, is_half, version 284 | 285 | if torch.cuda.is_available(): 286 | device = "cuda:0" 287 | elif torch.backends.mps.is_available(): 288 | device = "mps:0" 289 | else: 290 | print("Cuda or MPS not detected") 291 | 292 | if not verbose: 293 | logging.getLogger('fairseq').setLevel(logging.ERROR) 294 | logging.getLogger('rvc').setLevel(logging.ERROR) 295 | 296 | is_half = _is_half 297 | 298 | if output_dir_path == None: 299 | output_dir_path = "output" 300 | output_file_name = "out.wav" 301 | output_dir = os.getcwd() 302 | output_file_path = os.path.join(output_dir,output_dir_path, output_file_name) 303 | else: 304 | # Mainly for Jarod's Vivy project, specify entire path + wav name 305 | output_file_path = output_dir_path 306 | pass 307 | 308 | create_directory(output_dir_path) 309 | output_dir = get_path(output_dir_path) 310 | 311 | if(is_half.lower() == 'true'): 312 | is_half = True 313 | else: 314 | is_half = False 315 | 316 | config=Config(device,is_half) 317 | now_dir=os.getcwd() 318 | sys.path.append(now_dir) 319 | 320 | hubert_model=None 321 | 322 | get_vc(model_path) 323 | wav_opt=vc_single(0,input_path,f0_up_key,None,f0method,file_index,file_index2,index_rate,filter_radius,resample_sr,rms_mix_rate,protect) 324 | wavfile.write(output_file_path, tgt_sr, wav_opt) 325 | print(f"\nFile finished writing to: {output_file_path}") 326 | 327 | return output_file_path 328 | 329 | def main(): 330 | # Need to comment out yaml setting for input audio 331 | rvc_convert(model_path="models\\ado.pth", input_path="delilah.wav") 332 | 333 | if __name__ == "__main__": 334 | main() --------------------------------------------------------------------------------