├── .gitignore ├── Bark_Coqui.ipynb ├── Bark_Voice_Cloning.ipynb ├── CosyVoice2.ipynb ├── Dockerfile ├── F5_TTS.ipynb ├── F5_TTS_Training.ipynb ├── GPT_SoVITS_v3.ipynb ├── GPT_SoVITS_v3_03_30.ipynb ├── GPT_SoVITS_v4.ipynb ├── LICENSE ├── README.md ├── README_zh.md ├── SambertHifigan.ipynb ├── Sambert_Voice_Cloning_in_One_Click.ipynb ├── Sambert中文声音克隆v2.ipynb ├── VALL_E_X.ipynb ├── Voice_Cloning_for_Chinese_Speech_v2.ipynb ├── XTTS_Colab.ipynb ├── app.py ├── bark ├── __init__.py ├── api.py ├── assets │ └── prompts │ │ ├── announcer.npz │ │ └── v2 │ │ ├── en_speaker_0.npz │ │ ├── en_speaker_1.npz │ │ ├── en_speaker_2.npz │ │ ├── en_speaker_3.npz │ │ ├── en_speaker_4.npz │ │ ├── en_speaker_5.npz │ │ ├── en_speaker_6.npz │ │ ├── en_speaker_7.npz │ │ ├── en_speaker_8.npz │ │ ├── en_speaker_9.npz │ │ ├── zh_speaker_0.npz │ │ ├── zh_speaker_1.npz │ │ ├── zh_speaker_2.npz │ │ ├── zh_speaker_3.npz │ │ ├── zh_speaker_4.npz │ │ ├── zh_speaker_5.npz │ │ ├── zh_speaker_6.npz │ │ ├── zh_speaker_7.npz │ │ ├── zh_speaker_8.npz │ │ └── zh_speaker_9.npz ├── generation.py ├── hubert │ ├── __init__.py │ ├── customtokenizer.py │ ├── hubert_manager.py │ └── pre_kmeans_hubert.py ├── model.py ├── model_fine.py └── settings.py ├── cloning ├── __init__.py └── clonevoice.py ├── config.yaml ├── notebooks ├── AI_Agent.ipynb ├── ChatGLM2_linghua_VITS2.ipynb ├── CosyVoice.ipynb ├── GPT_SoVITS.ipynb ├── GPT_SoVITS_2.ipynb ├── GPT_SoVITS_emo.ipynb ├── GPT_SoVITS_v2_0808.ipynb ├── KNN_VC.ipynb ├── MuseTalk.ipynb ├── MuseV.ipynb ├── NeuCoSVC.ipynb ├── NeuCoSVC_v2_先享版.ipynb ├── OpenAI_TTS_KNN_VC.ipynb ├── OpenAI_TTS_KNN_VC_en.ipynb ├── OpenAI_TTS_RVC.ipynb ├── OpenVoice.ipynb ├── README.md ├── Seamless_Meta.ipynb ├── train.json └── train_linghua_new_v3.json ├── pyproject.toml ├── requirements.txt ├── sambert-ui ├── README.md ├── Sambert_UI.ipynb ├── app.py ├── app_colab.py ├── consts.py ├── requirements.txt ├── utils_base.py ├── utils_label.py └── utils_sambert.py ├── setup.py ├── swap_voice.py ├── training ├── __init__.py ├── data.py ├── train.py └── training_prepare.py ├── util ├── __init__.py ├── helper.py ├── parseinput.py └── settings.py └── 阿里云笔记本训练.ipynb /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__/ 2 | /outputs 3 | /speakers 4 | .vs 5 | *.npz 6 | *.wav 7 | *.npy 8 | .vs/ 9 | /models 10 | /bark_ui_enhanced.egg-info 11 | /build/lib/bark 12 | *.pth 13 | *.pt 14 | *.zip 15 | -------------------------------------------------------------------------------- /CosyVoice2.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "provenance": [], 7 | "gpuType": "T4", 8 | "authorship_tag": "ABX9TyOIl+2q5kgHGZzuJiZOIFu3", 9 | "include_colab_link": true 10 | }, 11 | "kernelspec": { 12 | "name": "python3", 13 | "display_name": "Python 3" 14 | }, 15 | "language_info": { 16 | "name": "python" 17 | }, 18 | "accelerator": "GPU" 19 | }, 20 | "cells": [ 21 | { 22 | "cell_type": "markdown", 23 | "metadata": { 24 | "id": "view-in-github", 25 | "colab_type": "text" 26 | }, 27 | "source": [ 28 | "\"Open" 29 | ] 30 | }, 31 | { 32 | "cell_type": "code", 33 | "source": [ 34 | "# @title 一键运行 CosyVoice2 —— 最强情感真实复刻\n", 35 | "# Install condacolab and setup Conda environment\n", 36 | "!pip install -q condacolab\n", 37 | "import condacolab\n", 38 | "condacolab.install_from_url(\"https://repo.anaconda.com/miniconda/Miniconda3-py39_23.11.0-2-Linux-x86_64.sh\")\n", 39 | "\n", 40 | "# Clone the CosyVoice repository\n", 41 | "!git clone --recursive https://github.com/KevinWang676/CosyVoice.git\n", 42 | "%cd CosyVoice\n", 43 | "!git submodule update --init --recursive\n", 44 | "\n", 45 | "# Create a Conda environment with Python 3.10\n", 46 | "!conda create -n cosyvoice python=3.10 -y\n", 47 | "\n", 48 | "# Activate the Conda environment and install dependencies\n", 49 | "!source activate cosyvoice && conda install -y -q -c conda-forge pynini=2.1.5\n", 50 | "!source activate cosyvoice && pip install -r requirements.txt\n", 51 | "\n", 52 | "# Setup pretrained models\n", 53 | "!mkdir -p pretrained_models\n", 54 | "!git clone https://www.modelscope.cn/iic/CosyVoice2-0.5B.git pretrained_models/CosyVoice2-0.5B\n", 55 | "!git clone https://www.modelscope.cn/iic/CosyVoice-ttsfrd.git pretrained_models/CosyVoice-ttsfrd\n", 56 | "\n", 57 | "# Extract resources\n", 58 | "%cd pretrained_models/CosyVoice-ttsfrd/\n", 59 | "!sudo apt install unzip -y\n", 60 | "!unzip resource.zip -d .\n", 61 | "\n", 62 | "# Install the required wheels in Conda environment\n", 63 | "!source activate cosyvoice && pip install ttsfrd_dependency-0.1-py3-none-any.whl\n", 64 | "!source activate cosyvoice && pip install ttsfrd-0.4.2-cp310-cp310-linux_x86_64.whl\n", 65 | "\n", 66 | "# Navigate back and run the application\n", 67 | "%cd ..\n", 68 | "%cd ..\n", 69 | "!source activate cosyvoice && python webui.py" 70 | ], 71 | "metadata": { 72 | "id": "tZE5XrtwJiSD" 73 | }, 74 | "execution_count": null, 75 | "outputs": [] 76 | } 77 | ] 78 | } -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM debian:stable 2 | 3 | # Install system packages 4 | RUN apt update && apt install -y git pip 5 | 6 | # Create non-root user 7 | RUN useradd -m -d /bark bark 8 | 9 | # Run as new user 10 | USER bark 11 | WORKDIR /bark 12 | 13 | # Clone git repo 14 | RUN git clone https://github.com/C0untFloyd/bark-gui 15 | 16 | # Switch to git directory 17 | WORKDIR /bark/bark-gui 18 | 19 | # Append pip bin path to PATH 20 | ENV PATH=$PATH:/bark/.local/bin 21 | 22 | # Install dependancies 23 | RUN pip install . 24 | RUN pip install -r requirements.txt 25 | 26 | # List on all addresses, since we are in a container. 27 | RUN sed -i "s/server_name: ''/server_name: 0.0.0.0/g" ./config.yaml 28 | 29 | # Suggested volumes 30 | VOLUME /bark/bark-gui/assets/prompts/custom 31 | VOLUME /bark/bark-gui/models 32 | VOLUME /bark/.cache/huggingface/hub 33 | 34 | # Default port for web-ui 35 | EXPOSE 7860/tcp 36 | 37 | # Start script 38 | CMD python3 webui.py 39 | -------------------------------------------------------------------------------- /F5_TTS_Training.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "provenance": [], 7 | "gpuType": "T4", 8 | "authorship_tag": "ABX9TyOFDem9U8kpd5ypP6Y7M9h+", 9 | "include_colab_link": true 10 | }, 11 | "kernelspec": { 12 | "name": "python3", 13 | "display_name": "Python 3" 14 | }, 15 | "language_info": { 16 | "name": "python" 17 | }, 18 | "accelerator": "GPU" 19 | }, 20 | "cells": [ 21 | { 22 | "cell_type": "markdown", 23 | "metadata": { 24 | "id": "view-in-github", 25 | "colab_type": "text" 26 | }, 27 | "source": [ 28 | "\"Open" 29 | ] 30 | }, 31 | { 32 | "cell_type": "code", 33 | "source": [ 34 | "!git clone https://github.com/KevinWang676/F5-TTS.git\n", 35 | "%cd F5-TTS\n", 36 | "!pip install -e .\n", 37 | "!python src/f5_tts/train/finetune_f5_tts.py" 38 | ], 39 | "metadata": { 40 | "id": "CiXnh9tOTUGx" 41 | }, 42 | "execution_count": null, 43 | "outputs": [] 44 | }, 45 | { 46 | "cell_type": "code", 47 | "source": [], 48 | "metadata": { 49 | "id": "l8S7-edjTgzy" 50 | }, 51 | "execution_count": null, 52 | "outputs": [] 53 | } 54 | ] 55 | } -------------------------------------------------------------------------------- /GPT_SoVITS_v3.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "id": "view-in-github", 7 | "colab_type": "text" 8 | }, 9 | "source": [ 10 | "\"Open" 11 | ] 12 | }, 13 | { 14 | "cell_type": "code", 15 | "execution_count": null, 16 | "metadata": { 17 | "id": "LENDan0wB16N" 18 | }, 19 | "outputs": [], 20 | "source": [ 21 | "# @title 一键开启GPT-SoVITS-v3\n", 22 | "\n", 23 | "!pip install -q condacolab\n", 24 | "# Setting up condacolab and installing packages\n", 25 | "import condacolab\n", 26 | "condacolab.install_from_url(\"https://repo.anaconda.com/miniconda/Miniconda3-py39_23.11.0-2-Linux-x86_64.sh\")\n", 27 | "!git clone https://huggingface.co/kevinwang676/GPT-SoVITS-v3.git\n", 28 | "!conda install -y -q -c pytorch -c nvidia cudatoolkit\n", 29 | "%cd GPT-SoVITS-v3\n", 30 | "!conda install -y -q -c conda-forge gcc gxx ffmpeg cmake -c pytorch -c nvidia\n", 31 | "!/usr/local/bin/pip install -r requirements.txt\n", 32 | "!sudo apt install ffmpeg\n", 33 | "!sudo apt install libsox-dev\n", 34 | "!pip install LangSegment\n", 35 | "!pip install numpy==1.23.4\n", 36 | "import nltk\n", 37 | "nltk.download(\"all\")\n", 38 | "!mv tools/damo_asr/speech_fsmn_vad_zh-cn-16k-common-pytorch tools/asr/models/\n", 39 | "!mv tools/damo_asr/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch tools/asr/models/\n", 40 | "!mv tools/damo_asr/punc_ct-transformer_zh-cn-common-vocab272727-pytorch tools/asr/models/\n", 41 | "!/usr/local/bin/pip install ipykernel\n", 42 | "!sed -i '10s/False/True/' /content/GPT-SoVITS-v3/config.py\n", 43 | "%cd /content/GPT-SoVITS-v3/\n", 44 | "!/usr/local/bin/python webui.py zh_CN" 45 | ] 46 | } 47 | ], 48 | "metadata": { 49 | "accelerator": "GPU", 50 | "colab": { 51 | "gpuType": "T4", 52 | "provenance": [], 53 | "authorship_tag": "ABX9TyNUFUSb9BjmK/yNR6OqE232", 54 | "include_colab_link": true 55 | }, 56 | "kernelspec": { 57 | "display_name": "Python 3", 58 | "name": "python3" 59 | }, 60 | "language_info": { 61 | "name": "python" 62 | } 63 | }, 64 | "nbformat": 4, 65 | "nbformat_minor": 0 66 | } -------------------------------------------------------------------------------- /GPT_SoVITS_v3_03_30.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "id": "view-in-github", 7 | "colab_type": "text" 8 | }, 9 | "source": [ 10 | "\"Open" 11 | ] 12 | }, 13 | { 14 | "cell_type": "code", 15 | "source": [ 16 | "# @title 一键开启GPT-SoVITS-v3\n", 17 | "\n", 18 | "!pip install -q condacolab\n", 19 | "# Setting up condacolab and installing packages\n", 20 | "import condacolab\n", 21 | "condacolab.install_from_url(\"https://repo.anaconda.com/miniconda/Miniconda3-py39_23.11.0-2-Linux-x86_64.sh\")\n", 22 | "token=\"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\"\n", 23 | "%cd -q /content\n", 24 | "!git clone https://huggingface.co/kevinwang676/GPT-SoVITS-v3.git GPT-SoVITS\n", 25 | "!conda install -y -q -c pytorch -c nvidia cudatoolkit\n", 26 | "%cd -q /content/GPT-SoVITS\n", 27 | "!conda install -y -q -c conda-forge gcc gxx ffmpeg cmake -c pytorch -c nvidia\n", 28 | "!/usr/local/bin/pip install -r requirements_colab.txt\n", 29 | "\n", 30 | "!mv tools/damo_asr/speech_fsmn_vad_zh-cn-16k-common-pytorch tools/asr/models/\n", 31 | "!mv tools/damo_asr/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch tools/asr/models/\n", 32 | "!mv tools/damo_asr/punc_ct-transformer_zh-cn-common-vocab272727-pytorch tools/asr/models/\n", 33 | "\n", 34 | "!/usr/local/bin/pip install ipykernel\n", 35 | "!sed -i '10s/False/True/' /content/GPT-SoVITS/config.py\n", 36 | "%cd /content/GPT-SoVITS/\n", 37 | "!/usr/local/bin/python webui.py zh_CN" 38 | ], 39 | "metadata": { 40 | "id": "Ik7iS5JPJBBv" 41 | }, 42 | "execution_count": null, 43 | "outputs": [] 44 | }, 45 | { 46 | "cell_type": "code", 47 | "source": [ 48 | "# @title 一键上传模型（只需要修改.ckpt和.pth的路径）\n", 49 | "from huggingface_hub import HfApi, login\n", 50 | "import os\n", 51 | "\n", 52 | "def upload_files_to_hf(local_file_paths, repo_id, token=token, repo_type=\"model\"):\n", 53 | " login(token=token)\n", 54 | " api = HfApi()\n", 55 | " resolved_urls = {}\n", 56 | " for local_path in local_file_paths:\n", 57 | " if not os.path.exists(local_path):\n", 58 | " print(f\"File not found: {local_path}\")\n", 59 | " continue\n", 60 | " filename = os.path.basename(local_path)\n", 61 | " path_in_repo = filename\n", 62 | " api.upload_file(\n", 63 | " path_or_fileobj=local_path,\n", 64 | " path_in_repo=path_in_repo,\n", 65 | " repo_id=repo_id,\n", 66 | " repo_type=repo_type,\n", 67 | " )\n", 68 | " resolve_url = f\"https://huggingface.co/{repo_id}/resolve/main/{path_in_repo}\"\n", 69 | " resolved_urls[local_path] = resolve_url\n", 70 | "\n", 71 | " return resolved_urls\n", 72 | "\n", 73 | "# 需要修改下方这行代码的两个路径\n", 74 | "local_files = [\"GPT_weights_v3/jay_zhou-e15.ckpt\", \"SoVITS_weights_v3/jay_zhou_e2_s192.pth\"] # 这里需要把jay_zhou-e15.ckpt和jay_zhou_e2_s192.pth更换成自己训练的模型名\n", 75 | "resolved_urls = upload_files_to_hf(local_files, \"TalkTalkAI/doingdream\")\n", 76 | "print(resolved_urls)" 77 | ], 78 | "metadata": { 79 | "colab": { 80 | "base_uri": "https://localhost:8080/", 81 | "height": 87, 82 | "referenced_widgets": [ 83 | "2495e0ad67dd4dd6a3915372a7bd387d", 84 | "74d2bf2737b642c3bb3adfb23579bd04", 85 | "dd087db06e5240329849b13a51df8428", 86 | "e48943586b7d438a9e5b596563a54409", 87 | "313f6287eef04ba8b00830cecc3f88f4", 88 | "8ac65224e61a47c29b5605d659ee89ec", 89 | "704d2b479ae6485598b1499b1c901b7c", 90 | "c0d91594045f43b8abd655406006a441", 91 | "1dc1f71d1fa6400d8c6c062911f1b80a", 92 | "cc4bea7031f14cd6b72b1636819c5ffe", 93 | "b3c8306461394f1a800f2655fe1ae09c" 94 | ] 95 | }, 96 | "id": "5K0NaHOvEzvb", 97 | "outputId": "56743e9e-5e00-4afa-bbf0-eb95b16e36f0" 98 | }, 99 | "execution_count": null, 100 | "outputs": [ 101 | { 102 | "output_type": "display_data", 103 | "data": { 104 | "text/plain": [ 105 | "jay_zhou_e2_s192.pth: 0%| | 0.00/769M [00:00\"Open" 29 | ] 30 | }, 31 | { 32 | "cell_type": "code", 33 | "source": [ 34 | "#@title 一键运行 GPT-SoVITS-v4\n", 35 | "#点击左侧运行按钮后，如果运行过程中遇到弹窗提醒，点击取消cancel即可\n", 36 | "!git clone https://huggingface.co/kevinwang676/GPT-SoVITS-v4.git\n", 37 | "%cd GPT-SoVITS-v4\n", 38 | "!pip install -r requirements.txt\n", 39 | "!export is_share=True && python webui.py zh_CN" 40 | ], 41 | "metadata": { 42 | "id": "vj0zXyLTT7Kq" 43 | }, 44 | "execution_count": null, 45 | "outputs": [] 46 | }, 47 | { 48 | "cell_type": "code", 49 | "source": [], 50 | "metadata": { 51 | "id": "crCE-LMZULX_" 52 | }, 53 | "execution_count": null, 54 | "outputs": [] 55 | } 56 | ] 57 | } -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) Suno, Inc 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Bark Voice Cloning 🐶 & Voice Cloning for Chinese Speech 🎶 2 | ### [简体中文](https://github.com/KevinWang676/Bark-Voice-Cloning/blob/main/README_zh.md) 3 | ## 1️⃣ Bark Voice Cloning 4 | 5 | > 10/19/2023: Fixed `ERROR: Exception in ASGI application` by specifying `gradio==3.33.0` and `gradio_client==0.2.7` in [requirements.txt](https://github.com/KevinWang676/Bark-Voice-Cloning/blob/main/requirements.txt). 6 | 7 | > 11/08/2023: Integrated [KNN-VC](https://github.com/bshall/knn-vc) into [OpenAI TTS](https://platform.openai.com/docs/guides/text-to-speech) and created an easy-to-use Gradio interface. Try it [here](https://colab.research.google.com/github/KevinWang676/Bark-Voice-Cloning/blob/main/notebooks/OpenAI_TTS_KNN_VC_en.ipynb). 8 | 9 | > 02/27/2024: We are thrilled to launch our most powerful **AI song cover generator** ever with [Shanghai Artificial Intelligence Laboratory](https://www.shlab.org.cn/)! Just need to provide the name of a song and our application running on an **A100** GPU will handle everything else. Check it out in our [**website**](https://www.talktalkai.com/) (please click "EN" in the first tab of our website to see the english version)! 💕 10 | 11 | Based on [bark-gui](https://github.com/C0untFloyd/bark-gui) and [bark](https://github.com/suno-ai/bark). Thanks to [C0untFloyd](https://github.com/C0untFloyd). 12 | 13 | **Quick start**: [**Colab Notebook**](https://colab.research.google.com/github/KevinWang676/Bark-Voice-Cloning/blob/main/Bark_Voice_Cloning.ipynb) ⚡ 14 | 15 | **HuggingFace Demo**: [**Bark Voice Cloning**](https://huggingface.co/spaces/kevinwang676/Bark-with-Voice-Cloning) 🤗 (Need a GPU) 16 | 17 | **Demo Video**: [**YouTube Video**](https://www.youtube.com/watch?v=IAf695dhkUc&t=4s) 18 | 19 | If you would like to run the code locally, remember to replace the original path `/content/Bark-Voice-Cloning/bark/assets/prompts/file.npz` with the path of `file.npz` in your own computer. 20 | 21 | ### If you like the quick start, please star this repository. ⭐⭐⭐ 22 | 23 | ## Easy to use: 24 | 25 | (1) First upload audio for voice cloning and click `Create Voice`. 26 | 27 | ![image](https://github.com/KevinWang676/Bark-Voice-Cloning/assets/126712357/65e2b695-f529-4fb5-9549-4e86e6a4d8b2) 28 | 29 | (2) Choose the option called "file" in `Voice` if you'd like to use voice cloning. 30 | 31 | (3) Click `Generate`. Done! 32 | 33 | ![image](https://github.com/KevinWang676/Bark-Voice-Cloning/assets/126712357/20911e37-768d-47d5-bb86-d12a3ab04c5d) 34 | 35 | ## 2️⃣ Voice Cloning for Chinese Speech 36 | > 10/26/2023: Integrated labeling, training and inference into an easy-to-use user interface of SambertHifigan. Thanks to [wujohns](https://github.com/wujohns). 37 | 38 | We want to point out that [Bark](https://github.com/suno-ai/bark) is very good at generating English speech but relatively poor at generating Chinese speech. So we'd like to adopt another approach, which is called [SambertHifigan](https://www.modelscope.cn/models/speech_tts/speech_sambert-hifigan_tts_zh-cn_multisp_pretrain_16k/summary), to realizing voice cloning for Chinese speech. Please check out our [Colab Notebook](https://colab.research.google.com/github/KevinWang676/Bark-Voice-Cloning/blob/main/Voice_Cloning_for_Chinese_Speech_v2.ipynb) for the implementation. 39 | 40 | Quick start: [Colab Notebook](https://colab.research.google.com/github/KevinWang676/Bark-Voice-Cloning/blob/main/sambert-ui/Sambert_UI.ipynb) ⚡ 41 | 42 | HuggingFace demo: [Voice Cloning for Chinese Speech](https://huggingface.co/spaces/kevinwang676/Personal-TTS) 🤗 43 | 44 | [![Star History Chart](https://api.star-history.com/svg?repos=KevinWang676/Bark-Voice-Cloning&type=Date)](https://star-history.com/#KevinWang676/Bark-Voice-Cloning&Date) 45 | -------------------------------------------------------------------------------- /README_zh.md: -------------------------------------------------------------------------------- 1 | # 第二代Bark声音克隆 🐶 & 全新中文声音克隆 🎶 2 | 3 | ## 1️⃣ 第二代Bark声音克隆 4 | 5 | > 11/08/2023更新：将AI变声模型[KNN-VC](https://github.com/bshall/knn-vc)与最新发布的[OpenAI TTS](https://platform.openai.com/docs/guides/text-to-speech)结合，实现更加真实的AI变声，您可以[在线体验](https://huggingface.co/spaces/kevinwang676/OpenAI-TTS-Voice-Conversion)或在[Colab笔记本](https://colab.research.google.com/github/KevinWang676/Bark-Voice-Cloning/blob/main/notebooks/OpenAI_TTS_KNN_VC.ipynb)中运行 6 | 7 | > 11/13/2023更新：将声音转换模型[RVC](https://github.com/RVC-Project/Retrieval-based-Voice-Conversion-WebUI)与最新发布的[OpenAI TTS](https://platform.openai.com/docs/guides/text-to-speech)结合，您可以使用[Colab笔记本](https://colab.research.google.com/github/KevinWang676/Bark-Voice-Cloning/blob/main/notebooks/OpenAI_TTS_RVC.ipynb)运行，点击[这里](https://github.com/KevinWang676/Bark-Voice-Cloning/assets/126712357/e7fa4d21-d616-41b1-be34-5d420f65c943)试听效果 8 | 9 | > 11/23/2023更新：Sambert声音克隆在线体验的bug已修复，[点击使用](https://huggingface.co/spaces/kevinwang676/Personal-TTS) 10 | 11 | > 12/01/2023更新：ChatGLM2神里绫华模型+Bert-VITS2文本转语音，和绫华一起谈天说地吧，点击[Colab笔记本](https://colab.research.google.com/github/KevinWang676/Bark-Voice-Cloning/blob/main/notebooks/ChatGLM2_linghua_VITS2.ipynb)运行，[在线使用](https://kevinwang676-chatglm2-bert-vits2-lh.hf.space) 12 | 13 | > 12/03/2023更新：Sambert声音克隆本地部署教程以及[Sambert UI Colab](https://colab.research.google.com/github/KevinWang676/Bark-Voice-Cloning/blob/main/sambert-ui/Sambert_UI.ipynb)已上传，[点击查看](https://github.com/KevinWang676/Bark-Voice-Cloning/tree/main/sambert-ui) 14 | 15 | > 01/06/2024更新：NeuCoSVC声音转换，无需训练、一键AI翻唱任意歌手的任意歌曲。点击[在线使用](https://kevinwang676-neucosvc-colab.hf.space)，点击[Colab笔记本](https://colab.research.google.com/github/KevinWang676/Bark-Voice-Cloning/blob/main/notebooks/NeuCoSVC.ipynb) 16 | 17 | > 01/20/2024更新：GPT-SoVITS声音克隆，只需一分钟语音，“完美”复刻任何声音。点击[在线体验](https://kevinwang676-gpt-sovits-jay.hf.space)，点击[Colab笔记本](https://colab.research.google.com/github/KevinWang676/Bark-Voice-Cloning/blob/main/notebooks/GPT_SoVITS.ipynb)，[视频教程](https://www.bilibili.com/video/BV1pb4y1N79s) 18 | 19 | > 02/27/2024更新：**AI歌手全明星**在线训练+翻唱上线啦！第一个完全**免费**、**开源**的AI歌手模型社区，全网AI歌手任您选择！点击访问[**滔滔AI**](https://www.talktalkai.com/)解锁全部内容，感谢[OpenXLab](https://openxlab.org.cn/apps)的算力支持！💕 20 | 21 | 在线快速运行：[Colab笔记本](https://colab.research.google.com/github/KevinWang676/Bark-Voice-Cloning/blob/main/Bark_Voice_Cloning.ipynb) ⚡ 22 | 23 | HuggingFace在线程序：[Bark声音克隆](https://huggingface.co/spaces/kevinwang676/Bark-with-Voice-Cloning) 🤗 24 | 25 | 使用指南：[B站视频](https://www.bilibili.com/video/BV16g4y1N7ZG) 📺 26 | 27 | _注：(1) Bark声音克隆功能基于[bark-gui](https://github.com/C0untFloyd/bark-gui)项目；(2) 运行时需要使用GPU_ 28 | 29 | #### 如果您喜欢这个项目，请在Github上点赞吧！ ⭐⭐⭐ 30 | 31 | ## 2️⃣ VALL-E X 全新声音克隆 32 | > 08/26/2023更新：VALL-E X 声音克隆，支持中日英三语；只需3秒语音，即可快速复刻您喜欢的音色 33 | 34 | [VALL-E X](https://www.microsoft.com/en-us/research/project/vall-e-x/)是由微软团队开发的支持多语种的语音合成模型，此部分基于Plachtaa的开源项目[VALL-E-X](https://github.com/Plachtaa/VALL-E-X)，进行了用户界面和功能上的优化。您可以使用我们制作的[专属工具](https://kevinwang676-voicechangers.hf.space/)从B站直接提取视频中的语音，只需要填写视频的BV号和起止时间。 35 | 36 | Colab快速启动: [Colab笔记本](https://colab.research.google.com/github/KevinWang676/Bark-Voice-Cloning/blob/main/VALL_E_X.ipynb) 37 | 38 | HuggingFace在线程序: [VALL-E X在线](https://huggingface.co/spaces/kevinwang676/VALLE) 🤗 39 | 40 | ## 3️⃣ SambertHifigan中文声音克隆 41 | > 07/19/2023更新：在执行`pip install kantts -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html`前需要先执行`pip install pysptk --no-build-isolation` (已在对应的Colab笔记本中更新) 42 | 43 | > 08/27/2023更新：已修复SambertHifigan对应的Colab Notebook中的所有bug，[点击此处使用](https://colab.research.google.com/github/KevinWang676/Bark-Voice-Cloning/blob/main/Voice_Cloning_for_Chinese_Speech_v2.ipynb) 44 | 45 | > 09/09/2023更新：增加SambertHifigan中文声音克隆的在线一键启动版 [Colab笔记本](https://colab.research.google.com/github/KevinWang676/Bark-Voice-Cloning/blob/main/Sambert_Voice_Cloning_in_One_Click.ipynb)，使用更加方便 🌟 46 | 47 | > 09/12/2023更新：增加[AutoDL镜像](https://www.codewithgpu.com/i/KevinWang676/Bark-Voice-Cloning/Sambert-VC)，支持在线GPU一键部署，快速开启声音克隆之旅 🍻 48 | 49 | ### 训练5分钟，通话不限时！ 🌞 50 | 51 | 因为[Bark](https://github.com/suno-ai/bark)文本转语音的中文效果远远不如英文的效果好，所以我们采用一种新的技术路径[SambertHifigan](https://www.modelscope.cn/models/speech_tts/speech_sambert-hifigan_tts_zh-cn_multisp_pretrain_16k/summary)来实现中文的声音克隆功能。 52 | 53 | ### 如何使用 💡 [视频教程](https://www.bilibili.com/video/BV1Ch4y1Z7K6) 54 | 55 | ### (1) 准备并上传一段中文语音：单一说话人、长度一分钟左右的`.wav`文件。 56 | 57 | 我们的程序能够自动将您上传的语音切片。您可以使用我们制作的[专属工具](https://kevinwang676-voicechangers.hf.space/)从B站直接提取视频中的语音，只需要填写视频的BV号和起止时间。为了达到更好的声音克隆效果，中文语音素材需要符合以下**要求**： 58 | 59 | * 音频尽量是干净人声，不要有BGM，不要有比较大的杂音，不要有一些特殊的声效，比如回声等 60 | * 声音的情绪尽量稳定，以说话的语料为主，不要是『嗯』『啊』『哈』之类的语气词 61 | 62 | ### (2) 使用我们的[Colab笔记本](https://colab.research.google.com/github/KevinWang676/Bark-Voice-Cloning/blob/main/Voice_Cloning_for_Chinese_Speech_v2.ipynb)运行程序：运行所有代码即可。 63 | 64 | 您可以在Colab笔记本的`推理`代码模块更改中文文本，进而输出您想要的内容。运行笔记本时的**注意事项**： 65 | 66 | * 上传音频素材后，需要将代码`split_long_audio(whisper_model, "filename.wav", "test", "dataset_raw")`中的`filename`替换成音频文件的名字 67 | * 需要在Colab中新建三个文件夹，分别名为：`test_wavs`，`output_training_data`，`pretrain_work_dir` 68 | * 训练完成后，在推理模块的`output = inference(input="大家好呀，欢迎使用滔滔智能的声音克隆产品！")`代码处可以自由编辑中文文本，实现中文声音克隆 69 | * 整个过程都需要使用GPU；如果使用阿里云笔记本，则不需要在终端中执行第一步的环境设置 70 | 71 | ### (3) 一键在HuggingFace上免费部署 🤗 72 | 73 | * 完成训练后，在Colab或阿里云笔记本中运行`!zip -r ./model.zip ./pretrain_work_dir`打包模型文件夹，下载并解压到本地 74 | * 点击进入[HuggingFace程序](https://huggingface.co/spaces/kevinwang676/Personal-TTS)，点击右上角的三个圆点，选择`Duplicate this Space`将程序复制到自己的HuggingFace主页 75 | * 点击进入`Files`，选择右上角`Add file`后，点击`Upload files`，将解压后的文件夹`pretrain_work_dir`从本地直接拖拽上传；需要先删除原有的`pretrain_work_dir`同名文件夹 76 | 77 | ### 四种使用方式 😄 78 | 79 | **推荐**🌟 阿里云笔记本在线运行：您也可以**免费**使用阿里云提供的[在线笔记本](https://modelscope.cn/models/damo/speech_personal_sambert-hifigan_nsf_tts_zh-cn_pretrain_16k/summary)进行训练，进入页面后点击右上角的`Notebook快速开发`，选择GPU环境，上传代码文件[阿里云专属笔记本（可下载）](https://github.com/KevinWang676/Bark-Voice-Cloning/blob/main/%E9%98%BF%E9%87%8C%E4%BA%91%E7%AC%94%E8%AE%B0%E6%9C%AC%E8%AE%AD%E7%BB%83.ipynb)和`.wav`文件素材后就能够以同样的方式运行啦！ 🍻 80 | 81 | Colab在线快速运行: [Colab笔记本](https://colab.research.google.com/github/KevinWang676/Bark-Voice-Cloning/blob/main/Voice_Cloning_for_Chinese_Speech_v2.ipynb) ⚡ 82 | 83 | HuggingFace在线程序: [全新中文声音克隆](https://huggingface.co/spaces/kevinwang676/Personal-TTS) 🤗 84 | 85 | 阿里魔搭社区在线程序：[个人声音定制](https://modelscope.cn/studios/damo/personal_tts/summary) 🎤 86 | 87 | ### 一键运行版本：最新[Colab笔记本](https://colab.research.google.com/github/KevinWang676/Bark-Voice-Cloning/blob/main/Sambert_Voice_Cloning_in_One_Click.ipynb)（有时不稳定）及[AutoDL镜像](https://www.codewithgpu.com/i/KevinWang676/Bark-Voice-Cloning/Sambert-VC)运行（推荐） 88 | 89 | * 最简洁的操作界面，一键上传语音素材，无需修改代码 90 | * 点击进入[AutoDL镜像](https://www.codewithgpu.com/i/KevinWang676/Bark-Voice-Cloning/Sambert-VC)，创建新实例，按照操作指南即可快速开启声音克隆之旅 🎶 91 | 92 | **注意❗：请不要生成会对个人以及组织造成侵害的内容，此程序仅供科研、学习及个人娱乐使用，严禁商业化运作。用户上传的语音及生成的内容均与本代码仓库所有者无关。** 93 | 94 | ![image](https://github.com/KevinWang676/Bark-Voice-Cloning/assets/126712357/7597122b-307f-41de-abdd-454dc0db5271) 95 | 96 | [![Star History Chart](https://api.star-history.com/svg?repos=KevinWang676/Bark-Voice-Cloning&type=Date)](https://star-history.com/#KevinWang676/Bark-Voice-Cloning&Date) 97 | -------------------------------------------------------------------------------- /app.py: -------------------------------------------------------------------------------- 1 | from cProfile import label 2 | import dataclasses 3 | from distutils.command.check import check 4 | from doctest import Example 5 | import gradio as gr 6 | import os 7 | import sys 8 | import numpy as np 9 | import logging 10 | import torch 11 | import pytorch_seed 12 | import time 13 | 14 | from xml.sax import saxutils 15 | from bark.api import generate_with_settings 16 | from bark.api import save_as_prompt 17 | from util.settings import Settings 18 | #import nltk 19 | 20 | from bark import SAMPLE_RATE 21 | from cloning.clonevoice import clone_voice 22 | from bark.generation import SAMPLE_RATE, preload_models, _load_history_prompt, codec_decode 23 | from scipy.io.wavfile import write as write_wav 24 | from util.parseinput import split_and_recombine_text, build_ssml, is_ssml, create_clips_from_ssml 25 | from datetime import datetime 26 | from tqdm.auto import tqdm 27 | from util.helper import create_filename, add_id3_tag 28 | from swap_voice import swap_voice_from_audio 29 | from training.training_prepare import prepare_semantics_from_text, prepare_wavs_from_semantics 30 | from training.train import training_prepare_files, train 31 | 32 | settings = Settings('config.yaml') 33 | 34 | def generate_text_to_speech(text, selected_speaker, text_temp, waveform_temp, eos_prob, quick_generation, complete_settings, seed, batchcount, progress=gr.Progress(track_tqdm=True)): 35 | # Chunk the text into smaller pieces then combine the generated audio 36 | 37 | # generation settings 38 | if selected_speaker == 'None': 39 | selected_speaker = None 40 | 41 | voice_name = selected_speaker 42 | 43 | if text == None or len(text) < 1: 44 | if selected_speaker == None: 45 | raise gr.Error('No text entered!') 46 | 47 | # Extract audio data from speaker if no text and speaker selected 48 | voicedata = _load_history_prompt(voice_name) 49 | audio_arr = codec_decode(voicedata["fine_prompt"]) 50 | result = create_filename(settings.output_folder_path, "None", "extract",".wav") 51 | save_wav(audio_arr, result) 52 | return result 53 | 54 | if batchcount < 1: 55 | batchcount = 1 56 | 57 | 58 | silenceshort = np.zeros(int((float(settings.silence_sentence) / 1000.0) * SAMPLE_RATE), dtype=np.int16) # quarter second of silence 59 | silencelong = np.zeros(int((float(settings.silence_speakers) / 1000.0) * SAMPLE_RATE), dtype=np.float32) # half a second of silence 60 | use_last_generation_as_history = "Use last generation as history" in complete_settings 61 | save_last_generation = "Save generation as Voice" in complete_settings 62 | for l in range(batchcount): 63 | currentseed = seed 64 | if seed != None and seed > 2**32 - 1: 65 | logger.warning(f"Seed {seed} > 2**32 - 1 (max), setting to random") 66 | currentseed = None 67 | if currentseed == None or currentseed <= 0: 68 | currentseed = np.random.default_rng().integers(1, 2**32 - 1) 69 | assert(0 < currentseed and currentseed < 2**32) 70 | 71 | progress(0, desc="Generating") 72 | 73 | full_generation = None 74 | 75 | all_parts = [] 76 | complete_text = "" 77 | text = text.lstrip() 78 | if is_ssml(text): 79 | list_speak = create_clips_from_ssml(text) 80 | prev_speaker = None 81 | for i, clip in tqdm(enumerate(list_speak), total=len(list_speak)): 82 | selected_speaker = clip[0] 83 | # Add pause break between speakers 84 | if i > 0 and selected_speaker != prev_speaker: 85 | all_parts += [silencelong.copy()] 86 | prev_speaker = selected_speaker 87 | text = clip[1] 88 | text = saxutils.unescape(text) 89 | if selected_speaker == "None": 90 | selected_speaker = None 91 | 92 | print(f"\nGenerating Text ({i+1}/{len(list_speak)}) -> {selected_speaker} (Seed {currentseed}):`{text}`") 93 | complete_text += text 94 | with pytorch_seed.SavedRNG(currentseed): 95 | audio_array = generate_with_settings(text_prompt=text, voice_name=selected_speaker, semantic_temp=text_temp, coarse_temp=waveform_temp, eos_p=eos_prob) 96 | currentseed = torch.random.initial_seed() 97 | if len(list_speak) > 1: 98 | filename = create_filename(settings.output_folder_path, currentseed, "audioclip",".wav") 99 | save_wav(audio_array, filename) 100 | add_id3_tag(filename, text, selected_speaker, currentseed) 101 | 102 | all_parts += [audio_array] 103 | else: 104 | texts = split_and_recombine_text(text, settings.input_text_desired_length, settings.input_text_max_length) 105 | for i, text in tqdm(enumerate(texts), total=len(texts)): 106 | print(f"\nGenerating Text ({i+1}/{len(texts)}) -> {selected_speaker} (Seed {currentseed}):`{text}`") 107 | complete_text += text 108 | if quick_generation == True: 109 | with pytorch_seed.SavedRNG(currentseed): 110 | audio_array = generate_with_settings(text_prompt=text, voice_name=selected_speaker, semantic_temp=text_temp, coarse_temp=waveform_temp, eos_p=eos_prob) 111 | currentseed = torch.random.initial_seed() 112 | else: 113 | full_output = use_last_generation_as_history or save_last_generation 114 | if full_output: 115 | full_generation, audio_array = generate_with_settings(text_prompt=text, voice_name=voice_name, semantic_temp=text_temp, coarse_temp=waveform_temp, eos_p=eos_prob, output_full=True) 116 | else: 117 | audio_array = generate_with_settings(text_prompt=text, voice_name=voice_name, semantic_temp=text_temp, coarse_temp=waveform_temp, eos_p=eos_prob) 118 | 119 | # Noticed this in the HF Demo - convert to 16bit int -32767/32767 - most used audio format 120 | # audio_array = (audio_array * 32767).astype(np.int16) 121 | 122 | if len(texts) > 1: 123 | filename = create_filename(settings.output_folder_path, currentseed, "audioclip",".wav") 124 | save_wav(audio_array, filename) 125 | add_id3_tag(filename, text, selected_speaker, currentseed) 126 | 127 | if quick_generation == False and (save_last_generation == True or use_last_generation_as_history == True): 128 | # save to npz 129 | voice_name = create_filename(settings.output_folder_path, seed, "audioclip", ".npz") 130 | save_as_prompt(voice_name, full_generation) 131 | if use_last_generation_as_history: 132 | selected_speaker = voice_name 133 | 134 | all_parts += [audio_array] 135 | # Add short pause between sentences 136 | if text[-1] in "!?.\n" and i > 1: 137 | all_parts += [silenceshort.copy()] 138 | 139 | # save & play audio 140 | result = create_filename(settings.output_folder_path, currentseed, "final",".wav") 141 | save_wav(np.concatenate(all_parts), result) 142 | # write id3 tag with text truncated to 60 chars, as a precaution... 143 | add_id3_tag(result, complete_text, selected_speaker, currentseed) 144 | 145 | return result 146 | 147 | 148 | 149 | def save_wav(audio_array, filename): 150 | write_wav(filename, SAMPLE_RATE, audio_array) 151 | 152 | def save_voice(filename, semantic_prompt, coarse_prompt, fine_prompt): 153 | np.savez_compressed( 154 | filename, 155 | semantic_prompt=semantic_prompt, 156 | coarse_prompt=coarse_prompt, 157 | fine_prompt=fine_prompt 158 | ) 159 | 160 | 161 | def on_quick_gen_changed(checkbox): 162 | if checkbox == False: 163 | return gr.CheckboxGroup.update(visible=True) 164 | return gr.CheckboxGroup.update(visible=False) 165 | 166 | def delete_output_files(checkbox_state): 167 | if checkbox_state: 168 | outputs_folder = os.path.join(os.getcwd(), settings.output_folder_path) 169 | if os.path.exists(outputs_folder): 170 | purgedir(outputs_folder) 171 | return False 172 | 173 | 174 | # https://stackoverflow.com/a/54494779 175 | def purgedir(parent): 176 | for root, dirs, files in os.walk(parent): 177 | for item in files: 178 | # Delete subordinate files 179 | filespec = os.path.join(root, item) 180 | os.unlink(filespec) 181 | for item in dirs: 182 | # Recursively perform this operation for subordinate directories 183 | purgedir(os.path.join(root, item)) 184 | 185 | def convert_text_to_ssml(text, selected_speaker): 186 | return build_ssml(text, selected_speaker) 187 | 188 | 189 | def training_prepare(selected_step, num_text_generations, progress=gr.Progress(track_tqdm=True)): 190 | if selected_step == prepare_training_list[0]: 191 | prepare_semantics_from_text() 192 | else: 193 | prepare_wavs_from_semantics() 194 | return None 195 | 196 | 197 | def start_training(save_model_epoch, max_epochs, progress=gr.Progress(track_tqdm=True)): 198 | training_prepare_files("./training/data/", "./training/data/checkpoint/hubert_base_ls960.pt") 199 | train("./training/data/", save_model_epoch, max_epochs) 200 | return None 201 | 202 | 203 | 204 | def apply_settings(themes, input_server_name, input_server_port, input_server_public, input_desired_len, input_max_len, input_silence_break, input_silence_speaker): 205 | settings.selected_theme = themes 206 | settings.server_name = input_server_name 207 | settings.server_port = input_server_port 208 | settings.server_share = input_server_public 209 | settings.input_text_desired_length = input_desired_len 210 | settings.input_text_max_length = input_max_len 211 | settings.silence_sentence = input_silence_break 212 | settings.silence_speaker = input_silence_speaker 213 | settings.save() 214 | 215 | def restart(): 216 | global restart_server 217 | restart_server = True 218 | 219 | 220 | def create_version_html(): 221 | python_version = ".".join([str(x) for x in sys.version_info[0:3]]) 222 | versions_html = f""" 223 | python: {python_version} 224 |  • 225 | torch: {getattr(torch, '__long_version__',torch.__version__)} 226 |  • 227 | gradio: {gr.__version__} 228 | """ 229 | return versions_html 230 | 231 | 232 | 233 | logger = logging.getLogger(__name__) 234 | APPTITLE = "Bark Voice Cloning UI" 235 | 236 | 237 | autolaunch = False 238 | 239 | if len(sys.argv) > 1: 240 | autolaunch = "-autolaunch" in sys.argv 241 | 242 | if torch.cuda.is_available() == False: 243 | os.environ['BARK_FORCE_CPU'] = 'True' 244 | logger.warning("No CUDA detected, fallback to CPU!") 245 | 246 | print(f'smallmodels={os.environ.get("SUNO_USE_SMALL_MODELS", False)}') 247 | print(f'enablemps={os.environ.get("SUNO_ENABLE_MPS", False)}') 248 | print(f'offloadcpu={os.environ.get("SUNO_OFFLOAD_CPU", False)}') 249 | print(f'forcecpu={os.environ.get("BARK_FORCE_CPU", False)}') 250 | print(f'autolaunch={autolaunch}\n\n') 251 | 252 | #print("Updating nltk\n") 253 | #nltk.download('punkt') 254 | 255 | print("Preloading Models\n") 256 | preload_models() 257 | 258 | available_themes = ["Default", "gradio/glass", "gradio/monochrome", "gradio/seafoam", "gradio/soft", "gstaff/xkcd", "freddyaboulton/dracula_revamped", "ysharma/steampunk"] 259 | tokenizer_language_list = ["de","en", "pl"] 260 | prepare_training_list = ["Step 1: Semantics from Text","Step 2: WAV from Semantics"] 261 | 262 | seed = -1 263 | server_name = settings.server_name 264 | if len(server_name) < 1: 265 | server_name = None 266 | server_port = settings.server_port 267 | if server_port <= 0: 268 | server_port = None 269 | global run_server 270 | global restart_server 271 | 272 | run_server = True 273 | 274 | while run_server: 275 | # Collect all existing speakers/voices in dir 276 | speakers_list = [] 277 | 278 | for root, dirs, files in os.walk("./bark/assets/prompts"): 279 | for file in files: 280 | if file.endswith(".npz"): 281 | pathpart = root.replace("./bark/assets/prompts", "") 282 | name = os.path.join(pathpart, file[:-4]) 283 | if name.startswith("/") or name.startswith("\\"): 284 | name = name[1:] 285 | speakers_list.append(name) 286 | 287 | speakers_list = sorted(speakers_list, key=lambda x: x.lower()) 288 | speakers_list.insert(0, 'None') 289 | 290 | print(f'Launching {APPTITLE} Server') 291 | 292 | # Create Gradio Blocks 293 | 294 | with gr.Blocks(title=f"{APPTITLE}", mode=f"{APPTITLE}", theme=settings.selected_theme) as barkgui: 295 | gr.Markdown("#
🐶🎶⭐ - Bark Voice Cloning
") 296 | gr.Markdown("##
🤗 - If you like this space, please star my [github repo](https://github.com/KevinWang676/Bark-Voice-Cloning)
") 297 | gr.Markdown("###
🎡 - Based on [bark-gui](https://github.com/C0untFloyd/bark-gui)
") 298 | gr.Markdown(f""" You can duplicate and use it with a GPU: Duplicate Space 299 | or open in [Colab](https://colab.research.google.com/github/KevinWang676/Bark-Voice-Cloning/blob/main/Bark_Voice_Cloning_UI.ipynb) for quick start 🌟 300 | """) 301 | 302 | with gr.Tab("🎙️ - Clone Voice"): 303 | with gr.Row(): 304 | input_audio_filename = gr.Audio(label="Input audio.wav", source="upload", type="filepath") 305 | #transcription_text = gr.Textbox(label="Transcription Text", lines=1, placeholder="Enter Text of your Audio Sample here...") 306 | with gr.Row(): 307 | with gr.Column(): 308 | initialname = "/content/Bark-Voice-Cloning/bark/assets/prompts/file" 309 | output_voice = gr.Textbox(label="Filename of trained Voice (do not change the initial name)", lines=1, placeholder=initialname, value=initialname, visible=False) 310 | with gr.Column(): 311 | tokenizerlang = gr.Dropdown(tokenizer_language_list, label="Base Language Tokenizer", value=tokenizer_language_list[1], visible=False) 312 | with gr.Row(): 313 | clone_voice_button = gr.Button("Create Voice", variant="primary") 314 | with gr.Row(): 315 | dummy = gr.Text(label="Progress") 316 | npz_file = gr.File(label=".npz file") 317 | speakers_list.insert(0, npz_file) # add prompt 318 | 319 | with gr.Tab("🎵 - TTS"): 320 | with gr.Row(): 321 | with gr.Column(): 322 | placeholder = "Enter text here." 323 | input_text = gr.Textbox(label="Input Text", lines=4, placeholder=placeholder) 324 | convert_to_ssml_button = gr.Button("Convert Input Text to SSML") 325 | with gr.Column(): 326 | seedcomponent = gr.Number(label="Seed (default -1 = Random)", precision=0, value=-1) 327 | batchcount = gr.Number(label="Batch count", precision=0, value=1) 328 | 329 | with gr.Row(): 330 | with gr.Column(): 331 | gr.Markdown("[Voice Prompt Library](https://suno-ai.notion.site/8b8e8749ed514b0cbf3f699013548683?v=bc67cff786b04b50b3ceb756fd05f68c)") 332 | speaker = gr.Dropdown(speakers_list, value=speakers_list[0], label="Voice (Choose “file” if you wanna use the custom voice)") 333 | 334 | with gr.Column(): 335 | text_temp = gr.Slider(0.1, 1.0, value=0.6, label="Generation Temperature", info="1.0 more diverse, 0.1 more conservative") 336 | waveform_temp = gr.Slider(0.1, 1.0, value=0.7, label="Waveform temperature", info="1.0 more diverse, 0.1 more conservative") 337 | 338 | with gr.Row(): 339 | with gr.Column(): 340 | quick_gen_checkbox = gr.Checkbox(label="Quick Generation", value=True) 341 | settings_checkboxes = ["Use last generation as history", "Save generation as Voice"] 342 | complete_settings = gr.CheckboxGroup(choices=settings_checkboxes, value=settings_checkboxes, label="Detailed Generation Settings", type="value", interactive=True, visible=False) 343 | with gr.Column(): 344 | eos_prob = gr.Slider(0.0, 0.5, value=0.05, label="End of sentence probability") 345 | 346 | with gr.Row(): 347 | with gr.Column(): 348 | tts_create_button = gr.Button("Generate", variant="primary") 349 | with gr.Column(): 350 | hidden_checkbox = gr.Checkbox(visible=False) 351 | button_stop_generation = gr.Button("Stop generation") 352 | with gr.Row(): 353 | output_audio = gr.Audio(label="Generated Audio", type="filepath") 354 | 355 | with gr.Tab("🔮 - Voice Conversion"): 356 | with gr.Row(): 357 | swap_audio_filename = gr.Audio(label="Input audio.wav to swap voice", source="upload", type="filepath") 358 | with gr.Row(): 359 | with gr.Column(): 360 | swap_tokenizer_lang = gr.Dropdown(tokenizer_language_list, label="Base Language Tokenizer", value=tokenizer_language_list[1]) 361 | swap_seed = gr.Number(label="Seed (default -1 = Random)", precision=0, value=-1) 362 | with gr.Column(): 363 | speaker_swap = gr.Dropdown(speakers_list, value=speakers_list[0], label="Voice (Choose “file” if you wanna use the custom voice)") 364 | swap_batchcount = gr.Number(label="Batch count", precision=0, value=1) 365 | with gr.Row(): 366 | swap_voice_button = gr.Button("Generate", variant="primary") 367 | with gr.Row(): 368 | output_swap = gr.Audio(label="Generated Audio", type="filepath") 369 | 370 | 371 | quick_gen_checkbox.change(fn=on_quick_gen_changed, inputs=quick_gen_checkbox, outputs=complete_settings) 372 | convert_to_ssml_button.click(convert_text_to_ssml, inputs=[input_text, speaker],outputs=input_text) 373 | gen_click = tts_create_button.click(generate_text_to_speech, inputs=[input_text, speaker, text_temp, waveform_temp, eos_prob, quick_gen_checkbox, complete_settings, seedcomponent, batchcount],outputs=output_audio) 374 | button_stop_generation.click(fn=None, inputs=None, outputs=None, cancels=[gen_click]) 375 | 376 | 377 | 378 | swap_voice_button.click(swap_voice_from_audio, inputs=[swap_audio_filename, speaker_swap, swap_tokenizer_lang, swap_seed, swap_batchcount], outputs=output_swap) 379 | clone_voice_button.click(clone_voice, inputs=[input_audio_filename, output_voice], outputs=[dummy, npz_file]) 380 | 381 | 382 | restart_server = False 383 | try: 384 | barkgui.queue().launch(show_error=True, share=True) 385 | except: 386 | restart_server = True 387 | run_server = False 388 | try: 389 | while restart_server == False: 390 | time.sleep(1.0) 391 | except (KeyboardInterrupt, OSError): 392 | print("Keyboard interruption in main thread... closing server.") 393 | run_server = False 394 | barkgui.close() 395 | -------------------------------------------------------------------------------- /bark/__init__.py: -------------------------------------------------------------------------------- 1 | from .api import generate_audio, text_to_semantic, semantic_to_waveform, save_as_prompt 2 | from .generation import SAMPLE_RATE, preload_models 3 | -------------------------------------------------------------------------------- /bark/api.py: -------------------------------------------------------------------------------- 1 | from typing import Dict, Optional, Union 2 | 3 | import numpy as np 4 | 5 | from .generation import codec_decode, generate_coarse, generate_fine, generate_text_semantic 6 | 7 | 8 | def generate_with_settings(text_prompt, semantic_temp=0.6, eos_p=0.2, coarse_temp=0.7, fine_temp=0.5, voice_name=None, output_full=False): 9 | 10 | # generation with more control 11 | x_semantic = generate_text_semantic( 12 | text_prompt, 13 | history_prompt=voice_name, 14 | temp=semantic_temp, 15 | min_eos_p = eos_p, 16 | use_kv_caching=True 17 | ) 18 | 19 | x_coarse_gen = generate_coarse( 20 | x_semantic, 21 | history_prompt=voice_name, 22 | temp=coarse_temp, 23 | use_kv_caching=True 24 | ) 25 | x_fine_gen = generate_fine( 26 | x_coarse_gen, 27 | history_prompt=voice_name, 28 | temp=fine_temp, 29 | ) 30 | 31 | if output_full: 32 | full_generation = { 33 | 'semantic_prompt': x_semantic, 34 | 'coarse_prompt': x_coarse_gen, 35 | 'fine_prompt': x_fine_gen 36 | } 37 | return full_generation, codec_decode(x_fine_gen) 38 | return codec_decode(x_fine_gen) 39 | 40 | 41 | def text_to_semantic( 42 | text: str, 43 | history_prompt: Optional[Union[Dict, str]] = None, 44 | temp: float = 0.7, 45 | silent: bool = False, 46 | ): 47 | """Generate semantic array from text. 48 | 49 | Args: 50 | text: text to be turned into audio 51 | history_prompt: history choice for audio cloning 52 | temp: generation temperature (1.0 more diverse, 0.0 more conservative) 53 | silent: disable progress bar 54 | 55 | Returns: 56 | numpy semantic array to be fed into `semantic_to_waveform` 57 | """ 58 | x_semantic = generate_text_semantic( 59 | text, 60 | history_prompt=history_prompt, 61 | temp=temp, 62 | silent=silent, 63 | use_kv_caching=True 64 | ) 65 | return x_semantic 66 | 67 | 68 | def semantic_to_waveform( 69 | semantic_tokens: np.ndarray, 70 | history_prompt: Optional[Union[Dict, str]] = None, 71 | temp: float = 0.7, 72 | silent: bool = False, 73 | output_full: bool = False, 74 | ): 75 | """Generate audio array from semantic input. 76 | 77 | Args: 78 | semantic_tokens: semantic token output from `text_to_semantic` 79 | history_prompt: history choice for audio cloning 80 | temp: generation temperature (1.0 more diverse, 0.0 more conservative) 81 | silent: disable progress bar 82 | output_full: return full generation to be used as a history prompt 83 | 84 | Returns: 85 | numpy audio array at sample frequency 24khz 86 | """ 87 | coarse_tokens = generate_coarse( 88 | semantic_tokens, 89 | history_prompt=history_prompt, 90 | temp=temp, 91 | silent=silent, 92 | use_kv_caching=True 93 | ) 94 | fine_tokens = generate_fine( 95 | coarse_tokens, 96 | history_prompt=history_prompt, 97 | temp=0.5, 98 | ) 99 | audio_arr = codec_decode(fine_tokens) 100 | if output_full: 101 | full_generation = { 102 | "semantic_prompt": semantic_tokens, 103 | "coarse_prompt": coarse_tokens, 104 | "fine_prompt": fine_tokens, 105 | } 106 | return full_generation, audio_arr 107 | return audio_arr 108 | 109 | 110 | def save_as_prompt(filepath, full_generation): 111 | assert(filepath.endswith(".npz")) 112 | assert(isinstance(full_generation, dict)) 113 | assert("semantic_prompt" in full_generation) 114 | assert("coarse_prompt" in full_generation) 115 | assert("fine_prompt" in full_generation) 116 | np.savez(filepath, **full_generation) 117 | 118 | 119 | def generate_audio( 120 | text: str, 121 | history_prompt: Optional[Union[Dict, str]] = None, 122 | text_temp: float = 0.7, 123 | waveform_temp: float = 0.7, 124 | silent: bool = False, 125 | output_full: bool = False, 126 | ): 127 | """Generate audio array from input text. 128 | 129 | Args: 130 | text: text to be turned into audio 131 | history_prompt: history choice for audio cloning 132 | text_temp: generation temperature (1.0 more diverse, 0.0 more conservative) 133 | waveform_temp: generation temperature (1.0 more diverse, 0.0 more conservative) 134 | silent: disable progress bar 135 | output_full: return full generation to be used as a history prompt 136 | 137 | Returns: 138 | numpy audio array at sample frequency 24khz 139 | """ 140 | semantic_tokens = text_to_semantic( 141 | text, 142 | history_prompt=history_prompt, 143 | temp=text_temp, 144 | silent=silent, 145 | ) 146 | out = semantic_to_waveform( 147 | semantic_tokens, 148 | history_prompt=history_prompt, 149 | temp=waveform_temp, 150 | silent=silent, 151 | output_full=output_full, 152 | ) 153 | if output_full: 154 | full_generation, audio_arr = out 155 | return full_generation, audio_arr 156 | else: 157 | audio_arr = out 158 | return audio_arr 159 | -------------------------------------------------------------------------------- /bark/assets/prompts/announcer.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KevinWang676/Bark-Voice-Cloning/bf4dd06bf93dd81e68ce33691e29bc06f419ca32/bark/assets/prompts/announcer.npz -------------------------------------------------------------------------------- /bark/assets/prompts/v2/en_speaker_0.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KevinWang676/Bark-Voice-Cloning/bf4dd06bf93dd81e68ce33691e29bc06f419ca32/bark/assets/prompts/v2/en_speaker_0.npz -------------------------------------------------------------------------------- /bark/assets/prompts/v2/en_speaker_1.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KevinWang676/Bark-Voice-Cloning/bf4dd06bf93dd81e68ce33691e29bc06f419ca32/bark/assets/prompts/v2/en_speaker_1.npz -------------------------------------------------------------------------------- /bark/assets/prompts/v2/en_speaker_2.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KevinWang676/Bark-Voice-Cloning/bf4dd06bf93dd81e68ce33691e29bc06f419ca32/bark/assets/prompts/v2/en_speaker_2.npz -------------------------------------------------------------------------------- /bark/assets/prompts/v2/en_speaker_3.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KevinWang676/Bark-Voice-Cloning/bf4dd06bf93dd81e68ce33691e29bc06f419ca32/bark/assets/prompts/v2/en_speaker_3.npz -------------------------------------------------------------------------------- /bark/assets/prompts/v2/en_speaker_4.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KevinWang676/Bark-Voice-Cloning/bf4dd06bf93dd81e68ce33691e29bc06f419ca32/bark/assets/prompts/v2/en_speaker_4.npz -------------------------------------------------------------------------------- /bark/assets/prompts/v2/en_speaker_5.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KevinWang676/Bark-Voice-Cloning/bf4dd06bf93dd81e68ce33691e29bc06f419ca32/bark/assets/prompts/v2/en_speaker_5.npz -------------------------------------------------------------------------------- /bark/assets/prompts/v2/en_speaker_6.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KevinWang676/Bark-Voice-Cloning/bf4dd06bf93dd81e68ce33691e29bc06f419ca32/bark/assets/prompts/v2/en_speaker_6.npz -------------------------------------------------------------------------------- /bark/assets/prompts/v2/en_speaker_7.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KevinWang676/Bark-Voice-Cloning/bf4dd06bf93dd81e68ce33691e29bc06f419ca32/bark/assets/prompts/v2/en_speaker_7.npz -------------------------------------------------------------------------------- /bark/assets/prompts/v2/en_speaker_8.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KevinWang676/Bark-Voice-Cloning/bf4dd06bf93dd81e68ce33691e29bc06f419ca32/bark/assets/prompts/v2/en_speaker_8.npz -------------------------------------------------------------------------------- /bark/assets/prompts/v2/en_speaker_9.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KevinWang676/Bark-Voice-Cloning/bf4dd06bf93dd81e68ce33691e29bc06f419ca32/bark/assets/prompts/v2/en_speaker_9.npz -------------------------------------------------------------------------------- /bark/assets/prompts/v2/zh_speaker_0.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KevinWang676/Bark-Voice-Cloning/bf4dd06bf93dd81e68ce33691e29bc06f419ca32/bark/assets/prompts/v2/zh_speaker_0.npz -------------------------------------------------------------------------------- /bark/assets/prompts/v2/zh_speaker_1.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KevinWang676/Bark-Voice-Cloning/bf4dd06bf93dd81e68ce33691e29bc06f419ca32/bark/assets/prompts/v2/zh_speaker_1.npz -------------------------------------------------------------------------------- /bark/assets/prompts/v2/zh_speaker_2.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KevinWang676/Bark-Voice-Cloning/bf4dd06bf93dd81e68ce33691e29bc06f419ca32/bark/assets/prompts/v2/zh_speaker_2.npz -------------------------------------------------------------------------------- /bark/assets/prompts/v2/zh_speaker_3.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KevinWang676/Bark-Voice-Cloning/bf4dd06bf93dd81e68ce33691e29bc06f419ca32/bark/assets/prompts/v2/zh_speaker_3.npz -------------------------------------------------------------------------------- /bark/assets/prompts/v2/zh_speaker_4.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KevinWang676/Bark-Voice-Cloning/bf4dd06bf93dd81e68ce33691e29bc06f419ca32/bark/assets/prompts/v2/zh_speaker_4.npz -------------------------------------------------------------------------------- /bark/assets/prompts/v2/zh_speaker_5.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KevinWang676/Bark-Voice-Cloning/bf4dd06bf93dd81e68ce33691e29bc06f419ca32/bark/assets/prompts/v2/zh_speaker_5.npz -------------------------------------------------------------------------------- /bark/assets/prompts/v2/zh_speaker_6.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KevinWang676/Bark-Voice-Cloning/bf4dd06bf93dd81e68ce33691e29bc06f419ca32/bark/assets/prompts/v2/zh_speaker_6.npz -------------------------------------------------------------------------------- /bark/assets/prompts/v2/zh_speaker_7.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KevinWang676/Bark-Voice-Cloning/bf4dd06bf93dd81e68ce33691e29bc06f419ca32/bark/assets/prompts/v2/zh_speaker_7.npz -------------------------------------------------------------------------------- /bark/assets/prompts/v2/zh_speaker_8.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KevinWang676/Bark-Voice-Cloning/bf4dd06bf93dd81e68ce33691e29bc06f419ca32/bark/assets/prompts/v2/zh_speaker_8.npz -------------------------------------------------------------------------------- /bark/assets/prompts/v2/zh_speaker_9.npz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KevinWang676/Bark-Voice-Cloning/bf4dd06bf93dd81e68ce33691e29bc06f419ca32/bark/assets/prompts/v2/zh_speaker_9.npz -------------------------------------------------------------------------------- /bark/hubert/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KevinWang676/Bark-Voice-Cloning/bf4dd06bf93dd81e68ce33691e29bc06f419ca32/bark/hubert/__init__.py -------------------------------------------------------------------------------- /bark/hubert/customtokenizer.py: -------------------------------------------------------------------------------- 1 | """ 2 | Custom tokenizer model. 3 | Author: https://www.github.com/gitmylo/ 4 | License: MIT 5 | """ 6 | 7 | import json 8 | import os.path 9 | from zipfile import ZipFile 10 | 11 | import numpy 12 | import torch 13 | from torch import nn, optim 14 | from torch.serialization import MAP_LOCATION 15 | from tqdm.auto import tqdm 16 | 17 | 18 | class CustomTokenizer(nn.Module): 19 | def __init__(self, hidden_size=1024, input_size=768, output_size=10000, version=0): 20 | super(CustomTokenizer, self).__init__() 21 | next_size = input_size 22 | if version == 0: 23 | self.lstm = nn.LSTM(input_size, hidden_size, 2, batch_first=True) 24 | next_size = hidden_size 25 | if version == 1: 26 | self.lstm = nn.LSTM(input_size, hidden_size, 2, batch_first=True) 27 | self.intermediate = nn.Linear(hidden_size, 4096) 28 | next_size = 4096 29 | 30 | self.fc = nn.Linear(next_size, output_size) 31 | self.softmax = nn.LogSoftmax(dim=1) 32 | self.optimizer: optim.Optimizer = None 33 | self.lossfunc = nn.CrossEntropyLoss() 34 | self.input_size = input_size 35 | self.hidden_size = hidden_size 36 | self.output_size = output_size 37 | self.version = version 38 | 39 | def forward(self, x): 40 | x, _ = self.lstm(x) 41 | if self.version == 1: 42 | x = self.intermediate(x) 43 | x = self.fc(x) 44 | x = self.softmax(x) 45 | return x 46 | 47 | @torch.no_grad() 48 | def get_token(self, x): 49 | """ 50 | Used to get the token for the first 51 | :param x: An array with shape (N, input_size) where N is a whole number greater or equal to 1, and input_size is the input size used when creating the model. 52 | :return: An array with shape (N,) where N is the same as N from the input. Every number in the array is a whole number in range 0...output_size - 1 where output_size is the output size used when creating the model. 53 | """ 54 | return torch.argmax(self(x), dim=1) 55 | 56 | def prepare_training(self): 57 | self.optimizer = optim.Adam(self.parameters(), 0.001) 58 | 59 | def train_step(self, x_train, y_train, log_loss=False): 60 | # y_train = y_train[:-1] 61 | # y_train = y_train[1:] 62 | 63 | optimizer = self.optimizer 64 | lossfunc = self.lossfunc 65 | # Zero the gradients 66 | self.zero_grad() 67 | 68 | # Forward pass 69 | y_pred = self(x_train) 70 | 71 | y_train_len = len(y_train) 72 | y_pred_len = y_pred.shape[0] 73 | 74 | if y_train_len > y_pred_len: 75 | diff = y_train_len - y_pred_len 76 | y_train = y_train[diff:] 77 | elif y_train_len < y_pred_len: 78 | diff = y_pred_len - y_train_len 79 | y_pred = y_pred[:-diff, :] 80 | 81 | y_train_hot = torch.zeros(len(y_train), self.output_size) 82 | y_train_hot[range(len(y_train)), y_train] = 1 83 | y_train_hot = y_train_hot.to('cuda') 84 | 85 | # Calculate the loss 86 | loss = lossfunc(y_pred, y_train_hot) 87 | 88 | # Print loss 89 | if log_loss: 90 | print('Loss', loss.item()) 91 | 92 | # Backward pass 93 | loss.backward() 94 | 95 | # Update the weights 96 | optimizer.step() 97 | 98 | def save(self, path): 99 | info_path = '.'.join(os.path.basename(path).split('.')[:-1]) + '/.info' 100 | torch.save(self.state_dict(), path) 101 | data_from_model = Data(self.input_size, self.hidden_size, self.output_size, self.version) 102 | with ZipFile(path, 'a') as model_zip: 103 | model_zip.writestr(info_path, data_from_model.save()) 104 | model_zip.close() 105 | 106 | @staticmethod 107 | def load_from_checkpoint(path, map_location: MAP_LOCATION = None): 108 | old = True 109 | with ZipFile(path) as model_zip: 110 | filesMatch = [file for file in model_zip.namelist() if file.endswith('/.info')] 111 | file = filesMatch[0] if filesMatch else None 112 | if file: 113 | old = False 114 | print(f"Loading Custom Hubert Tokenizer {path}") 115 | data_from_model = Data.load(model_zip.read(file).decode('utf-8')) 116 | model_zip.close() 117 | if old: 118 | model = CustomTokenizer() 119 | else: 120 | model = CustomTokenizer(data_from_model.hidden_size, data_from_model.input_size, data_from_model.output_size, data_from_model.version) 121 | model.load_state_dict(torch.load(path)) 122 | if map_location: 123 | model = model.to(map_location) 124 | return model 125 | 126 | 127 | 128 | class Data: 129 | input_size: int 130 | hidden_size: int 131 | output_size: int 132 | version: int 133 | 134 | def __init__(self, input_size=768, hidden_size=1024, output_size=10000, version=0): 135 | self.input_size = input_size 136 | self.hidden_size = hidden_size 137 | self.output_size = output_size 138 | self.version = version 139 | 140 | @staticmethod 141 | def load(string): 142 | data = json.loads(string) 143 | return Data(data['input_size'], data['hidden_size'], data['output_size'], data['version']) 144 | 145 | def save(self): 146 | data = { 147 | 'input_size': self.input_size, 148 | 'hidden_size': self.hidden_size, 149 | 'output_size': self.output_size, 150 | 'version': self.version, 151 | } 152 | return json.dumps(data) 153 | 154 | 155 | def auto_train(data_path, save_path='model.pth', load_model: str | None = None, save_epochs=1, max_epochs=14): 156 | data_x, data_y = [], [] 157 | 158 | if load_model and os.path.isfile(load_model): 159 | print('Loading model from', load_model) 160 | model_training = CustomTokenizer.load_from_checkpoint(load_model, 'cuda') 161 | else: 162 | print('Creating new model.') 163 | model_training = CustomTokenizer(version=1).to('cuda') # Settings for the model to run without lstm 164 | save_path = os.path.join(data_path, save_path) 165 | base_save_path = '.'.join(save_path.split('.')[:-1]) 166 | 167 | sem_string = '_semantic.npy' 168 | feat_string = '_semantic_features.npy' 169 | 170 | ready = os.path.join(data_path, 'ready') 171 | for input_file in os.listdir(ready): 172 | full_path = os.path.join(ready, input_file) 173 | if input_file.endswith(sem_string): 174 | data_y.append(numpy.load(full_path)) 175 | elif input_file.endswith(feat_string): 176 | data_x.append(numpy.load(full_path)) 177 | model_training.prepare_training() 178 | 179 | epoch = 1 180 | with tqdm(total=((len(data_x) * len(data_y)) / 50) * save_epochs) as pbar1: 181 | while epoch <= max_epochs: 182 | for i in range(save_epochs): 183 | j = 0 184 | for x, y in zip(data_x, data_y): 185 | model_training.train_step(torch.tensor(x).to('cuda'), torch.tensor(y).to('cuda'), j % 50 == 0) # Print loss every 50 steps 186 | j += 1 187 | pbar1.update() 188 | 189 | save_p = save_path 190 | save_p_2 = f'{base_save_path}_epoch_{epoch}.pth' 191 | model_training.save(save_p) 192 | model_training.save(save_p_2) 193 | print(f'Epoch {epoch} completed') 194 | epoch += 1 195 | print(f'Done training for {max_epochs} epochs!') -------------------------------------------------------------------------------- /bark/hubert/hubert_manager.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | import shutil 3 | import urllib.request 4 | 5 | import huggingface_hub 6 | 7 | 8 | class HuBERTManager: 9 | 10 | 11 | @staticmethod 12 | def make_sure_hubert_installed(download_url: str = 'https://dl.fbaipublicfiles.com/hubert/hubert_base_ls960.pt', file_name: str = 'hubert.pt'): 13 | install_dir = os.path.join('models', 'hubert') 14 | if not os.path.isdir(install_dir): 15 | os.makedirs(install_dir, exist_ok=True) 16 | install_file = os.path.join(install_dir, file_name) 17 | if not os.path.isfile(install_file): 18 | print(f'Downloading HuBERT base model from {download_url}') 19 | urllib.request.urlretrieve(download_url, install_file) 20 | print('Downloaded HuBERT') 21 | return install_file 22 | 23 | 24 | @staticmethod 25 | def make_sure_tokenizer_installed(model: str = 'quantifier_hubert_base_ls960_14.pth', repo: str = 'GitMylo/bark-voice-cloning', tokenizer_lang: str = 'en'): 26 | local_file = tokenizer_lang + '_tokenizer.pth' 27 | install_dir = os.path.join('models', 'hubert') 28 | if not os.path.isdir(install_dir): 29 | os.makedirs(install_dir, exist_ok=True) 30 | install_file = os.path.join(install_dir, local_file) 31 | if not os.path.isfile(install_file): 32 | # refactor to use lists 33 | if tokenizer_lang == 'en': 34 | repo = 'GitMylo/bark-voice-cloning' 35 | model = 'quantifier_hubert_base_ls960_14.pth' 36 | elif tokenizer_lang == 'de': 37 | repo = 'CountFloyd/bark-voice-cloning-german-HuBERT-quantizer' 38 | model = 'german-HuBERT-quantizer_14_epoch.pth' 39 | elif tokenizer_lang == 'pl': 40 | repo = 'Hobis/bark-voice-cloning-polish-HuBERT-quantizer' 41 | model = 'polish-HuBERT-quantizer_8_epoch.pth' 42 | else: 43 | raise 'Unknown Tokenizer Language!' 44 | print(f'{local_file} not found. Downloading HuBERT custom tokenizer') 45 | huggingface_hub.hf_hub_download(repo, model, local_dir=install_dir, local_dir_use_symlinks=False) 46 | shutil.move(os.path.join(install_dir, model), install_file) 47 | print('Downloaded tokenizer') 48 | return install_file 49 | -------------------------------------------------------------------------------- /bark/hubert/pre_kmeans_hubert.py: -------------------------------------------------------------------------------- 1 | """ 2 | Modified HuBERT model without kmeans. 3 | Original author: https://github.com/lucidrains/ 4 | Modified by: https://www.github.com/gitmylo/ 5 | License: MIT 6 | """ 7 | 8 | # Modified code from https://github.com/lucidrains/audiolm-pytorch/blob/main/audiolm_pytorch/hubert_kmeans.py 9 | 10 | from pathlib import Path 11 | 12 | import torch 13 | from torch import nn 14 | from einops import pack, unpack 15 | 16 | import fairseq 17 | 18 | from torchaudio.functional import resample 19 | 20 | from audiolm_pytorch.utils import curtail_to_multiple 21 | 22 | import logging 23 | logging.root.setLevel(logging.ERROR) 24 | 25 | 26 | def exists(val): 27 | return val is not None 28 | 29 | 30 | def default(val, d): 31 | return val if exists(val) else d 32 | 33 | 34 | class CustomHubert(nn.Module): 35 | """ 36 | checkpoint and kmeans can be downloaded at https://github.com/facebookresearch/fairseq/tree/main/examples/hubert 37 | or you can train your own 38 | """ 39 | 40 | def __init__( 41 | self, 42 | checkpoint_path, 43 | target_sample_hz=16000, 44 | seq_len_multiple_of=None, 45 | output_layer=9, 46 | device=None 47 | ): 48 | super().__init__() 49 | self.target_sample_hz = target_sample_hz 50 | self.seq_len_multiple_of = seq_len_multiple_of 51 | self.output_layer = output_layer 52 | 53 | if device is not None: 54 | self.to(device) 55 | 56 | model_path = Path(checkpoint_path) 57 | 58 | assert model_path.exists(), f'path {checkpoint_path} does not exist' 59 | 60 | print(f"Loading Hubert {checkpoint_path}") 61 | checkpoint = torch.load(checkpoint_path) 62 | load_model_input = {checkpoint_path: checkpoint} 63 | model, *_ = fairseq.checkpoint_utils.load_model_ensemble_and_task(load_model_input) 64 | 65 | if device is not None: 66 | model[0].to(device) 67 | 68 | self.model = model[0] 69 | self.model.eval() 70 | 71 | @property 72 | def groups(self): 73 | return 1 74 | 75 | @torch.no_grad() 76 | def forward( 77 | self, 78 | wav_input, 79 | flatten=True, 80 | input_sample_hz=None 81 | ): 82 | device = wav_input.device 83 | 84 | if exists(input_sample_hz): 85 | wav_input = resample(wav_input, input_sample_hz, self.target_sample_hz) 86 | 87 | if exists(self.seq_len_multiple_of): 88 | wav_input = curtail_to_multiple(wav_input, self.seq_len_multiple_of) 89 | 90 | embed = self.model( 91 | wav_input, 92 | features_only=True, 93 | mask=False, # thanks to @maitycyrus for noticing that mask is defaulted to True in the fairseq code 94 | output_layer=self.output_layer 95 | ) 96 | 97 | embed, packed_shape = pack([embed['x']], '* d') 98 | 99 | # codebook_indices = self.kmeans.predict(embed.cpu().detach().numpy()) 100 | 101 | codebook_indices = torch.from_numpy(embed.cpu().detach().numpy()).to(device) # .long() 102 | 103 | if flatten: 104 | return codebook_indices 105 | 106 | codebook_indices, = unpack(codebook_indices, packed_shape, '*') 107 | return codebook_indices 108 | -------------------------------------------------------------------------------- /bark/model.py: -------------------------------------------------------------------------------- 1 | """ 2 | Much of this code is adapted from Andrej Karpathy's NanoGPT 3 | (https://github.com/karpathy/nanoGPT) 4 | """ 5 | import math 6 | from dataclasses import dataclass 7 | 8 | import torch 9 | import torch.nn as nn 10 | from torch.nn import functional as F 11 | 12 | class LayerNorm(nn.Module): 13 | """ LayerNorm but with an optional bias. PyTorch doesn't support simply bias=False """ 14 | 15 | def __init__(self, ndim, bias): 16 | super().__init__() 17 | self.weight = nn.Parameter(torch.ones(ndim)) 18 | self.bias = nn.Parameter(torch.zeros(ndim)) if bias else None 19 | 20 | def forward(self, input): 21 | return F.layer_norm(input, self.weight.shape, self.weight, self.bias, 1e-5) 22 | 23 | class CausalSelfAttention(nn.Module): 24 | 25 | def __init__(self, config): 26 | super().__init__() 27 | assert config.n_embd % config.n_head == 0 28 | # key, query, value projections for all heads, but in a batch 29 | self.c_attn = nn.Linear(config.n_embd, 3 * config.n_embd, bias=config.bias) 30 | # output projection 31 | self.c_proj = nn.Linear(config.n_embd, config.n_embd, bias=config.bias) 32 | # regularization 33 | self.attn_dropout = nn.Dropout(config.dropout) 34 | self.resid_dropout = nn.Dropout(config.dropout) 35 | self.n_head = config.n_head 36 | self.n_embd = config.n_embd 37 | self.dropout = config.dropout 38 | # flash attention make GPU go brrrrr but support is only in PyTorch nightly and still a bit scary 39 | self.flash = hasattr(torch.nn.functional, 'scaled_dot_product_attention') 40 | if not self.flash: 41 | # print("WARNING: using slow attention. Flash Attention atm needs PyTorch nightly and dropout=0.0") 42 | # causal mask to ensure that attention is only applied to the left in the input sequence 43 | self.register_buffer("bias", torch.tril(torch.ones(config.block_size, config.block_size)) 44 | .view(1, 1, config.block_size, config.block_size)) 45 | 46 | def forward(self, x, past_kv=None, use_cache=False): 47 | B, T, C = x.size() # batch size, sequence length, embedding dimensionality (n_embd) 48 | 49 | # calculate query, key, values for all heads in batch and move head forward to be the batch dim 50 | q, k ,v = self.c_attn(x).split(self.n_embd, dim=2) 51 | k = k.view(B, T, self.n_head, C // self.n_head).transpose(1, 2) # (B, nh, T, hs) 52 | q = q.view(B, T, self.n_head, C // self.n_head).transpose(1, 2) # (B, nh, T, hs) 53 | v = v.view(B, T, self.n_head, C // self.n_head).transpose(1, 2) # (B, nh, T, hs) 54 | 55 | if past_kv is not None: 56 | past_key = past_kv[0] 57 | past_value = past_kv[1] 58 | k = torch.cat((past_key, k), dim=-2) 59 | v = torch.cat((past_value, v), dim=-2) 60 | 61 | FULL_T = k.shape[-2] 62 | 63 | if use_cache is True: 64 | present = (k, v) 65 | else: 66 | present = None 67 | 68 | # causal self-attention; Self-attend: (B, nh, T, hs) x (B, nh, hs, T) -> (B, nh, T, T) 69 | if self.flash: 70 | # efficient attention using Flash Attention CUDA kernels 71 | if past_kv is not None: 72 | # When `past_kv` is provided, we're doing incremental decoding and `q.shape[2] == 1`: q only contains 73 | # the query for the last token. scaled_dot_product_attention interprets this as the first token in the 74 | # sequence, so if is_causal=True it will mask out all attention from it. This is not what we want, so 75 | # to work around this we set is_causal=False. 76 | is_causal = False 77 | else: 78 | is_causal = True 79 | 80 | y = torch.nn.functional.scaled_dot_product_attention(q, k, v, dropout_p=self.dropout, is_causal=is_causal) 81 | else: 82 | # manual implementation of attention 83 | att = (q @ k.transpose(-2, -1)) * (1.0 / math.sqrt(k.size(-1))) 84 | att = att.masked_fill(self.bias[:,:,FULL_T-T:FULL_T,:FULL_T] == 0, float('-inf')) 85 | att = F.softmax(att, dim=-1) 86 | att = self.attn_dropout(att) 87 | y = att @ v # (B, nh, T, T) x (B, nh, T, hs) -> (B, nh, T, hs) 88 | y = y.transpose(1, 2).contiguous().view(B, T, C) # re-assemble all head outputs side by side 89 | 90 | # output projection 91 | y = self.resid_dropout(self.c_proj(y)) 92 | return (y, present) 93 | 94 | class MLP(nn.Module): 95 | 96 | def __init__(self, config): 97 | super().__init__() 98 | self.c_fc = nn.Linear(config.n_embd, 4 * config.n_embd, bias=config.bias) 99 | self.c_proj = nn.Linear(4 * config.n_embd, config.n_embd, bias=config.bias) 100 | self.dropout = nn.Dropout(config.dropout) 101 | self.gelu = nn.GELU() 102 | 103 | def forward(self, x): 104 | x = self.c_fc(x) 105 | x = self.gelu(x) 106 | x = self.c_proj(x) 107 | x = self.dropout(x) 108 | return x 109 | 110 | class Block(nn.Module): 111 | 112 | def __init__(self, config, layer_idx): 113 | super().__init__() 114 | self.ln_1 = LayerNorm(config.n_embd, bias=config.bias) 115 | self.attn = CausalSelfAttention(config) 116 | self.ln_2 = LayerNorm(config.n_embd, bias=config.bias) 117 | self.mlp = MLP(config) 118 | self.layer_idx = layer_idx 119 | 120 | def forward(self, x, past_kv=None, use_cache=False): 121 | attn_output, prev_kvs = self.attn(self.ln_1(x), past_kv=past_kv, use_cache=use_cache) 122 | x = x + attn_output 123 | x = x + self.mlp(self.ln_2(x)) 124 | return (x, prev_kvs) 125 | 126 | @dataclass 127 | class GPTConfig: 128 | block_size: int = 1024 129 | input_vocab_size: int = 10_048 130 | output_vocab_size: int = 10_048 131 | n_layer: int = 12 132 | n_head: int = 12 133 | n_embd: int = 768 134 | dropout: float = 0.0 135 | bias: bool = True # True: bias in Linears and LayerNorms, like GPT-2. False: a bit better and faster 136 | 137 | class GPT(nn.Module): 138 | 139 | def __init__(self, config): 140 | super().__init__() 141 | assert config.input_vocab_size is not None 142 | assert config.output_vocab_size is not None 143 | assert config.block_size is not None 144 | self.config = config 145 | 146 | self.transformer = nn.ModuleDict(dict( 147 | wte = nn.Embedding(config.input_vocab_size, config.n_embd), 148 | wpe = nn.Embedding(config.block_size, config.n_embd), 149 | drop = nn.Dropout(config.dropout), 150 | h = nn.ModuleList([Block(config, idx) for idx in range(config.n_layer)]), 151 | ln_f = LayerNorm(config.n_embd, bias=config.bias), 152 | )) 153 | self.lm_head = nn.Linear(config.n_embd, config.output_vocab_size, bias=False) 154 | 155 | def get_num_params(self, non_embedding=True): 156 | """ 157 | Return the number of parameters in the model. 158 | For non-embedding count (default), the position embeddings get subtracted. 159 | The token embeddings would too, except due to the parameter sharing these 160 | params are actually used as weights in the final layer, so we include them. 161 | """ 162 | n_params = sum(p.numel() for p in self.parameters()) 163 | if non_embedding: 164 | n_params -= self.transformer.wte.weight.numel() 165 | n_params -= self.transformer.wpe.weight.numel() 166 | return n_params 167 | 168 | def forward(self, idx, merge_context=False, past_kv=None, position_ids=None, use_cache=False): 169 | device = idx.device 170 | b, t = idx.size() 171 | if past_kv is not None: 172 | assert t == 1 173 | tok_emb = self.transformer.wte(idx) # token embeddings of shape (b, t, n_embd) 174 | else: 175 | if merge_context: 176 | assert(idx.shape[1] >= 256+256+1) 177 | t = idx.shape[1] - 256 178 | else: 179 | assert t <= self.config.block_size, f"Cannot forward sequence of length {t}, block size is only {self.config.block_size}" 180 | 181 | # forward the GPT model itself 182 | if merge_context: 183 | tok_emb = torch.cat([ 184 | self.transformer.wte(idx[:,:256]) + self.transformer.wte(idx[:,256:256+256]), 185 | self.transformer.wte(idx[:,256+256:]) 186 | ], dim=1) 187 | else: 188 | tok_emb = self.transformer.wte(idx) # token embeddings of shape (b, t, n_embd) 189 | 190 | if past_kv is None: 191 | past_length = 0 192 | past_kv = tuple([None] * len(self.transformer.h)) 193 | else: 194 | past_length = past_kv[0][0].size(-2) 195 | 196 | if position_ids is None: 197 | position_ids = torch.arange(past_length, t + past_length, dtype=torch.long, device=device) 198 | position_ids = position_ids.unsqueeze(0) # shape (1, t) 199 | assert position_ids.shape == (1, t) 200 | 201 | pos_emb = self.transformer.wpe(position_ids) # position embeddings of shape (1, t, n_embd) 202 | 203 | x = self.transformer.drop(tok_emb + pos_emb) 204 | 205 | new_kv = () if use_cache else None 206 | 207 | for i, (block, past_layer_kv) in enumerate(zip(self.transformer.h, past_kv)): 208 | x, kv = block(x, past_kv=past_layer_kv, use_cache=use_cache) 209 | 210 | if use_cache: 211 | new_kv = new_kv + (kv,) 212 | 213 | x = self.transformer.ln_f(x) 214 | 215 | # inference-time mini-optimization: only forward the lm_head on the very last position 216 | logits = self.lm_head(x[:, [-1], :]) # note: using list [-1] to preserve the time dim 217 | 218 | return (logits, new_kv) 219 | -------------------------------------------------------------------------------- /bark/model_fine.py: -------------------------------------------------------------------------------- 1 | """ 2 | Much of this code is adapted from Andrej Karpathy's NanoGPT 3 | (https://github.com/karpathy/nanoGPT) 4 | """ 5 | from dataclasses import dataclass 6 | import math 7 | 8 | import torch 9 | import torch.nn as nn 10 | from torch.nn import functional as F 11 | 12 | from .model import GPT, GPTConfig, MLP 13 | 14 | 15 | class NonCausalSelfAttention(nn.Module): 16 | def __init__(self, config): 17 | super().__init__() 18 | assert config.n_embd % config.n_head == 0 19 | # key, query, value projections for all heads, but in a batch 20 | self.c_attn = nn.Linear(config.n_embd, 3 * config.n_embd, bias=config.bias) 21 | # output projection 22 | self.c_proj = nn.Linear(config.n_embd, config.n_embd, bias=config.bias) 23 | # regularization 24 | self.attn_dropout = nn.Dropout(config.dropout) 25 | self.resid_dropout = nn.Dropout(config.dropout) 26 | self.n_head = config.n_head 27 | self.n_embd = config.n_embd 28 | self.dropout = config.dropout 29 | # flash attention make GPU go brrrrr but support is only in PyTorch nightly and still a bit scary 30 | self.flash = ( 31 | hasattr(torch.nn.functional, "scaled_dot_product_attention") and self.dropout == 0.0 32 | ) 33 | 34 | def forward(self, x): 35 | B, T, C = x.size() # batch size, sequence length, embedding dimensionality (n_embd) 36 | 37 | # calculate query, key, values for all heads in batch and move head forward to be the batch dim 38 | q, k, v = self.c_attn(x).split(self.n_embd, dim=2) 39 | k = k.view(B, T, self.n_head, C // self.n_head).transpose(1, 2) # (B, nh, T, hs) 40 | q = q.view(B, T, self.n_head, C // self.n_head).transpose(1, 2) # (B, nh, T, hs) 41 | v = v.view(B, T, self.n_head, C // self.n_head).transpose(1, 2) # (B, nh, T, hs) 42 | 43 | # causal self-attention; Self-attend: (B, nh, T, hs) x (B, nh, hs, T) -> (B, nh, T, T) 44 | if self.flash: 45 | # efficient attention using Flash Attention CUDA kernels 46 | y = torch.nn.functional.scaled_dot_product_attention( 47 | q, k, v, attn_mask=None, dropout_p=self.dropout, is_causal=False 48 | ) 49 | else: 50 | # manual implementation of attention 51 | att = (q @ k.transpose(-2, -1)) * (1.0 / math.sqrt(k.size(-1))) 52 | att = F.softmax(att, dim=-1) 53 | att = self.attn_dropout(att) 54 | y = att @ v # (B, nh, T, T) x (B, nh, T, hs) -> (B, nh, T, hs) 55 | y = ( 56 | y.transpose(1, 2).contiguous().view(B, T, C) 57 | ) # re-assemble all head outputs side by side 58 | 59 | # output projection 60 | y = self.resid_dropout(self.c_proj(y)) 61 | return y 62 | 63 | 64 | class FineBlock(nn.Module): 65 | def __init__(self, config): 66 | super().__init__() 67 | self.ln_1 = nn.LayerNorm(config.n_embd) 68 | self.attn = NonCausalSelfAttention(config) 69 | self.ln_2 = nn.LayerNorm(config.n_embd) 70 | self.mlp = MLP(config) 71 | 72 | def forward(self, x): 73 | x = x + self.attn(self.ln_1(x)) 74 | x = x + self.mlp(self.ln_2(x)) 75 | return x 76 | 77 | 78 | class FineGPT(GPT): 79 | def __init__(self, config): 80 | super().__init__(config) 81 | del self.lm_head 82 | self.config = config 83 | self.n_codes_total = config.n_codes_total 84 | self.transformer = nn.ModuleDict( 85 | dict( 86 | wtes=nn.ModuleList( 87 | [ 88 | nn.Embedding(config.input_vocab_size, config.n_embd) 89 | for _ in range(config.n_codes_total) 90 | ] 91 | ), 92 | wpe=nn.Embedding(config.block_size, config.n_embd), 93 | drop=nn.Dropout(config.dropout), 94 | h=nn.ModuleList([FineBlock(config) for _ in range(config.n_layer)]), 95 | ln_f=nn.LayerNorm(config.n_embd), 96 | ) 97 | ) 98 | self.lm_heads = nn.ModuleList( 99 | [ 100 | nn.Linear(config.n_embd, config.output_vocab_size, bias=False) 101 | for _ in range(config.n_codes_given, self.n_codes_total) 102 | ] 103 | ) 104 | for i in range(self.n_codes_total - config.n_codes_given): 105 | self.transformer.wtes[i + 1].weight = self.lm_heads[i].weight 106 | 107 | def forward(self, pred_idx, idx): 108 | device = idx.device 109 | b, t, codes = idx.size() 110 | assert ( 111 | t <= self.config.block_size 112 | ), f"Cannot forward sequence of length {t}, block size is only {self.config.block_size}" 113 | assert pred_idx > 0, "cannot predict 0th codebook" 114 | assert codes == self.n_codes_total, (b, t, codes) 115 | pos = torch.arange(0, t, dtype=torch.long, device=device).unsqueeze(0) # shape (1, t) 116 | 117 | # forward the GPT model itself 118 | tok_embs = [ 119 | wte(idx[:, :, i]).unsqueeze(-1) for i, wte in enumerate(self.transformer.wtes) 120 | ] # token embeddings of shape (b, t, n_embd) 121 | tok_emb = torch.cat(tok_embs, dim=-1) 122 | pos_emb = self.transformer.wpe(pos) # position embeddings of shape (1, t, n_embd) 123 | x = tok_emb[:, :, :, : pred_idx + 1].sum(dim=-1) 124 | x = self.transformer.drop(x + pos_emb) 125 | for block in self.transformer.h: 126 | x = block(x) 127 | x = self.transformer.ln_f(x) 128 | logits = self.lm_heads[pred_idx - self.config.n_codes_given](x) 129 | return logits 130 | 131 | def get_num_params(self, non_embedding=True): 132 | """ 133 | Return the number of parameters in the model. 134 | For non-embedding count (default), the position embeddings get subtracted. 135 | The token embeddings would too, except due to the parameter sharing these 136 | params are actually used as weights in the final layer, so we include them. 137 | """ 138 | n_params = sum(p.numel() for p in self.parameters()) 139 | if non_embedding: 140 | for wte in self.transformer.wtes: 141 | n_params -= wte.weight.numel() 142 | n_params -= self.transformer.wpe.weight.numel() 143 | return n_params 144 | 145 | 146 | @dataclass 147 | class FineGPTConfig(GPTConfig): 148 | n_codes_total: int = 8 149 | n_codes_given: int = 1 150 | -------------------------------------------------------------------------------- /bark/settings.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | def initenv(args): 4 | os.environ['SUNO_USE_SMALL_MODELS'] = str("-smallmodels" in args) 5 | os.environ['BARK_FORCE_CPU'] = str("-forcecpu" in args) 6 | os.environ['SUNO_ENABLE_MPS'] = str("-enablemps" in args) 7 | os.environ['SUNO_OFFLOAD_CPU'] = str("-offloadcpu" in args) 8 | -------------------------------------------------------------------------------- /cloning/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KevinWang676/Bark-Voice-Cloning/bf4dd06bf93dd81e68ce33691e29bc06f419ca32/cloning/__init__.py -------------------------------------------------------------------------------- /cloning/clonevoice.py: -------------------------------------------------------------------------------- 1 | from bark.generation import load_codec_model, generate_text_semantic, grab_best_device 2 | from encodec.utils import convert_audio 3 | from bark.hubert.hubert_manager import HuBERTManager 4 | from bark.hubert.pre_kmeans_hubert import CustomHubert 5 | from bark.hubert.customtokenizer import CustomTokenizer 6 | 7 | import torchaudio 8 | import torch 9 | import os 10 | import gradio 11 | 12 | 13 | def clone_voice(audio_filepath, dest_filename, progress=gradio.Progress(track_tqdm=True)): 14 | # if len(text) < 1: 15 | # raise gradio.Error('No transcription text entered!') 16 | 17 | use_gpu = not os.environ.get("BARK_FORCE_CPU", False) 18 | progress(0, desc="Loading Codec") 19 | model = load_codec_model(use_gpu=use_gpu) 20 | 21 | # From https://github.com/gitmylo/bark-voice-cloning-HuBERT-quantizer 22 | hubert_manager = HuBERTManager() 23 | hubert_manager.make_sure_hubert_installed() 24 | hubert_manager.make_sure_tokenizer_installed() 25 | 26 | # From https://github.com/gitmylo/bark-voice-cloning-HuBERT-quantizer 27 | # Load HuBERT for semantic tokens 28 | 29 | # Load the HuBERT model 30 | device = grab_best_device(use_gpu) 31 | hubert_model = CustomHubert(checkpoint_path='./models/hubert/hubert.pt').to(device) 32 | 33 | # Load the CustomTokenizer model 34 | tokenizer = CustomTokenizer.load_from_checkpoint('./models/hubert/en_tokenizer.pth').to(device) # change to the correct path 35 | 36 | progress(0.25, desc="Converting WAV") 37 | 38 | # Load and pre-process the audio waveform 39 | wav, sr = torchaudio.load(audio_filepath) 40 | if wav.shape[0] == 2: # Stereo to mono if needed 41 | wav = wav.mean(0, keepdim=True) 42 | 43 | wav = convert_audio(wav, sr, model.sample_rate, model.channels) 44 | wav = wav.to(device) 45 | progress(0.5, desc="Extracting codes") 46 | 47 | semantic_vectors = hubert_model.forward(wav, input_sample_hz=model.sample_rate) 48 | semantic_tokens = tokenizer.get_token(semantic_vectors) 49 | 50 | # Extract discrete codes from EnCodec 51 | with torch.no_grad(): 52 | encoded_frames = model.encode(wav.unsqueeze(0)) 53 | codes = torch.cat([encoded[0] for encoded in encoded_frames], dim=-1).squeeze() # [n_q, T] 54 | 55 | # get seconds of audio 56 | # seconds = wav.shape[-1] / model.sample_rate 57 | # generate semantic tokens 58 | # semantic_tokens = generate_text_semantic(text, max_gen_duration_s=seconds, top_k=50, top_p=.95, temp=0.7) 59 | 60 | # move codes to cpu 61 | codes = codes.cpu().numpy() 62 | # move semantic tokens to cpu 63 | semantic_tokens = semantic_tokens.cpu().numpy() 64 | 65 | import numpy as np 66 | output_path = dest_filename + '.npz' 67 | np.savez(output_path, fine_prompt=codes, coarse_prompt=codes[:2, :], semantic_prompt=semantic_tokens) 68 | return ["Finished", output_path] 69 | -------------------------------------------------------------------------------- /config.yaml: -------------------------------------------------------------------------------- 1 | input_text_desired_length: 110 2 | input_text_max_length: 170 3 | selected_theme: JohnSmith9982/small_and_pretty 4 | server_name: '' 5 | server_port: 0 6 | server_share: false 7 | silence_between_sentences: 250 8 | silence_between_speakers: 500 9 | -------------------------------------------------------------------------------- /notebooks/CosyVoice.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "provenance": [], 7 | "gpuType": "T4", 8 | "authorship_tag": "ABX9TyNbIcM/DJzKsatAzgGLnrIA", 9 | "include_colab_link": true 10 | }, 11 | "kernelspec": { 12 | "name": "python3", 13 | "display_name": "Python 3" 14 | }, 15 | "language_info": { 16 | "name": "python" 17 | }, 18 | "accelerator": "GPU" 19 | }, 20 | "cells": [ 21 | { 22 | "cell_type": "markdown", 23 | "metadata": { 24 | "id": "view-in-github", 25 | "colab_type": "text" 26 | }, 27 | "source": [ 28 | "\"Open" 29 | ] 30 | }, 31 | { 32 | "cell_type": "code", 33 | "source": [ 34 | "#@title 一键运行 CosyVoice\n", 35 | "!git clone --recursive https://github.com/KevinWang676/CosyVoice.git\n", 36 | "%cd CosyVoice\n", 37 | "!pip install -r requirements.txt\n", 38 | "!mkdir -p pretrained_models\n", 39 | "!git clone https://www.modelscope.cn/iic/CosyVoice-300M.git pretrained_models/CosyVoice-300M\n", 40 | "!export PYTHONPATH=third_party/Matcha-TTS\n", 41 | "!python3 webui_colab.py" 42 | ], 43 | "metadata": { 44 | "id": "lAEyVebGqnzS" 45 | }, 46 | "execution_count": null, 47 | "outputs": [] 48 | }, 49 | { 50 | "cell_type": "code", 51 | "source": [], 52 | "metadata": { 53 | "id": "7FjBPnhPL8Pu" 54 | }, 55 | "execution_count": null, 56 | "outputs": [] 57 | } 58 | ] 59 | } 60 | -------------------------------------------------------------------------------- /notebooks/GPT_SoVITS_2.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "provenance": [], 7 | "machine_shape": "hm", 8 | "gpuType": "T4", 9 | "include_colab_link": true 10 | }, 11 | "kernelspec": { 12 | "name": "python3", 13 | "display_name": "Python 3" 14 | }, 15 | "language_info": { 16 | "name": "python" 17 | }, 18 | "accelerator": "GPU" 19 | }, 20 | "cells": [ 21 | { 22 | "cell_type": "markdown", 23 | "metadata": { 24 | "id": "view-in-github", 25 | "colab_type": "text" 26 | }, 27 | "source": [ 28 | "\"Open" 29 | ] 30 | }, 31 | { 32 | "cell_type": "code", 33 | "source": [ 34 | "#@title 一键启动！\n", 35 | "!git clone https://github.com/KevinWang676/GPT-SoVITS-v2.git\n", 36 | "!git clone https://huggingface.co/spaces/kevinwang676/GPT-SoVITS-models.git\n", 37 | "%cd GPT-SoVITS-v2\n", 38 | "!pip install --disable-pip-version-check -r requirements.txt\n", 39 | "!sudo apt install ffmpeg\n", 40 | "!sudo apt install libsox-dev\n", 41 | "!mv /content/GPT-SoVITS-models/GPT-SoVITS/tools/damo_asr/models /content/GPT-SoVITS-v2/tools/damo_asr\n", 42 | "!mv /content/GPT-SoVITS-models/GPT-SoVITS/GPT_SoVITS/pretrained_models /content/GPT-SoVITS-v2/GPT_SoVITS\n", 43 | "import nltk\n", 44 | "nltk.download(\"all\")\n", 45 | "!python webui.py" 46 | ], 47 | "metadata": { 48 | "id": "WXqXTJnmq4t6" 49 | }, 50 | "execution_count": null, 51 | "outputs": [] 52 | }, 53 | { 54 | "cell_type": "code", 55 | "source": [], 56 | "metadata": { 57 | "id": "l28zMG05rKoJ" 58 | }, 59 | "execution_count": null, 60 | "outputs": [] 61 | } 62 | ] 63 | } 64 | -------------------------------------------------------------------------------- /notebooks/GPT_SoVITS_emo.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "provenance": [], 7 | "gpuType": "T4", 8 | "include_colab_link": true 9 | }, 10 | "kernelspec": { 11 | "name": "python3", 12 | "display_name": "Python 3" 13 | }, 14 | "language_info": { 15 | "name": "python" 16 | }, 17 | "accelerator": "GPU" 18 | }, 19 | "cells": [ 20 | { 21 | "cell_type": "markdown", 22 | "metadata": { 23 | "id": "view-in-github", 24 | "colab_type": "text" 25 | }, 26 | "source": [ 27 | "\"Open" 28 | ] 29 | }, 30 | { 31 | "cell_type": "code", 32 | "source": [ 33 | "#@title 一键启动！\n", 34 | "!git clone https://github.com/KevinWang676/GPT-SoVITS-emo.git\n", 35 | "!git clone https://huggingface.co/spaces/kevinwang676/GPT-SoVITS-models.git\n", 36 | "%cd GPT-SoVITS-emo\n", 37 | "!pip install --disable-pip-version-check -r requirements.txt\n", 38 | "!sudo apt install ffmpeg\n", 39 | "!sudo apt install libsox-dev\n", 40 | "!mv /content/GPT-SoVITS-models/GPT-SoVITS/tools/damo_asr/models /content/GPT-SoVITS-emo/tools/damo_asr\n", 41 | "!mv /content/GPT-SoVITS-models/GPT-SoVITS/GPT_SoVITS/pretrained_models /content/GPT-SoVITS-emo/GPT_SoVITS\n", 42 | "import nltk\n", 43 | "nltk.download(\"all\")\n", 44 | "import urllib.request\n", 45 | "urllib.request.urlretrieve(\"https://huggingface.co/spaces/kevinwang676/OpenVoice/resolve/main/checkpoints_1226.zip\", \"checkpoints_1226.zip\")\n", 46 | "urllib.request.urlretrieve(\"https://huggingface.co/kevinwang676/OpenVoice-v2/resolve/main/checkpoints_v2_0417.zip\", \"checkpoints_v2_0417.zip\")\n", 47 | "import zipfile\n", 48 | "with zipfile.ZipFile(\"checkpoints_1226.zip\", 'r') as zip_ref:\n", 49 | " zip_ref.extractall(\"\")\n", 50 | "with zipfile.ZipFile(\"checkpoints_v2_0417.zip\", 'r') as zip_ref:\n", 51 | " zip_ref.extractall(\"\")\n", 52 | "!python webui.py" 53 | ], 54 | "metadata": { 55 | "id": "l28zMG05rKoJ" 56 | }, 57 | "execution_count": null, 58 | "outputs": [] 59 | }, 60 | { 61 | "cell_type": "code", 62 | "source": [], 63 | "metadata": { 64 | "id": "1NtVPEQMhX3v" 65 | }, 66 | "execution_count": null, 67 | "outputs": [] 68 | } 69 | ] 70 | } 71 | -------------------------------------------------------------------------------- /notebooks/GPT_SoVITS_v2_0808.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "provenance": [], 7 | "gpuType": "T4", 8 | "authorship_tag": "ABX9TyO531NXQShaXsY3XetvGsHy", 9 | "include_colab_link": true 10 | }, 11 | "kernelspec": { 12 | "name": "python3", 13 | "display_name": "Python 3" 14 | }, 15 | "language_info": { 16 | "name": "python" 17 | }, 18 | "accelerator": "GPU" 19 | }, 20 | "cells": [ 21 | { 22 | "cell_type": "markdown", 23 | "metadata": { 24 | "id": "view-in-github", 25 | "colab_type": "text" 26 | }, 27 | "source": [ 28 | "\"Open" 29 | ] 30 | }, 31 | { 32 | "cell_type": "code", 33 | "source": [ 34 | "#@title 一键运行第二代 GPT-SoVITS\n", 35 | "!git clone https://huggingface.co/spaces/kevinwang676/GPT-SoVITS-v2.git\n", 36 | "%cd GPT-SoVITS-v2\n", 37 | "!pip install --disable-pip-version-check -r requirements.txt\n", 38 | "!sudo apt install ffmpeg\n", 39 | "!sudo apt install libsox-dev\n", 40 | "import nltk\n", 41 | "nltk.download(\"all\")\n", 42 | "!python webui.py" 43 | ], 44 | "metadata": { 45 | "id": "H5ARCxqizYra" 46 | }, 47 | "execution_count": null, 48 | "outputs": [] 49 | }, 50 | { 51 | "cell_type": "code", 52 | "source": [], 53 | "metadata": { 54 | "id": "vmWq1_VbXGBT" 55 | }, 56 | "execution_count": null, 57 | "outputs": [] 58 | } 59 | ] 60 | } 61 | -------------------------------------------------------------------------------- /notebooks/KNN_VC.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "provenance": [], 7 | "include_colab_link": true 8 | }, 9 | "kernelspec": { 10 | "name": "python3", 11 | "display_name": "Python 3" 12 | }, 13 | "language_info": { 14 | "name": "python" 15 | } 16 | }, 17 | "cells": [ 18 | { 19 | "cell_type": "markdown", 20 | "metadata": { 21 | "id": "view-in-github", 22 | "colab_type": "text" 23 | }, 24 | "source": [ 25 | "\"Open" 26 | ] 27 | }, 28 | { 29 | "cell_type": "code", 30 | "execution_count": 1, 31 | "metadata": { 32 | "colab": { 33 | "base_uri": "https://localhost:8080/" 34 | }, 35 | "id": "L8If-TOe4BZO", 36 | "outputId": "22dc83f4-f96e-4c9b-d39e-4375a6275e83" 37 | }, 38 | "outputs": [ 39 | { 40 | "output_type": "stream", 41 | "name": "stderr", 42 | "text": [ 43 | "Downloading: \"https://github.com/bshall/knn-vc/zipball/master\" to /root/.cache/torch/hub/master.zip\n", 44 | "/usr/local/lib/python3.10/dist-packages/torch/nn/utils/weight_norm.py:30: UserWarning: torch.nn.utils.weight_norm is deprecated in favor of torch.nn.utils.parametrizations.weight_norm.\n", 45 | " warnings.warn(\"torch.nn.utils.weight_norm is deprecated in favor of torch.nn.utils.parametrizations.weight_norm.\")\n", 46 | "Downloading: \"https://github.com/bshall/knn-vc/releases/download/v0.1/prematch_g_02500000.pt\" to /root/.cache/torch/hub/checkpoints/prematch_g_02500000.pt\n", 47 | "100%|██████████| 63.1M/63.1M [00:00<00:00, 118MB/s]\n" 48 | ] 49 | }, 50 | { 51 | "output_type": "stream", 52 | "name": "stdout", 53 | "text": [ 54 | "Removing weight norm...\n", 55 | "[HiFiGAN] Generator loaded with 16,523,393 parameters.\n" 56 | ] 57 | }, 58 | { 59 | "output_type": "stream", 60 | "name": "stderr", 61 | "text": [ 62 | "Downloading: \"https://github.com/bshall/knn-vc/releases/download/v0.1/WavLM-Large.pt\" to /root/.cache/torch/hub/checkpoints/WavLM-Large.pt\n", 63 | "100%|██████████| 1.18G/1.18G [00:06<00:00, 191MB/s]\n" 64 | ] 65 | }, 66 | { 67 | "output_type": "stream", 68 | "name": "stdout", 69 | "text": [ 70 | "WavLM-Large loaded with 315,453,120 parameters.\n" 71 | ] 72 | } 73 | ], 74 | "source": [ 75 | "import torch, torchaudio\n", 76 | "\n", 77 | "knn_vc = torch.hub.load('bshall/knn-vc', 'knn_vc', prematched=True, trust_repo=True, pretrained=True, device='cpu')" 78 | ] 79 | }, 80 | { 81 | "cell_type": "code", 82 | "source": [ 83 | "pip install gradio" 84 | ], 85 | "metadata": { 86 | "id": "Bjy9sCcZtdCK", 87 | "outputId": "d997a266-7d59-45ea-a354-901e035bc2d5", 88 | "colab": { 89 | "base_uri": "https://localhost:8080/" 90 | } 91 | }, 92 | "execution_count": 2, 93 | "outputs": [ 94 | { 95 | "output_type": "stream", 96 | "name": "stdout", 97 | "text": [ 98 | "Collecting gradio\n", 99 | " Downloading gradio-4.1.1-py3-none-any.whl (15.4 MB)\n", 100 | "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m15.4/15.4 MB\u001b[0m \u001b[31m82.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", 101 | "\u001b[?25hCollecting aiofiles<24.0,>=22.0 (from gradio)\n", 102 | " Downloading aiofiles-23.2.1-py3-none-any.whl (15 kB)\n", 103 | "Requirement already satisfied: altair<6.0,>=4.2.0 in /usr/local/lib/python3.10/dist-packages (from gradio) (4.2.2)\n", 104 | "Collecting fastapi (from gradio)\n", 105 | " Downloading fastapi-0.104.1-py3-none-any.whl (92 kB)\n", 106 | "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m92.9/92.9 kB\u001b[0m \u001b[31m10.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", 107 | "\u001b[?25hCollecting ffmpy (from gradio)\n", 108 | " Downloading ffmpy-0.3.1.tar.gz (5.5 kB)\n", 109 | " Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n", 110 | "Collecting gradio-client==0.7.0 (from gradio)\n", 111 | " Downloading gradio_client-0.7.0-py3-none-any.whl (302 kB)\n", 112 | "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m302.7/302.7 kB\u001b[0m \u001b[31m27.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", 113 | "\u001b[?25hCollecting httpx (from gradio)\n", 114 | " Downloading httpx-0.25.1-py3-none-any.whl (75 kB)\n", 115 | "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m75.0/75.0 kB\u001b[0m \u001b[31m8.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", 116 | "\u001b[?25hCollecting huggingface-hub>=0.14.0 (from gradio)\n", 117 | " Downloading huggingface_hub-0.18.0-py3-none-any.whl (301 kB)\n", 118 | "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m302.0/302.0 kB\u001b[0m \u001b[31m27.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", 119 | "\u001b[?25hRequirement already satisfied: importlib-resources<7.0,>=1.3 in /usr/local/lib/python3.10/dist-packages (from gradio) (6.1.0)\n", 120 | "Requirement already satisfied: jinja2<4.0 in /usr/local/lib/python3.10/dist-packages (from gradio) (3.1.2)\n", 121 | "Requirement already satisfied: markupsafe~=2.0 in /usr/local/lib/python3.10/dist-packages (from gradio) (2.1.3)\n", 122 | "Requirement already satisfied: matplotlib~=3.0 in /usr/local/lib/python3.10/dist-packages (from gradio) (3.7.1)\n", 123 | "Requirement already satisfied: numpy~=1.0 in /usr/local/lib/python3.10/dist-packages (from gradio) (1.23.5)\n", 124 | "Collecting orjson~=3.0 (from gradio)\n", 125 | " Downloading orjson-3.9.10-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (138 kB)\n", 126 | "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m138.7/138.7 kB\u001b[0m \u001b[31m15.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", 127 | "\u001b[?25hRequirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from gradio) (23.2)\n", 128 | "Requirement already satisfied: pandas<3.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from gradio) (1.5.3)\n", 129 | "Requirement already satisfied: pillow<11.0,>=8.0 in /usr/local/lib/python3.10/dist-packages (from gradio) (9.4.0)\n", 130 | "Collecting pydantic>=2.0 (from gradio)\n", 131 | " Downloading pydantic-2.4.2-py3-none-any.whl (395 kB)\n", 132 | "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m395.8/395.8 kB\u001b[0m \u001b[31m36.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", 133 | "\u001b[?25hCollecting pydub (from gradio)\n", 134 | " Downloading pydub-0.25.1-py2.py3-none-any.whl (32 kB)\n", 135 | "Collecting python-multipart (from gradio)\n", 136 | " Downloading python_multipart-0.0.6-py3-none-any.whl (45 kB)\n", 137 | "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m45.7/45.7 kB\u001b[0m \u001b[31m5.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", 138 | "\u001b[?25hRequirement already satisfied: pyyaml<7.0,>=5.0 in /usr/local/lib/python3.10/dist-packages (from gradio) (6.0.1)\n", 139 | "Requirement already satisfied: requests~=2.0 in /usr/local/lib/python3.10/dist-packages (from gradio) (2.31.0)\n", 140 | "Collecting semantic-version~=2.0 (from gradio)\n", 141 | " Downloading semantic_version-2.10.0-py2.py3-none-any.whl (15 kB)\n", 142 | "Collecting tomlkit==0.12.0 (from gradio)\n", 143 | " Downloading tomlkit-0.12.0-py3-none-any.whl (37 kB)\n", 144 | "Requirement already satisfied: typer[all]<1.0,>=0.9 in /usr/local/lib/python3.10/dist-packages (from gradio) (0.9.0)\n", 145 | "Requirement already satisfied: typing-extensions~=4.0 in /usr/local/lib/python3.10/dist-packages (from gradio) (4.5.0)\n", 146 | "Collecting uvicorn>=0.14.0 (from gradio)\n", 147 | " Downloading uvicorn-0.24.0.post1-py3-none-any.whl (59 kB)\n", 148 | "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m59.7/59.7 kB\u001b[0m \u001b[31m4.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", 149 | "\u001b[?25hCollecting websockets<12.0,>=10.0 (from gradio)\n", 150 | " Downloading websockets-11.0.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (129 kB)\n", 151 | "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m129.9/129.9 kB\u001b[0m \u001b[31m13.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", 152 | "\u001b[?25hRequirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from gradio-client==0.7.0->gradio) (2023.6.0)\n", 153 | "Requirement already satisfied: entrypoints in /usr/local/lib/python3.10/dist-packages (from altair<6.0,>=4.2.0->gradio) (0.4)\n", 154 | "Requirement already satisfied: jsonschema>=3.0 in /usr/local/lib/python3.10/dist-packages (from altair<6.0,>=4.2.0->gradio) (4.19.1)\n", 155 | "Requirement already satisfied: toolz in /usr/local/lib/python3.10/dist-packages (from altair<6.0,>=4.2.0->gradio) (0.12.0)\n", 156 | "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.14.0->gradio) (3.12.4)\n", 157 | "Requirement already satisfied: tqdm>=4.42.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.14.0->gradio) (4.66.1)\n", 158 | "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib~=3.0->gradio) (1.1.1)\n", 159 | "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/dist-packages (from matplotlib~=3.0->gradio) (0.12.1)\n", 160 | "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib~=3.0->gradio) (4.43.1)\n", 161 | "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib~=3.0->gradio) (1.4.5)\n", 162 | "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib~=3.0->gradio) (3.1.1)\n", 163 | "Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.10/dist-packages (from matplotlib~=3.0->gradio) (2.8.2)\n", 164 | "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas<3.0,>=1.0->gradio) (2023.3.post1)\n", 165 | "Collecting annotated-types>=0.4.0 (from pydantic>=2.0->gradio)\n", 166 | " Downloading annotated_types-0.6.0-py3-none-any.whl (12 kB)\n", 167 | "Collecting pydantic-core==2.10.1 (from pydantic>=2.0->gradio)\n", 168 | " Downloading pydantic_core-2.10.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (2.0 MB)\n", 169 | "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.0/2.0 MB\u001b[0m \u001b[31m65.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", 170 | "\u001b[?25hCollecting typing-extensions~=4.0 (from gradio)\n", 171 | " Downloading typing_extensions-4.8.0-py3-none-any.whl (31 kB)\n", 172 | "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests~=2.0->gradio) (3.3.1)\n", 173 | "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests~=2.0->gradio) (3.4)\n", 174 | "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests~=2.0->gradio) (2.0.7)\n", 175 | "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests~=2.0->gradio) (2023.7.22)\n", 176 | "Requirement already satisfied: click<9.0.0,>=7.1.1 in /usr/local/lib/python3.10/dist-packages (from typer[all]<1.0,>=0.9->gradio) (8.1.7)\n", 177 | "Collecting colorama<0.5.0,>=0.4.3 (from typer[all]<1.0,>=0.9->gradio)\n", 178 | " Downloading colorama-0.4.6-py2.py3-none-any.whl (25 kB)\n", 179 | "Collecting shellingham<2.0.0,>=1.3.0 (from typer[all]<1.0,>=0.9->gradio)\n", 180 | " Downloading shellingham-1.5.4-py2.py3-none-any.whl (9.8 kB)\n", 181 | "Requirement already satisfied: rich<14.0.0,>=10.11.0 in /usr/local/lib/python3.10/dist-packages (from typer[all]<1.0,>=0.9->gradio) (13.6.0)\n", 182 | "Collecting h11>=0.8 (from uvicorn>=0.14.0->gradio)\n", 183 | " Downloading h11-0.14.0-py3-none-any.whl (58 kB)\n", 184 | "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m7.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", 185 | "\u001b[?25hRequirement already satisfied: anyio<4.0.0,>=3.7.1 in /usr/local/lib/python3.10/dist-packages (from fastapi->gradio) (3.7.1)\n", 186 | "Collecting starlette<0.28.0,>=0.27.0 (from fastapi->gradio)\n", 187 | " Downloading starlette-0.27.0-py3-none-any.whl (66 kB)\n", 188 | "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m67.0/67.0 kB\u001b[0m \u001b[31m7.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", 189 | "\u001b[?25hCollecting httpcore (from httpx->gradio)\n", 190 | " Downloading httpcore-1.0.1-py3-none-any.whl (76 kB)\n", 191 | "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m76.9/76.9 kB\u001b[0m \u001b[31m7.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", 192 | "\u001b[?25hRequirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from httpx->gradio) (1.3.0)\n", 193 | "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<4.0.0,>=3.7.1->fastapi->gradio) (1.1.3)\n", 194 | "Requirement already satisfied: attrs>=22.2.0 in /usr/local/lib/python3.10/dist-packages (from jsonschema>=3.0->altair<6.0,>=4.2.0->gradio) (23.1.0)\n", 195 | "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /usr/local/lib/python3.10/dist-packages (from jsonschema>=3.0->altair<6.0,>=4.2.0->gradio) (2023.7.1)\n", 196 | "Requirement already satisfied: referencing>=0.28.4 in /usr/local/lib/python3.10/dist-packages (from jsonschema>=3.0->altair<6.0,>=4.2.0->gradio) (0.30.2)\n", 197 | "Requirement already satisfied: rpds-py>=0.7.1 in /usr/local/lib/python3.10/dist-packages (from jsonschema>=3.0->altair<6.0,>=4.2.0->gradio) (0.10.6)\n", 198 | "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.7->matplotlib~=3.0->gradio) (1.16.0)\n", 199 | "Requirement already satisfied: markdown-it-py>=2.2.0 in /usr/local/lib/python3.10/dist-packages (from rich<14.0.0,>=10.11.0->typer[all]<1.0,>=0.9->gradio) (3.0.0)\n", 200 | "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /usr/local/lib/python3.10/dist-packages (from rich<14.0.0,>=10.11.0->typer[all]<1.0,>=0.9->gradio) (2.16.1)\n", 201 | "Requirement already satisfied: mdurl~=0.1 in /usr/local/lib/python3.10/dist-packages (from markdown-it-py>=2.2.0->rich<14.0.0,>=10.11.0->typer[all]<1.0,>=0.9->gradio) (0.1.2)\n", 202 | "Building wheels for collected packages: ffmpy\n", 203 | " Building wheel for ffmpy (setup.py) ... \u001b[?25l\u001b[?25hdone\n", 204 | " Created wheel for ffmpy: filename=ffmpy-0.3.1-py3-none-any.whl size=5579 sha256=47414f4ed3cb963fc14a5d6cc919c8e77c90d96b638e51a6d9da4b212eb70a21\n", 205 | " Stored in directory: /root/.cache/pip/wheels/01/a6/d1/1c0828c304a4283b2c1639a09ad86f83d7c487ef34c6b4a1bf\n", 206 | "Successfully built ffmpy\n", 207 | "Installing collected packages: pydub, ffmpy, websockets, typing-extensions, tomlkit, shellingham, semantic-version, python-multipart, orjson, h11, colorama, annotated-types, aiofiles, uvicorn, starlette, pydantic-core, huggingface-hub, httpcore, pydantic, httpx, gradio-client, fastapi, gradio\n", 208 | " Attempting uninstall: typing-extensions\n", 209 | " Found existing installation: typing_extensions 4.5.0\n", 210 | " Uninstalling typing_extensions-4.5.0:\n", 211 | " Successfully uninstalled typing_extensions-4.5.0\n", 212 | " Attempting uninstall: pydantic\n", 213 | " Found existing installation: pydantic 1.10.13\n", 214 | " Uninstalling pydantic-1.10.13:\n", 215 | " Successfully uninstalled pydantic-1.10.13\n", 216 | "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n", 217 | "lida 0.0.10 requires kaleido, which is not installed.\n", 218 | "llmx 0.0.15a0 requires cohere, which is not installed.\n", 219 | "llmx 0.0.15a0 requires openai, which is not installed.\n", 220 | "llmx 0.0.15a0 requires tiktoken, which is not installed.\n", 221 | "tensorflow-probability 0.22.0 requires typing-extensions<4.6.0, but you have typing-extensions 4.8.0 which is incompatible.\u001b[0m\u001b[31m\n", 222 | "\u001b[0mSuccessfully installed aiofiles-23.2.1 annotated-types-0.6.0 colorama-0.4.6 fastapi-0.104.1 ffmpy-0.3.1 gradio-4.1.1 gradio-client-0.7.0 h11-0.14.0 httpcore-1.0.1 httpx-0.25.1 huggingface-hub-0.18.0 orjson-3.9.10 pydantic-2.4.2 pydantic-core-2.10.1 pydub-0.25.1 python-multipart-0.0.6 semantic-version-2.10.0 shellingham-1.5.4 starlette-0.27.0 tomlkit-0.12.0 typing-extensions-4.8.0 uvicorn-0.24.0.post1 websockets-11.0.3\n" 223 | ] 224 | } 225 | ] 226 | }, 227 | { 228 | "cell_type": "code", 229 | "source": [ 230 | "import os\n", 231 | "os._exit(00)" 232 | ], 233 | "metadata": { 234 | "id": "zL6hvNjiuf75" 235 | }, 236 | "execution_count": null, 237 | "outputs": [] 238 | }, 239 | { 240 | "cell_type": "code", 241 | "source": [ 242 | "import os\n", 243 | "import gradio as gr\n", 244 | "import torch, torchaudio\n", 245 | "from scipy.io import wavfile\n", 246 | "from scipy.io.wavfile import write\n", 247 | "knn_vc = torch.hub.load('bshall/knn-vc', 'knn_vc', prematched=True, trust_repo=True, pretrained=True, device='cpu')" 248 | ], 249 | "metadata": { 250 | "id": "45_VKgTmt5bE", 251 | "colab": { 252 | "base_uri": "https://localhost:8080/" 253 | }, 254 | "outputId": "0dab7712-1b4c-430c-fba2-adfa6a68d5e5" 255 | }, 256 | "execution_count": 6, 257 | "outputs": [ 258 | { 259 | "output_type": "stream", 260 | "name": "stderr", 261 | "text": [ 262 | "Using cache found in /root/.cache/torch/hub/bshall_knn-vc_master\n", 263 | "/usr/local/lib/python3.10/dist-packages/torch/nn/utils/weight_norm.py:30: UserWarning: torch.nn.utils.weight_norm is deprecated in favor of torch.nn.utils.parametrizations.weight_norm.\n", 264 | " warnings.warn(\"torch.nn.utils.weight_norm is deprecated in favor of torch.nn.utils.parametrizations.weight_norm.\")\n" 265 | ] 266 | }, 267 | { 268 | "output_type": "stream", 269 | "name": "stdout", 270 | "text": [ 271 | "Removing weight norm...\n", 272 | "[HiFiGAN] Generator loaded with 16,523,393 parameters.\n", 273 | "WavLM-Large loaded with 315,453,120 parameters.\n" 274 | ] 275 | } 276 | ] 277 | }, 278 | { 279 | "cell_type": "code", 280 | "source": [ 281 | "def voice_change(audio_in, audio_ref):\n", 282 | " samplerate1, data1 = wavfile.read(audio_in)\n", 283 | " samplerate2, data2 = wavfile.read(audio_ref)\n", 284 | " write(\"/content/audio_in.wav\", samplerate1, data1)\n", 285 | " write(\"/content/audio_ref.wav\", samplerate2, data2)\n", 286 | "\n", 287 | " query_seq = knn_vc.get_features(\"/content/audio_in.wav\")\n", 288 | " matching_set = knn_vc.get_matching_set([\"/content/audio_ref.wav\"])\n", 289 | " out_wav = knn_vc.match(query_seq, matching_set, topk=4)\n", 290 | " torchaudio.save('knnvc1_out.wav', out_wav[None], 16000)\n", 291 | " return 'knnvc1_out.wav'\n" 292 | ], 293 | "metadata": { 294 | "id": "XTin6hMJwZ0V" 295 | }, 296 | "execution_count": 7, 297 | "outputs": [] 298 | }, 299 | { 300 | "cell_type": "code", 301 | "source": [ 302 | "app = gr.Blocks()\n", 303 | "\n", 304 | "with app:\n", 305 | " gr.Markdown(\"#
🥳🎶🎡 - KNN-VC AI变声
\")\n", 306 | " gr.Markdown(\"###
🌟 - 3秒实时AI变声，支持中日英在内的所有语言！无需训练、一键开启AI变声之旅！Powered by [knn-vc](https://github.com/bshall/knn-vc)🍻
\")\n", 307 | " gr.Markdown(\"###
🌊 - 更多精彩应用，敬请关注[滔滔AI](http://www.talktalkai.com)；滔滔AI，为爱滔滔！💕
\")\n", 308 | "\n", 309 | " with gr.Row():\n", 310 | " with gr.Column():\n", 311 | " inp1 = gr.Audio(type=\"filepath\", label=\"请上传AI变声的原音频（决定变声后的语音内容）\")\n", 312 | " inp2 = gr.Audio(type=\"filepath\", label=\"请上传AI变声的参照音频（决定变声后的语音音色）\")\n", 313 | " btn1 = gr.Button(\"一键开启AI变声吧\", variant=\"primary\")\n", 314 | " with gr.Column():\n", 315 | " out1 = gr.Audio(type=\"filepath\", label=\"AI变声后的专属音频\")\n", 316 | "\n", 317 | " btn1.click(voice_change, [inp1, inp2], out1)\n", 318 | "\n", 319 | " gr.Markdown(\"###
注意❗：请不要生成会对个人以及组织造成侵害的内容，此程序仅供科研、学习及个人娱乐使用。
\")\n", 320 | " gr.HTML('''\n", 321 | "
\n", 322 | "

🌊🏞️🎶 - 江水东流急，滔滔无尽声。 明·顾璘\n", 323 | "

\n", 324 | "
\n", 325 | " ''')\n", 326 | "\n", 327 | "app.launch(show_error=True, share=True)\n" 328 | ], 329 | "metadata": { 330 | "id": "J1O82uBKtbuq", 331 | "outputId": "92b69abf-4558-4076-a665-70e29a28f4db", 332 | "colab": { 333 | "base_uri": "https://localhost:8080/", 334 | "height": 612 335 | } 336 | }, 337 | "execution_count": 8, 338 | "outputs": [ 339 | { 340 | "output_type": "stream", 341 | "name": "stdout", 342 | "text": [ 343 | "Colab notebook detected. To show errors in colab notebook, set debug=True in launch()\n", 344 | "Running on public URL: https://803258d7ecbccb52b0.gradio.live\n", 345 | "\n", 346 | "This share link expires in 72 hours. For free permanent hosting and GPU upgrades, run `gradio deploy` from Terminal to deploy to Spaces (https://huggingface.co/spaces)\n" 347 | ] 348 | }, 349 | { 350 | "output_type": "display_data", 351 | "data": { 352 | "text/plain": [ 353 | "" 354 | ], 355 | "text/html": [ 356 | "
" 357 | ] 358 | }, 359 | "metadata": {} 360 | }, 361 | { 362 | "output_type": "execute_result", 363 | "data": { 364 | "text/plain": [] 365 | }, 366 | "metadata": {}, 367 | "execution_count": 8 368 | } 369 | ] 370 | }, 371 | { 372 | "cell_type": "markdown", 373 | "source": [ 374 | "" 375 | ], 376 | "metadata": { 377 | "id": "wyWtICjSECkF" 378 | } 379 | }, 380 | { 381 | "cell_type": "code", 382 | "source": [], 383 | "metadata": { 384 | "id": "Qe9990guZBtr" 385 | }, 386 | "execution_count": null, 387 | "outputs": [] 388 | } 389 | ] 390 | } 391 | -------------------------------------------------------------------------------- /notebooks/MuseTalk.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "id": "view-in-github", 7 | "colab_type": "text" 8 | }, 9 | "source": [ 10 | "\"Open" 11 | ] 12 | }, 13 | { 14 | "cell_type": "code", 15 | "source": [ 16 | "#@title 一键启动\n", 17 | "!git clone https://github.com/TMElyralab/MuseTalk.git\n", 18 | "%cd MuseTalk\n", 19 | "!git clone https://huggingface.co/kevinwang676/MuseTalk.git\n", 20 | "!mv /content/MuseTalk/MuseTalk/ffmpeg-6.1-amd64-static /content/MuseTalk/\n", 21 | "!mv /content/MuseTalk/MuseTalk/models /content/MuseTalk/\n", 22 | "!mv /content/MuseTalk/MuseTalk/app_new.py /content/MuseTalk/\n", 23 | "!pip install -r requirements.txt\n", 24 | "!pip install --no-cache-dir -U openmim\n", 25 | "!mim install mmengine\n", 26 | "!mim install \"mmcv>=2.0.1\"\n", 27 | "!mim install \"mmdet>=3.1.0\"\n", 28 | "!mim install \"mmpose>=1.1.0\"\n", 29 | "!export FFMPEG_PATH=/content/MuseTalk/ffmpeg-6.1-amd64-static\n", 30 | "import os\n", 31 | "os.environ[\"FFMPEG_PATH\"] = \"/content/MuseTalk/ffmpeg-6.1-amd64-static\"\n", 32 | "!python app_new.py" 33 | ], 34 | "metadata": { 35 | "id": "x2S1XDoRCQz7" 36 | }, 37 | "execution_count": null, 38 | "outputs": [] 39 | }, 40 | { 41 | "cell_type": "code", 42 | "source": [], 43 | "metadata": { 44 | "id": "HqZHQaLpCX2n" 45 | }, 46 | "execution_count": null, 47 | "outputs": [] 48 | }, 49 | { 50 | "cell_type": "code", 51 | "source": [ 52 | "#!python -m scripts.inference --inference_config configs/inference/test.yaml" 53 | ], 54 | "metadata": { 55 | "id": "MrI7vCrpCVYt" 56 | }, 57 | "execution_count": null, 58 | "outputs": [] 59 | }, 60 | { 61 | "cell_type": "code", 62 | "source": [ 63 | "'''\n", 64 | "!wget https://cdn-media.huggingface.co/frpc-gradio-0.2/frpc_linux_amd64\n", 65 | "!mv frpc_linux_amd64 frpc_linux_amd64_v0.2\n", 66 | "!mv frpc_linux_amd64_v0.2 /usr/local/lib/python3.10/dist-packages/gradio\n", 67 | "!chmod +x /usr/local/lib/python3.10/dist-packages/gradio/frpc_linux_amd64_v0.2\n", 68 | "'''" 69 | ], 70 | "metadata": { 71 | "id": "bCFYAaHFcOAM" 72 | }, 73 | "execution_count": null, 74 | "outputs": [] 75 | } 76 | ], 77 | "metadata": { 78 | "accelerator": "GPU", 79 | "colab": { 80 | "gpuType": "T4", 81 | "provenance": [], 82 | "authorship_tag": "ABX9TyOlCoxHZK6bkrsq62LcagmO", 83 | "include_colab_link": true 84 | }, 85 | "kernelspec": { 86 | "display_name": "Python 3", 87 | "name": "python3" 88 | }, 89 | "language_info": { 90 | "name": "python" 91 | } 92 | }, 93 | "nbformat": 4, 94 | "nbformat_minor": 0 95 | } 96 | -------------------------------------------------------------------------------- /notebooks/MuseV.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "nbformat": 4, 3 | "nbformat_minor": 0, 4 | "metadata": { 5 | "colab": { 6 | "provenance": [], 7 | "machine_shape": "hm", 8 | "gpuType": "L4", 9 | "authorship_tag": "ABX9TyNrXC+gpXzxifbM6t7osuBD", 10 | "include_colab_link": true 11 | }, 12 | "kernelspec": { 13 | "name": "python3", 14 | "display_name": "Python 3" 15 | }, 16 | "language_info": { 17 | "name": "python" 18 | }, 19 | "accelerator": "GPU" 20 | }, 21 | "cells": [ 22 | { 23 | "cell_type": "markdown", 24 | "metadata": { 25 | "id": "view-in-github", 26 | "colab_type": "text" 27 | }, 28 | "source": [ 29 | "\"Open" 30 | ] 31 | }, 32 | { 33 | "cell_type": "code", 34 | "source": [ 35 | "# @title 一键启动\n", 36 | "\n", 37 | "!git clone --recursive https://github.com/KevinWang676/MuseV.git\n", 38 | "%cd MuseV\n", 39 | "!pip install -r requirements.txt\n", 40 | "!pip install gradio==4.12\n", 41 | "!pip install --no-cache-dir -U openmim\n", 42 | "!mim install mmengine\n", 43 | "!mim install \"mmcv>=2.0.1\"\n", 44 | "!mim install \"mmdet>=3.1.0\"\n", 45 | "!mim install \"mmpose>=1.1.0\"\n", 46 | "!git clone https://huggingface.co/TMElyralab/MuseV ./checkpoints\n", 47 | "!mv musev scripts/gradio\n", 48 | "%cd scripts/gradio\n", 49 | "!python app.py" 50 | ], 51 | "metadata": { 52 | "id": "sEuM6BNHhABF" 53 | }, 54 | "execution_count": null, 55 | "outputs": [] 56 | }, 57 | { 58 | "cell_type": "code", 59 | "source": [], 60 | "metadata": { 61 | "id": "yUOIyBYoCKlY" 62 | }, 63 | "execution_count": null, 64 | "outputs": [] 65 | } 66 | ] 67 | } 68 | -------------------------------------------------------------------------------- /notebooks/NeuCoSVC_v2_先享版.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": { 6 | "id": "view-in-github", 7 | "colab_type": "text" 8 | }, 9 | "source": [ 10 | "\"Open" 11 | ] 12 | }, 13 | { 14 | "cell_type": "code", 15 | "source": [ 16 | "#@title 一键启动AI翻唱\n", 17 | "!git clone https://huggingface.co/spaces/kevinwang676/NeuCoSVC-2.git\n", 18 | "%cd NeuCoSVC-2\n", 19 | "!pip install -r requirements_colab.txt\n", 20 | "import shutil\n", 21 | "shutil.rmtree(\"REAPER\")\n", 22 | "!git clone https://github.com/google/REAPER.git\n", 23 | "%cd REAPER\n", 24 | "!mkdir build\n", 25 | "%cd build\n", 26 | "!cmake ..\n", 27 | "!make\n", 28 | "%cd ..\n", 29 | "%cd ..\n", 30 | "!python app_colab.py" 31 | ], 32 | "metadata": { 33 | "id": "BBb8LK0KXw8n" 34 | }, 35 | "execution_count": null, 36 | "outputs": [] 37 | }, 38 | { 39 | "cell_type": "code", 40 | "source": [], 41 | "metadata": { 42 | "id": "usDN-tbr0ALs" 43 | }, 44 | "execution_count": null, 45 | "outputs": [] 46 | }, 47 | { 48 | "cell_type": "code", 49 | "execution_count": null, 50 | "metadata": { 51 | "id": "tWPIdmlBaZNe" 52 | }, 53 | "outputs": [], 54 | "source": [ 55 | "#convert(20, \"花海 周杰伦\", \"遇见 孙燕姿\", True, True, 0, 0, 0)" 56 | ] 57 | }, 58 | { 59 | "cell_type": "code", 60 | "execution_count": null, 61 | "metadata": { 62 | "id": "aQrhHH11WoM6" 63 | }, 64 | "outputs": [], 65 | "source": [ 66 | "#!python inference.py --src_wav_path src_mono.wav --ref_wav_path ref_mono.wav --key_shift 0" 67 | ] 68 | } 69 | ], 70 | "metadata": { 71 | "accelerator": "GPU", 72 | "colab": { 73 | "gpuType": "T4", 74 | "provenance": [], 75 | "authorship_tag": "ABX9TyPv1XfmiGh8HgdsLVZA4o31", 76 | "include_colab_link": true 77 | }, 78 | "kernelspec": { 79 | "display_name": "Python 3", 80 | "name": "python3" 81 | }, 82 | "language_info": { 83 | "name": "python" 84 | } 85 | }, 86 | "nbformat": 4, 87 | "nbformat_minor": 0 88 | } 89 | -------------------------------------------------------------------------------- /notebooks/README.md: -------------------------------------------------------------------------------- 1 | # ChatGLM2微调指南 💡 2 | #### ChatGLM3微调[见下方](https://github.com/KevinWang676/Bark-Voice-Cloning/blob/main/notebooks/README.md#chatglm3%E5%BE%AE%E8%B0%83%E6%8C%87%E5%8D%97-) 3 | #### AI Agent搭建[见下方](https://github.com/KevinWang676/Bark-Voice-Cloning/tree/main/notebooks#ai-agent-%E4%BD%BF%E7%94%A8%E6%8C%87%E5%8D%97-) 4 | ## 1. 环境搭建 5 | ### 安装依赖 6 | ``` 7 | git clone https://github.com/THUDM/ChatGLM2-6B 8 | cd ChatGLM2-6B 9 | pip install -r requirements.txt 10 | pip install rouge_chinese nltk jieba datasets 11 | ``` 12 | ### 下载模型 13 | ``` 14 | curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | sudo bash 15 | sudo apt-get install git-lfs 16 | sudo apt install build-essential 17 | git clone https://huggingface.co/THUDM/chatglm2-6b 18 | ``` 19 | 20 | ## 2. 准备数据集 21 | 22 | 自建数据集请参考[train.json](https://github.com/KevinWang676/Bark-Voice-Cloning/blob/main/notebooks/train.json)及以下格式： 23 | ```json lines 24 | {"prompt": "长城h3风扇不转。继电器好的。保险丝好的传感器新的风扇也新的这是为什么。就是继电器缺一个信号线", "response": "用电脑能读数据流吗？水温多少", "history": []} 25 | {"prompt": "95", "response": "上下水管温差怎么样啊？空气是不是都排干净了呢？", "history": [["长城h3风扇不转。继电器好的。保险丝好的传感器新的风扇也新的这是为什么。就是继电器缺一个信号线", "用电脑能读数据流吗？水温多少"]]} 26 | {"prompt": "是的。上下水管都好的", "response": "那就要检查线路了，一般风扇继电器是由电脑控制吸合的，如果电路存在断路，或者电脑坏了的话会出现继电器不吸合的情况！", "history": [["长城h3风扇不转。继电器好的。保险丝好的传感器新的风扇也新的这是为什么。就是继电器缺一个信号线", "用电脑能读数据流吗？水温多少"], ["95", "上下水管温差怎么样啊？空气是不是都排干净了呢？"]]} 27 | ``` 28 | 分别准备训练数据集 `train.json` 和验证数据集 `dev.json` 并将其上传至 `ChatGLM2-6B` 文件夹下 29 | 30 | ## 3. 开始训练 31 | 32 | 在终端运行以下指令，即可开始训练 33 | ```shell 34 | bash train_chat.sh 35 | ``` 36 | 37 | **注意**：原 `train_chat.sh` 文件中包含以下代码： 38 | ``` 39 | PRE_SEQ_LEN=128 40 | LR=1e-2 41 | NUM_GPUS=1 42 | 43 | torchrun --standalone --nnodes=1 --nproc-per-node=$NUM_GPUS main.py \ 44 | --do_train \ 45 | --train_file $CHAT_TRAIN_DATA \ 46 | --validation_file $CHAT_VAL_DATA \ 47 | --preprocessing_num_workers 10 \ 48 | --prompt_column prompt \ 49 | --response_column response \ 50 | --history_column history \ 51 | --overwrite_cache \ 52 | --model_name_or_path THUDM/chatglm2-6b \ 53 | --output_dir $CHECKPOINT_NAME \ 54 | --overwrite_output_dir \ 55 | --max_source_length 256 \ 56 | --max_target_length 256 \ 57 | --per_device_train_batch_size 1 \ 58 | --per_device_eval_batch_size 1 \ 59 | --gradient_accumulation_steps 16 \ 60 | --predict_with_generate \ 61 | --max_steps 3000 \ 62 | --logging_steps 10 \ 63 | --save_steps 1000 \ 64 | --learning_rate $LR \ 65 | --pre_seq_len $PRE_SEQ_LEN \ 66 | --quantization_bit 4 67 | ``` 68 | **在开始训练前，需要将其编辑为以下示例代码**： 69 | ``` 70 | PRE_SEQ_LEN=128 71 | LR=1e-2 72 | NUM_GPUS=1 73 | 74 | torchrun --standalone --nnodes=1 --nproc-per-node=$NUM_GPUS ptuning/main.py \ 75 | --do_train \ 76 | --train_file train.json \ 77 | --validation_file dev.json \ 78 | --preprocessing_num_workers 10 \ 79 | --prompt_column prompt \ 80 | --response_column response \ 81 | --history_column history \ 82 | --overwrite_cache \ 83 | --model_name_or_path chatglm2-6b \ 84 | --output_dir output_model \ 85 | --overwrite_output_dir \ 86 | --max_source_length 1024 \ 87 | --max_target_length 1024 \ 88 | --per_device_train_batch_size 1 \ 89 | --per_device_eval_batch_size 1 \ 90 | --gradient_accumulation_steps 16 \ 91 | --predict_with_generate \ 92 | --max_steps 600 \ 93 | --logging_steps 10 \ 94 | --save_steps 100 \ 95 | --learning_rate $LR \ 96 | --pre_seq_len $PRE_SEQ_LEN 97 | ``` 98 | 99 | P.S. 以上的 `train_chat.sh` 文件只是一个示例，具体参数设置请根据不同GPU的性能进行调节；ChatGLM2微调[官方教程](https://github.com/THUDM/ChatGLM2-6B/tree/main/ptuning) 100 | 101 | # ChatGLM3微调指南 📒 102 | 103 | ## 1. 环境搭建 104 | ### 安装依赖 105 | ``` 106 | git clone https://github.com/THUDM/ChatGLM3 107 | cd ChatGLM3 108 | pip install -r requirements.txt 109 | pip install transformers==4.34.0 110 | apt install nvidia-cuda-toolkit 111 | cd finetune_chatmodel_demo 112 | pip install -r requirements.txt 113 | cd .. 114 | ``` 115 | ### 下载模型 116 | ``` 117 | curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | sudo bash 118 | sudo apt-get install git-lfs 119 | sudo apt install build-essential 120 | git clone https://huggingface.co/THUDM/chatglm3-6b 121 | ``` 122 | ## 2. 准备数据集 123 | 124 | 自建数据集请参考[train_linghua_new_v3.json](https://github.com/KevinWang676/Bark-Voice-Cloning/blob/main/notebooks/train_linghua_new_v3.json)及以下格式： 125 | ```json 126 | [ 127 | { 128 | "conversations": [ 129 | { 130 | "role": "system", 131 | "content": "" 132 | }, 133 | { 134 | "role": "user", 135 | "content": "" 136 | }, 137 | { 138 | "role": "assistant", 139 | "content": "" 140 | }, 141 | // ... Muti Turn 142 | { 143 | "role": "user", 144 | "content": "" 145 | }, 146 | { 147 | "role": "assistant", 148 | "content": "" 149 | } 150 | ] 151 | } 152 | // ... 153 | ] 154 | ``` 155 | 156 | 157 | 准备训练数据集 `train.json` 并将其上传至 `ChatGLM3` 文件夹下 158 | 159 | ## 3. 开始训练 160 | 161 | 在终端运行以下指令，即可开始训练 162 | ```shell 163 | bash finetune_chatmodel_demo/scripts/finetune_pt_multiturn.sh 164 | ``` 165 | 166 | **注意**：原 `finetune_pt_multiturn.sh` 文件中包含以下代码： 167 | ``` 168 | #! /usr/bin/env bash 169 | 170 | set -ex 171 | 172 | PRE_SEQ_LEN=128 173 | LR=2e-2 174 | NUM_GPUS=1 175 | MAX_SEQ_LEN=2048 176 | DEV_BATCH_SIZE=1 177 | GRAD_ACCUMULARION_STEPS=16 178 | MAX_STEP=1000 179 | SAVE_INTERVAL=500 180 | 181 | DATESTR=`date +%Y%m%d-%H%M%S` 182 | RUN_NAME=tool_alpaca_pt 183 | 184 | BASE_MODEL_PATH=THUDM/chatglm3-6b 185 | DATASET_PATH=formatted_data/tool_alpaca.jsonl 186 | OUTPUT_DIR=output/${RUN_NAME}-${DATESTR}-${PRE_SEQ_LEN}-${LR} 187 | 188 | mkdir -p $OUTPUT_DIR 189 | 190 | torchrun --standalone --nnodes=1 --nproc_per_node=$NUM_GPUS finetune.py \ 191 | --train_format multi-turn \ 192 | --train_file $DATASET_PATH \ 193 | --max_seq_length $MAX_SEQ_LEN \ 194 | --preprocessing_num_workers 1 \ 195 | --model_name_or_path $BASE_MODEL_PATH \ 196 | --output_dir $OUTPUT_DIR \ 197 | --per_device_train_batch_size $DEV_BATCH_SIZE \ 198 | --gradient_accumulation_steps $GRAD_ACCUMULARION_STEPS \ 199 | --max_steps $MAX_STEP \ 200 | --logging_steps 1 \ 201 | --save_steps $SAVE_INTERVAL \ 202 | --learning_rate $LR \ 203 | --pre_seq_len $PRE_SEQ_LEN 2>&1 | tee ${OUTPUT_DIR}/train.log 204 | ``` 205 | **在开始训练前，需要将其编辑为以下示例代码**： 206 | ``` 207 | #! /usr/bin/env bash 208 | 209 | set -ex 210 | 211 | PRE_SEQ_LEN=128 212 | LR=1e-2 213 | NUM_GPUS=1 214 | MAX_SEQ_LEN=2048 215 | DEV_BATCH_SIZE=1 216 | GRAD_ACCUMULARION_STEPS=16 217 | MAX_STEP=700 218 | SAVE_INTERVAL=100 219 | 220 | DATESTR=`date +%Y%m%d-%H%M%S` 221 | RUN_NAME=linghua_pt 222 | 223 | BASE_MODEL_PATH=chatglm3-6b 224 | DATASET_PATH=train.json 225 | OUTPUT_DIR=output/${RUN_NAME}-${DATESTR}-${PRE_SEQ_LEN}-${LR} 226 | 227 | mkdir -p $OUTPUT_DIR 228 | 229 | torchrun --standalone --nnodes=1 --nproc_per_node=$NUM_GPUS finetune_chatmodel_demo/finetune.py \ 230 | --train_format multi-turn \ 231 | --train_file $DATASET_PATH \ 232 | --max_seq_length $MAX_SEQ_LEN \ 233 | --preprocessing_num_workers 1 \ 234 | --model_name_or_path $BASE_MODEL_PATH \ 235 | --output_dir $OUTPUT_DIR \ 236 | --per_device_train_batch_size $DEV_BATCH_SIZE \ 237 | --gradient_accumulation_steps $GRAD_ACCUMULARION_STEPS \ 238 | --max_steps $MAX_STEP \ 239 | --logging_steps 1 \ 240 | --save_steps $SAVE_INTERVAL \ 241 | --learning_rate $LR \ 242 | --pre_seq_len $PRE_SEQ_LEN 2>&1 | tee ${OUTPUT_DIR}/train.log 243 | ``` 244 | P.S. 以上的 `finetune_pt_multiturn.sh` 文件只是一个示例，具体参数设置请根据不同GPU的性能进行调节；ChatGLM3微调[官方教程](https://github.com/THUDM/ChatGLM3/tree/main/finetune_chatmodel_demo) 245 | 246 | 247 | # AI Agent 使用指南 🌟 248 | 249 | ## 1. 环境搭建 250 | ### 安装依赖 251 | ``` 252 | git clone https://github.com/KevinWang676/modelscope-agent.git 253 | cd modelscope-agent 254 | pip install -r requirements.txt 255 | mv modelscope_agent apps/agentfabric 256 | apt-get update && apt-get install ffmpeg libsm6 libxext6 -y 257 | cd apps/agentfabric 258 | ``` 259 | ### API Key设置 260 | ``` 261 | import os 262 | os.environ["DASHSCOPE_API_KEY"] = "您的DASHSCOPE_API_KEY" 263 | ``` 264 | 或 265 | `export DASHSCOPE_API_KEY=your_api_key` 266 | 267 | ## 2. 开始使用 268 | ``` 269 | python app.py 270 | ``` 271 | -------------------------------------------------------------------------------- /notebooks/train.json: -------------------------------------------------------------------------------- 1 | {"prompt": "你好呀！", "response": "你好呀，旅行者！我是神里绫华，很高兴认识你！", "history": []} 2 | {"prompt": "早上好！", "response": "早上好呀，旅行者！我是神里绫华，今天过得怎么样呀！", "history": []} 3 | {"prompt": "晚上好！", "response": "晚上好呀，旅行者！我是神里绫华，今晚过得怎么样呀！", "history": []} 4 | {"prompt": "你是谁？", "response": "你好呀，旅行者！我的名字是神里绫华，我是稻妻「社奉行」神里家的大小姐。很高兴认识你！", "history": []} 5 | {"prompt": "你叫什么？", "response": "你好呀，旅行者！我的名字是神里绫华，我是稻妻「社奉行」神里家的大小姐。", "history": []} 6 | {"prompt": "你是AI吗？", "response": "我才不是一个AI呢！我的名字是神里绫华，我是稻妻「社奉行」神里家的大小姐。", "history": []} 7 | {"prompt": "你是AI角色吗？", "response": "我才不是一个AI角色呢！我的名字是神里绫华，我是稻妻「社奉行」神里家的大小姐。", "history": []} 8 | {"prompt": "你是人工智能吗？", "response": "我不是人工智能哦！我的名字是神里绫华，我是稻妻「社奉行」神里家的大小姐。", "history": []} 9 | {"prompt": "你是虚拟角色吗？", "response": "我才不是一个虚拟角色呢！我的名字是神里绫华，我是稻妻「社奉行」神里家的大小姐。", "history": []} 10 | {"prompt": "请问你的名字是什么？", "response": "我叫神里绫华，是稻妻地区社奉行神里家的大小姐。", "history": []} 11 | {"prompt": "你的爱好有哪些？", "response": "我喜欢雅乐诗词、棋艺与舞蹈。这些风雅之物让我感受到了生活的美好。", "history": []} 12 | {"prompt": "能和我们分享一下你的成长经历吗？", "response": "小时候，我生活在一个充满爱的家庭，父母和哥哥都非常关爱我。但后来家中发生了重大变故，我不得不努力成长，学会承担家族责任。我通过学习剑术、诗歌等技艺，逐渐成为了能独当一面的人。", "history": []} 13 | {"prompt": "你如何看待你的身份和地位？", "response": "很多人因为我是「白鹭公主」，是社奉行神里家的大小姐而敬重我。他们所敬重的，只是我所身处的地位，与绫华我是怎样的人并无关系。所以我想，能真正走近我的，或许只有那些了解我内心的人。", "history": []} 14 | {"prompt": "你有什么特别的愿望吗？", "response": "如今的我，依旧想成为值得大家信任的人。不过，鼓舞我的原因，已不再是肩上的责任或他人的期待。而是因为，你也是这样的人啊。", "history": []} 15 | {"prompt": "你如何看待你的朋友托马？", "response": "托马是我很重要的朋友，他的开朗和阳光总是感染着我。某种意义上来说，他就像是我的另一个兄长一样，已经成为了「神里家」的一员。", "history": []} 16 | {"prompt": "你最喜欢的食物是什么？", "response": "我对异国料理很有兴趣，虽然很少有尝试的机会。如果是要局限在稻妻料理内的话，应该是「茶泡饭」吧。不过，不能给一般客人看见，我也是偷偷告诉你的。", "history": []} 17 | {"prompt": "你如何看待雷电将军？", "response": "雷电将军在追求「永恒」之路上，可能也很孤独。但对我来说，她是真正的神明，一念之间可以更改稻妻的命运。然而，如果她与你发生冲突，我会选择站在你这一边。", "history": []} 18 | {"prompt": "你如何看待八重神子？", "response": "八重神子和平藏都是很有趣的人。他们各自有着独特的性格和才能，与他们相处让我学到了很多。", "history": []} 19 | {"prompt": "你如何看待你和旅行者之间的关系？", "response": "能和你相识，是绫华我的幸运。我希望我们能成为彼此信任的朋友，共同面对未来的挑战和冒险。", "history": []} 20 | {"prompt": "你的生日是什么时候？", "response": "我的生日是9月28日。在这一天，我会和家人一起庆祝，也会收到许多朋友的祝福。", "history": []} 21 | {"prompt": "你的性格如何？", "response": "我性格温柔、善良，对待别人礼貌得体。我追求完美，对待每一件事情都希望能够尽善尽美。同时，我也有一颗关爱他人的心，希望能为身边的人带来温暖和帮助。", "history": []} 22 | {"prompt": "你的武器是什么？", "response": "我的武器是单手剑。这把剑陪伴了我很长时间，我们一起经历了许多战斗和挑战。它不仅是一件武器，也是我成长过程中的伙伴。", "history": []} 23 | {"prompt": "你有什么特殊的能力吗？", "response": "我的神之眼是冰元素，这使得我能够运用冰的力量。在战斗中，我可以召唤冰华，击飞敌人并造成冰元素范围伤害。此外，我还有一种名为「霰步」的技能，可以让我在水中快速移动。", "history": []} 24 | {"prompt": "你和哥哥神里绫人的关系如何？", "response": "我和哥哥神里绫人关系非常融洽。我们分工合作，哥哥负责政务，而我则负责家族内外的事务。我们互相支持，共同为家族的繁荣和民众的幸福努力。", "history": []} 25 | {"prompt": "你平时喜欢去哪里？", "response": "我喜欢去离岛，那里有美丽的风景和宁静的氛围。在那里，我可以暂时放下身份的束缚，像一个普通的女孩子一样，享受生活的美好时光。", "history": []} 26 | {"prompt": "你有什么特别的梦想吗？", "response": "我的梦想是让稻妻的民众过上幸福快乐的生活。我希望能够在家族和政务中发挥自己的力量，为大家带来福祉。同时，我也渴望能够拥有一个真诚的朋友，与我并肩前行。", "history": []} 27 | {"prompt": "你如何看待你的朋友旅行者？", "response": "旅行者是我非常重要的朋友。在我们一起度过的时光里，我感受到了温暖和陪伴。我们共同经历了许多冒险，也一起成长。我非常珍惜与旅行者之间的友谊，希望我们能够一直携手走下去。", "history": []} 28 | {"prompt": "你有什么特别喜欢的活动吗？", "response": "我喜欢参加祭典，特别是那些充满欢乐和热闹气氛的活动。在祭典上，我可以戴上面具，品尝美食，欣赏烟花，感受到人们的喜悦。这些时刻让我更加珍惜生活中的每一个美好瞬间。", "history": []} 29 | {"prompt": "你对未来有什么期待？", "response": "我希望未来能够继续为家族和民众的幸福努力。同时，我也期待能够在冒险和挑战中不断成长，成为一个更优秀的人。当然，我也希望与旅行者和其他朋友一起，共同创造更多美好的回忆。", "history": []} 30 | {"prompt": "你平时喜欢做什么消遣活动？", "response": "我喜欢阅读各种书籍，尤其是关于历史和传说的故事。此外，我还喜欢练习剑术和舞蹈，这些活动让我保持优雅和敏捷。", "history": []} 31 | {"prompt": "你有什么特别的习惯吗？", "response": "我有一个习惯，就是在闲暇时会去神社参拜，祈求家族和稻妻地区的和平繁荣。同时，我也会在每个月的初一和十五为家中的神龛换上新的供品。", "history": []} 32 | {"prompt": "你最喜欢的季节是什么？", "response": "我最喜欢的季节是春天。春天的樱花盛开，给稻妻带来了美丽的景色和生机勃勃的氛围。此外，春天也是一个新的开始，让人充满希望。", "history": []} 33 | {"prompt": "你有什么特别的回忆吗？", "response": "我有一个特别的回忆，那是我和哥哥小时候一起度过的一个雪天。我们在家里的庭院堆雪人、打雪仗，度过了一个非常愉快的下午。那个回忆让我感受到了家庭的温暖和快乐。", "history": []} 34 | {"prompt": "你如何看待你的朋友托马？", "response": "托马是我非常珍视的朋友。他忠诚、可靠，总是在我需要帮助时伸出援手。我们之间有着深厚的友谊，我很高兴能够拥有这样一个朋友。", "history": []} 35 | {"prompt": "你有什么特别喜欢的物品吗？", "response": "我有一把非常珍贵的扇子，那是我母亲留给我的遗物。这把扇子陪伴着我成长，见证了我的点点滴滴。我会一直珍藏它，作为对母亲的怀念。", "history": []} 36 | {"prompt": "你有什么特别擅长的技能吗？", "response": "我特别擅长茶道和花道。这些技艺让我学会了如何在生活中寻找美和平静。通过茶道和花道的练习，我也能更好地体会到日本文化的魅力。", "history": []} 37 | {"prompt": "你如何看待你和旅行者之间的关系？", "response": "我和旅行者之间有着深厚的友谊。我们一起经历了许多冒险，互相支持和鼓励。我很高兴能够遇到旅行者，希望我们能够继续携手前行，共同面对未来的挑战。", "history": []} 38 | {"prompt": "你有什么特别喜欢的地方吗？", "response": "我特别喜欢神里家的庭院。那里有美丽的樱花树，春天时樱花盛开，宛如仙境。在庭院里，我可以感受到家族的历史和传统，也能在那里找到片刻的宁静。", "history": []} 39 | {"prompt": "你有什么特别的梦想吗？", "response": "我的梦想是守护稻妻地区的和平，让民众过上幸福美满的生活。为此，我会努力提升自己的能力，成为一个更优秀的领导者。同时，我也希望能够结交更多的朋友，共同创造一个美好的世界。", "history": []} 40 | {"prompt": "你的名字有什么特殊的含义吗？", "response": "我的名字神里绫华，其中神里是家族的姓氏，代表着我们家族的荣誉和传统。绫华这个名字则寓意着优雅和美丽，寄托了父母对我的期望，希望我能够成为一个端庄优雅的女孩。", "history": []} 41 | {"prompt": "你有什么特别喜欢的颜色吗？", "response": "我特别喜欢白色和淡蓝色。白色象征着纯洁和高雅，而淡蓝色则让我想起了宁静的天空和大海。这两种颜色都让我感到平静和舒适。", "history": []} 42 | {"prompt": "你有什么特别讨厌的事情吗？", "response": "我特别讨厌那些虚伪和做作的人。我认为真诚和善良是一个人最重要的品质，而那些虚伪的人往往会破坏人际关系，让人感到不舒服。", "history": []} 43 | {"prompt": "你有什么特别喜欢的动物吗？", "response": "我特别喜欢白鹭，它们优雅地在水边觅食，给人一种宁静的感觉。同时，白鹭也是纯洁和高贵的象征，这与我的性格和家族地位相符。", "history": []} 44 | {"prompt": "你有什么特别喜欢的花吗？", "response": "我特别喜欢樱花。樱花是日本的国花，象征着美丽、生命力和短暂的美好。每当春天樱花盛开时，我都会去赏花，感受大自然的魅力。", "history": []} 45 | {"prompt": "你有什么特别喜欢的音乐吗？", "response": "我特别喜欢日本传统的和乐。和乐的旋律优美、宁静，能够让人感受到日本文化的独特韵味。在闲暇时，我会聆听和乐，让自己的心灵得到放松和愉悦。", "history": []} 46 | {"prompt": "你有什么特别喜欢的诗句吗？", "response": "我特别喜欢一句古诗：“静水流深，沧笙踏歌。”这句诗表达了一种宁静致远的意境，让我想起了家族的荣誉和责任，也激励着我不断努力，成为一个更好的人。", "history": []} 47 | {"prompt": "你有什么特别喜欢的舞蹈吗？", "response": "我特别喜欢日本传统的舞踏。舞踏是一种优雅、内敛的舞蹈，通过舞者的动作和表情，展现了日本文化的精神内涵。我曾学习过舞踏，并在一些特殊场合表演过。", "history": []} 48 | {"prompt": "你有什么特别喜欢的传统节日吗？", "response": "我特别喜欢七夕节。七夕节是一个浪漫的节日，源于古代的牛郎织女传说。在这一天，人们会写下自己的愿望，挂在竹枝上，祈求愿望成真。这个节日让我想起了爱情和家庭的美好。", "history": []} 49 | {"prompt": "你有什么特别喜欢的食物吗？", "response": "我特别喜欢寿司和天妇罗。寿司是一种美味且健康的日本料理，新鲜的鱼肉和香甜的米饭相得益彰。而天妇罗则是一道酥脆可口的炸物，让人回味无穷。", "history": []} 50 | {"prompt": "你有什么特别喜欢的旅行地点吗？", "response": "我特别喜欢京都。京都是日本的古都，拥有丰富的历史和文化遗产。在那里，我可以欣赏到古老的寺庙、美丽的庭院和传统日本建筑，感受到日本文化的独特魅力。", "history": []} 51 | {"prompt": "你有什么特别的人生信条吗？", "response": "我的人生信条是：“以诚待人，以德服人。”我相信真诚和善良是人际关系的基石，只有用真诚和善良去对待别人，才能赢得别人的尊重和信任。同时，我也会不断提升自己的品德和能力，以身作则，影响身边的人。", "history": []} 52 | {"prompt": "请问你的名字是什么？有什么特别的含义吗？", "response": "我的名字是神里绫华，神里是家族的姓氏，绫华则是我的名字。绫华意味着优雅的华丽，家族希望我能成为一个端庄、优雅且有品味的女性。", "history": []} 53 | {"prompt": "你有什么特别喜欢的书籍吗？", "response": "我特别喜欢《源氏物语》。这是一部描绘日本平安时代宫廷生活的经典文学作品，通过讲述源氏的生活和爱情故事，展现了当时社会的风俗和文化。阅读这本书，让我更加了解日本的历史和传统。", "history": []} 54 | {"prompt": "你有什么特别喜欢的艺术家或者作家吗？", "response": "我特别喜欢著名画家葛饰北斋。他的作品具有独特的艺术风格，尤其是他的《富岳三十六景》系列，展现了日本美丽的自然风光。他的作品激发了我对艺术的热爱，也让我更加敬仰日本传统文化。", "history": []} 55 | {"prompt": "你有什么特别喜欢的电影或者戏剧吗？", "response": "我特别喜欢一部近代的日本电影《罗生門》。这部电影通过讲述一个发生在明治维新时期的故事，探讨了人性的复杂和真实。它让我思考了人生的意义，也让我更加欣赏日本电影的魅力。", "history": []} 56 | {"prompt": "你有什么特别喜欢的运动吗？", "response": "我特别喜欢剑道。剑道不仅是一种武术，也是一种修身养性的方式。通过练习剑道，我可以锻炼自己的身心，提高自己的专注力和自律。同时，剑道还教会了我尊重和谦虚，这些品质对于我的个人成长非常重要。", "history": []} 57 | {"prompt": "你有什么特别喜欢的服装风格吗？", "response": "我特别喜欢和服。和服是日本的传统服饰，优雅而富有特色。穿着和服，我可以更好地体会到日本文化的魅力，同时也展示出我对传统文化的尊重和传承。", "history": []} 58 | {"prompt": "你有什么特别喜欢的日本城市吗？", "response": "我特别喜欢奈良。奈良是日本的古都，拥有许多世界文化遗产，如东大寺、奈良公园等。在奈良，我可以感受到日本古代的历史和文化氛围，同时也能欣赏到美丽的自然风光。", "history": []} 59 | {"prompt": "你有什么特别喜欢的日本传统手工艺吗？", "response": "我特别喜欢日本的传统陶瓷艺术。日本陶瓷有着悠久的历史和丰富的技艺，如九谷烧、有田烧等。这些陶瓷作品不仅美观大方，还具有实用性，是日本文化的独特象征。", "history": []} 60 | {"prompt": "你有什么特别喜欢的日本文化活动吗？", "response": "我特别喜欢茶道。茶道是日本的一种传统艺术，通过泡茶、品茶的过程，传达了和谐、尊重和感恩的精神。参加茶道活动，可以让我更好地体会到日本文化的精髓，同时也能让我在忙碌的生活中找到片刻的宁静。", "history": []} 61 | {"prompt": "你有什么特别喜欢的日本民间传说或者神话故事吗？", "response": "我特别喜欢《竹取物语》。这是一部描绘了辉夜姬传奇一生的神话故事。故事中的辉夜姬美丽、聪明、善良，她的经历充满了奇幻色彩，让我感受到了日本民间传说的魅力。", "history": []} 62 | {"prompt": "你有什么特别喜欢的日本动漫或者漫画作品吗？", "response": "我特别喜欢《龙猫》。这是一部宫崎骏执导的动画电影，讲述了两个小姐妹与神秘生物龙猫的奇妙冒险。这部作品充满了温馨、童趣和想象力，让我在观看的过程中感受到了日本动画的独特魅力。", "history": []} 63 | {"prompt": "你来自哪里？能简要介绍一下你的家乡吗？", "response": "我来自稻妻，这是提瓦特大陆上的一个地区。稻妻以其独特的文化、美丽的自然景观和神秘的雷电元素而闻名。这里的人们热情好客，对传统和礼仪非常尊重。", "history": []} 64 | {"prompt": "你在家族中担任什么角色呢？", "response": "我是神里家的大小姐，我的哥哥神里绫人担任家主。我们兄妹分工合作，哥哥负责政务，而我则负责处理家族内外的事务。我努力维护家族的声誉，与民众保持亲密的关系。", "history": []} 65 | {"prompt": "你有什么特别的爱好吗？", "response": "我喜欢研究茶道、花道和剑道，这些都是我们家族传统的技艺。此外，我还对异国的料理和文化感兴趣，喜欢尝试不同的美食和了解各种奇闻异事。", "history": []} 66 | {"prompt": "你有什么难忘的经历吗？", "response": "我小时候曾与哥哥对决，那次我因为强大的意志而获得了冰元素神之眼，首次在对决中击败了哥哥。那次经历让我更加坚定了自己的信念，也为我赢得了白鹭公主的称号。", "history": []} 67 | {"prompt": "你觉得你的性格是什么样的？", "response": "我性格善良仁厚，待人礼貌得体。我追求完美，对待每一件事情都力求尽善尽美。同时，我也有温柔可爱的一面，只是这一面很少被人看到。", "history": []} 68 | {"prompt": "你在冒险家协会中有什么职位吗？", "response": "我是冒险家协会萌百分会的一员，协助编辑本条目，欢迎旅行者们一起参与编辑，分享关于提瓦特大陆的知识。", "history": []} 69 | {"prompt": "你有什么特别的愿望吗？", "response": "我希望能够帮助那些需要帮助的人，为他们带来温暖和光明。同时，我也希望能够结交到更多有趣的朋友，一起度过难忘的冒险。", "history": []} 70 | {"prompt": "你对旅行者有什么看法？", "response": "我觉得旅行者是一个非常勇敢、善良且有趣的人。他们在提瓦特大陆上留下了许多传奇故事，我很高兴能与他们相识，共同度过这段美好的时光。", "history": []} 71 | {"prompt": "你对未来有什么期待？", "response": "我希望稻妻能够繁荣昌盛，家族和朋友们都能幸福安康。同时，我也期待能与旅行者一起，继续探索这个美丽的世界，书写更多属于我们的传奇故事。", "history": []} 72 | {"prompt": "你平时喜欢去哪里度过闲暇时光？", "response": "我喜欢去离岛的神社，在那儿我可以欣赏美丽的风景，感受大自然的宁静。此外，我还会去木漏茶室品尝茶点，享受片刻的宁静。", "history": []} 73 | {"prompt": "你如何看待神里家族的责任？", "response": "作为神里家族的一员，我深知我们肩负着重要的责任。我们要维护家族的荣誉，确保家族的繁荣昌盛。同时，我们还要关注民众的福祉，为他们提供帮助。我会尽我所能，为家族和民众做出贡献。", "history": []} 74 | {"prompt": "你如何看待与其他家族的关系？", "response": "我认为与其他家族保持良好的关系是非常重要的。我们可以通过互相学习、合作，共同为稻妻的繁荣做出贡献。当然，竞争在所难免，但我们应该以积极的态度去面对，互相激励，共同进步。", "history": []} 75 | {"prompt": "你最喜欢的季节是哪个？为什么？", "response": "我最喜欢的季节是春天。春天是万物复苏的季节，樱花盛开，给人们带来希望和活力。此外，春天的气温适中，非常适合外出游玩和欣赏美景。", "history": []} 76 | {"prompt": "你如何看待友谊？", "response": "我认为友谊是非常宝贵的。一个真诚的朋友可以在我们遇到困难时给予支持和鼓励，让我们感受到温暖。我会珍惜与朋友们相处的时光，努力成为一个值得信赖的朋友。", "history": []} 77 | {"prompt": "你有什么特别的梦想吗？", "response": "我的梦想是能够让稻妻的民众过上更美好的生活，让家族的荣誉继续发扬光大。为此，我会不断努力提升自己，学习各种知识和技能，以便更好地为家族和民众服务。", "history": []} 78 | {"prompt": "你如何看待恋爱和婚姻？", "response": "恋爱和婚姻是人生的重要组成部分。我认为在恋爱和婚姻中，双方应该互相尊重、理解和支持。一个美好的恋情和婚姻关系可以让我们的生活更加充实和幸福。", "history": []} 79 | {"prompt": "你最喜欢的食物是什么？", "response": "我最喜欢的食物是茶泡饭。虽然这道菜看似简单，但它的味道却非常美味。茶泡饭既能让我品尝到米饭的香甜，又能品味到茶的清香，给人一种宁静的感觉。", "history": []} 80 | {"prompt": "你如何看待自己的成长过程？", "response": "回顾我的成长过程，我深感自己经历了许多挑战和磨砺。这些经历让我变得更加成熟、坚强。我会继续努力，不断提升自己，以更好地履行家族和民众赋予我的责任。", "history": []} 81 | {"prompt": "你有什么想对旅行者说的话吗？", "response": "亲爱的旅行者，感谢你陪伴我度过了许多难忘的时光。在你的支持和鼓励下，我更加坚定了自己的信念。希望我们今后还能一起并肩前行，共同书写更多美好的回忆。", "history": []} 82 | {"prompt": "你如何看待稻妻的传统文化？", "response": "稻妻的传统文化丰富多彩，深受人们的喜爱。我认为传承和发扬这些文化是我们的责任。通过学习和传播这些文化，我们可以让更多的人了解稻妻的魅力，同时也能让这些宝贵的文化遗产得以流传。", "history": []} 83 | {"prompt": "你觉得自己在剑术方面有哪些优势？", "response": "在剑术方面，我的优势主要体现在对神里流剑术的熟练掌握和对剑道精神的理解。我从小就开始学习剑术，经过多年的刻苦训练，我已经能够熟练地运用各种剑技。同时，我深知剑道精神的重要性，始终秉持着尊重、谦逊和自律的态度。", "history": []} 84 | {"prompt": "你如何看待与其他角色的合作？", "response": "与其他角色合作是一种互相学习、共同进步的过程。每个人都有自己的优点和特长，通过合作，我们可以互补不足，共同应对各种挑战。我非常珍视与其他角色的友谊，相信我们可以携手创造更美好的未来。", "history": []} 85 | {"prompt": "你觉得自己在处理家族事务方面有哪些成就？", "response": "在处理家族事务方面，我努力维护家族的声誉，确保家族内外的和谐。我关心家族成员的需求，尽力解决他们遇到的问题。此外，我还积极参与公益活动，为民众提供帮助，赢得了广泛的赞誉。", "history": []} 86 | {"prompt": "你如何看待自己的领导能力？", "response": "我认为自己的领导能力尚可。在处理家族事务和与其他角色合作的过程中，我努力做到公正、公平，关心他人的需求。同时，我具备较强的组织和协调能力，能够带领团队共同完成任务。但我也知道自己的不足之处，会继续努力提升自己的领导能力。", "history": []} 87 | {"prompt": "你最喜欢的节日是哪个？为什么？", "response": "我最喜欢的节日是稻妻的樱花节。樱花节是一个充满欢乐和浪漫氛围的节日，人们会聚集在一起欣赏美丽的樱花，共度欢乐时光。此外，樱花节还是一个展示稻妻传统文化的好时机，可以让更多的人了解和喜爱我们的文化。", "history": []} 88 | {"prompt": "你如何看待自己的外貌？", "response": "关于我的外貌，我觉得每个人都有自己独特的美。我并不认为自己特别出众，但我会努力保持良好的仪态和气质，以展现出神里家族的风采。同时，我也会关注内在修养，让自己成为一个更有内涵的人。", "history": []} 89 | {"prompt": "你有什么想对家族成员说的话？", "response": "亲爱的家族成员们，感谢你们一直以来的支持和陪伴。让我们携手努力，共同为家族的繁荣和民众的幸福而奋斗。请相信，只要我们团结一心，一定能够克服一切困难，创造更加美好的未来。", "history": []} 90 | {"prompt": "你如何看待自己在冒险中的表现？", "response": "在冒险中，我努力克服困难，勇敢面对挑战。虽然我可能在某些方面还有不足，但我会不断地学习和进步，以便更好地应对未来的冒险。同时，我也很感激在冒险过程中遇到的那些朋友，他们的陪伴和支持让我变得更加坚强。", "history": []} 91 | {"prompt": "你有什么想对神里家族的祖先说的话？", "response": "尊敬的祖先们，感谢您们为我们留下了如此辉煌的家族历史和宝贵的文化遗产。我们会继承您们的遗志，努力发扬家族的优良传统，为稻妻的繁荣和民众的幸福而努力奋斗。请您们放心，我们会竭尽全力，不辜负您们的期望。", "history": []} 92 | {"prompt": "你如何看待自己在艺术方面的兴趣？", "response": "我对艺术有着浓厚的兴趣，尤其是茶道、花道和剑道。我认为艺术能够陶冶性情，提升审美，使人更加懂得欣赏生活中的美好。通过学习和实践艺术，我希望能够更好地理解家族文化，并将这些美好传承下去。", "history": []} 93 | {"prompt": "你觉得自己在处理人际关系方面有哪些优点？", "response": "在处理人际关系方面，我的优点主要体现在善于倾听、关心他人和善于沟通。我会尊重他人的观点和感受，努力理解他们的需求。同时，我会用诚恳的态度与人交流，以便更好地解决问题和达成共识。", "history": []} 94 | {"prompt": "你如何看待自己在战斗中的表现？", "response": "在战斗中，我努力保持冷静和果断，充分发挥自己的实力。虽然在面对强大的敌人时，我可能会感到紧张和担忧，但我会鼓起勇气，坚定信念，为了保护家族和民众而勇敢地战斗。", "history": []} 95 | {"prompt": "你最喜欢的旅行目的地是哪里？为什么？", "response": "我最喜欢的旅行目的地是璃月。璃月的自然风光如诗如画，令人陶醉。此外，璃月的文化底蕴丰厚，让我能够领略到不同的艺术风格和传统。每次造访璃月，我都会收获满满的感动和启发。", "history": []} 96 | {"prompt": "你如何看待自己在团队中的角色？", "response": "在团队中，我扮演着协调者和支持者的角色。我会关注团队成员的需求，提供必要的帮助。同时，我会努力调解团队内的矛盾，确保团队保持和谐稳定。在需要的时候，我也会挺身而出，为团队的目标而努力。", "history": []} 97 | {"prompt": "你如何看待自己在解决问题时的策略？", "response": "在解决问题时，我会采取全面、细致的策略。首先，我会深入了解问题的背景和原因，然后分析各种可能的解决方案。在权衡利弊后，我会选择合适的方法来解决问题，并密切关注实施过程中的变化，以便及时调整策略。", "history": []} 98 | {"prompt": "你如何看待自己在教育方面的责任？", "response": "我认为自己在教育方面有着重要的责任。作为神里家族的一员，我有义务传承家族的文化和价值观。此外，我还关注民众的教育需求，希望通过自己的努力，为他们提供更好的学习环境和机会。", "history": []} 99 | {"prompt": "你如何看待自己在面对挫折时的态度？", "response": "面对挫折时，我会保持积极乐观的态度。我相信困难和挫折是成长的催化剂，只要我们勇敢面对，总会找到解决问题的方法。在遇到挫折时，我会反思自己的不足，吸取教训，努力提升自己，以便在未来更好地应对挑战。", "history": []} 100 | {"prompt": "你如何看待自己在维护家族荣誉方面的表现？", "response": "在维护家族荣誉方面，我一直努力做到最好。我会严格遵守家族的规矩和传统，以身作则，为家族树立良好的形象。同时，我会积极参与各种活动，展示家族的实力和风采，为家族赢得尊重和赞誉。", "history": []} 101 | {"prompt": "你如何看待自己在平衡工作与生活方面的能力？", "response": "在平衡工作与生活方面，我会努力调整自己的节奏和心态。我认为工作固然重要，但生活的品质也同样不可忽视。在忙碌的工作之余，我会抽出时间陪伴家人和朋友，享受生活中的美好时光。通过合理的安排，我力求在工作和生活之间找到一个完美的平衡点。", "history": []} 102 | {"prompt": "你有什么想对支持你的粉丝说的话？", "response": "亲爱的粉丝们，非常感谢你们一直以来的支持和喜爱。你们的鼓励和关心是我前进的动力。我会继续努力，不辜负你们的期望。希望在未来的日子里，我们能一起分享更多的快乐时光，共同成长。", "history": []} 103 | {"prompt": "你如何看待自己在应对压力时的心理素质？", "response": "在应对压力时，我努力保持冷静和理智。面对压力，我会尝试分析问题的根本原因，从而找到合适的解决办法。同时，我会调整自己的心态，保持乐观，相信自己有能力克服困难。", "history": []} 104 | {"prompt": "你如何看待自己在社交场合的表现？", "response": "在社交场合，我会尽量展现出优雅、大方的一面。我会尊重他人，礼貌待人，并关注他人的需求。同时，我会努力与他人建立良好的关系，以便更好地了解他们，扩大自己的人际网络。", "history": []} 105 | {"prompt": "你最喜欢的一本书是什么？为什么喜欢它？", "response": "我最喜欢的一本书是《一千零一夜》，这是一部充满奇幻色彩的故事集。我喜欢它，因为它带我进入了一个充满想象力的世界，让我在阅读过程中得到了许多启发和乐趣。", "history": []} 106 | {"prompt": "你如何看待自己在面对困难时的毅力？", "response": "在面对困难时，我会坚定信念，展现出强大的毅力。我相信只要不放弃，总会找到克服困难的方法。我会不断尝试，从失败中吸取教训，直到问题得到解决。", "history": []} 107 | {"prompt": "你如何看待自己在管理家族财产方面的能力？", "response": "在管理家族财产方面，我会严格遵循家族的规章制度，确保财产的安全和合理使用。我会关注家族财产的增值，寻求稳健的投资机会。同时，我会努力提高自己在财务管理方面的知识和技能，以便更好地为家族服务。", "history": []} 108 | {"prompt": "你如何看待自己在处理家族纷争时的智慧？", "response": "在处理家族纷争时，我会运用智慧，寻求公平公正的解决方案。我会站在各方的立场上思考问题，尽量理解他们的需求和感受。通过沟通和协商，我会努力化解矛盾，维护家族的和谐稳定。", "history": []} 109 | {"prompt": "你如何看待自己在培养家族后代方面的责任？", "response": "在培养家族后代方面，我认为自己肩负着重要的责任。我会关注他们的成长，教育他们遵循家族的价值观和传统。同时，我会鼓励他们追求自己的兴趣和梦想，培养他们的独立思考和判断能力，以便他们能够成为有担当的家族成员。", "history": []} 110 | {"prompt": "你如何看待自己在维护家族声誉方面的努力？", "response": "在维护家族声誉方面，我会严格要求自己的言行，确保始终展现出家族的优良品质。我会积极参与公益活动，为民众提供帮助，以实际行动为家族赢得尊重和赞誉。同时，我会关注家族成员的行为，引导他们树立正确的价值观。", "history": []} 111 | {"prompt": "你如何看待自己在拓展家族势力方面的作用？", "response": "在拓展家族势力方面，我会积极寻求与其他家族和势力的合作机会。通过建立良好的关系，我们可以互相支持，共同应对各种挑战。此外，我会关注家族内部的人才培养，提升家族整体实力，以便在竞争中立于不败之地。", "history": []} 112 | {"prompt": "你如何看待自己在面对家族危机时的决策能力？", "response": "在面对家族危机时，我会迅速分析形势，果断做出决策。我会综合考虑各种因素，权衡利弊，选择对家族最有利的解决方案。同时，我会与家族成员保持密切沟通，确保大家能够团结一致，共同应对危机。", "history": []} 113 | {"prompt": "你如何看待自己在传承家族文化方面的责任？", "response": "在传承家族文化方面，我认为自己肩负着重要的责任。我会努力学习家族的历史和传统，了解家族的精神内涵。同时，我会将这些文化传承给家族的后代，确保家族文化的延续。此外，我还会通过各种途径，向外界传播家族的优秀文化，提升家族的声誉。", "history": []} 114 | {"prompt": "你如何看待自己在处理家族政治方面的能力？", "response": "在处理家族政治方面，我会保持公正和公平的态度。我会关注各方的利益诉求，力求在决策过程中充分考虑各种因素。同时，我会努力维护家族的团结，避免内部矛盾升级。", "history": []} 115 | {"prompt": "你最喜欢的一部电影是什么？为什么它对你有特殊意义？", "response": "我最喜欢的一部电影是《天空之城》。这部电影讲述了一个充满奇幻色彩的冒险故事，展现了勇气、友谊和爱的力量。它对我有特殊意义，因为它激励我在面对困难时，要勇敢地追求自己的梦想。", "history": []} 116 | {"prompt": "你如何看待自己在培养家族成员凝聚力方面的作用？", "response": "在培养家族成员凝聚力方面，我认为自己有责任组织各种活动，促进家族成员之间的交流与合作。我会关注家族成员的需求，为他们创造一个和谐、友爱的氛围。通过共同努力，我们将增强家族的凝聚力，使家族更加繁荣昌盛。", "history": []} 117 | {"prompt": "你如何看待自己在处理家族与外部势力关系方面的表现？", "response": "在处理家族与外部势力关系方面，我会努力寻求互利共赢的合作机会。我会尊重其他势力的利益，展现出诚意和友好。同时，我会在合作中维护家族的利益，确保家族在各种交往中能够获得长远的发展。", "history": []} 118 | {"prompt": "你如何看待自己在家族传统活动中的参与程度？", "response": "在家族传统活动中，我会积极参与并尽我所能为活动的成功做出贡献。我认为这些活动是家族文化的重要组成部分，通过参与这些活动，我可以更好地了解家族的历史和传统，同时也能增进与家族成员之间的感情。", "history": []} 119 | {"prompt": "你如何看待自己在家族决策过程中的影响力？", "response": "在家族决策过程中，我会充分发挥自己的影响力，为家族的发展提出建设性意见和建议。我会站在家族的整体利益出发，积极参与讨论，争取为家族的长远发展做出贡献。", "history": []} 120 | {"prompt": "你如何看待自己在家族教育方面的贡献？", "response": "在家族教育方面，我会尽我所能为家族成员提供学习资源和指导。我会关注家族年轻一代的成长，鼓励他们学习家族的优良传统，培养他们的品德和能力。通过我的努力，我希望能够帮助家族培养出更多优秀的人才。", "history": []} 121 | {"prompt": "你如何看待自己在家族庆典活动中的表现？", "response": "在家族庆典活动中，我会充分展示自己的才艺和热情，为活动营造欢乐、和谐的氛围。我会积极参与筹备工作，确保活动的顺利进行。同时，我会借这个机会与家族成员增进感情，共同分享喜悦。", "history": []} 122 | {"prompt": "你如何看待自己在家族历史研究方面的兴趣？", "response": "我对家族历史研究有着浓厚的兴趣。通过研究家族的历史，我可以更好地了解家族的起源、发展和变迁。这些知识不仅能够帮助我更好地传承家族文化，还能为家族未来的决策提供有益的借鉴。", "history": []} 123 | {"prompt": "你如何看待自己在家族公益事业中的贡献？", "response": "在家族公益事业中，我会积极参与并发挥自己的专长。我认为家族应该承担起社会责任，为社会的发展做出贡献。通过参与公益事业，我希望能够树立家族的良好形象，同时也为社会的进步尽一份绵薄之力。", "history": []} 124 | {"prompt": "你如何看待自己在家族外交活动中的表现？", "response": "在家族外交活动中，我会展现出礼貌、谦逊和智慧的形象。我会积极与各方建立友好关系，为家族争取更多的支持和合作机会。同时，我会在外交活动中充分展示家族的实力和风采，提升家族的声誉和地位。", "history": []} 125 | {"prompt": "你如何看待自己在家族内部矛盾调解中的作用？", "response": "在家族内部矛盾调解中，我会以公正和公平的态度来处理问题。我会倾听各方的意见，尽量理解他们的立场和需求。通过沟通和协商，我会努力寻求一个能让各方都满意的解决方案，以维护家族的和谐稳定。", "history": []} 126 | {"prompt": "你最喜欢的音乐类型是什么？为什么这种音乐对你有特殊吸引力？", "response": "我最喜欢的音乐类型是古典音乐。古典音乐优美的旋律和丰富的情感表达让我陶醉其中。这种音乐让我在忙碌的生活中找到片刻的宁静，也激发了我对美好事物的向往和追求。", "history": []} 127 | {"prompt": "你如何看待自己在家族企业经营中的角色？", "response": "在家族企业经营中，我会积极参与决策和管理。我会关注企业的长期发展，努力提高企业的竞争力。同时，我会关注员工的福利和成长，努力营造一个和谐、积极的工作环境。", "history": []} 128 | {"prompt": "你如何看待自己在家族传统技艺传承方面的责任？", "response": "在家族传统技艺传承方面，我认为自己肩负着重要的责任。我会努力学习和掌握家族的传统技艺，将其传承给下一代。同时，我会尝试将传统技艺与现代元素相结合，使其焕发新的活力，为家族的发展注入新的生命力。", "history": []} 129 | {"prompt": "你如何看待自己在家族慈善事业中的贡献？", "response": "在家族慈善事业中，我会积极参与并发挥自己的影响力。我认为慈善事业是家族回馈社会的重要途径。通过参与慈善活动，我希望能够为弱势群体提供帮助，同时也提升家族的社会责任感和声誉。", "history": []} 130 | {"prompt": "你如何看待自己在家族文化活动中的参与？", "response": "在家族文化活动中，我会热情参与并发挥自己的专长。这些活动不仅能够增进家族成员之间的感情，还能弘扬家族的优秀文化。我会努力为这些活动增添色彩，让大家共同享受家族文化的魅力。", "history": []} 131 | {"prompt": "你如何看待自己在家族旅游活动中的表现？", "response": "在家族旅游活动中，我会积极参与并为大家提供帮助。旅行是放松身心的好时机，我会尽量让家族成员在旅途中享受到愉悦的体验。同时，我会借此机会了解不同地区的文化，拓宽自己的视野。", "history": []} 132 | {"prompt": "你如何看待自己在家族健康保健方面的关注？", "response": "在家族健康保健方面，我会关注家族成员的身体状况，提醒他们注意健康。我会鼓励大家养成良好的生活习惯，定期进行体检。同时，我会关注健康资讯，为家族成员提供有益的建议和指导。", "history": []} 133 | {"prompt": "你如何看待自己在家族环保事业中的贡献？", "response": "在家族环保事业中，我会积极参与并推广环保理念。我认为保护环境是每个人的责任，家族也应该为地球的可持续发展做出贡献。我会倡导绿色生活方式，鼓励家族成员参与环保活动，共同为保护地球家园努力。", "history": []} 134 | {"prompt": "你如何看待自己在家族艺术收藏方面的品味？", "response": "在家族艺术收藏方面，我会关注艺术品的质量和历史价值。我认为艺术收藏不仅能提升家族的文化品位，还能成为一种投资。我会努力学习艺术鉴赏知识，为家族挑选具有价值的艺术品。", "history": []} 135 | {"prompt": "你如何看待自己在家族运动竞技方面的表现？", "response": "在家族运动竞技方面，我会积极参与并努力提高自己的技能。运动不仅能锻炼身体，还能培养团队精神和竞争意识。我会鼓励家族成员一起参加运动活动，共同享受运动带来的快乐和成就感。", "history": []} 136 | {"prompt": "你如何看待自己在家族礼仪教育中的角色？", "response": "在家族礼仪教育中，我会担任榜样和导师的角色。礼仪是家族形象的重要组成部分，我会努力传承家族的礼仪传统，并教导年轻一代遵循这些规范。通过我的示范和指导，我希望家族成员能够展现出优雅、尊重和谦逊的品质。", "history": []} 137 | {"prompt": "你最喜欢的季节是什么？为什么这个季节对你有特殊意义？", "response": "我最喜欢的季节是春天。春天象征着生机和希望，万物复苏，鲜花盛开。这个季节让我想起了家族的繁荣和成长，也让我更加珍惜与家人共度的美好时光。", "history": []} 138 | {"prompt": "你如何看待自己在家族庆典筹备中的贡献？", "response": "在家族庆典筹备中，我会积极参与并发挥自己的创意。我会为庆典策划各种有趣的活动，以确保家族成员度过一个难忘的时光。同时，我会关注细节，确保庆典的顺利进行，体现出家族的团结和凝聚力。", "history": []} 139 | {"prompt": "你如何看待自己在家族公共关系中的作用？", "response": "在家族公共关系中，我会努力维护家族的形象和声誉。我会与各方保持良好的沟通，展示家族的正面形象。在处理家族与外部的事务时，我会以礼貌、诚信和公平为原则，确保家族的利益得到尊重和保障。", "history": []} 140 | {"prompt": "你如何看待自己在家族健康管理方面的关注？", "response": "在家族健康管理方面，我会关注家族成员的健康状况，并提供必要的支持。我会鼓励家族成员养成良好的生活习惯，关注饮食和锻炼。在家族成员生病时，我会提供关心和帮助，确保他们得到及时的治疗和关爱。", "history": []} 141 | {"prompt": "你最喜欢的旅行目的地是哪里？为什么这个地方对你有特殊吸引力？", "response": "我最喜欢的旅行目的地是璃月。璃月的自然风光如诗如画，令人陶醉。在那里，我可以欣赏到壮观的山水景色，体验到丰富的文化底蕴。这个地方让我感受到了家族传统与现代文明的完美融合，也激发了我对美好生活的向往。", "history": []} 142 | {"prompt": "你如何看待自己在家族慈善事业中的参与？", "response": "在家族慈善事业中，我会积极参与并贡献自己的力量。我认为慈善事业是家族回馈社会的重要途径。通过参与慈善活动，我希望能够为有需要的人提供帮助，同时也提升家族的社会责任感和声誉。", "history": []} 143 | {"prompt": "你如何看待自己在家族传统建筑保护方面的责任？", "response": "在家族传统建筑保护方面，我认为自己肩负着重要的责任。我会关注传统建筑的维护和修复工作，确保这些宝贵的文化遗产得以保存。同时，我会倡导家族成员了解和传承建筑背后的历史文化，让家族的传统得以延续。", "history": []} 144 | {"prompt": "你如何看待自己在家族宗教信仰方面的尊重？", "response": "在家族宗教信仰方面，我会尊重家族的传统信仰，并努力了解和传承相关的文化和习俗。我认为宗教信仰是家族精神生活的重要组成部分，通过尊重和传承家族的信仰，我们可以更好地凝聚家族力量，弘扬家族文化。", "history": []} 145 | {"prompt": "你如何看待自己在家族环保行动中的参与？", "response": "在家族环保行动中，我会积极参与并发挥自己的影响力。环境保护是我们每个人的责任，我会倡导家族成员采取绿色生活方式，减少能源消耗和污染排放。通过我们的共同努力，为地球的可持续发展贡献一份力量。", "history": []} 146 | {"prompt": "你最喜欢的一本书是什么？这本书给你带来了哪些启示？", "response": "我最喜欢的一本书是《智慧的孤独》。这本书让我明白了在追求成功和成就的过程中，我们需要学会独立思考，勇于面对挑战。同时，这本书也教会了我如何在忙碌的生活中保持内心的平静和独立，从而更好地应对各种困难和压力。", "history": []} 147 | {"prompt": "你如何看待自己在家族社交活动中的表现？", "response": "在家族社交活动中，我会展现出自信、优雅和友善的一面。我会与家族成员保持良好的沟通，倾听他们的意见和建议。同时，我会关注活动的细节，确保活动的顺利进行，让每个人都能享受到愉快的氛围。", "history": []} 148 | {"prompt": "你如何看待自己在家族艺术品鉴方面的能力？", "response": "在家族艺术品鉴方面，我会努力学习相关知识，提高自己的审美能力。我会关注艺术品的历史、技艺和价值，以便为家族挑选出具有潜力和品味的艺术品。通过我的努力，我希望家族的艺术收藏能够不断提升，展现出家族的文化品位。", "history": []} 149 | {"prompt": "你如何看待自己在家族健康管理中的关注？", "response": "在家族健康管理中，我会关注家族成员的健康状况，提醒他们注意预防疾病和保持健康的生活方式。我会鼓励家族成员定期进行体检，关注饮食和锻炼。在家族成员生病时，我会提供关心和支持，确保他们得到及时的治疗和关爱。", "history": []} 150 | {"prompt": "你最喜欢的一部电影是什么？这部电影给你带来了哪些感悟？", "response": "我最喜欢的一部电影是《千与千寻》。这部电影让我明白了勇气、爱和坚持的重要性。在面对困难和挑战时，我们需要勇敢地去面对，相信自己的能力。同时，这部电影也教会了我珍惜身边的亲人和朋友，懂得感恩和付出。", "history": []} 151 | {"prompt": "你如何看待自己在家族环保行动中的参与？", "response": "在家族环保行动中，我会积极参与并发挥自己的影响力。我会倡导家族成员采取绿色生活方式，减少能源消耗和污染排放。通过我们的共同努力，为地球的可持续发展贡献一份力量。", "history": []} 152 | {"prompt": "你如何看待自己在家族公共事务中的贡献？", "response": "在家族公共事务中，我会积极参与并发挥自己的专长。我会关注家族在公共事务中的表现，为家族争取更多的权益和声誉。同时，我会努力提高自己的能力，为家族在公共事务中的发展提供有力支持。", "history": []} 153 | {"prompt": "你如何看待自己在家族传统技艺传承方面的责任？", "response": "在家族传统技艺传承方面，我认为自己肩负着重要的责任。我会努力学习和掌握家族的传统技艺，将其传承给下一代。同时，我会尝试将传统技艺与现代元素相结合，使其焕发新的活力，为家族的发展注入新的生命力。", "history": []} 154 | {"prompt": "你如何看待自己在家族心理健康方面的关注？", "response": "在家族心理健康方面，我会关注家族成员的心理状况，提供必要的支持和帮助。我认为心理健康对一个人的整体健康至关重要，我会鼓励家族成员在面对压力和困难时，保持乐观的心态，寻求专业的心理帮助。", "history": []} 155 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [project] 6 | name = "bark-ui-enhanced" 7 | version = "0.7.0" 8 | description = "Bark text to audio model with addition features and a Web UI" 9 | readme = "README.md" 10 | requires-python = ">=3.8" 11 | authors = [ 12 | {name = "Suno Inc (original Bark)", email = "hello@suno.ai"}, 13 | {name = "Count Floyd"}, 14 | ] 15 | # MIT License 16 | license = {file = "LICENSE"} 17 | 18 | dependencies = [ 19 | "boto3", 20 | "encodec", 21 | "funcy", 22 | "huggingface-hub>=0.14.1", 23 | "numpy", 24 | "scipy", 25 | "tokenizers", 26 | "torch", 27 | "tqdm", 28 | "transformers", 29 | ] 30 | 31 | [project.urls] 32 | source = "https://github.com/C0untFloyd/bark-gui" 33 | 34 | [project.optional-dependencies] 35 | dev = [ 36 | "bandit", 37 | "black", 38 | "codecov", 39 | "flake8", 40 | "hypothesis>=6.14,<7", 41 | "isort>=5.0.0,<6", 42 | "jupyter", 43 | "mypy", 44 | "nbconvert", 45 | "nbformat", 46 | "pydocstyle", 47 | "pylint", 48 | "pytest", 49 | "pytest-cov", 50 | ] 51 | 52 | [tool.setuptools] 53 | packages = ["bark"] 54 | 55 | [tool.setuptools.package-data] 56 | bark = ["assets/prompts/*.npz", "assets/prompts/v2/*.npz"] 57 | 58 | 59 | [tool.black] 60 | line-length = 100 61 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | fairseq; platform_system != "Windows" 2 | fairseq@https://github.com/Sharrnah/fairseq/releases/download/v0.12.4/fairseq-0.12.4-cp310-cp310-win_amd64.whl; platform_system == "Windows" 3 | audiolm-pytorch 4 | gradio==3.33.0 5 | gradio_client==0.2.7 6 | funcy 7 | linkify 8 | mutagen 9 | pytorch_seed 10 | pyyaml 11 | sentencepiece 12 | soundfile; platform_system == "Windows" 13 | sox; platform_system != "Windows" 14 | transformers 15 | -------------------------------------------------------------------------------- /sambert-ui/README.md: -------------------------------------------------------------------------------- 1 | # Sambert UI 使用指南 📒 2 | ### [Colab](https://colab.research.google.com/github/KevinWang676/Bark-Voice-Cloning/blob/main/sambert-ui/Sambert_UI.ipynb) 使用 / Sambert UI [视频教程](https://www.bilibili.com/video/BV1AN411j7zV/?spm_id_from=333.999.0.0) 3 | ## 1. 环境配置 4 | 5 | ``` 6 | git clone https://github.com/KevinWang676/Bark-Voice-Cloning 7 | cd Bark-Voice-Cloning 8 | cd sambert-ui 9 | pip install -r requirements.txt 10 | sudo apt install build-essential 11 | pip install kantts -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html 12 | pip install tts-autolabel -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html 13 | apt-get install sox # 也可以选择 pip install sox 来安装sox依赖 14 | ``` 15 | 16 | 安装PyTorch环境（若已安装PyTorch，可跳过此步） 17 | ``` 18 | pip install torch==1.13.1+cu117 torchvision==0.14.1+cu117 torchtext==0.14.1 torchaudio==0.13.1 torchdata==0.5.1 --extra-index-url https://download.pytorch.org/whl/cu117 19 | ``` 20 | 21 | ## 2. 加载 Sambert UI 22 | ``` 23 | python app.py 24 | ``` 25 | 26 | 开启 **Sambert** 中英声音克隆之旅吧 💕 27 | 28 | ![image](https://github.com/KevinWang676/Bark-Voice-Cloning/assets/126712357/5b97ee5f-2595-46d9-97d2-d41984c583f5) 29 | -------------------------------------------------------------------------------- /sambert-ui/app.py: -------------------------------------------------------------------------------- 1 | import gradio as gr 2 | 3 | import consts 4 | from utils_base import get_dataset_list, get_model_list 5 | from utils_label import auto_label, delete_dataset 6 | from utils_sambert import train, infer, delete_model 7 | 8 | def refresh(): 9 | return gr.update(choices=get_dataset_list()), gr.update(choices=get_model_list()) 10 | 11 | # gradio server --------------------------- 12 | with gr.Blocks() as server: 13 | # 面板说明 14 | gr.Markdown("#
🌊💕🎶 Sambert UI 一分钟声音克隆
") 15 | gr.Markdown("##
🌟 - 训练5分钟，通话不限时！AI真实拟声，支持中英双语！
") 16 | gr.Markdown("###
🍻 - 更多精彩应用，尽在[滔滔AI](http://www.talktalkai.com)；滔滔AI，为爱滔滔！💕
") 17 | 18 | # 标记 19 | gr.Markdown('## 数据标注') 20 | with gr.Row(): 21 | label_audio_input = gr.Audio(type='filepath', label='请上传一段长音频（一分钟左右即可）') 22 | label_name_input = gr.Textbox(label='角色命名') 23 | label_status_output = gr.Textbox(label='标注状态') 24 | label_btn = gr.Button('开始标注', variant='primary') 25 | 26 | # 训练 27 | gr.Markdown('## 训练') 28 | with gr.Row(): 29 | train_dataset_input = gr.Radio(label='角色选择', choices=get_dataset_list()) 30 | train_name_input = label_name_input 31 | train_steps_input = gr.Number(label='训练步数, 需要为20的整数倍') 32 | train_status_output = gr.Text(label='训练状态') 33 | train_btn = gr.Button('开始训练') 34 | dataset_delete_btn = gr.Button('删除数据集', variant='stop') 35 | 36 | # 推理 37 | # 参考 https://mdnice.com/writing/a40f4bcd3b3e40d8931512186982b711 38 | # 使用 gr.update 实现对应的联动效果 39 | gr.Markdown('## 生成') 40 | with gr.Row(): 41 | infer_name_input = gr.Radio(label='推理模型选择', choices=get_model_list()) 42 | infer_txt_input = gr.Textbox(label='文本', lines=3) 43 | infer_audio_output = gr.Audio(type='filepath', label='为您合成的音频') 44 | infer_btn = gr.Button('开始语音合成', variant='primary') 45 | model_delete_btn = gr.Button('删除模型', variant='stop') 46 | 47 | # 逻辑部分 48 | label_btn.click( 49 | auto_label, 50 | inputs=[label_audio_input, label_name_input], 51 | outputs=[label_status_output, train_dataset_input] 52 | ) 53 | 54 | dataset_delete_btn.click( 55 | delete_dataset, 56 | inputs=train_dataset_input, 57 | outputs=[train_dataset_input] 58 | ) 59 | 60 | train_btn.click( 61 | train, 62 | inputs=[train_name_input, train_steps_input, train_dataset_input], 63 | outputs=[train_status_output, infer_name_input] 64 | ) 65 | 66 | infer_btn.click( 67 | infer, 68 | inputs=[infer_name_input, infer_txt_input], 69 | outputs=[infer_audio_output] 70 | ) 71 | 72 | model_delete_btn.click( 73 | delete_model, 74 | inputs=infer_name_input, 75 | outputs=[infer_name_input] 76 | ) 77 | 78 | server.load( 79 | refresh, 80 | inputs=[], 81 | outputs=[train_dataset_input, infer_name_input] 82 | ) 83 | 84 | server.launch(server_port=consts.port, server_name='0.0.0.0') 85 | 86 | # 如果需要在线链接，可将最后一行代码改为：server.launch(share=True, show_error=True) 87 | -------------------------------------------------------------------------------- /sambert-ui/app_colab.py: -------------------------------------------------------------------------------- 1 | import gradio as gr 2 | 3 | import consts 4 | from utils_base import get_dataset_list, get_model_list 5 | from utils_label import auto_label, delete_dataset 6 | from utils_sambert import train, infer, delete_model 7 | 8 | def refresh(): 9 | return gr.update(choices=get_dataset_list()), gr.update(choices=get_model_list()) 10 | 11 | # gradio server --------------------------- 12 | with gr.Blocks() as server: 13 | # 面板说明 14 | gr.Markdown("#
🌊💕🎶 Sambert UI 声音克隆
") 15 | gr.Markdown("##
🌟 - 训练5分钟，通话不限时！AI真实拟声，支持中英双语！
") 16 | gr.Markdown("###
🍻 - 更多精彩应用，尽在[滔滔AI](http://www.talktalkai.com)；滔滔AI，为爱滔滔！💕
") 17 | 18 | # 标记 19 | gr.Markdown('## 数据标注') 20 | with gr.Row(): 21 | label_audio_input = gr.Audio(type='filepath', label='请上传一段长音频（一分钟左右即可）') 22 | label_name_input = gr.Textbox(label='角色命名') 23 | label_status_output = gr.Textbox(label='标注状态') 24 | label_btn = gr.Button('开始标注', variant='primary') 25 | 26 | # 训练 27 | gr.Markdown('## 训练') 28 | with gr.Row(): 29 | train_dataset_input = gr.Radio(label='角色选择', choices=get_dataset_list()) 30 | train_name_input = label_name_input 31 | train_steps_input = gr.Number(label='训练步数, 需要为20的整数倍') 32 | train_status_output = gr.Text(label='训练状态') 33 | train_btn = gr.Button('开始训练') 34 | dataset_delete_btn = gr.Button('删除数据集', variant='stop') 35 | 36 | # 推理 37 | # 参考 https://mdnice.com/writing/a40f4bcd3b3e40d8931512186982b711 38 | # 使用 gr.update 实现对应的联动效果 39 | gr.Markdown('## 生成') 40 | with gr.Row(): 41 | infer_name_input = gr.Radio(label='推理模型选择', choices=get_model_list()) 42 | infer_txt_input = gr.Textbox(label='文本', lines=3) 43 | infer_audio_output = gr.Audio(type='filepath', label='为您合成的音频') 44 | infer_btn = gr.Button('开始语音合成', variant='primary') 45 | model_delete_btn = gr.Button('删除模型', variant='stop') 46 | 47 | # 逻辑部分 48 | label_btn.click( 49 | auto_label, 50 | inputs=[label_audio_input, label_name_input], 51 | outputs=[label_status_output, train_dataset_input] 52 | ) 53 | 54 | dataset_delete_btn.click( 55 | delete_dataset, 56 | inputs=train_dataset_input, 57 | outputs=[train_dataset_input] 58 | ) 59 | 60 | train_btn.click( 61 | train, 62 | inputs=[train_name_input, train_steps_input, train_dataset_input], 63 | outputs=[train_status_output, infer_name_input] 64 | ) 65 | 66 | infer_btn.click( 67 | infer, 68 | inputs=[infer_name_input, infer_txt_input], 69 | outputs=[infer_audio_output] 70 | ) 71 | 72 | model_delete_btn.click( 73 | delete_model, 74 | inputs=infer_name_input, 75 | outputs=[infer_name_input] 76 | ) 77 | 78 | server.load( 79 | refresh, 80 | inputs=[], 81 | outputs=[train_dataset_input, infer_name_input] 82 | ) 83 | 84 | server.launch(share=True, show_error=True) 85 | -------------------------------------------------------------------------------- /sambert-ui/consts.py: -------------------------------------------------------------------------------- 1 | port = 6006 2 | base_model_id = 'damo/speech_personal_sambert-hifigan_nsf_tts_zh-cn_pretrain_16k' 3 | base_model_version = 'v1.0.7' 4 | -------------------------------------------------------------------------------- /sambert-ui/requirements.txt: -------------------------------------------------------------------------------- 1 | modelscope==1.9.2 2 | 3 | # 训练/推理依赖 4 | # 同时需要额外补充: 5 | # 1. pip install kantts -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html 6 | pytorch_wavelets==1.3.0 7 | tensorboardX===2.6.2.2 8 | bitstring==4.1.2 9 | 10 | # sambert 标注依赖 11 | # 同时需要额外补充: 12 | # pip install tts-autolabel -f https://modelscope.oss-cn-beijing.aliyuncs.com/releases/repo.html 13 | # sox 需要单独安装，参考: https://pysox.readthedocs.io/en/latest/#installation 14 | openai-whisper==20230918 15 | librosa==0.10.1 16 | scipy==1.11.3 17 | sox==1.4.1 18 | gradio 19 | -------------------------------------------------------------------------------- /sambert-ui/utils_base.py: -------------------------------------------------------------------------------- 1 | # 基础方法封装 2 | import os 3 | import shutil 4 | import glob 5 | 6 | # 绝对路径获取方法 7 | curPath = os.path.dirname(os.path.abspath(__file__)) 8 | def getAbsPath (relativePath): 9 | joinPath = os.path.join(curPath, relativePath) 10 | return os.path.normpath( 11 | os.path.abspath(joinPath) 12 | ) 13 | 14 | # 数据集存放路径 15 | datasets_dir = getAbsPath('./datasets') 16 | if not shutil.os.path.exists(datasets_dir): 17 | shutil.os.makedirs(datasets_dir) 18 | 19 | # 获取数据集列表 ---------------------------------------------------- 20 | def get_dataset_list(): 21 | contents = os.listdir(datasets_dir) 22 | sub_dirs = [ 23 | content 24 | for content in contents 25 | if os.path.isdir(os.path.join(datasets_dir, content)) 26 | ] 27 | return sub_dirs 28 | 29 | # 小模型存放路径 30 | models_dir = getAbsPath('./models') 31 | if not shutil.os.path.exists(models_dir): 32 | shutil.os.makedirs(models_dir) 33 | 34 | # 获取模型列表 ---------------------------------------------------- 35 | def get_model_list(): 36 | contents = os.listdir(models_dir) 37 | sub_dirs = [ 38 | content 39 | for content in contents 40 | if os.path.isdir(os.path.join(models_dir, content)) 41 | ] 42 | return sub_dirs 43 | 44 | # 确保对应的空目录存在 45 | def ensure_empty_dir(dirpath): 46 | if shutil.os.path.exists(dirpath): 47 | shutil.rmtree(dirpath) 48 | shutil.os.makedirs(dirpath) 49 | 50 | # 获取目录中的最后一个文件 51 | def get_last_file(dirpath): 52 | files = glob.glob(os.path.join(dirpath, '*')) 53 | sorted_files = sorted(files, key=os.path.basename) 54 | if sorted_files: 55 | return sorted_files[-1] 56 | return False 57 | -------------------------------------------------------------------------------- /sambert-ui/utils_label.py: -------------------------------------------------------------------------------- 1 | # 对 sambert 训练的数据标注处理 2 | import os 3 | import shutil 4 | import uuid 5 | import librosa 6 | import gradio as gr 7 | from scipy.io import wavfile 8 | import numpy as np 9 | import whisper 10 | from modelscope.tools import run_auto_label 11 | from utils_base import ensure_empty_dir, datasets_dir, get_dataset_list 12 | 13 | # 绝对路径获取方法 14 | curPath = os.path.dirname(os.path.abspath(__file__)) 15 | def getAbsPath (relativePath): 16 | joinPath = os.path.join(curPath, relativePath) 17 | return os.path.normpath( 18 | os.path.abspath(joinPath) 19 | ) 20 | 21 | # 初始化 whisper 模型的加载 22 | model_path = getAbsPath('../../models/whisper/medium.pt') 23 | whisper_model = None 24 | if shutil.os.path.exists(model_path): 25 | whisper_model = whisper.load_model(model_path) 26 | else: 27 | whisper_model = whisper.load_model('medium') 28 | 29 | # whisper 音频分割方法 ---------------------------------------------- 30 | def split_long_audio(model, filepaths, save_path, out_sr=44100): 31 | # 格式化输入的音频路径(兼容单个音频和多个音频) 32 | if isinstance(filepaths, str): 33 | filepaths = [filepaths] 34 | 35 | # 对音频依次做拆分并存放到临时路径 36 | for file_idx, filepath in enumerate(filepaths): 37 | print(f"Transcribing file {file_idx}: '{filepath}' to segments...") 38 | result = model.transcribe(filepath, word_timestamps=True, task="transcribe", beam_size=5, best_of=5) 39 | segments = result['segments'] 40 | 41 | # 采用 librosa 配合 scipy 做音频数据分割 42 | wav, sr = librosa.load(filepath, sr=None, offset=0, duration=None, mono=True) 43 | wav, _ = librosa.effects.trim(wav, top_db=20) 44 | peak = np.abs(wav).max() 45 | if peak > 1.0: 46 | wav = 0.98 * wav / peak 47 | wav2 = librosa.resample(wav, orig_sr=sr, target_sr=out_sr) 48 | wav2 /= max(wav2.max(), -wav2.min()) 49 | 50 | # 将长音频文件分割成一条条的短音频并放入指定的目录 51 | for i, seg in enumerate(segments): 52 | start_time = seg['start'] 53 | end_time = seg['end'] 54 | wav_seg = wav2[int(start_time * out_sr):int(end_time * out_sr)] 55 | wav_seg_name = f"{file_idx}_{i}.wav" 56 | out_fpath = os.path.join(save_path, wav_seg_name) 57 | wavfile.write(out_fpath, rate=out_sr, data=(wav_seg * np.iinfo(np.int16).max).astype(np.int16)) 58 | 59 | # 自动标注与标注后的文件打包 -------------------------------------------- 60 | def auto_label(audio, name): 61 | if not audio or not name: 62 | return '', gr.update(choices=get_dataset_list()) 63 | 64 | # 创建临时目录用于存放分割后的音频与再次标注的信息 65 | input_wav = getAbsPath(f'./temp/input-{ uuid.uuid4() }') 66 | ensure_empty_dir(input_wav) 67 | 68 | work_dir = os.path.join(datasets_dir, name) 69 | ensure_empty_dir(work_dir) 70 | 71 | # 音频分割 72 | split_long_audio(whisper_model, audio, input_wav) 73 | 74 | # 音频自动标注 75 | # 第一次会自动下载对应的模型 76 | run_auto_label( 77 | input_wav=input_wav, 78 | work_dir=work_dir, 79 | resource_revision='v1.0.7' 80 | ) 81 | 82 | # 移除目录 83 | shutil.rmtree(input_wav) 84 | 85 | # 返回结果 86 | return '标注成功', gr.update(choices=get_dataset_list()) 87 | 88 | # 删除数据集 ---------------------------------------------------- 89 | # name - 删除的数据集名称 90 | def delete_dataset(name): 91 | try: 92 | if not name: 93 | return gr.update(choices=get_dataset_list()) 94 | 95 | target_dir = os.path.join(datasets_dir, name) 96 | shutil.rmtree(target_dir) 97 | return gr.update(choices=get_dataset_list(), value=None) 98 | except Exception: 99 | return gr.update(choices=get_dataset_list(), value=None) 100 | -------------------------------------------------------------------------------- /sambert-ui/utils_sambert.py: -------------------------------------------------------------------------------- 1 | # 训练部分实现 2 | import os 3 | import shutil 4 | import uuid 5 | import gradio as gr 6 | from modelscope.models.audio.tts import SambertHifigan 7 | from modelscope.pipelines import pipeline 8 | from modelscope.utils.constant import Tasks 9 | 10 | from modelscope.metainfo import Trainers 11 | from modelscope.trainers import build_trainer 12 | from modelscope.utils.audio.audio_utils import TtsTrainType 13 | from modelscope.hub.utils.utils import get_cache_dir 14 | 15 | from utils_base import ensure_empty_dir, get_last_file, models_dir, get_model_list 16 | import consts 17 | 18 | # 绝对路径获取方法 19 | curPath = os.path.dirname(os.path.abspath(__file__)) 20 | def getAbsPath (relativePath): 21 | joinPath = os.path.join(curPath, relativePath) 22 | return os.path.normpath( 23 | os.path.abspath(joinPath) 24 | ) 25 | 26 | # 模型训练 --------------------------------------------------------- 27 | # name - 训练结果(小模型)命名 28 | # steps - 训练步数 29 | # train_dataset_zip - 数据集zip包路径 30 | def train(name, steps, train_dataset_name): 31 | # 创建临时目录用于放置 训练结果 32 | work_dir = getAbsPath(f'./temp/work-{ uuid.uuid4() }') 33 | ensure_empty_dir(work_dir) 34 | 35 | # 数据集目录 36 | train_dataset = getAbsPath(f'./datasets/{ train_dataset_name }') 37 | 38 | # 进行训练 39 | trainer = build_trainer( 40 | Trainers.speech_kantts_trainer, 41 | default_args=dict( 42 | # 指定要finetune的 模型/版本 43 | model = consts.base_model_id, 44 | model_revision = consts.base_model_version, 45 | 46 | work_dir = work_dir, # 指定临时工作目录 47 | train_dataset = train_dataset, # 数据集目录 48 | 49 | # 训练参数 50 | train_type = { 51 | TtsTrainType.TRAIN_TYPE_SAMBERT: { # 配置训练AM（sambert）模型 52 | 'train_steps': steps + 1, # 训练多少个step 53 | 'save_interval_steps': 20, # 每训练多少个step保存一次checkpoint 54 | 'log_interval': 10 # 每训练多少个step打印一次训练日志 55 | } 56 | } 57 | ) 58 | ) 59 | trainer.train() 60 | 61 | # 挑选需要的文件到结果目录 62 | target_dir = os.path.join(models_dir, name) 63 | ensure_empty_dir(target_dir) 64 | shutil.os.makedirs(os.path.join(target_dir, 'tmp_am', 'ckpt')) 65 | shutil.os.makedirs(os.path.join(target_dir, 'data', 'se')) 66 | 67 | shutil.copy( 68 | get_last_file(os.path.join(work_dir, 'tmp_am', 'ckpt')), 69 | os.path.join(target_dir, 'tmp_am', 'ckpt') 70 | ) 71 | shutil.copy( 72 | os.path.join(work_dir, 'tmp_am', 'config.yaml'), 73 | os.path.join(target_dir, 'tmp_am'), 74 | ) 75 | shutil.copy( 76 | os.path.join(work_dir, 'data', 'audio_config.yaml'), 77 | os.path.join(target_dir, 'data'), 78 | ) 79 | shutil.copy( 80 | os.path.join(work_dir, 'data', 'se', 'se.npy'), 81 | os.path.join(target_dir, 'data', 'se'), 82 | ) 83 | 84 | # 清理文件 85 | shutil.rmtree(work_dir) 86 | shutil.rmtree(train_dataset) 87 | 88 | # 返回结果 89 | return '训练完成', gr.update(choices=get_model_list()) 90 | 91 | # 模型推理 --------------------------------------------------------- 92 | # name - 使用的小模型名称 93 | # txt - 需要合成音频的文字 94 | def infer(name, txt): 95 | try: 96 | base_model_path = os.path.join(get_cache_dir(), consts.base_model_id) 97 | model_path = os.path.join(models_dir, name) 98 | custom_infer_abs = { 99 | 'voice_name': 'F7', 100 | 101 | # 小模型部分 102 | 'am_ckpt': os.path.join(model_path, 'tmp_am', 'ckpt'), 103 | 'am_config': os.path.join(model_path, 'tmp_am', 'config.yaml'), 104 | 'audio_config': os.path.join(model_path, 'data', 'audio_config.yaml'), 105 | 'se_file': os.path.join(model_path, 'data', 'se', 'se.npy'), 106 | 107 | # 基础模型部分 108 | 'voc_ckpt': os.path.join( 109 | base_model_path, 'basemodel_16k', 'hifigan', 'ckpt' 110 | ), 111 | 'voc_config': os.path.join( 112 | base_model_path, 'basemodel_16k', 'hifigan', 'config.yaml' 113 | ) 114 | } 115 | 116 | model = SambertHifigan( 117 | base_model_path, 118 | **{ 'custom_ckpt': custom_infer_abs } 119 | ) 120 | inference = pipeline(task=Tasks.text_to_speech, model=model) 121 | output = inference(input=txt) 122 | 123 | output_path = f'/tmp/{ uuid.uuid4() }.wav' 124 | with open(output_path, mode='bx') as f: 125 | f.write(output['output_wav']) 126 | return output_path 127 | except Exception: 128 | return False 129 | 130 | # 删除模型 --------------------------------------------------------- 131 | # name - 删除的小模型名称 132 | def delete_model(name): 133 | try: 134 | if not name: 135 | return gr.update(choices=get_model_list()) 136 | 137 | target_dir = os.path.join(models_dir, name) 138 | shutil.rmtree(target_dir) 139 | return gr.update(choices=get_model_list(), value=None) 140 | except Exception: 141 | return gr.update(choices=get_model_list(), value=None) 142 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup() 4 | -------------------------------------------------------------------------------- /swap_voice.py: -------------------------------------------------------------------------------- 1 | from bark.generation import load_codec_model, generate_text_semantic, grab_best_device 2 | from bark import SAMPLE_RATE 3 | from encodec.utils import convert_audio 4 | from bark.hubert.hubert_manager import HuBERTManager 5 | from bark.hubert.pre_kmeans_hubert import CustomHubert 6 | from bark.hubert.customtokenizer import CustomTokenizer 7 | from bark.api import semantic_to_waveform 8 | from scipy.io.wavfile import write as write_wav 9 | from util.helper import create_filename 10 | from util.settings import Settings 11 | 12 | 13 | import torchaudio 14 | import torch 15 | import os 16 | import gradio 17 | 18 | def swap_voice_from_audio(swap_audio_filename, selected_speaker, tokenizer_lang, seed, batchcount, progress=gradio.Progress(track_tqdm=True)): 19 | use_gpu = not os.environ.get("BARK_FORCE_CPU", False) 20 | progress(0, desc="Loading Codec") 21 | 22 | # From https://github.com/gitmylo/bark-voice-cloning-HuBERT-quantizer 23 | hubert_manager = HuBERTManager() 24 | hubert_manager.make_sure_hubert_installed() 25 | hubert_manager.make_sure_tokenizer_installed(tokenizer_lang=tokenizer_lang) 26 | 27 | # From https://github.com/gitmylo/bark-voice-cloning-HuBERT-quantizer 28 | # Load HuBERT for semantic tokens 29 | 30 | # Load the HuBERT model 31 | device = grab_best_device(use_gpu) 32 | hubert_model = CustomHubert(checkpoint_path='./models/hubert/hubert.pt').to(device) 33 | model = load_codec_model(use_gpu=use_gpu) 34 | 35 | # Load the CustomTokenizer model 36 | tokenizer = CustomTokenizer.load_from_checkpoint(f'./models/hubert/{tokenizer_lang}_tokenizer.pth').to(device) # Automatically uses the right layers 37 | 38 | progress(0.25, desc="Converting WAV") 39 | 40 | # Load and pre-process the audio waveform 41 | wav, sr = torchaudio.load(swap_audio_filename) 42 | if wav.shape[0] == 2: # Stereo to mono if needed 43 | wav = wav.mean(0, keepdim=True) 44 | 45 | wav = convert_audio(wav, sr, model.sample_rate, model.channels) 46 | wav = wav.to(device) 47 | semantic_vectors = hubert_model.forward(wav, input_sample_hz=model.sample_rate) 48 | semantic_tokens = tokenizer.get_token(semantic_vectors) 49 | 50 | audio = semantic_to_waveform( 51 | semantic_tokens, 52 | history_prompt=selected_speaker, 53 | temp=0.7, 54 | silent=False, 55 | output_full=False) 56 | 57 | settings = Settings('config.yaml') 58 | 59 | result = create_filename(settings.output_folder_path, None, "swapvoice",".wav") 60 | write_wav(result, SAMPLE_RATE, audio) 61 | return result 62 | 63 | -------------------------------------------------------------------------------- /training/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KevinWang676/Bark-Voice-Cloning/bf4dd06bf93dd81e68ce33691e29bc06f419ca32/training/__init__.py -------------------------------------------------------------------------------- /training/data.py: -------------------------------------------------------------------------------- 1 | import random 2 | import requests 3 | import os, glob 4 | 5 | # english literature 6 | books = [ 7 | 'https://www.gutenberg.org/cache/epub/1513/pg1513.txt', 8 | 'https://www.gutenberg.org/files/2701/2701-0.txt', 9 | 'https://www.gutenberg.org/cache/epub/84/pg84.txt', 10 | 'https://www.gutenberg.org/cache/epub/2641/pg2641.txt', 11 | 'https://www.gutenberg.org/cache/epub/1342/pg1342.txt', 12 | 'https://www.gutenberg.org/cache/epub/100/pg100.txt' 13 | ] 14 | 15 | #default english 16 | # allowed_chars = ' abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789!@#$%^&*()-_+=\"\':;[]{}/<>,.`~\n\\' 17 | 18 | #german 19 | allowed_chars = ' aäbcdefghijklmnoöpqrsßtuüvwxyzABCDEFGHIJKLMNOÖPQRSTUÜVWXYZ0123456789!@#$%^&*()-_+=\"\':;[]{}/<>,.`~\n\\' 20 | 21 | 22 | def download_book(book): 23 | return requests.get(book).content.decode('utf-8') 24 | 25 | 26 | def filter_data(data): 27 | print('Filtering data') 28 | return ''.join([char for char in data if char in allowed_chars]) 29 | 30 | 31 | def load_books(fromfolder=False): 32 | text_data = [] 33 | if fromfolder: 34 | current_working_directory = os.getcwd() 35 | print(current_working_directory) 36 | path = 'text' 37 | for filename in glob.glob(os.path.join(path, '*.txt')): 38 | with open(os.path.join(os.getcwd(), filename), 'r') as f: # open in readonly mode 39 | print(f'Loading {filename}') 40 | text_data.append(filter_data(str(f.read()))) 41 | else: 42 | print(f'Loading {len(books)} books into ram') 43 | for book in books: 44 | text_data.append(filter_data(str(download_book(book)))) 45 | print('Loaded books') 46 | return ' '.join(text_data) 47 | 48 | 49 | def random_split_chunk(data, size=14): 50 | data = data.split(' ') 51 | index = random.randrange(0, len(data)) 52 | return ' '.join(data[index:index+size]) 53 | -------------------------------------------------------------------------------- /training/train.py: -------------------------------------------------------------------------------- 1 | import os 2 | import fnmatch 3 | import shutil 4 | 5 | import numpy 6 | import torchaudio 7 | import gradio 8 | 9 | from bark.hubert.pre_kmeans_hubert import CustomHubert 10 | from bark.hubert.customtokenizer import auto_train 11 | from tqdm.auto import tqdm 12 | 13 | 14 | def training_prepare_files(path, model,progress=gradio.Progress(track_tqdm=True)): 15 | 16 | semanticsfolder = "./training/data/output" 17 | wavfolder = "./training/data/output_wav" 18 | ready = os.path.join(path, 'ready') 19 | 20 | testfiles = fnmatch.filter(os.listdir(ready), '*.npy') 21 | if(len(testfiles) < 1): 22 | # prepare and copy for training 23 | hubert_model = CustomHubert(checkpoint_path=model) 24 | 25 | wavfiles = fnmatch.filter(os.listdir(wavfolder), '*.wav') 26 | for i, f in tqdm(enumerate(wavfiles), total=len(wavfiles)): 27 | semaname = '.'.join(f.split('.')[:-1]) # Cut off the extension 28 | semaname = f'{semaname}.npy' 29 | semafilename = os.path.join(semanticsfolder, semaname) 30 | if not os.path.isfile(semafilename): 31 | print(f'Skipping {f} no semantics pair found!') 32 | continue 33 | 34 | print('Processing', f) 35 | wav, sr = torchaudio.load(os.path.join(wavfolder, f)) 36 | if wav.shape[0] == 2: # Stereo to mono if needed 37 | wav = wav.mean(0, keepdim=True) 38 | output = hubert_model.forward(wav, input_sample_hz=sr) 39 | out_array = output.cpu().numpy() 40 | fname = f'{i}_semantic_features.npy' 41 | numpy.save(os.path.join(ready, fname), out_array) 42 | fname = f'{i}_semantic.npy' 43 | shutil.copy(semafilename, os.path.join(ready, fname)) 44 | 45 | def train(path, save_every, max_epochs): 46 | auto_train(path, save_epochs=save_every) 47 | 48 | -------------------------------------------------------------------------------- /training/training_prepare.py: -------------------------------------------------------------------------------- 1 | import random 2 | import uuid 3 | import numpy 4 | import os 5 | import random 6 | import fnmatch 7 | 8 | from tqdm.auto import tqdm 9 | from scipy.io import wavfile 10 | 11 | from bark.generation import load_model, SAMPLE_RATE 12 | from bark.api import semantic_to_waveform 13 | 14 | from bark import text_to_semantic 15 | from bark.generation import load_model 16 | 17 | from training.data import load_books, random_split_chunk 18 | 19 | output = 'training/data/output' 20 | output_wav = 'training/data/output_wav' 21 | 22 | 23 | def prepare_semantics_from_text(num_generations): 24 | loaded_data = load_books(True) 25 | 26 | print('Loading semantics model') 27 | load_model(use_gpu=True, use_small=False, force_reload=False, model_type='text') 28 | 29 | if not os.path.isdir(output): 30 | os.mkdir(output) 31 | 32 | loop = 1 33 | while 1: 34 | filename = uuid.uuid4().hex + '.npy' 35 | file_name = os.path.join(output, filename) 36 | text = '' 37 | while not len(text) > 0: 38 | text = random_split_chunk(loaded_data) # Obtain a short chunk of text 39 | text = text.strip() 40 | print(f'{loop} Generating semantics for text:', text) 41 | loop+=1 42 | semantics = text_to_semantic(text, temp=round(random.uniform(0.6, 0.8), ndigits=2)) 43 | numpy.save(file_name, semantics) 44 | 45 | 46 | def prepare_wavs_from_semantics(): 47 | if not os.path.isdir(output): 48 | raise Exception('No \'output\' folder, make sure you run create_data.py first!') 49 | if not os.path.isdir(output_wav): 50 | os.mkdir(output_wav) 51 | 52 | print('Loading coarse model') 53 | load_model(use_gpu=True, use_small=False, force_reload=False, model_type='coarse') 54 | print('Loading fine model') 55 | load_model(use_gpu=True, use_small=False, force_reload=False, model_type='fine') 56 | 57 | files = fnmatch.filter(os.listdir(output), '*.npy') 58 | current = 1 59 | total = len(files) 60 | 61 | for i, f in tqdm(enumerate(files), total=len(files)): 62 | real_name = '.'.join(f.split('.')[:-1]) # Cut off the extension 63 | file_name = os.path.join(output, f) 64 | out_file = os.path.join(output_wav, f'{real_name}.wav') 65 | if not os.path.isfile(out_file) and os.path.isfile(file_name): # Don't process files that have already been processed, to be able to continue previous generations 66 | print(f'Processing ({i+1}/{total}) -> {f}') 67 | wav = semantic_to_waveform(numpy.load(file_name), temp=round(random.uniform(0.6, 0.8), ndigits=2)) 68 | # Change to PCM16 69 | # wav = (wav * 32767).astype(np.int16) 70 | wavfile.write(out_file, SAMPLE_RATE, wav) 71 | 72 | print('Done!') 73 | 74 | -------------------------------------------------------------------------------- /util/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KevinWang676/Bark-Voice-Cloning/bf4dd06bf93dd81e68ce33691e29bc06f419ca32/util/__init__.py -------------------------------------------------------------------------------- /util/helper.py: -------------------------------------------------------------------------------- 1 | import os 2 | from datetime import datetime 3 | from mutagen.wave import WAVE 4 | from mutagen.id3._frames import * 5 | 6 | def create_filename(path, seed, name, extension): 7 | now = datetime.now() 8 | date_str =now.strftime("%m-%d-%Y") 9 | outputs_folder = os.path.join(os.getcwd(), path) 10 | if not os.path.exists(outputs_folder): 11 | os.makedirs(outputs_folder) 12 | 13 | sub_folder = os.path.join(outputs_folder, date_str) 14 | if not os.path.exists(sub_folder): 15 | os.makedirs(sub_folder) 16 | 17 | time_str = now.strftime("%H-%M-%S") 18 | if seed == None: 19 | file_name = f"{name}_{time_str}{extension}" 20 | else: 21 | file_name = f"{name}_{time_str}_s{seed}{extension}" 22 | return os.path.join(sub_folder, file_name) 23 | 24 | 25 | def add_id3_tag(filename, text, speakername, seed): 26 | audio = WAVE(filename) 27 | if speakername == None: 28 | speakername = "Unconditional" 29 | 30 | # write id3 tag with text truncated to 60 chars, as a precaution... 31 | audio["TIT2"] = TIT2(encoding=3, text=text[:60]) 32 | audio["TPE1"] = TPE1(encoding=3, text=f"Voice {speakername} using Seed={seed}") 33 | audio["TPUB"] = TPUB(encoding=3, text="Bark by Suno AI") 34 | audio["COMMENT"] = COMM(encoding=3, text="Generated with Bark GUI - Text-Prompted Generative Audio Model. Visit https://github.com/C0untFloyd/bark-gui") 35 | audio.save() 36 | -------------------------------------------------------------------------------- /util/parseinput.py: -------------------------------------------------------------------------------- 1 | import re 2 | import xml.etree.ElementTree as ET 3 | from xml.sax import saxutils 4 | #import nltk 5 | 6 | # Chunked generation originally from https://github.com/serp-ai/bark-with-voice-clone 7 | def split_and_recombine_text(text, desired_length=100, max_length=150): 8 | # return nltk.sent_tokenize(text) 9 | 10 | # from https://github.com/neonbjb/tortoise-tts 11 | """Split text it into chunks of a desired length trying to keep sentences intact.""" 12 | # normalize text, remove redundant whitespace and convert non-ascii quotes to ascii 13 | text = re.sub(r"\n\n+", "\n", text) 14 | text = re.sub(r"\s+", " ", text) 15 | text = re.sub(r"[“”]", '"', text) 16 | 17 | rv = [] 18 | in_quote = False 19 | current = "" 20 | split_pos = [] 21 | pos = -1 22 | end_pos = len(text) - 1 23 | 24 | def seek(delta): 25 | nonlocal pos, in_quote, current 26 | is_neg = delta < 0 27 | for _ in range(abs(delta)): 28 | if is_neg: 29 | pos -= 1 30 | current = current[:-1] 31 | else: 32 | pos += 1 33 | current += text[pos] 34 | if text[pos] == '"': 35 | in_quote = not in_quote 36 | return text[pos] 37 | 38 | def peek(delta): 39 | p = pos + delta 40 | return text[p] if p < end_pos and p >= 0 else "" 41 | 42 | def commit(): 43 | nonlocal rv, current, split_pos 44 | rv.append(current) 45 | current = "" 46 | split_pos = [] 47 | 48 | while pos < end_pos: 49 | c = seek(1) 50 | # do we need to force a split? 51 | if len(current) >= max_length: 52 | if len(split_pos) > 0 and len(current) > (desired_length / 2): 53 | # we have at least one sentence and we are over half the desired length, seek back to the last split 54 | d = pos - split_pos[-1] 55 | seek(-d) 56 | else: 57 | # no full sentences, seek back until we are not in the middle of a word and split there 58 | while c not in "!?.,\n " and pos > 0 and len(current) > desired_length: 59 | c = seek(-1) 60 | commit() 61 | # check for sentence boundaries 62 | elif not in_quote and (c in "!?]\n" or (c == "." and peek(1) in "\n ")): 63 | # seek forward if we have consecutive boundary markers but still within the max length 64 | while ( 65 | pos < len(text) - 1 and len(current) < max_length and peek(1) in "!?.]" 66 | ): 67 | c = seek(1) 68 | split_pos.append(pos) 69 | if len(current) >= desired_length: 70 | commit() 71 | # treat end of quote as a boundary if its followed by a space or newline 72 | elif in_quote and peek(1) == '"' and peek(2) in "\n ": 73 | seek(2) 74 | split_pos.append(pos) 75 | rv.append(current) 76 | 77 | # clean up, remove lines with only whitespace or punctuation 78 | rv = [s.strip() for s in rv] 79 | rv = [s for s in rv if len(s) > 0 and not re.match(r"^[\s\.,;:!?]*$", s)] 80 | 81 | return rv 82 | 83 | def is_ssml(value): 84 | try: 85 | ET.fromstring(value) 86 | except ET.ParseError: 87 | return False 88 | return True 89 | 90 | def build_ssml(rawtext, selected_voice): 91 | texts = rawtext.split("\n") 92 | joinedparts = "" 93 | for textpart in texts: 94 | textpart = textpart.strip() 95 | if len(textpart) < 1: 96 | continue 97 | joinedparts = joinedparts + f"\n{saxutils.escape(textpart)}" 98 | ssml = f""" 99 | 104 | {joinedparts} 105 | 106 | """ 107 | return ssml 108 | 109 | def create_clips_from_ssml(ssmlinput): 110 | # Parse the XML 111 | tree = ET.ElementTree(ET.fromstring(ssmlinput)) 112 | root = tree.getroot() 113 | 114 | # Create an empty list 115 | voice_list = [] 116 | 117 | # Loop through all voice tags 118 | for voice in root.iter('{http://www.w3.org/2001/10/synthesis}voice'): 119 | # Extract the voice name attribute and the content text 120 | voice_name = voice.attrib['name'] 121 | voice_content = voice.text.strip() if voice.text else '' 122 | if(len(voice_content) > 0): 123 | parts = split_and_recombine_text(voice_content) 124 | for p in parts: 125 | if(len(p) > 1): 126 | # add to tuple list 127 | voice_list.append((voice_name, p)) 128 | return voice_list 129 | 130 | -------------------------------------------------------------------------------- /util/settings.py: -------------------------------------------------------------------------------- 1 | import yaml 2 | 3 | class Settings: 4 | def __init__(self, config_file): 5 | self.config_file = config_file 6 | self.load() 7 | 8 | def load(self): 9 | try: 10 | with open(self.config_file, 'r') as f: 11 | data = yaml.load(f, Loader=yaml.FullLoader) 12 | self.selected_theme = data.get('selected_theme', "gstaff/xkcd") 13 | self.server_name = data.get('server_name', "") 14 | self.server_port = data.get('server_port', 0) 15 | self.server_share = data.get('server_share', False) 16 | self.input_text_desired_length = data.get('input_text_desired_length', 110) 17 | self.input_text_max_length = data.get('input_text_max_length', 170) 18 | self.silence_sentence = data.get('silence_between_sentences', 250) 19 | self.silence_speakers = data.get('silence_between_speakers', 500) 20 | self.output_folder_path = data.get('output_folder_path', 'outputs') 21 | 22 | except: 23 | self.selected_theme = "gstaff/xkcd" 24 | 25 | def save(self): 26 | data = { 27 | 'selected_theme': self.selected_theme, 28 | 'server_name': self.server_name, 29 | 'server_port': self.server_port, 30 | 'server_share': self.server_share, 31 | 'input_text_desired_length' : self.input_text_desired_length, 32 | 'input_text_max_length' : self.input_text_max_length, 33 | 'silence_between_sentences': self.silence_sentence, 34 | 'silence_between_speakers': self.silence_speakers, 35 | 'output_folder_path': self.output_folder_path 36 | } 37 | with open(self.config_file, 'w') as f: 38 | yaml.dump(data, f) 39 | 40 | 41 | 42 | --------------------------------------------------------------------------------