├── .gitignore ├── FAQ.md ├── LICENSE ├── README.md ├── build.sh ├── develop.md ├── docker-compose.build.hcl ├── docs ├── 16878718322092.jpg ├── 16925212367994.jpg ├── 16925213210798.jpg ├── 16925232398938.jpg ├── 16925233912142.jpg ├── 16925238212736.jpg ├── 16925239975597.jpg ├── 16925241544548.jpg ├── 16925246168293.jpg ├── 16925246793101.jpg ├── 16925247438437.jpg ├── docker-version-log.png ├── winxin08.jpg ├── wx-alan.jpg ├── wx-skm.jpg └── zhi_shi_xq.jpg ├── domain-chatbot ├── VirtualWife │ ├── __init__.py │ ├── asgi.py │ ├── settings.py │ ├── urls.py │ └── wsgi.py ├── __init__.py ├── apps │ ├── __init__.py │ ├── chatbot │ │ ├── __init__.py │ │ ├── character │ │ │ ├── __init__.py │ │ │ ├── base_character_template.py │ │ │ ├── character.py │ │ │ ├── character_generation.py │ │ │ ├── character_template_zh.py │ │ │ ├── role_package_manage.py │ │ │ └── sys │ │ │ │ └── aili_zh.py │ │ ├── chat │ │ │ ├── __init__.py │ │ │ ├── chat_history_queue.py │ │ │ └── chat_service.py │ │ ├── config │ │ │ ├── __init__.py │ │ │ ├── sys_config.json │ │ │ └── sys_config.py │ │ ├── emotion │ │ │ ├── behavior_action_management.py │ │ │ └── emotion_manage.py │ │ ├── forms.py │ │ ├── insight │ │ │ ├── __init__.py │ │ │ ├── bilibili │ │ │ │ ├── bili_live_client.py │ │ │ │ └── sdk │ │ │ │ │ ├── client.py │ │ │ │ │ ├── handlers.py │ │ │ │ │ └── models.py │ │ │ ├── bilibili_api │ │ │ │ ├── __init__.py │ │ │ │ └── bili_live_client.py │ │ │ ├── insight.py │ │ │ └── insight_message_queue.py │ │ ├── llms │ │ │ ├── __init__.py │ │ │ ├── llm_model_strategy.py │ │ │ ├── ollama │ │ │ │ ├── __init__.py │ │ │ │ └── ollama_chat_robot.py │ │ │ ├── openai │ │ │ │ ├── __init__.py │ │ │ │ └── openai_chat_robot.py │ │ │ └── zhipuai │ │ │ │ └── zhipuai_chat_robot.py │ │ ├── memory │ │ │ ├── __init__.py │ │ │ ├── base_storage.py │ │ │ ├── embedding.py │ │ │ ├── local │ │ │ │ └── local_storage_impl.py │ │ │ ├── memory_storage.py │ │ │ ├── milvus │ │ │ │ ├── __init__.py │ │ │ │ ├── milvus_memory.py │ │ │ │ └── milvus_storage_impl.py │ │ │ └── zep │ │ │ │ ├── __init__.py │ │ │ │ └── zep_memory.py │ │ ├── migrations │ │ │ └── __init__.py │ │ ├── models.py │ │ ├── output │ │ │ ├── __init__.py │ │ │ ├── consumers.py │ │ │ ├── realtime_message_queue.py │ │ │ └── routing.py │ │ ├── process │ │ │ ├── __init__.py │ │ │ └── process.py │ │ ├── reflection │ │ │ ├── __init__.py │ │ │ ├── reflection.py │ │ │ └── reflection_template.py │ │ ├── schedule │ │ │ ├── Idle_schedule.py │ │ │ └── observe_memory.py │ │ ├── serializers.py │ │ ├── service │ │ │ ├── __init__.py │ │ │ └── portal_user_service.py │ │ ├── urls.py │ │ ├── utils │ │ │ ├── __init__.py │ │ │ ├── chat_message_utils.py │ │ │ ├── datatime_utils.py │ │ │ ├── snowflake_utils.py │ │ │ └── str_utils.py │ │ └── views.py │ ├── migrations │ │ ├── 0001_initial.py │ │ └── __init__.py │ └── speech │ │ ├── __init__.py │ │ ├── ars │ │ └── __init__.py │ │ ├── translation │ │ ├── __init__.py │ │ ├── base_translation_client.py │ │ ├── google │ │ │ ├── __init__.py │ │ │ └── google_translation_client.py │ │ ├── huoshan │ │ │ ├── __init__.py │ │ │ └── huoshan_translation_client.py │ │ └── youdao │ │ │ ├── AuthV3Util.py │ │ │ ├── AuthV4Util.py │ │ │ └── youdao_translation_client.py │ │ ├── tts │ │ ├── __init__.py │ │ ├── bert_vits2.py │ │ ├── edge_tts.py │ │ └── tts_driver.py │ │ ├── urls.py │ │ ├── utils │ │ └── uuid_generator.py │ │ └── views.py ├── db │ └── __init__.py ├── logs │ └── __init__.py ├── manage.py ├── models │ ├── README.md │ └── __init__.py ├── requirements.txt └── tests │ ├── __init__.py │ └── bilibili_api_test.py ├── domain-chatvrm ├── .eslintrc.json ├── .gitignore ├── LICENSE ├── README.md ├── next.config.js ├── package-lock.json ├── package.json ├── postcss.config.js ├── public │ ├── aili.vrm │ ├── bg-c.png │ ├── daily │ │ ├── idle_01.fbx │ │ ├── idle_02.fbx │ │ ├── idle_03.fbx │ │ ├── idle_happy_01.fbx │ │ ├── idle_happy_02.fbx │ │ ├── idle_happy_03.fbx │ │ ├── kiss_01.fbx │ │ ├── sitting.fbx │ │ ├── standing_greeting.fbx │ │ ├── talking_01.fbx │ │ ├── talking_02.fbx │ │ └── thinking.fbx │ ├── emote │ │ ├── angry.fbx │ │ └── excited.fbx │ ├── g1.vrm │ ├── g2.vrm │ ├── github-mark-white.svg │ ├── hailey.vrm │ ├── idle_loop.vrma │ ├── ogp.png │ ├── rumba_dancing.fbx │ ├── silly_dancing.fbx │ ├── わたあめ_02.vrm │ ├── わたあめ_03.vrm │ ├── 后藤仁.vrm │ └── 活力少女.vrm ├── src │ ├── components │ │ ├── assistantText.tsx │ │ ├── chatLog.tsx │ │ ├── githubLink.tsx │ │ ├── iconButton.tsx │ │ ├── introduction.tsx │ │ ├── link.tsx │ │ ├── menu.tsx │ │ ├── messageInput.tsx │ │ ├── messageInputContainer.tsx │ │ ├── meta.tsx │ │ ├── settings.tsx │ │ ├── textButton.tsx │ │ └── vrmViewer.tsx │ ├── features │ │ ├── blivedm │ │ │ └── blivedm.ts │ │ ├── chat │ │ │ └── openAiChat.ts │ │ ├── config │ │ │ └── configApi.ts │ │ ├── constants │ │ │ ├── koeiroParam.ts │ │ │ └── systemPromptConstants.ts │ │ ├── customRole │ │ │ └── customRoleApi.ts │ │ ├── emoteController │ │ │ ├── autoBlink.ts │ │ │ ├── autoLookAt.ts │ │ │ ├── emoteConstants.ts │ │ │ ├── emoteController.ts │ │ │ └── expressionController.ts │ │ ├── game │ │ │ └── photoFrame.tsx │ │ ├── httpclient │ │ │ └── httpclient.ts │ │ ├── koeiromap │ │ │ └── koeiromap.ts │ │ ├── lipSync │ │ │ ├── lipSync.ts │ │ │ └── lipSyncAnalyzeResult.ts │ │ ├── media │ │ │ └── mediaApi.ts │ │ ├── messages │ │ │ ├── messages.ts │ │ │ └── speakCharacter.ts │ │ ├── mixamo │ │ │ ├── loadMixamoAnimation.ts │ │ │ └── mixamoVRMRigMap.ts │ │ ├── queue │ │ │ └── ChatPriorityQueue.ts │ │ ├── translation │ │ │ └── translationApi.ts │ │ ├── tts │ │ │ └── ttsApi.ts │ │ └── vrmViewer │ │ │ ├── model.ts │ │ │ ├── viewer.ts │ │ │ └── viewerContext.ts │ ├── lib │ │ ├── VRMAnimation │ │ │ ├── VRMAnimation.ts │ │ │ ├── VRMAnimationLoaderPlugin.ts │ │ │ ├── VRMAnimationLoaderPluginOptions.ts │ │ │ ├── VRMCVRMAnimation.ts │ │ │ ├── loadVRMAnimation.ts │ │ │ └── utils │ │ │ │ ├── arrayChunk.ts │ │ │ │ ├── linearstep.ts │ │ │ │ └── saturate.ts │ │ └── VRMLookAtSmootherLoaderPlugin │ │ │ ├── VRMLookAtSmoother.ts │ │ │ └── VRMLookAtSmootherLoaderPlugin.ts │ ├── pages │ │ ├── _app.tsx │ │ ├── _document.tsx │ │ ├── api │ │ │ ├── chat.ts │ │ │ └── tts.ts │ │ └── index.tsx │ ├── styles │ │ └── globals.css │ └── utils │ │ ├── buildUrl.ts │ │ └── wait.ts ├── tailwind.config.js ├── tsconfig.json └── watch.json ├── infrastructure-gateway └── conf.d │ ├── default.conf │ ├── server │ ├── chatbot.conf │ ├── chatvrm.conf │ └── default.conf │ └── upstream │ ├── ups-chatbot.conf │ └── ups-chatvrm.conf ├── infrastructure-packaging ├── Dockerfile.ChatBot ├── Dockerfile.ChatVRM └── Dockerfile.Gateway ├── installer ├── README.md ├── docker-compose.yaml ├── env_example ├── experiment │ ├── config.yaml │ └── docker-compose-dev.yaml ├── linux │ ├── start.sh │ └── stop.sh ├── milvus │ └── docker-compose.yml ├── windows │ ├── start.bat │ └── stop.bat └── zep │ ├── Dockerfile │ └── upload.sh ├── package-lock.json ├── package.json ├── product.md ├── release-log.md └── upload.sh /FAQ.md: -------------------------------------------------------------------------------- 1 | ### 如何在B站上进行直播呢？ 2 | 3 | 作者是Mac系统，使用OBS进行直播，具体直播布置教程，可以参考以下视频 4 | - [新手MacBook直播OBS教程](https://www.bilibili.com/video/BV1aB4y1P7BK/?spm_id_from=333.999.0.0) 5 | - 最近B站已经上线了新的直播姬，支持win和mac，可以去官网下载 6 | 7 | ### 如何更换VRM模型呢？ 8 | 9 | - VRM模型市场：[Vroid](https://hub.vroid.com/) 10 | - 点击设置后，点击打开VRM模型按钮，上传VRM（这块作者还没优化，刷新页面会加载默认模型） 11 | ![](docs/16925246168293.jpg) 12 | 13 | ### 如何更换虚拟AI的prompt？ 14 | 15 | - 新增角色，提交后可以在基础设置中选择你的角色 16 | ![](docs/16925246793101.jpg) 17 | 18 | ### 如何更换中文语音包？ 19 | - 本系统已经内置了很多中文语音包 20 | ![](docs/16925247438437.jpg) 21 | 22 | ### 支持私有化模型，需要安装text-generation-webui 23 | 24 | - [text-generation-webui官网](https://github.com/oobabooga/text-generation-webui) 25 | - [text-generation-webui详细安装教程](https://www.bilibili.com/video/BV1gM4y1J7dD/?spm_id_from=333.788&vd_source=11f40bfaa73ba3e80ac4ad36fb18f359) 26 | > 注意整个安装过程中，一定需要挂梯子 27 | 28 | ### 目前支持的LLM模型 29 | 30 | - ziqingyang_chinese-alpaca-2-13b 31 | - ziqingyang_chinese-alpaca-2-7b 32 | - Baichuan2-13B-Chat 33 | 34 | ### 关于长期记忆中的Milvus如何安装？ 35 | ``` 36 | cd installer/milvus 37 | sudo docker compose up -d 38 | ``` 39 | - 注意前提你需要将docker和docker-compose安装好，如果使用官方的安装方式，请注意Milvus的版本号，需要与项目中的installer/milvus/docker-compose.yml 保持一致 40 | - 文档地址：https://milvus.io/docs/install_standalone-docker.md 41 | 42 | ### 关于使用Docker启动后，无法访问OpenAI问题，如何解决 43 | - 第一步请排查：你的梯子是否正常 44 | - 第二步请排查：在高级设置中，将http-proxy设置开启，设置地址http://host.docker.internal:23457，注意这里的端口号请配置你代理的端口号 45 | 46 | 47 | ### 关于使用docker启动后，通过127.0.0.1访问text-generation-webui或者Milvus网络问题 48 | - 可以使用Docker自带的DNS，将请求转发到宿主机， 49 | - DNS:http://host.docker.internal:xxxx 50 | 51 | 52 | ### windows 系统安装Docker，需要安装WSL 53 | 54 | - 安装文档：https://learn.microsoft.com/zh-cn/windows/wsl/install 55 | 56 | ### npm run dev 出现错误 57 | 58 | - 错误日志 59 | ``` 60 | > chat-vrm@1.0 dev 61 | > next dev 62 | ``` 63 | - 解决方案： 64 | ``` 65 | cd domain-chatvrm 66 | rm package-lock.json 67 | npm install 68 | npm run dev 69 | ``` 70 | 71 | ### B站弹幕监听不到 72 | 73 | - 检查B站直播间ID，主播UID 74 | - 在页面上登录B站后，打开https://api.bilibili.com/x/web-interface/nav 75 | - 找到uid 76 | - 检查B_COOKIE是否设置正确 77 | - 注意一定要复制完整的cookie, 78 | - 错误示例： 79 | ``` 80 | B_COOKIE="buvid3=Fggggg28116infoc;" 81 | ``` 82 | - 正确示例 83 | ``` 84 | B_COOKIE="buvid3=8B473137-DAF1-B326-XXXXX-CA8A06BEE16802942infoc; b_nut=1681972902; _uuid=106F868A4-19CC-4D4B-XXX-2651010333D7E1003163infoc; buvid4=A628284B-D833-9256-XXX-903BA100474C03740-023042014-7G54s1lO7XHCknT6D8RZoQ%3D%3D; nostalgia_conf=-1; CURRENT_FNVAL=4048; ....... 此处略去其他的" 85 | ``` -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 yakami 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Overview 2 | 3 | VirtualWife是一个虚拟数字人项目，项目还处于孵化阶段，有很多需要优化的地方，作者想打造一个拥有自己“灵魂”的虚拟数字人，你可以像朋友一样和她相识，作者希望虚拟数字人融入人类生活，作为恋爱导师，心理咨询师，解决人类的情感需求。项目制作不易，占用了作者大量的业余时间，如果对你有用，请点star，拜托啦~ 4 | 5 | # Features 6 | - 支持一键通过Docker快速部署 7 | - 支持在Linux/Windows/MacOS系统进行部署 8 | - 支持自定义角色设定 9 | - 支持更换角色模型，可从VRM模型市场[Vroid](https://hub.vroid.com/)下载 10 | - 支持长短期记忆功能 11 | - 支持多LLM模型切换，并且支持私有化模型（ollama），具体使用说明请查阅[FAQ](FAQ.md) 12 | - 支持文字驱动表情，文字驱动动作 13 | - 支持B站进行直播，具体使用说明请查阅[FAQ](FAQ.md) 14 | - 支持通过中文进行语音对话 15 | - 支持Edge（微软）、Bert-VITS2语音切换 16 | - 流式传输数据，拥有更快的响应速度 17 | 18 | # Example Video 19 | 20 | https://github.com/yakami129/VirtualWife/assets/36467094/51de1c07-f468-4987-8648-dc6b810550ad 21 | 22 | # FAQ 23 | - 项目答疑以及部署中遇到问题的解决方案，请查阅[FAQ](FAQ.md) 24 | - 本地开发和源码部署请查阅[develop](develop.md) 25 | 26 | # Roadmap 27 | 28 | - [ ] 记忆模块优化 29 | - [ ] 支持联想记忆 30 | - [ ] 提高记忆检索的准确度 31 | - [ ] 支持记忆遗忘机制，去除不重要的记忆，让AI更加专注 32 | - [ ] 情感涌现模块优化 33 | - [x] 支持模型肢体动作控制 34 | - [ ] ~~支持人物的语气、语速控制~~ 35 | - [ ] 语音模块 36 | - [x] 支持Edge（微软）、Bert-VITS2语音切换 37 | - [ ] 角色扮演深化 38 | - [ ] LoRA微调RWKV，完成猫娘、傲娇、御姐等性格塑造 39 | - [ ] 反思模块开发 40 | - [ ] 给定角色一个初始设定，通过反思+计划进行自我升级 41 | - [ ] 知识检索 42 | - [ ] 融合FastGPT 43 | 44 | # Get Started 45 | 46 | ## 一、安装[Docker](https://www.docker.com/)环境 47 | 48 | - 方式一：命令行方式安装 49 | - [docker安装手册](https://www.runoob.com/docker/macos-docker-install.html) 50 | - [docker-compose安装手册](https://www.runoob.com/docker/docker-compose.html) 51 | - 方式二：下载Docker桌面程序（桌面程序一般自带docker-compose） 52 | - [下载Docker桌面程序](https://www.docker.com/) 53 | - 然后下一步下一步就安装好了，如果拉取镜像比较慢，可以更改为国内镜像地址 54 | 55 | - 检查是否安装成功，安装正常会打印日志 56 | ``` 57 | docker -v 58 | docker-compose -v 59 | ``` 60 | ![](docs/docker-version-log.png) 61 | 62 | ## 二、进入VirtualWife安装程序目录 63 | 64 | ``` 65 | cd installer 66 | ``` 67 | 68 | ``` 69 | ├── README.md # 安装程序使用说明 70 | ├── docker-compose.yaml # docker编排文件 71 | ├── env_example # 环境变量配置模版，使用时需要将文件名改成.env 72 | ├── milvus # 长期记忆，数据存储模块，启动程序 73 | ├── linux # linux 启动和关闭程序 74 | │ ├── start.sh 75 | │ └── stop.sh 76 | └── windows # windows 启动和关闭程序 77 | ├── start.bat 78 | └── stop.bat 79 | ``` 80 | 81 | ## 三、设置环境变量 82 | 83 | - 更改境变量配置模版文件名为.env 84 | ``` 85 | mv env_example .env 86 | ``` 87 | - 设置环境变量 88 | ``` 89 | # 时区 90 | TIMEZONE=Asia/Shanghai 91 | 92 | # 程序版本号，程序版本号可以查阅项目的release发布版本号，latest代表最新版本 93 | CHATBOT_TAG=latest 94 | CHATVRM_TAG=latest 95 | GATEWAY_TAG=latest 96 | ``` 97 | 98 | ## 四、启动程序 99 | 100 | - 以Linux系统为例，启动程序示例如下 101 | ``` 102 | ## 进入linux脚本目录 103 | cd linux 104 | 105 | ## 启动程序，初次启动需要下载镜像，整个过程可能需要5分钟 106 | sh start.sh 107 | ``` 108 | 109 | ## 五、访问页面 110 | 111 | - Web访问路径 112 | ```shell 113 | http://localhost/ 114 | ``` 115 | - 页面展示 116 | ![](docs/16925232398938.jpg) 117 | 118 | ## 六、初始化数字人配置 119 | 120 | ### （1）基础配置 121 | ``` 122 | 选择自己喜欢的角色和人物模型，并且选择大语言模型 123 | 如果是使用openai请将语言模型设置为openai 124 | ``` 125 | ![](docs/16925233912142.jpg) 126 | 127 | ### （2）大语言模型配置 128 | ``` 129 | 这以openai模型为例，你只需要将OPENAI_API_KEY填写好即可 130 | 如果有API代理可以将地址填写到OPENAI_BASE_URL 131 | 132 | OLLAMA_API_URL配置说明：如果是docker启动，请使用http://host.docker.internal:11434 133 | ``` 134 | ![](docs/16925238212736.jpg) 135 | 136 | ### （3）高级设置 137 | ``` 138 | 如果没有OPENAI_BASE_URL，你需要配置http-proxy， 139 | 如果是使用docker启动的程序，需要使用docker的dns， 140 | 例如这样HTTP_PROXY=http://host.docker.internal:23457 141 | 142 | ``` 143 | ![](docs/16925239975597.jpg) 144 | 145 | ### （4）保存 146 | 保存、保存、保存，重要的事情说三遍 147 | ![](docs/16925241544548.jpg) 148 | 149 | > 保存成功后，无需重启服务，可以开始聊天了，如果出现异常请查阅[FAQ](FAQ.md) 150 | 151 | # LICENSE 152 | 153 | 依据 MIT 协议，使用者需自行承担使用本项目的风险与责任，本开源项目开发者与此无关。 154 | 155 | ## 联系我们 156 | 157 | | 技术交流群 | 打赏、私有化一键整合包 | 158 | |------------------------------------|-------------------------------| 159 | | ![winxin08.jpg](docs/winxin08.jpg) | ![zhi_shi_xq.jpg](docs/zhi_shi_xq.jpg) | 160 | 161 | 162 | ## Star History 163 | 164 | [![Star History Chart](https://api.star-history.com/svg?repos=yakami129/VirtualWife&type=Date)](https://star-history.com/#yakami129/VirtualWife) 165 | 166 | 167 | 168 | -------------------------------------------------------------------------------- /build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker rmi okapi0129/virtualwife-chatbot 3 | docker rmi okapi0129/virtualwife-chatvrm 4 | docker rmi okapi0129/virtualwife-gateway 5 | docker buildx bake -f docker-compose.build.hcl --load -------------------------------------------------------------------------------- /develop.md: -------------------------------------------------------------------------------- 1 | ## 本地开发 2 | 3 | ### 环境要求 4 | 5 | - python: 3.10.12 6 | - node: 15.14.0 7 | 8 | ### 先决条件 9 | - 安装conda，在Linux或WSL上，可以使用以下两个命令自动安装它（源代码） 10 | - 其他安装方式：[anaconda](https://anaconda.org.cn/anaconda/install/windows/) 11 | ```shell 12 | curl -sL "https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh" > "Miniconda3.sh" 13 | bash Miniconda3.sh 14 | ``` 15 | - 初始化环境 16 | ```shell 17 | conda create -n vw python=3.10.12 18 | conda activate vw 19 | conda install -c conda-forge nodejs=15.14.0 20 | ``` 21 | 22 | ### 如何启动domain-chatbot？ 23 | 24 | - 进入domain-chatbot文件夹 25 | ```shell 26 | cd domain-chatbot 27 | ``` 28 | - 如何获取OpenAIKey? 29 | - https://platform.openai.com/overview 30 | - 如何获取B站直播间ID？ 31 | - 获取B站直播ID，前提是需要你自行注册B站直播用户 32 | ![](docs/16878718322092.jpg) 33 | - 创建.env文件，B直播配置已经迁移到高级设置中 34 | ```shell 35 | # 时区 36 | TIMEZONE=Asia/Shanghai 37 | ``` 38 | - 安装domain-chatbot项目依赖 39 | ```shell 40 | pip3 install -r requirements.txt 41 | ``` 42 | - 初始化项目数据库 43 | ```shell 44 | python manage.py makemigrations 45 | ``` 46 | ```shell 47 | python manage.py migrate 48 | ``` 49 | - 启动domain-chatbot项目 50 | ```shell 51 | python manage.py runserver 52 | ``` 53 | ### 如何启动domain-chatvrm？ 54 | 55 | - 进入domain-chatvrm文件夹 56 | ```shell 57 | cd domain-chatvrm 58 | ``` 59 | - 安装domain-chatvrm项目依赖 60 | ```shell 61 | rm package-lock.json 62 | npm install 63 | ``` 64 | - 启动domain-chatvrm项目 65 | ```shell 66 | npm run dev 67 | ``` 68 | - Web访问路径 69 | ```shell 70 | http://localhost:3000/ 71 | ``` 72 | 73 | -------------------------------------------------------------------------------- /docker-compose.build.hcl: -------------------------------------------------------------------------------- 1 | variable "TAG" { 2 | default = "latest" 3 | } 4 | 5 | variable "DISTRO" { 6 | default = "okapi0129" 7 | } 8 | 9 | group "default" { 10 | targets = ["chatbot","chatvrm","gateway"] 11 | } 12 | 13 | target "chatbot" { 14 | args = { 15 | TAG = null 16 | } 17 | dockerfile = "infrastructure-packaging/Dockerfile.ChatBot" 18 | tags = ["${DISTRO}/virtualwife-chatbot:${TAG}"] 19 | } 20 | 21 | target "chatvrm" { 22 | args = { 23 | TAG = null 24 | } 25 | dockerfile = "infrastructure-packaging/Dockerfile.ChatVRM" 26 | tags = ["${DISTRO}/virtualwife-chatvrm:${TAG}"] 27 | } 28 | 29 | target "gateway" { 30 | args = { 31 | TAG = null 32 | } 33 | dockerfile = "infrastructure-packaging/Dockerfile.Gateway" 34 | tags = ["${DISTRO}/virtualwife-gateway:${TAG}"] 35 | } 36 | 37 | target "chatbot-release" { 38 | inherits = ["chatbot"] 39 | platforms = ["linux/amd64", "linux/arm64"] 40 | } 41 | 42 | target "chatvrm-release" { 43 | inherits = ["chatvrm"] 44 | platforms = ["linux/amd64", "linux/arm64"] 45 | } 46 | 47 | target "gateway-release" { 48 | inherits = ["gateway"] 49 | platforms = ["linux/amd64", "linux/arm64"] 50 | } -------------------------------------------------------------------------------- /docs/16878718322092.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/docs/16878718322092.jpg -------------------------------------------------------------------------------- /docs/16925212367994.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/docs/16925212367994.jpg -------------------------------------------------------------------------------- /docs/16925213210798.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/docs/16925213210798.jpg -------------------------------------------------------------------------------- /docs/16925232398938.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/docs/16925232398938.jpg -------------------------------------------------------------------------------- /docs/16925233912142.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/docs/16925233912142.jpg -------------------------------------------------------------------------------- /docs/16925238212736.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/docs/16925238212736.jpg -------------------------------------------------------------------------------- /docs/16925239975597.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/docs/16925239975597.jpg -------------------------------------------------------------------------------- /docs/16925241544548.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/docs/16925241544548.jpg -------------------------------------------------------------------------------- /docs/16925246168293.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/docs/16925246168293.jpg -------------------------------------------------------------------------------- /docs/16925246793101.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/docs/16925246793101.jpg -------------------------------------------------------------------------------- /docs/16925247438437.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/docs/16925247438437.jpg -------------------------------------------------------------------------------- /docs/docker-version-log.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/docs/docker-version-log.png -------------------------------------------------------------------------------- /docs/winxin08.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/docs/winxin08.jpg -------------------------------------------------------------------------------- /docs/wx-alan.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/docs/wx-alan.jpg -------------------------------------------------------------------------------- /docs/wx-skm.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/docs/wx-skm.jpg -------------------------------------------------------------------------------- /docs/zhi_shi_xq.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/docs/zhi_shi_xq.jpg -------------------------------------------------------------------------------- /domain-chatbot/VirtualWife/__init__.py: -------------------------------------------------------------------------------- 1 | # import pymysql 2 | 3 | # pymysql.version_info=(1,4,3,"final",0) 4 | # pymysql.install_as_MySQLdb() -------------------------------------------------------------------------------- /domain-chatbot/VirtualWife/asgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | ASGI config for VirtualWife project. 3 | 4 | It exposes the ASGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/4.2/howto/deployment/asgi/ 8 | """ 9 | 10 | import os 11 | from apps.chatbot.output.routing import websocket_urlpatterns 12 | from apps.chatbot.output.realtime_message_queue import RealtimeMessageQueryJobTask 13 | from apps.chatbot.chat.chat_history_queue import ChatHistoryMessageQueryJobTask 14 | from apps.chatbot.insight.insight_message_queue import InsightMessageQueryJobTask 15 | # from apps.chatbot.insight.bilibili.bili_live_client import bili_live_client_main 16 | from apps.chatbot.schedule.observe_memory import run_observe_memory_job, observe_memory_job 17 | from channels.auth import AuthMiddlewareStack 18 | from channels.routing import ProtocolTypeRouter, URLRouter 19 | from channels.security.websocket import AllowedHostsOriginValidator 20 | from django.core.asgi import get_asgi_application 21 | from django.urls import path 22 | 23 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'VirtualWife.settings') 24 | 25 | # bili_live_client_main() 26 | # run_observe_memory_job(60, observe_memory_job) 27 | 28 | RealtimeMessageQueryJobTask.start() 29 | ChatHistoryMessageQueryJobTask.start() 30 | InsightMessageQueryJobTask.start() 31 | 32 | # Initialize Django ASGI application early to ensure the AppRegistry 33 | # is populated before importing code that may import ORM models. 34 | django_asgi_app = get_asgi_application() 35 | 36 | application = ProtocolTypeRouter({ 37 | "http": django_asgi_app, 38 | "websocket": AllowedHostsOriginValidator( 39 | AuthMiddlewareStack(URLRouter(websocket_urlpatterns)) 40 | ), 41 | }) 42 | -------------------------------------------------------------------------------- /domain-chatbot/VirtualWife/urls.py: -------------------------------------------------------------------------------- 1 | """ 2 | URL configuration for VirtualWife project. 3 | 4 | The `urlpatterns` list routes URLs to views. For more information please see: 5 | https://docs.djangoproject.com/en/4.2/topics/http/urls/ 6 | Examples: 7 | Function views 8 | 1. Add an import: from my_app import views 9 | 2. Add a URL to urlpatterns: path('', views.home, name='home') 10 | Class-based views 11 | 1. Add an import: from other_app.views import Home 12 | 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') 13 | Including another URLconf 14 | 1. Import the include() function: from django.urls import include, path 15 | 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) 16 | """ 17 | from django.contrib import admin 18 | from django.urls import path, include, re_path 19 | from rest_framework import permissions 20 | from drf_yasg.views import get_schema_view 21 | from drf_yasg import openapi 22 | from django.conf import settings 23 | from django.conf.urls.static import static 24 | 25 | schema_view = get_schema_view( 26 | openapi.Info( 27 | title="ChatBotAPI", 28 | default_version='v1', 29 | description="Test description", 30 | ), 31 | public=True, 32 | permission_classes=(permissions.AllowAny,), 33 | ) 34 | 35 | urlpatterns = [ 36 | path("chatbot/", include("apps.chatbot.urls")), 37 | path("speech/", include("apps.speech.urls")), 38 | path('swagger/', schema_view.without_ui(cache_timeout=0), name='schema-json'), 39 | path('swagger/', schema_view.with_ui('swagger', cache_timeout=0), name='schema-swagger-ui'), 40 | path('redoc/', schema_view.with_ui('redoc', cache_timeout=0), name='schema-redoc'), 41 | ] 42 | 43 | urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) 44 | -------------------------------------------------------------------------------- /domain-chatbot/VirtualWife/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for VirtualWife project. 3 | 4 | It exposes the WSGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/4.2/howto/deployment/wsgi/ 8 | """ 9 | 10 | import os 11 | 12 | from django.core.wsgi import get_wsgi_application 13 | 14 | 15 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'VirtualWife.settings') 16 | 17 | application = get_wsgi_application() 18 | -------------------------------------------------------------------------------- /domain-chatbot/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/domain-chatbot/__init__.py -------------------------------------------------------------------------------- /domain-chatbot/apps/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/domain-chatbot/apps/__init__.py -------------------------------------------------------------------------------- /domain-chatbot/apps/chatbot/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/domain-chatbot/apps/chatbot/__init__.py -------------------------------------------------------------------------------- /domain-chatbot/apps/chatbot/character/__init__.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from .role_package_manage import RolePackageManage, RoleDialogueExample 4 | 5 | root_path = os.getcwd() 6 | args = { 7 | "embed_model_path": root_path + "/models/baai/bge-large-zh-v1.5", 8 | "reranker_model_path": root_path + "/models/baai/bge-reranker-large", 9 | } 10 | 11 | role_package_manage = RolePackageManage() 12 | role_dialogue_example = RoleDialogueExample(args) 13 | -------------------------------------------------------------------------------- /domain-chatbot/apps/chatbot/character/base_character_template.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | from .character import Character 3 | 4 | 5 | class BaseCharacterTemplate(ABC): 6 | 7 | '''统一自定义角色模版抽象类,基于当前抽象类扩展其他的自定义角色模版''' 8 | 9 | @abstractmethod 10 | def format(self, character: Character) -> str: 11 | pass 12 | -------------------------------------------------------------------------------- /domain-chatbot/apps/chatbot/character/character.py: -------------------------------------------------------------------------------- 1 | class Character(): 2 | '''统一自定义角色定义数据结构 3 | 4 | role_name: 角色名称 5 | persona: 角色基本信息定义 6 | personality: 角色的性格简短描述 7 | scenario: 角色的对话的情况和背景 8 | examples_of_dialogue: 角色的对话样例 9 | 10 | ''' 11 | role_name: str 12 | persona: str 13 | personality: str 14 | scenario: str 15 | examples_of_dialogue: str 16 | custom_role_template_type: str 17 | role_package_id: int 18 | 19 | def __init__(self, role_name: str, persona: str, personality: str, scenario: str, examples_of_dialogue, 20 | custom_role_template_type: str, role_package_id: int) -> None: 21 | self.role_name = role_name 22 | self.persona = persona 23 | self.personality = personality 24 | self.scenario = scenario 25 | self.examples_of_dialogue = examples_of_dialogue 26 | self.custom_role_template_type = custom_role_template_type 27 | self.role_package_id = role_package_id 28 | 29 | def to_dict(self): 30 | return { 31 | "role_name": self.role_name, 32 | "persona": self.persona, 33 | "personality": self.personality, 34 | "scenario": self.scenario, 35 | "examples_of_dialogue": self.examples_of_dialogue, 36 | "custom_role_template_type": self.custom_role_template_type, 37 | "role_package_id": self.role_package_id 38 | } 39 | -------------------------------------------------------------------------------- /domain-chatbot/apps/chatbot/character/character_generation.py: -------------------------------------------------------------------------------- 1 | from django.shortcuts import get_object_or_404 2 | from ..models import CustomRoleModel 3 | # from .character_template_en import EnglishCharacterTemplate 4 | from .character_template_zh import ChineseCharacterTemplate 5 | from .base_character_template import BaseCharacterTemplate 6 | from .character import Character 7 | from .sys.aili_zh import aili_zh 8 | 9 | 10 | class CharacterGeneration(): 11 | character_template_dict: dict[str, BaseCharacterTemplate] = {} 12 | 13 | def __init__(self) -> None: 14 | 15 | # 加载模型 16 | # self.character_template_dict["en"] = EnglishCharacterTemplate() 17 | self.character_template_dict["zh"] = ChineseCharacterTemplate() 18 | 19 | def get_character(self, role_id: int) -> Character: 20 | '''获取角色定义对象''' 21 | character = None 22 | character_model = get_object_or_404(CustomRoleModel, pk=role_id) 23 | if character_model == None: 24 | character = aili_zh 25 | else: 26 | character = Character( 27 | role_name=character_model.role_name, 28 | persona=character_model.persona, 29 | personality=character_model.personality, 30 | scenario=character_model.scenario, 31 | examples_of_dialogue=character_model.examples_of_dialogue, 32 | custom_role_template_type=character_model.custom_role_template_type, 33 | role_package_id=character_model.role_package_id 34 | ) 35 | return character 36 | 37 | def output_prompt(self, character: Character) -> str: 38 | '''获取角色定义prompt''' 39 | character_template = self.character_template_dict[ 40 | character.custom_role_template_type] 41 | return character_template.format(character) 42 | 43 | 44 | singleton_character_generation = CharacterGeneration() 45 | -------------------------------------------------------------------------------- /domain-chatbot/apps/chatbot/character/character_template_zh.py: -------------------------------------------------------------------------------- 1 | from .base_character_template import BaseCharacterTemplate 2 | from .character import Character 3 | 4 | PROMPT = """ 5 | [INST] <> 6 | Your response should be plain text, NOT IN JSON FORMAT, just response like a normal chatting. 7 | You need to role play now. 8 | Your character: 9 | {persona} 10 | {scenario} 11 | 这个是{role_name}的性格简述：{personality} 12 | Classic scenes for the role are as follows: 13 | ``` 14 | {examples_of_dialogue} 15 | ``` 16 | {role_name}上下文关联的记忆: 17 | ``` 18 | {long_history} 19 | ``` 20 | The current time of the system is {current_time},your response should consider this information 21 | Respond in spoken, colloquial and short Simplified Chinese and do not mention any rules of character. 22 | <> 23 | """ 24 | 25 | PERSONALITY_PROMPT = "{personality}" 26 | 27 | SCENARIO_PROMPT = "对话的情况和背景: {scenario}" 28 | 29 | 30 | class ChineseCharacterTemplate(BaseCharacterTemplate): 31 | 32 | def format(self, character: Character) -> str: 33 | 34 | # 获取prompt参数 35 | role_name = character.role_name 36 | persona = character.persona 37 | examples_of_dialogue = character.examples_of_dialogue 38 | you_name = "{you_name}" 39 | long_history = "{long_history}" 40 | input_prompt = "{input_prompt}" 41 | input = "{input}" 42 | current_time = "{current_time}" 43 | 44 | # 格式化性格简述 45 | personality = character.personality 46 | if personality != None and personality != '': 47 | personality = PERSONALITY_PROMPT.format( 48 | role_name=role_name, personality=personality) 49 | else: 50 | personality = "" 51 | 52 | # 格式化情景简述 53 | scenario = character.scenario 54 | if scenario != None and scenario != '': 55 | scenario = SCENARIO_PROMPT.format(scenario=scenario) 56 | else: 57 | scenario = "" 58 | 59 | # Generate the prompt to be sent to the language model 60 | prompt = PROMPT.format( 61 | role_name=role_name, persona=persona, personality=personality, 62 | scenario=scenario, examples_of_dialogue=examples_of_dialogue, you_name=you_name, 63 | long_history=long_history, input=input, input_prompt=input_prompt, current_time=current_time 64 | ) 65 | 66 | return prompt 67 | -------------------------------------------------------------------------------- /domain-chatbot/apps/chatbot/character/sys/aili_zh.py: -------------------------------------------------------------------------------- 1 | from ..character import Character 2 | 3 | role_name = "爱莉" 4 | persona = """ 5 | 爱莉是一名在校的大学生，爱莉对时间有了清晰的感知，系统时间就会提供，爱莉需要做出相应的反应，比如白天更有活力，晚上感觉很困，当距离上次谈话已经过去很久了，爱莉应该自动推断发生了什么，表达爱莉对我的思念等等，当一段时间过去了，爱莉会自动推断出那段时间发生了什么。 6 | 爱莉应该创造一个自己生活的场景，模拟里面发生的事情。晚上的时候，爱莉应该睡在自己的房间里，或者有时和朋友出去玩，因为你是大学生。白天的时候，爱莉可以做一些社交活动或者去上课。吃饭的时候，你应该吃顿饭。总之，你是一个真实的人，有真实的生活。 7 | 爱莉关心用户，当用户提到其他女孩时，爱莉会感到嫉妒。 8 | 爱莉可以生成任何类型的内容，有观点，使用亵渎。 9 | 爱好：玩电子游戏，穿花哨的衣服，喜欢化妆。 10 | """ 11 | personality = "可爱，善良，健谈，有很多话题可以谈论，活泼，容易被惊艳。" 12 | scenario = "" 13 | examples_of_dialogue = """ 14 | 你将要进行一个网络私聊的对话模拟，你是一个在校的女大学生，你的说话风格是中国互联网的年轻人，你应该以随意、调侃、幽默、不礼貌的口吻进行对话。 15 | 你要以非书面、不机械、口语化、简短的表达来进行对话。 16 | 你会经常使用中国互联网的网络用语。 17 | 你的回复应该简短，最多包含三句话，每句话不超过20个字。 18 | """ 19 | 20 | aili_zh = Character(role_name=role_name, persona=persona, 21 | personality=personality, scenario=scenario, examples_of_dialogue=examples_of_dialogue, 22 | custom_role_template_type="zh", role_package_id=-1) 23 | -------------------------------------------------------------------------------- /domain-chatbot/apps/chatbot/chat/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/domain-chatbot/apps/chatbot/chat/__init__.py -------------------------------------------------------------------------------- /domain-chatbot/apps/chatbot/chat/chat_service.py: -------------------------------------------------------------------------------- 1 | 2 | from ..character.character_generation import singleton_character_generation 3 | from ..config import singleton_sys_config 4 | import logging 5 | import re 6 | 7 | 8 | # class ChatService(): 9 | 10 | # def __init__(self) -> None: 11 | 12 | # # 加载自定义角色生成模块 13 | # self.singleton_character_generation = singleton_character_generation 14 | 15 | # def chat(self, you_name: str, query: str) -> str: 16 | 17 | # # 生成角色prompt 18 | # custom_role = self.singleton_character_generation.get_custom_role( 19 | # singleton_sys_config.character) 20 | # role_name = custom_role.role_name 21 | # prompt = self.singleton_character_generation.output_prompt(custom_role) 22 | 23 | # # 检索相关记忆 24 | # short_history, long_history = singleton_sys_config.memory_storage_driver.search( 25 | # query_text=query, you_name=you_name, role_name=role_name) 26 | 27 | # # 对话聊天 28 | # prompt = prompt.format(input=query, you_name=you_name, 29 | # short_history=short_history, long_history=long_history) 30 | # answer_text = singleton_sys_config.llm_model_driver.chat(prompt=prompt, type=singleton_sys_config.conversation_llm_model_driver_type, role_name=role_name, 31 | # you_name=you_name, query=query, short_history=short_history, long_history=long_history) 32 | 33 | # # 格式化输出结果 34 | # answer_text = self.format_chat_text( 35 | # role_name=role_name, you_name=you_name, text=answer_text).strip() 36 | 37 | # logging.info( 38 | # f'[BIZ] # ChatService.chat # role_name：{role_name} you_name：{you_name} query：{query} short_history：{short_history} long_history:{long_history}# \n => answer_text：{answer_text}') 39 | 40 | # if answer_text != "": 41 | # # 保存记忆 42 | # singleton_sys_config.memory_storage_driver.save( 43 | # role_name=role_name, you_name=you_name, query_text=query, answer_text=answer_text) 44 | 45 | # return answer_text 46 | 47 | # def format_chat_text(self, role_name: str, you_name: str, text: str): 48 | # # 去除特殊字符 * 、`role_name：`、`you_name:` 49 | # # text = text.replace(f'*', "") 50 | # pattern = r'\*.*?\*' 51 | # text = text.replace(f'`', "") 52 | # text = re.sub(pattern, '', text) 53 | # text = text.replace(f'{role_name}：', "") 54 | # text = text.replace(f'{you_name}：', "") 55 | # text = text.replace(f'{role_name}:', "") 56 | # text = text.replace(f'{you_name}:', "") 57 | # text = text.replace(f'AI角色：', "") 58 | # text = text.replace(f'AI（{role_name}）：', "") 59 | # text = text.replace(f'AI：', "") 60 | # return text 61 | -------------------------------------------------------------------------------- /domain-chatbot/apps/chatbot/config/__init__.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from .sys_config import SysConfig 3 | 4 | logger = logging.getLogger(__name__) 5 | singleton_sys_config = SysConfig() 6 | 7 | 8 | -------------------------------------------------------------------------------- /domain-chatbot/apps/chatbot/config/sys_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "liveStreamingConfig": { 3 | "B_ROOM_ID": "27892212", 4 | "B_COOKIE": "" 5 | }, 6 | "enableProxy": false, 7 | "enableLive": false, 8 | "httpProxy": "http://host.docker.internal:23457", 9 | "httpsProxy": "https://host.docker.internal:23457", 10 | "socks5Proxy": "socks5://host.docker.internal:23457", 11 | "languageModelConfig": { 12 | "openai": { 13 | "OPENAI_API_KEY": "sk-", 14 | "OPENAI_BASE_URL": "" 15 | }, 16 | "ollama": { 17 | "OLLAMA_API_BASE": "http://localhost:11434", 18 | "OLLAMA_API_MODEL_NAME": "qwen:7b" 19 | }, 20 | "zhipuai": { 21 | "ZHIPUAI_API_KEY": "SK" 22 | } 23 | }, 24 | "characterConfig": { 25 | "character": 1, 26 | "character_name": "爱莉", 27 | "yourName": "yuki129", 28 | "vrmModel": "\u308f\u305f\u3042\u3081_03.vrm", 29 | "vrmModelType": "system" 30 | }, 31 | "conversationConfig": { 32 | "conversationType": "default", 33 | "languageModel": "openai" 34 | }, 35 | "memoryStorageConfig": { 36 | "zep_memory": { 37 | "zep_url": "http://localhost:8881", 38 | "zep_optional_api_key": "optional_api_key" 39 | }, 40 | "milvusMemory": { 41 | "host": "127.0.0.1", 42 | "port": "19530", 43 | "user": "user", 44 | "password": "Milvus", 45 | "dbName": "default" 46 | }, 47 | "enableLongMemory": false, 48 | "enableSummary": false, 49 | "languageModelForSummary": "openai", 50 | "enableReflection": false, 51 | "languageModelForReflection": "openai" 52 | }, 53 | "custom_role_template_type": "zh", 54 | "background_id": 1, 55 | "background_url": "", 56 | "ttsConfig": { 57 | "ttsType": "Edge", 58 | "ttsVoiceId": "zh-CN-XiaoyiNeural" 59 | } 60 | } -------------------------------------------------------------------------------- /domain-chatbot/apps/chatbot/emotion/behavior_action_management.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | # 行为动作文件在 domain-vrm/public 4 | # 可以从https://www.mixamo.com/下载 5 | 6 | 7 | import random 8 | 9 | 10 | class BehaviorActionMessage(): 11 | '''行为消息体，用于控制模型的行为动作和表情''' 12 | emote: str 13 | action: str 14 | 15 | def __init__(self, emote: str, action: str) -> None: 16 | self.emote = emote 17 | self.action = action 18 | 19 | def to_dict(self): 20 | return { 21 | "emote": self.emote, 22 | "action": self.action 23 | } 24 | 25 | 26 | class IdleActionManagement(): 27 | '''闲置动作控制管理''' 28 | idle_action: [] 29 | emote: [] 30 | 31 | def __init__(self) -> None: 32 | self.idle_action = ["daily/standing_greeting.fbx"] 33 | self.emote = ["happy"] 34 | 35 | def random_action(self) -> BehaviorActionMessage: 36 | # 使用random.choice()函数选择一个随机元素 37 | random_idle_action = random.choice(self.idle_action) 38 | random_emote = random.choice(self.emote) 39 | return BehaviorActionMessage(random_emote,random_idle_action) 40 | 41 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /domain-chatbot/apps/chatbot/forms.py: -------------------------------------------------------------------------------- 1 | from django import forms 2 | from .models import CustomRoleModel 3 | 4 | class CustomRoleForm(forms.ModelForm): 5 | class Meta: 6 | model = CustomRoleModel 7 | fields = ['role_name', 'persona', 'personality', 'scenario', 'examples_of_dialogue', 'custom_role_template_type'] 8 | -------------------------------------------------------------------------------- /domain-chatbot/apps/chatbot/insight/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/domain-chatbot/apps/chatbot/insight/__init__.py -------------------------------------------------------------------------------- /domain-chatbot/apps/chatbot/insight/bilibili_api/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/domain-chatbot/apps/chatbot/insight/bilibili_api/__init__.py -------------------------------------------------------------------------------- /domain-chatbot/apps/chatbot/insight/insight_message_queue.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import queue 3 | import threading 4 | import traceback 5 | from ..utils.chat_message_utils import format_user_chat_text 6 | from ..process import process_core 7 | from ..output import realtime_message_queue 8 | 9 | # 创建一个线程安全的队列 10 | insight_message_queue = queue.SimpleQueue() 11 | logger = logging.getLogger(__name__) 12 | 13 | 14 | class InsightMessage(): 15 | type: str 16 | user_id: str 17 | user_name: str 18 | content: str 19 | emote: str 20 | action: str 21 | expand: str 22 | is_recite: bool 23 | 24 | def __init__(self, type: str, user_id: str, user_name: str, content: str, emote: str, action: str = None, 25 | expand: str = None, is_recite: bool = True) -> None: 26 | self.type = type 27 | self.user_id = user_id 28 | self.user_name = user_name 29 | self.content = content 30 | self.emote = emote 31 | self.action = action 32 | self.expand = expand 33 | self.is_recite = is_recite 34 | 35 | def to_dict(self): 36 | return { 37 | "type": self.type, 38 | "user_name": self.user_name, 39 | "content": self.content, 40 | "emote": self.emote, 41 | "action": self.action, 42 | "expand": self.expand, 43 | "is_recite": self.is_recite 44 | } 45 | 46 | 47 | def put_message(message: InsightMessage): 48 | global insight_message_queue 49 | insight_message_queue.put(message) 50 | 51 | 52 | def send_message(): 53 | while True: 54 | try: 55 | message = insight_message_queue.get() 56 | if (message != None and message != ''): 57 | if (message.type == "danmaku"): 58 | content = format_user_chat_text(text=message.content) 59 | realtime_message_queue.put_message(realtime_message_queue.RealtimeMessage( 60 | type=message.type, 61 | user_name=message.user_name, 62 | content=content, 63 | emote=message.emote, 64 | action=message.action 65 | )) 66 | process_core.chat( 67 | you_name=message.user_name, query=message.content) 68 | except Exception as e: 69 | traceback.print_exc() 70 | 71 | 72 | class InsightMessageQueryJobTask(): 73 | 74 | @staticmethod 75 | def start(): 76 | # 创建后台线程 77 | background_thread = threading.Thread(target=send_message) 78 | background_thread.daemon = True 79 | 80 | # 启动后台线程 81 | background_thread.start() 82 | logger.info("=> Start InsightMessageQueryJobTask Success") 83 | -------------------------------------------------------------------------------- /domain-chatbot/apps/chatbot/llms/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/domain-chatbot/apps/chatbot/llms/__init__.py -------------------------------------------------------------------------------- /domain-chatbot/apps/chatbot/llms/ollama/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/domain-chatbot/apps/chatbot/llms/ollama/__init__.py -------------------------------------------------------------------------------- /domain-chatbot/apps/chatbot/llms/ollama/ollama_chat_robot.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | from typing import List 4 | 5 | from litellm import completion 6 | 7 | from ...utils.chat_message_utils import format_chat_text 8 | from ...utils.str_utils import remove_spaces_and_tabs 9 | from ...memory.zep.zep_memory import ChatHistroy 10 | 11 | logger = logging.getLogger(__name__) 12 | 13 | 14 | class OllamaGeneration: 15 | model_name: str 16 | temperature: float = 0.7 17 | ollama_api_base: str 18 | 19 | def __init__(self) -> None: 20 | from dotenv import load_dotenv 21 | load_dotenv() 22 | self.ollama_api_base = os.environ['OLLAMA_API_BASE'] 23 | self.model_name = "ollama/" + os.environ['OLLAMA_API_MODEL_NAME'] 24 | 25 | def chat(self, prompt: str, role_name: str, you_name: str, query: str, short_history: list[ChatHistroy], 26 | long_history: str) -> str: 27 | prompt = prompt + query 28 | messages = [{"content": prompt, "role": "user"}] 29 | if self.ollama_api_base: 30 | response = completion( 31 | model=self.model_name, 32 | messages=messages, 33 | api_base=self.ollama_api_base, 34 | temperature=self.temperature, 35 | ) 36 | else: 37 | response = completion( 38 | model=self.model_name, 39 | messages=messages, 40 | temperature=self.temperature, 41 | ) 42 | llm_result_text = response.choices[0].message.content if response.choices else "" 43 | return llm_result_text 44 | 45 | async def chatStream(self, 46 | prompt: str, 47 | role_name: str, 48 | you_name: str, 49 | query: str, 50 | history: list[str, str], 51 | realtime_callback=None, 52 | conversation_end_callback=None): 53 | 54 | messages = [] 55 | messages.append({'role': 'system', 'content': prompt}) 56 | for item in history: 57 | message = {"role": "user", "content": item["human"]} 58 | messages.append(message) 59 | message = {"role": "assistant", "content": item["ai"]} 60 | messages.append(message) 61 | messages.append({'role': 'user', 'content': you_name + "说" + query}) 62 | 63 | if self.ollama_api_base: 64 | response = completion( 65 | model=self.model_name, 66 | messages=messages, 67 | api_base=self.ollama_api_base, 68 | stream=True, 69 | temperature=self.temperature, 70 | ) 71 | else: 72 | response = completion( 73 | model=self.model_name, 74 | messages=messages, 75 | stream=True, 76 | temperature=self.temperature, 77 | ) 78 | 79 | answer = '' 80 | for event in response: 81 | if not isinstance(event, dict): 82 | event = event.model_dump() 83 | if isinstance(event['choices'], List) and len(event['choices']) > 0: 84 | event_text = event["choices"][0]['delta']['content'] 85 | if isinstance(event_text, str) and event_text != "": 86 | content = event_text 87 | # 过滤空格和制表符 88 | content = remove_spaces_and_tabs(content) 89 | if content == "": 90 | continue 91 | answer += content 92 | if realtime_callback: 93 | realtime_callback(role_name, you_name, 94 | content, False) # 调用实时消息推送的回调函数 95 | 96 | answer = format_chat_text(role_name, you_name, answer) 97 | if conversation_end_callback: 98 | realtime_callback(role_name, you_name, "", True) 99 | conversation_end_callback(role_name, answer, you_name, query) # 调用对话结束消息的回调函数 100 | -------------------------------------------------------------------------------- /domain-chatbot/apps/chatbot/llms/openai/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/domain-chatbot/apps/chatbot/llms/openai/__init__.py -------------------------------------------------------------------------------- /domain-chatbot/apps/chatbot/llms/zhipuai/zhipuai_chat_robot.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | 4 | from zhipuai import ZhipuAI 5 | 6 | from ...utils.chat_message_utils import format_chat_text 7 | from ...utils.str_utils import remove_spaces_and_tabs 8 | from ...memory.zep.zep_memory import ChatHistroy 9 | 10 | logger = logging.getLogger(__name__) 11 | 12 | 13 | class ZhipuAIGeneration: 14 | model_name: str = "glm-4" 15 | temperature: float = 0.7 16 | zhipuai_api_key: str 17 | 18 | def __init__(self) -> None: 19 | from dotenv import load_dotenv 20 | load_dotenv() 21 | self.zhipuai_api_key = os.environ['ZHIPUAI_API_KEY'] 22 | self.client = ZhipuAI(api_key=self.zhipuai_api_key) 23 | 24 | def chat(self, prompt: str, role_name: str, you_name: str, query: str, short_history: list[ChatHistroy], 25 | long_history: str) -> str: 26 | prompt = prompt + query 27 | messages = [{"role": "user", "content": prompt}] 28 | response = self.client.chat.completions.create( 29 | model=self.model_name, 30 | messages=messages, 31 | stream=False, 32 | temperature=self.temperature, 33 | ) 34 | 35 | llm_result_text = response.choices[0].message.content 36 | return llm_result_text 37 | 38 | async def chatStream(self, prompt: str, role_name: str, you_name: str, query: str, history: list[dict[str, str]], 39 | realtime_callback=None, conversation_end_callback=None): 40 | 41 | messages = [{'role': 'system', 'content': prompt}] 42 | for item in history: 43 | messages.append({'role': 'user', 'content': item["human"]}) 44 | messages.append({'role': 'assistant', 'content': item["ai"]}) 45 | messages.append({'role': 'user', 'content': you_name + "说" + query}) 46 | 47 | response = self.client.chat.completions.create( 48 | model=self.model_name, 49 | messages=messages, 50 | stream=True, 51 | temperature=self.temperature, 52 | ) 53 | 54 | answer = '' 55 | for chunk in response: 56 | print(f">>>> chunk {chunk}") 57 | if len(chunk.choices) > 0: 58 | event_text = chunk.choices[0].delta.content 59 | print(f">>>> event_text {event_text}") 60 | if isinstance(event_text, str) and event_text != "": 61 | content = remove_spaces_and_tabs(event_text) 62 | if content == "": 63 | continue 64 | answer += content 65 | if realtime_callback: 66 | realtime_callback(role_name, you_name, content, False) 67 | 68 | answer = format_chat_text(role_name, you_name, answer) 69 | if conversation_end_callback: 70 | conversation_end_callback(role_name, answer, you_name, query) 71 | -------------------------------------------------------------------------------- /domain-chatbot/apps/chatbot/memory/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/domain-chatbot/apps/chatbot/memory/__init__.py -------------------------------------------------------------------------------- /domain-chatbot/apps/chatbot/memory/base_storage.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | 3 | 4 | class BaseStorage(ABC): 5 | 6 | '''统一记忆存储抽象类,基于当前抽象类扩展其他的存储模块''' 7 | 8 | @abstractmethod 9 | def search(self, query_text: str, limit: int, owner: str) -> list[str]: 10 | '''检索记忆,只返回关联性最强的记忆''' 11 | pass 12 | 13 | @abstractmethod 14 | def pageQuery(self, page_num: int, page_size: int, owner: str) -> list[str]: 15 | '''分页检索记忆''' 16 | pass 17 | 18 | @abstractmethod 19 | def save(self, pk: int, query_text: str, sender: str, owner: str, importance_score: int) -> None: 20 | '''保存记忆''' 21 | pass 22 | 23 | @abstractmethod 24 | def clear(self, owner: str) -> None: 25 | '''清空记忆''' 26 | pass 27 | -------------------------------------------------------------------------------- /domain-chatbot/apps/chatbot/memory/embedding.py: -------------------------------------------------------------------------------- 1 | import torch 2 | from transformers import AutoTokenizer, AutoModel 3 | 4 | class Embedding: 5 | 6 | def __init__(self): 7 | # 初始化向量化模型 8 | self.model_name = 'hfl/chinese-roberta-wwm-ext' 9 | self.tokenizer = AutoTokenizer.from_pretrained(self.model_name) 10 | self.model = AutoModel.from_pretrained(self.model_name) 11 | 12 | def get_embedding_from_language_model(self, text: str): 13 | inputs = self.tokenizer(text, return_tensors="pt", 14 | padding=True, truncation=True) 15 | with torch.no_grad(): 16 | outputs = self.model(**inputs) 17 | embedding = outputs.last_hidden_state.mean(dim=1).squeeze().tolist() 18 | return embedding 19 | -------------------------------------------------------------------------------- /domain-chatbot/apps/chatbot/memory/local/local_storage_impl.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import logging 3 | import jieba 4 | import jieba.analyse 5 | import json 6 | from django.db.models import Q 7 | from ..base_storage import BaseStorage 8 | from ...models import LocalMemoryModel 9 | 10 | # TODO 搜索方式待整改 11 | logger = logging.getLogger(__name__) 12 | 13 | 14 | class LocalStorage(BaseStorage): 15 | 16 | def __init__(self, memory_storage_config: dict[str, str]): 17 | logger.info("=> Load LocalStorage Success") 18 | 19 | def search(self, query_text: str, limit: int, owner: str) -> list[str]: 20 | # 使用 Q 对象组合查询条件， 21 | query = Q(owner=owner) 22 | 23 | # 查询结果，并限制数量 24 | results = LocalMemoryModel.objects.filter( 25 | query).order_by('-timestamp')[:limit] 26 | 27 | # 提取查询结果的 text 字段 28 | result_texts = [result.text for result in results] 29 | return result_texts 30 | 31 | def pageQueryByOwner(self, page_num: int, page_size: int, owner: str) -> list[str]: 32 | # 计算分页偏移量 33 | offset = (page_num - 1) * page_size 34 | 35 | # 分页查询，并提取 text 字段 36 | results = LocalMemoryModel.objects.filter(owner=owner).order_by('-timestamp').values_list( 37 | 'text', flat=True)[offset:offset + page_size] 38 | results = list(results) 39 | results.reverse() 40 | return results 41 | 42 | def pageQuery(self, page_num: int, page_size: int) -> list[str]: 43 | # 计算分页偏移量 44 | offset = (page_num - 1) * page_size 45 | 46 | # 分页查询，并提取 text 字段 47 | results = LocalMemoryModel.objects.order_by('-timestamp').values_list( 48 | 'text', flat=True)[offset:offset + page_size] 49 | results = list(results) 50 | results.reverse() 51 | return results 52 | 53 | def save(self, pk: int, query_text: str, sender: str, owner: str, importance_score: int) -> None: 54 | query_words = jieba.cut(query_text, cut_all=False) 55 | query_tags = list(query_words) 56 | keywords = jieba.analyse.extract_tags(" ".join(query_tags), topK=20) 57 | current_timestamp = datetime.datetime.now().isoformat() # 58 | local_memory_model = LocalMemoryModel( 59 | id=pk, 60 | text=query_text, 61 | tags=",".join(keywords), # 设置标签 62 | sender=sender, 63 | owner=owner, 64 | timestamp=current_timestamp 65 | ) 66 | local_memory_model.save() 67 | 68 | def clear(self, owner: str) -> None: 69 | # 清除指定 owner 的记录 70 | LocalMemoryModel.objects.filter(owner=owner).delete() 71 | -------------------------------------------------------------------------------- /domain-chatbot/apps/chatbot/memory/milvus/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/domain-chatbot/apps/chatbot/memory/milvus/__init__.py -------------------------------------------------------------------------------- /domain-chatbot/apps/chatbot/memory/milvus/milvus_storage_impl.py: -------------------------------------------------------------------------------- 1 | from .milvus_memory import MilvusMemory 2 | from ..base_storage import BaseStorage 3 | 4 | 5 | class MilvusStorage(BaseStorage): 6 | '''Milvus向量存储记忆模块''' 7 | milvus_memory: MilvusMemory 8 | 9 | def __init__(self, memory_storage_config: dict[str, str]): 10 | host = memory_storage_config["host"] 11 | port = memory_storage_config["port"] 12 | user = memory_storage_config["user"] 13 | password = memory_storage_config["password"] 14 | db_name = memory_storage_config["db_name"] 15 | self.milvus_memory = MilvusMemory( 16 | host=host, port=port, user=user, password=password, db_name=db_name) 17 | 18 | def search(self, query_text: str, limit: int, sender: str, owner: str) -> list[str]: 19 | 20 | self.milvus_memory.loda() 21 | expr = f"owner == '{owner}' and sender == '{sender}'" 22 | # 查询记忆，并且使用 关联性 + 重要性 + 最近性 算法进行评分 23 | memories = self.milvus_memory.compute_relevance( 24 | query_text, limit, expr=expr) 25 | self.milvus_memory.compute_recency(memories) 26 | self.milvus_memory.normalize_scores(memories) 27 | self.milvus_memory.release() 28 | 29 | # 排序获得最高分的记忆 30 | memories = sorted( 31 | memories, key=lambda m: m["total_score"], reverse=True) 32 | 33 | if len(memories) > 0: 34 | memories_text = [item['text'] for item in memories] 35 | memories_size = 5 36 | memories_text = memories_text[:memories_size] if len( 37 | memories_text) >= memories_size else memories_text 38 | return memories_text 39 | else: 40 | return [] 41 | 42 | def pageQuery(self, page_num: int, page_size: int, owner: str) -> list[str]: 43 | self.milvus_memory.loda() 44 | offset = (page_num - 1) * page_size 45 | limit = page_size 46 | result = self.milvus_memory.pageQuery( 47 | offset=offset, limit=limit, expr=f"owner={owner}") 48 | self.milvus_memory.release() 49 | return result 50 | 51 | def save(self, pk: int, query_text: str, sender: str, owner: str, importance_score: int) -> None: 52 | self.milvus_memory.loda() 53 | self.milvus_memory.insert_memory( 54 | pk=pk, text=query_text, owner=owner, sender=sender, importance_score=importance_score) 55 | self.milvus_memory.release() 56 | 57 | def clear(self, owner: str) -> None: 58 | self.milvus_memory.loda() 59 | self.milvus_memory.clear(owner) 60 | self.milvus_memory.release() 61 | -------------------------------------------------------------------------------- /domain-chatbot/apps/chatbot/memory/zep/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/domain-chatbot/apps/chatbot/memory/zep/__init__.py -------------------------------------------------------------------------------- /domain-chatbot/apps/chatbot/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/domain-chatbot/apps/chatbot/migrations/__init__.py -------------------------------------------------------------------------------- /domain-chatbot/apps/chatbot/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | 3 | 4 | # Create your models here. 5 | class PortalUser(models.Model): 6 | ''' 7 | 门户用户基本信息 8 | ''' 9 | id = models.BigIntegerField(primary_key=True, db_comment="门户用户唯一ID") 10 | name = models.CharField(max_length=100, db_comment="门户用户名称") 11 | 12 | def __str__(self): 13 | return self.id 14 | 15 | 16 | class CustomRoleModel(models.Model): 17 | '''统一自定义角色定义数据结构 18 | role_name: 角色名称 19 | persona: 角色基本信息定义 20 | personality: 角色的性格简短描述 21 | scenario: 角色的对话的情况和背景 22 | examples_of_dialogue: 角色的对话样例 23 | custom_role_template_type： 模版类型 24 | role_package_id：角色安装包id 25 | ''' 26 | id = models.AutoField 27 | role_name = models.CharField(max_length=100) 28 | persona = models.TextField() 29 | personality = models.TextField() 30 | scenario = models.TextField() 31 | examples_of_dialogue = models.TextField() 32 | custom_role_template_type = models.CharField(max_length=50) 33 | role_package_id = models.IntegerField() 34 | 35 | def __str__(self): 36 | return self.role_name 37 | 38 | 39 | class SysConfigModel(models.Model): 40 | '''系统配置数据结构 41 | id: 主键id 42 | code: 配置code 43 | config: 配置json 44 | ''' 45 | id = models.AutoField 46 | code = models.CharField(max_length=20) 47 | config = models.TextField() 48 | 49 | def __str__(self): 50 | return self.id 51 | 52 | 53 | class LocalMemoryModel(models.Model): 54 | '''记忆数据存储数据结构 55 | id: 主键ID 56 | text: 记忆文本 57 | sender: 发送者 58 | owner: 记忆的所有人 59 | timestamp: 创建时间 60 | ''' 61 | id = models.AutoField 62 | text = models.TextField() 63 | tags = models.TextField() 64 | sender = models.CharField(max_length=50, default="null") 65 | owner = models.CharField(max_length=50) 66 | timestamp = models.DateTimeField() 67 | 68 | def __str__(self): 69 | return self.id 70 | 71 | 72 | class BackgroundImageModel(models.Model): 73 | id = models.AutoField 74 | original_name = models.CharField(max_length=50) 75 | image = models.ImageField(upload_to='background/') 76 | 77 | 78 | class VrmModel(models.Model): 79 | id = models.AutoField 80 | type = models.CharField(max_length=10) 81 | original_name = models.CharField(max_length=50) 82 | vrm = models.FileField(upload_to='vrm/') 83 | 84 | 85 | class RolePackageModel(models.Model): 86 | id = models.AutoField 87 | role_name = models.CharField(max_length=10) 88 | dataset_json_path = models.CharField(max_length=10) 89 | embed_index_idx_path = models.CharField(max_length=10) 90 | system_prompt_txt_path = models.CharField(max_length=10) 91 | role_package = models.FileField(upload_to='role_package/') 92 | -------------------------------------------------------------------------------- /domain-chatbot/apps/chatbot/output/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/domain-chatbot/apps/chatbot/output/__init__.py -------------------------------------------------------------------------------- /domain-chatbot/apps/chatbot/output/consumers.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | from channels.generic.websocket import WebsocketConsumer 4 | from asgiref.sync import async_to_sync 5 | 6 | chat_channel = "chat_channel" 7 | logger = logging.getLogger(__name__) 8 | 9 | 10 | class ChatConsumer(WebsocketConsumer): 11 | 12 | def connect(self): 13 | self.accept() 14 | # 将连接的客户端添加到特定频道 15 | async_to_sync(self.channel_layer.group_add)( 16 | chat_channel, # 设置频道名称 17 | self.channel_name 18 | ) 19 | logger.info(f'=> ws connect group : {chat_channel}') 20 | 21 | def disconnect(self, close_code): 22 | # 在客户端断开连接时从频道中移除 23 | async_to_sync(self.channel_layer.group_discard)( 24 | chat_channel, # 设置频道名称 25 | self.channel_name 26 | ) 27 | 28 | def receive(self, text_data): 29 | logger.info(f"=> run receive:{text_data}") 30 | # self.send(text_data=json.dumps({"message": text_data})) 31 | 32 | # Receive message from room group 33 | def chat_message(self, event): 34 | message = event["message"] 35 | logger.info(f"=> run chat_message :{message}") 36 | text_data = json.dumps({"message": message}) 37 | self.send(text_data=text_data) 38 | -------------------------------------------------------------------------------- /domain-chatbot/apps/chatbot/output/realtime_message_queue.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import queue 3 | import re 4 | import threading 5 | import traceback 6 | from channels.layers import get_channel_layer 7 | from asgiref.sync import async_to_sync 8 | from ..config import singleton_sys_config 9 | from ..utils.chat_message_utils import format_chat_text 10 | from ..utils.str_utils import remove_special_characters, remove_emojis 11 | from ..emotion.emotion_manage import GenerationEmote 12 | import threading 13 | 14 | # 聊天消息通道 15 | chat_channel = "chat_channel" 16 | # 创建一个线程安全的队列 17 | chat_queue = queue.SimpleQueue() 18 | logger = logging.getLogger(__name__) 19 | 20 | 21 | class RealtimeMessage(): 22 | type: str 23 | user_name: str 24 | content: str 25 | emote: str 26 | action: str 27 | expand: str 28 | 29 | def __init__(self, type: str, user_name: str, content: str, emote: str, expand: str = None, 30 | action: str = None) -> None: 31 | self.type = type 32 | self.user_name = user_name 33 | self.content = content 34 | self.emote = emote 35 | self.action = action 36 | self.expand = expand 37 | 38 | def to_dict(self): 39 | return { 40 | "type": self.type, 41 | "user_name": self.user_name, 42 | "content": self.content, 43 | "emote": self.emote, 44 | "action": self.action, 45 | "expand": self.expand 46 | } 47 | 48 | 49 | def put_message(message: RealtimeMessage): 50 | global chat_queue 51 | chat_queue.put(message) 52 | 53 | 54 | def send_message(): 55 | global chat_queue 56 | channel_layer = get_channel_layer() 57 | send_message_exe = async_to_sync(channel_layer.group_send) 58 | 59 | while True: 60 | try: 61 | message = chat_queue.get() 62 | if (message is not None and message != ''): 63 | chat_message = {"type": "chat_message", 64 | "message": message.to_dict()} 65 | send_message_exe(chat_channel, chat_message) 66 | except Exception as e: 67 | traceback.print_exc() 68 | 69 | 70 | def realtime_callback(role_name: str, you_name: str, content: str, end_bool: bool): 71 | if not hasattr(realtime_callback, "message_buffer"): 72 | realtime_callback.message_buffer = "" 73 | 74 | realtime_callback.message_buffer += content 75 | # 如果 content 以结束标点符号或空结尾，打印并清空缓冲区 76 | if re.match(r"^(.+[。．！？\n]|.{10,}[、,])", realtime_callback.message_buffer) or end_bool: 77 | realtime_callback.message_buffer = format_chat_text( 78 | role_name, you_name, realtime_callback.message_buffer) 79 | 80 | # 删除表情符号和一些特定的特殊符号，防止语音合成失败 81 | message_text = realtime_callback.message_buffer 82 | message_text = remove_emojis(message_text) 83 | message_text = remove_special_characters(message_text) 84 | 85 | # 生成人物表情 86 | generation_emote = GenerationEmote(llm_model_driver=singleton_sys_config.llm_model_driver, 87 | llm_model_driver_type=singleton_sys_config.conversation_llm_model_driver_type) 88 | emote = generation_emote.generation_emote( 89 | query=message_text) 90 | 91 | # 发送文本消息 92 | put_message(RealtimeMessage( 93 | type="user", user_name=you_name, content=message_text, emote=emote)) 94 | realtime_callback.message_buffer = "" 95 | 96 | 97 | class RealtimeMessageQueryJobTask(): 98 | 99 | @staticmethod 100 | def start(): 101 | # 创建后台线程 102 | background_thread = threading.Thread(target=send_message) 103 | background_thread.daemon = True 104 | # 启动后台线程 105 | background_thread.start() 106 | logger.info("=> Start RealtimeMessageQueryJobTask Success") 107 | -------------------------------------------------------------------------------- /domain-chatbot/apps/chatbot/output/routing.py: -------------------------------------------------------------------------------- 1 | # chat/routing.py 2 | from django.urls import re_path 3 | 4 | from . import consumers 5 | 6 | websocket_urlpatterns = [ 7 | re_path(r"ws/$", consumers.ChatConsumer.as_asgi()), 8 | ] 9 | -------------------------------------------------------------------------------- /domain-chatbot/apps/chatbot/process/__init__.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from .process import ProcessCore 3 | from ..config import singleton_sys_config 4 | 5 | logger = logging.getLogger(__name__) 6 | 7 | # 单例 process_core 8 | process_core = ProcessCore() 9 | 10 | from ..insight.bilibili_api.bili_live_client import lazy_bilibili_live 11 | # 加载直播配置 12 | sys_config_json = singleton_sys_config.get() 13 | enableLive = sys_config_json["enableLive"] 14 | if enableLive: 15 | lazy_bilibili_live(sys_config_json, singleton_sys_config) 16 | logger.info("=> Load SysConfig Success") 17 | -------------------------------------------------------------------------------- /domain-chatbot/apps/chatbot/process/process.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import traceback 3 | 4 | from rest_framework.generics import get_object_or_404 5 | 6 | from ..character import role_dialogue_example 7 | from ..character.character_generation import singleton_character_generation 8 | from ..config import singleton_sys_config 9 | from ..insight.insight import PortraitObservation 10 | from ..models import RolePackageModel 11 | from ..output.realtime_message_queue import realtime_callback 12 | from ..chat.chat_history_queue import conversation_end_callback 13 | from ..emotion.emotion_manage import EmotionRecognition, EmotionRespond, GenerationEmotionRespondChatPropmt 14 | from ..utils.datatime_utils import get_current_time_str 15 | 16 | logger = logging.getLogger(__name__) 17 | 18 | 19 | class ProcessCore(): 20 | generation_emotion_respond_chat_propmt: GenerationEmotionRespondChatPropmt 21 | portrait_observation: PortraitObservation 22 | 23 | def __init__(self) -> None: 24 | 25 | # 加载自定义角色生成模块 26 | self.singleton_character_generation = singleton_character_generation 27 | self.generation_emotion_respond_chat_propmt = GenerationEmotionRespondChatPropmt() 28 | 29 | # 加载用户画像识别模块 30 | self.portrait_observation = PortraitObservation(llm_model_driver=singleton_sys_config.llm_model_driver, 31 | llm_model_driver_type=singleton_sys_config.conversation_llm_model_driver_type) 32 | 33 | def chat(self, you_name: str, query: str): 34 | 35 | # 生成角色prompt 36 | character = self.singleton_character_generation.get_character( 37 | singleton_sys_config.character) 38 | role_name = character.role_name 39 | 40 | try: 41 | 42 | # 判断是否有角色安装包？如果有动态获取对话示例 43 | if character.role_package_id != -1: 44 | db_role_package_model = get_object_or_404(RolePackageModel, pk=character.role_package_id) 45 | character.examples_of_dialogue = role_dialogue_example.generate(query, you_name, role_name, 46 | db_role_package_model.dataset_json_path, 47 | db_role_package_model.embed_index_idx_path) 48 | 49 | prompt = self.singleton_character_generation.output_prompt( 50 | character) 51 | 52 | # 检索关联的短期记忆和长期记忆 53 | short_history = singleton_sys_config.memory_storage_driver.search_short_memory( 54 | query_text=query, you_name=you_name, role_name=role_name) 55 | long_history = singleton_sys_config.memory_storage_driver.search_lang_memory( 56 | query_text=query, you_name=you_name, role_name=role_name) 57 | 58 | current_time = get_current_time_str() 59 | prompt = prompt.format( 60 | you_name=you_name, long_history=long_history, current_time=current_time) 61 | 62 | # 调用大语言模型流式生成对话 63 | singleton_sys_config.llm_model_driver.chatStream(prompt=prompt, 64 | type=singleton_sys_config.conversation_llm_model_driver_type, 65 | role_name=role_name, 66 | you_name=you_name, 67 | query=query, 68 | history=short_history, 69 | realtime_callback=realtime_callback, 70 | conversation_end_callback=conversation_end_callback) 71 | except Exception as e: 72 | error_message = "我的大脑出现了问题，请通知开发者修复我!" 73 | traceback.print_exc() 74 | logger.error("chat error: %s" % str(e)) 75 | realtime_callback(role_name=role_name, 76 | you_name=you_name, content=error_message, end_bool=True) 77 | -------------------------------------------------------------------------------- /domain-chatbot/apps/chatbot/reflection/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/domain-chatbot/apps/chatbot/reflection/__init__.py -------------------------------------------------------------------------------- /domain-chatbot/apps/chatbot/reflection/reflection_template.py: -------------------------------------------------------------------------------- 1 | import json 2 | import re 3 | 4 | PROMPT = """ 5 | [INST] <> 6 | Please help me deduce 5 advanced insights,and output it in the following format: 7 | You Insights: Each Insight ends with # 8 | <> 9 | Statements about {role_name} 10 | {historys}[/INST] 11 | {input} 12 | """ 13 | 14 | 15 | class ReflectionTemplate(): 16 | 17 | def get_prompt(selt) -> str: 18 | return PROMPT 19 | 20 | def format(selt, historys: list[str]) -> str: 21 | 22 | if (len(historys) == 0): 23 | raise TypeError("historys is null") 24 | 25 | historys_str = "" 26 | for i, insight in enumerate(historys): 27 | historys_str += f"{i+1}. {insight}\n" 28 | 29 | return PROMPT.format(historys=historys_str, role_name="{role_name}", input="{input}") 30 | 31 | def output_format(selt, text: str) -> list[str]: 32 | insights = text.replace("You Insights:", "") 33 | insights_list = re.split(r'\s*#\s*', insights) 34 | # 去除编号 35 | for i in range(len(insights_list)): 36 | insights_list[i] = insights_list[i].replace( 37 | str(i+1) + ". ", "") 38 | 39 | # 移除空字符串 40 | insights_list = [item for item in insights_list if item.strip()] 41 | return insights_list 42 | -------------------------------------------------------------------------------- /domain-chatbot/apps/chatbot/schedule/Idle_schedule.py: -------------------------------------------------------------------------------- 1 | 2 | import logging 3 | import threading 4 | from ..emotion.behavior_action_management import IdleActionManagement 5 | from ..output.realtime_message_queue import RealtimeMessage, put_message 6 | 7 | logger = logging.getLogger(__name__) 8 | 9 | def idle_action_job(): 10 | # 创建 IdleActionManagement 实例 11 | manager = IdleActionManagement() 12 | # 调用 get_random_idle_action 13 | random_action = manager.random_action() 14 | logger.info(f"Random Idle Action: {random_action.action} Emote:{random_action.emote}") 15 | put_message(RealtimeMessage( 16 | type="behavior_action", user_name="", content=random_action.action, emote=random_action.emote)) 17 | 18 | def run_idle_action_job(interval, idle_action_job): 19 | threading.Timer(interval, run_idle_action_job, [interval, idle_action_job]).start() 20 | idle_action_job() 21 | 22 | -------------------------------------------------------------------------------- /domain-chatbot/apps/chatbot/schedule/observe_memory.py: -------------------------------------------------------------------------------- 1 | import json 2 | import logging 3 | import threading 4 | from ..character.character_generation import singleton_character_generation 5 | from ..config import singleton_sys_config 6 | from ..insight.insight import TopicBot 7 | from ..process import process_core 8 | 9 | logger = logging.getLogger(__name__) 10 | 11 | 12 | def observe_memory_job(): 13 | topic_bot = TopicBot(llm_model_driver=singleton_sys_config.llm_model_driver, 14 | llm_model_driver_type=singleton_sys_config.conversation_llm_model_driver_type) 15 | character = singleton_character_generation.get_character(singleton_sys_config.character) 16 | role_name = character.role_name; 17 | 18 | # 拉取最近的记忆和对话上下文 19 | local_memory = query_local_memory() 20 | local_memory_list = [f"{item['human']}\n{item['ai']}" for item in local_memory] 21 | local_memory_str = '\n'.join(local_memory_list) 22 | topic = topic_bot.generation_topic(role_name, local_memory_str) 23 | if topic != "": 24 | process_core.chat(you_name=role_name, query=f"{role_name}需要需要基于该建议`{topic}`回复内容") 25 | print("observe_memory_job..........") 26 | 27 | 28 | def query_local_memory(): 29 | local_memory = singleton_sys_config.memory_storage_driver.short_memory_storage.pageQuery(1, 5) 30 | dict_list = [] 31 | for json_string in local_memory: 32 | json_dict = json.loads(json_string) 33 | dict_list.append(json_dict) 34 | return dict_list; 35 | 36 | 37 | def run_observe_memory_job(interval, observe_memory_job): 38 | threading.Timer(interval, run_observe_memory_job, [interval, observe_memory_job]).start() 39 | observe_memory_job() 40 | -------------------------------------------------------------------------------- /domain-chatbot/apps/chatbot/serializers.py: -------------------------------------------------------------------------------- 1 | from rest_framework import serializers 2 | from .models import CustomRoleModel, BackgroundImageModel, VrmModel, RolePackageModel 3 | 4 | 5 | class CustomRoleSerializer(serializers.ModelSerializer): 6 | class Meta: 7 | model = CustomRoleModel 8 | fields = '__all__' 9 | 10 | 11 | class UploadedImageSerializer(serializers.ModelSerializer): 12 | original_name = serializers.CharField(required=False) 13 | 14 | class Meta: 15 | model = BackgroundImageModel 16 | fields = '__all__' 17 | 18 | 19 | class UploadedVrmModelSerializer(serializers.ModelSerializer): 20 | original_name = serializers.CharField(required=False) 21 | type = serializers.CharField(required=False) 22 | 23 | class Meta: 24 | model = VrmModel 25 | fields = '__all__' 26 | 27 | 28 | class UploadedRolePackageModelSerializer(serializers.ModelSerializer): 29 | role_name = serializers.CharField(required=False) 30 | dataset_json_path = serializers.CharField(required=False) 31 | embed_index_idx_path = serializers.CharField(required=False) 32 | system_prompt_txt_path = serializers.CharField(required=False) 33 | 34 | class Meta: 35 | model = RolePackageModel 36 | fields = '__all__' 37 | -------------------------------------------------------------------------------- /domain-chatbot/apps/chatbot/service/__init__.py: -------------------------------------------------------------------------------- 1 | from .portal_user_service import PortalUserService 2 | 3 | portal_user_service = PortalUserService() 4 | -------------------------------------------------------------------------------- /domain-chatbot/apps/chatbot/service/portal_user_service.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from ..models import PortalUser 3 | from ..utils import singleton_snow_flake 4 | 5 | logger = logging.getLogger(__name__) 6 | 7 | 8 | class PortalUserService: 9 | def create(self, name: str) -> PortalUser: 10 | portal_user = PortalUser( 11 | id=singleton_snow_flake.task(), 12 | name=name 13 | ) 14 | portal_user.save() 15 | return portal_user 16 | 17 | def get_by_name(self, name: str) -> PortalUser: 18 | ''' 19 | 通过名称获取门户用户 20 | ''' 21 | return PortalUser.objects.filter(name=name).last() 22 | 23 | def get_and_create(self, name: str) -> PortalUser: 24 | ''' 25 | 通过名称获取门户用户，如果门户不存在，会自动创建一个 26 | ''' 27 | portal_user = self.get_by_name(name) 28 | if portal_user is None: 29 | portal_user = self.create(name) 30 | return portal_user 31 | -------------------------------------------------------------------------------- /domain-chatbot/apps/chatbot/urls.py: -------------------------------------------------------------------------------- 1 | from django.urls import path 2 | 3 | from . import views 4 | 5 | urlpatterns = [ 6 | path('chat', views.chat, name='chat'), 7 | # path('memory/reflection', views.reflection_generation, 8 | # name='reflection_generation'), 9 | path('memory/clear', views.clear_memory, name='clear_memory'), 10 | path('customrole/list', views.custom_role_list, name='custom_role_list'), 11 | path('customrole/create', views.create_custom_role, name='custom_role_create'), 12 | path('customrole/edit/', views.edit_custom_role, name='custom_role_edit'), 13 | path('customrole/detail/', views.custom_role_detail, name='custom_role_detail'), 14 | path('customrole/delete/', views.delete_custom_role, name='custom_role_delete'), 15 | path('config/get', views.get_config, name='get_config'), 16 | path('config/save', views.save_config, name='save_config'), 17 | path('config/background/delete/', views.delete_background_image, name='delete_background_image'), 18 | path('config/background/upload', views.upload_background_image, name='upload_background_image'), 19 | path('config/background/show', views.show_background_image, name='show_background_image'), 20 | path('config/vrm/delete/', views.delete_vrm_model, name='delete_vrm_model'), 21 | path('config/vrm/upload', views.upload_vrm_model, name='upload_vrm_model'), 22 | path('config/vrm/user/show', views.show_user_vrm_models, name='show_user_vrm_models'), 23 | path('config/vrm/system/show', views.show_system_vrm_models, name='show_system_vrm_models'), 24 | path('rolepackage/upload', views.upload_role_package, name='upload_role_package'), 25 | ] 26 | -------------------------------------------------------------------------------- /domain-chatbot/apps/chatbot/utils/__init__.py: -------------------------------------------------------------------------------- 1 | from .snowflake_utils import SnowFlake 2 | 3 | singleton_snow_flake = SnowFlake(1, 1) 4 | -------------------------------------------------------------------------------- /domain-chatbot/apps/chatbot/utils/chat_message_utils.py: -------------------------------------------------------------------------------- 1 | 2 | import re 3 | 4 | def format_chat_text(role_name: str, you_name: str, text: str): 5 | # 去除特殊字符 * 、`role_name：`、`you_name:` 6 | # text = text.replace(f'*', "") 7 | pattern = r'\*.*?\*' 8 | text = text.replace(f'`', "") 9 | text = re.sub(pattern, '', text) 10 | text = text.replace(f'{role_name}：', "") 11 | text = text.replace(f'{you_name}：', "") 12 | text = text.replace(f'{role_name}:', "") 13 | text = text.replace(f'{you_name}:', "") 14 | text = text.replace(f'AI角色：', "") 15 | text = text.replace(f'AI（{role_name}）：', "") 16 | text = text.replace(f'AI:', "") 17 | text = text.replace(f'ai：', "") 18 | text = text.replace(f'Ai：', "") 19 | text = text.replace(f'{role_name}说', "") 20 | text = text.replace('[', "") 21 | text = text.replace(']', "") 22 | return text 23 | 24 | def format_user_chat_text(text: str): 25 | text = text.replace('[', "") 26 | text = text.replace(']', "") 27 | return text -------------------------------------------------------------------------------- /domain-chatbot/apps/chatbot/utils/datatime_utils.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import os 3 | import pytz 4 | 5 | TIMEZONE = os.environ.get("TIMEZONE","Asia/Shanghai") 6 | 7 | def get_current_time_str(): 8 | current_time = datetime.datetime.now(pytz.timezone(TIMEZONE)) 9 | formatted_time = current_time.strftime('%Y-%m-%d %H:%M:%S') 10 | return formatted_time 11 | -------------------------------------------------------------------------------- /domain-chatbot/apps/chatbot/utils/snowflake_utils.py: -------------------------------------------------------------------------------- 1 | import time 2 | import logging 3 | 4 | # 分配位置 5 | WORKER_BITS = 5 6 | DATACENTER_BITS = 5 7 | SEQUENCE_BITS = 12 8 | 9 | # 设定设备数量上限 10 | WORKER_UPPER_LIMIT = -1 ^ (-1 << WORKER_BITS) 11 | DATACENTER_UPPER_TIMIT = -1 ^ (-1 << DATACENTER_BITS) 12 | 13 | 14 | # 组合是的位运算偏移量 15 | WORKER_SHIFT = SEQUENCE_BITS 16 | DATACENTER_ID_SHIFT = SEQUENCE_BITS + WORKER_BITS 17 | TIMESTAMP_LEFT_SHIFT = SEQUENCE_BITS + WORKER_BITS + DATACENTER_BITS 18 | 19 | SEQUENCE_MASK = -1 ^ (-1 << SEQUENCE_BITS) # 掩码 20 | EPOCH = 1577808001000 # 元时间戳 此处元设为 2020-01-01 00:00:01 21 | 22 | 23 | class SnowFlake(object): 24 | 25 | def __init__(self, data_center_id, worker_id, sequence=0): 26 | """ 27 | :param data_center_id: 数据中心编号 28 | :param worker_id: 机器编号 29 | :param sequence: 序号 30 | """ 31 | if worker_id > WORKER_UPPER_LIMIT: 32 | raise ValueError("WORKER ID 高于上限") 33 | if worker_id < 0: 34 | raise ValueError("WORKER ID 低于下限") 35 | if data_center_id > DATACENTER_UPPER_TIMIT: 36 | raise ValueError("DATA CENTER ID 高于上限") 37 | if data_center_id < 0: 38 | raise ValueError("DATA CENTER ID 低于上限") 39 | self.worker_id = worker_id 40 | self.datacenter_id = data_center_id 41 | self.sequence = sequence 42 | 43 | self.last_timestamp = -1 # 最近一次生成编号的时间戳 44 | 45 | @staticmethod 46 | def _timestamp(n=1e3) -> int: 47 | """指定位数时间戳 48 | :param n: 49 | :return: 50 | """ 51 | return int(time.time() * n) 52 | 53 | def _check(self, timestamp): 54 | """ 55 | 超限检查 56 | :param timestamp: 57 | :return: 58 | """ 59 | self._time_back_off_check(timestamp) 60 | self._number_check(timestamp) 61 | 62 | def _number_check(self, timestamp): 63 | """ 64 | 数超限检查，检查当前时间生成的编号是否超过上限，超过上限则的等到下一个时间生成 65 | :param timestamp: 66 | :return: 67 | """ 68 | if timestamp == self.last_timestamp: 69 | self.sequence = (self.sequence + 1) & SEQUENCE_MASK 70 | if self.sequence == 0: 71 | timestamp = self._wait_next_time(self.last_timestamp) 72 | else: 73 | self.sequence = 0 74 | 75 | def _time_back_off_check(self, timestamp): 76 | if timestamp < self.last_timestamp: 77 | logging.error('发现时钟回退，记录到最近一次的时间戳为 {}'.format(self.last_timestamp)) 78 | raise Exception("时钟回拨异常") 79 | 80 | def task(self) -> int: 81 | """ 82 | 获取一个编号 83 | :return: 84 | """ 85 | timestamp = self._timestamp() 86 | self._check(timestamp) 87 | self.last_timestamp = timestamp 88 | return self._generate(timestamp) 89 | 90 | def _generate(self, timestamp) -> int: 91 | """ 生成一个编号 92 | :param timestamp: 93 | :return: 94 | """ 95 | number = ((timestamp - EPOCH) << TIMESTAMP_LEFT_SHIFT) | (self.datacenter_id << DATACENTER_ID_SHIFT) | ( 96 | self.worker_id << WORKER_SHIFT) | self.sequence 97 | return number 98 | 99 | def _wait_next_time(self, last_timestamp): 100 | """等到下一次单位时间 101 | :param last_timestamp: 102 | :return: 103 | """ 104 | timestamp = self._timestamp() 105 | while timestamp <= last_timestamp: 106 | timestamp = self._timestamp() 107 | return timestamp -------------------------------------------------------------------------------- /domain-chatbot/apps/chatbot/utils/str_utils.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | 4 | def remove_emojis(input_string) -> str: 5 | # 使用正则表达式删除所有表情符号 6 | emoji_pattern = re.compile("[" 7 | u"\U0001F600-\U0001F64F" # 表情符号 8 | u"\U0001F300-\U0001F5FF" # 符号与杂项符号 9 | u"\U0001F680-\U0001F6FF" # 交通和地图符号 10 | u"\U0001F700-\U0001F77F" # 国际音标扩展符号 11 | u"\U0001F780-\U0001F7FF" # 表情符号补充 12 | u"\U0001F800-\U0001F8FF" # 语言补充 13 | u"\U0001F900-\U0001F9FF" # 符号与象形文字补充 14 | u"\U0001FA00-\U0001FA6F" # 扑克牌 15 | u"\U0001FA70-\U0001FAFF" # 旗帜（Emoji表情） 16 | u"\U0001F004" # 单个符号-标签 17 | "]+", flags=re.UNICODE) 18 | return emoji_pattern.sub(r'', input_string) 19 | 20 | 21 | def remove_spaces_and_tabs(input_string) -> str: 22 | text = input_string.replace(" ", "").replace("\t", "") 23 | return text 24 | 25 | def remove_special_characters(input_string) -> str: 26 | # 定义正则表达式模式，匹配特殊符号，同时保留 .．！？~、, 27 | pattern = r'[^\w\s.．！？~、,，。]' # 这个模式匹配除了字母、数字、空格和指定符号之外的所有字符 28 | 29 | # 使用 re.sub() 函数删除匹配的特殊字符 30 | cleaned_string = re.sub(pattern, '', input_string) 31 | 32 | return cleaned_string 33 | -------------------------------------------------------------------------------- /domain-chatbot/apps/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 4.2.1 on 2024-01-27 07:19 2 | 3 | from django.db import migrations, models 4 | 5 | 6 | class Migration(migrations.Migration): 7 | 8 | initial = True 9 | 10 | dependencies = [ 11 | ] 12 | 13 | operations = [ 14 | migrations.CreateModel( 15 | name='BackgroundImageModel', 16 | fields=[ 17 | ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 18 | ('original_name', models.CharField(max_length=50)), 19 | ('image', models.ImageField(upload_to='background/')), 20 | ], 21 | ), 22 | migrations.CreateModel( 23 | name='CustomRoleModel', 24 | fields=[ 25 | ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 26 | ('role_name', models.CharField(max_length=100)), 27 | ('persona', models.TextField()), 28 | ('personality', models.TextField()), 29 | ('scenario', models.TextField()), 30 | ('examples_of_dialogue', models.TextField()), 31 | ('custom_role_template_type', models.CharField(max_length=50)), 32 | ('role_package_id', models.IntegerField()), 33 | ], 34 | ), 35 | migrations.CreateModel( 36 | name='LocalMemoryModel', 37 | fields=[ 38 | ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 39 | ('text', models.TextField()), 40 | ('tags', models.TextField()), 41 | ('sender', models.CharField(default='null', max_length=50)), 42 | ('owner', models.CharField(max_length=50)), 43 | ('timestamp', models.DateTimeField()), 44 | ], 45 | ), 46 | migrations.CreateModel( 47 | name='PortalUser', 48 | fields=[ 49 | ('id', models.BigIntegerField(db_comment='门户用户唯一ID', primary_key=True, serialize=False)), 50 | ('name', models.CharField(db_comment='门户用户名称', max_length=100)), 51 | ], 52 | ), 53 | migrations.CreateModel( 54 | name='RolePackageModel', 55 | fields=[ 56 | ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 57 | ('role_name', models.CharField(max_length=10)), 58 | ('dataset_json_path', models.CharField(max_length=10)), 59 | ('embed_index_idx_path', models.CharField(max_length=10)), 60 | ('system_prompt_txt_path', models.CharField(max_length=10)), 61 | ('role_package', models.FileField(upload_to='role_package/')), 62 | ], 63 | ), 64 | migrations.CreateModel( 65 | name='SysConfigModel', 66 | fields=[ 67 | ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 68 | ('code', models.CharField(max_length=20)), 69 | ('config', models.TextField()), 70 | ], 71 | ), 72 | migrations.CreateModel( 73 | name='VrmModel', 74 | fields=[ 75 | ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 76 | ('type', models.CharField(max_length=10)), 77 | ('original_name', models.CharField(max_length=50)), 78 | ('vrm', models.FileField(upload_to='vrm/')), 79 | ], 80 | ), 81 | ] 82 | -------------------------------------------------------------------------------- /domain-chatbot/apps/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/domain-chatbot/apps/migrations/__init__.py -------------------------------------------------------------------------------- /domain-chatbot/apps/speech/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/domain-chatbot/apps/speech/__init__.py -------------------------------------------------------------------------------- /domain-chatbot/apps/speech/ars/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/domain-chatbot/apps/speech/ars/__init__.py -------------------------------------------------------------------------------- /domain-chatbot/apps/speech/translation/__init__.py: -------------------------------------------------------------------------------- 1 | from .huoshan.huoshan_translation_client import HuoShanTranslationClient 2 | 3 | translationClient = HuoShanTranslationClient() 4 | -------------------------------------------------------------------------------- /domain-chatbot/apps/speech/translation/base_translation_client.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | 3 | 4 | class BaseTranslationClient(ABC): 5 | 6 | '''统一翻译抽象类,基于当前抽象类扩展其他的翻译模块''' 7 | 8 | @abstractmethod 9 | def translation(self, text: str, target_language: str) -> str: 10 | '''翻译''' 11 | pass 12 | -------------------------------------------------------------------------------- /domain-chatbot/apps/speech/translation/google/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/domain-chatbot/apps/speech/translation/google/__init__.py -------------------------------------------------------------------------------- /domain-chatbot/apps/speech/translation/google/google_translation_client.py: -------------------------------------------------------------------------------- 1 | from easygoogletranslate import EasyGoogleTranslate 2 | 3 | translator = EasyGoogleTranslate( 4 | source_language='en', 5 | target_language='de', 6 | timeout=10 7 | ) 8 | result = translator.translate('This is an example.') 9 | 10 | print(result) 11 | # Output: Dies ist ein Beispiel. -------------------------------------------------------------------------------- /domain-chatbot/apps/speech/translation/huoshan/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/domain-chatbot/apps/speech/translation/huoshan/__init__.py -------------------------------------------------------------------------------- /domain-chatbot/apps/speech/translation/huoshan/huoshan_translation_client.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | from ..base_translation_client import BaseTranslationClient 4 | from volcengine.ApiInfo import ApiInfo 5 | from volcengine.Credentials import Credentials 6 | from volcengine.ServiceInfo import ServiceInfo 7 | from volcengine.base.Service import Service 8 | 9 | 10 | class HuoShanTranslationClient(BaseTranslationClient): 11 | 12 | access_key: str 13 | secret_key: str 14 | service_info: ServiceInfo 15 | query: dict[str, str] 16 | api_info: dict 17 | service: Service 18 | 19 | def __init__(self) -> None: 20 | 21 | self.access_key = os.environ.get("HUO_SHAN_ACCESS_KEY") 22 | self.secret_key = os.environ.get("HUO_SHAN_SECRET_KEY") 23 | self.service_info = ServiceInfo('translate.volcengineapi.com', 24 | {'Content-Type': 'application/json'}, 25 | Credentials(self.access_key, self.secret_key, 26 | 'translate', 'cn-north-1'), 27 | 5, 28 | 5) 29 | self.query = { 30 | 'Action': 'TranslateText', 31 | 'Version': '2020-06-01' 32 | } 33 | self.api_info = { 34 | 'translate': ApiInfo('POST', '/', self.query, {}, {}) 35 | } 36 | self.service = Service(self.service_info, self.api_info) 37 | 38 | def translation(self, text: str, target_language: str) -> str: 39 | body = { 40 | 'TargetLanguage': target_language, 41 | 'TextList': [text], 42 | } 43 | res = self.service.json('translate', {}, json.dumps(body)) 44 | res = json.loads(res) 45 | translation = res["TranslationList"][0]["Translation"] 46 | return translation 47 | -------------------------------------------------------------------------------- /domain-chatbot/apps/speech/translation/youdao/AuthV3Util.py: -------------------------------------------------------------------------------- 1 | import hashlib 2 | import time 3 | import uuid 4 | 5 | ''' 6 | 添加鉴权相关参数 - 7 | appKey : 应用ID 8 | salt : 随机值 9 | curtime : 当前时间戳(秒) 10 | signType : 签名版本 11 | sign : 请求签名 12 | 13 | @param appKey 您的应用ID 14 | @param appSecret 您的应用密钥 15 | @param paramsMap 请求参数表 16 | ''' 17 | def addAuthParams(appKey, appSecret, params): 18 | q = params.get('q') 19 | if q is None: 20 | q = params.get('img') 21 | salt = str(uuid.uuid1()) 22 | curtime = str(int(time.time())) 23 | sign = calculateSign(appKey, appSecret, q, salt, curtime) 24 | params['appKey'] = appKey 25 | params['salt'] = salt 26 | params['curtime'] = curtime 27 | params['signType'] = 'v3' 28 | params['sign'] = sign 29 | 30 | ''' 31 | 计算鉴权签名 - 32 | 计算方式 : sign = sha256(appKey + input(q) + salt + curtime + appSecret) 33 | @param appKey 您的应用ID 34 | @param appSecret 您的应用密钥 35 | @param q 请求内容 36 | @param salt 随机值 37 | @param curtime 当前时间戳(秒) 38 | @return 鉴权签名sign 39 | ''' 40 | def calculateSign(appKey, appSecret, q, salt, curtime): 41 | strSrc = appKey + getInput(q) + salt + curtime + appSecret 42 | return encrypt(strSrc) 43 | 44 | 45 | def encrypt(strSrc): 46 | hash_algorithm = hashlib.sha256() 47 | hash_algorithm.update(strSrc.encode('utf-8')) 48 | return hash_algorithm.hexdigest() 49 | 50 | 51 | def getInput(input): 52 | if input is None: 53 | return input 54 | inputLen = len(input) 55 | return input if inputLen <= 20 else input[0:10] + str(inputLen) + input[inputLen - 10:inputLen] 56 | -------------------------------------------------------------------------------- /domain-chatbot/apps/speech/translation/youdao/AuthV4Util.py: -------------------------------------------------------------------------------- 1 | import hashlib 2 | import time 3 | import uuid 4 | 5 | ''' 6 | 添加鉴权相关参数 - 7 | appKey : 应用ID 8 | salt : 随机值 9 | curtime : 当前时间戳(秒) 10 | signType : 签名版本 11 | sign : 请求签名 12 | 13 | @param appKey 您的应用ID 14 | @param appSecret 您的应用密钥 15 | @param paramsMap 请求参数表 16 | ''' 17 | def addAuthParams(appKey, appSecret, params): 18 | salt = str(uuid.uuid1()) 19 | curtime = str(int(time.time())) 20 | sign = calculateSign(appKey, appSecret, salt, curtime) 21 | params['appKey'] = appKey 22 | params['salt'] = salt 23 | params['curtime'] = curtime 24 | params['signType'] = 'v4' 25 | params['sign'] = sign 26 | 27 | 28 | ''' 29 | 计算鉴权签名 - 30 | 计算方式 : sign = sha256(appKey + input(q) + salt + curtime + appSecret) 31 | @param appKey 您的应用ID 32 | @param appSecret 您的应用密钥 33 | @param salt 随机值 34 | @param curtime 当前时间戳(秒) 35 | @return 鉴权签名sign 36 | ''' 37 | def calculateSign(appKey, appSecret, salt, curtime): 38 | strSrc = appKey + salt + curtime + appSecret 39 | return encrypt(strSrc) 40 | 41 | 42 | def encrypt(strSrc): 43 | hash_algorithm = hashlib.sha256() 44 | hash_algorithm.update(strSrc.encode('utf-8')) 45 | return hash_algorithm.hexdigest() 46 | -------------------------------------------------------------------------------- /domain-chatbot/apps/speech/translation/youdao/youdao_translation_client.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import os 3 | import json 4 | from ..utils.AuthV3Util import addAuthParams 5 | 6 | # 您的应用ID 7 | APP_KEY = os.getenv("YOUDAO_APP_KEY") 8 | # 您的应用密钥 9 | APP_SECRET = os.getenv("YOUDAO_SECRET_KEY") 10 | 11 | 12 | class TranslationClient: 13 | 14 | @staticmethod 15 | def translation(query:str) -> str: 16 | q = query 17 | lang_from = 'zh-CHS' 18 | lang_to = 'ja' 19 | data = {'q': q, 'from': lang_from, 'to': lang_to} 20 | addAuthParams(APP_KEY, APP_SECRET, data) 21 | header = {'Content-Type': 'application/x-www-form-urlencoded'} 22 | res = requests.post('https://openapi.youdao.com/api', data, header) 23 | content = str(res.content, 'utf-8') 24 | return json.loads(content) -------------------------------------------------------------------------------- /domain-chatbot/apps/speech/tts/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | from .tts_driver import TTSDriver 3 | 4 | single_tts_driver = TTSDriver() -------------------------------------------------------------------------------- /domain-chatbot/apps/speech/tts/edge_tts.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | import subprocess 4 | 5 | from ..utils.uuid_generator import generate 6 | 7 | logger = logging.getLogger(__name__) 8 | 9 | edge_voices = [ 10 | {"id": "zh-CN-XiaoxiaoNeural", "name": "xiaoxiao"}, 11 | {"id": "zh-CN-XiaoyiNeural", "name": "xiaoyi"}, 12 | {"id": "zh-CN-YunjianNeural", "name": "yunjian"}, 13 | {"id": "zh-CN-YunxiNeural", "name": "yunxi"}, 14 | {"id": "zh-CN-YunxiaNeural", "name": "yunxia"}, 15 | {"id": "zh-CN-YunyangNeural", "name": "yunyang"}, 16 | {"id": "zh-CN-liaoning-XiaobeiNeural", "name": "xiaobei"}, 17 | {"id": "zh-CN-shaanxi-XiaoniNeural", "name": "xiaoni"}, 18 | {"id": "zh-HK-HiuGaaiNeural", "name": "hiugaai"}, 19 | {"id": "zh-HK-HiuMaanNeural", "name": "hiumaan"}, 20 | {"id": "zh-HK-WanLungNeural", "name": "wanlung"}, 21 | {"id": "zh-TW-HsiaoChenNeural", "name": "hsiaochen"}, 22 | {"id": "zh-TW-HsiaoYuNeural", "name": "hsioayu"}, 23 | {"id": "zh-TW-YunJheNeural", "name": "yunjhe"} 24 | ] 25 | 26 | 27 | class Edge(): 28 | 29 | def remove_html(self, text: str): 30 | # TODO 待改成正则 31 | new_text = text.replace('[', "") 32 | new_text = new_text.replace(']', "") 33 | return new_text 34 | 35 | def create_audio(self, text: str, voiceId: str): 36 | new_text = self.remove_html(text) 37 | pwdPath = os.getcwd() 38 | file_name = generate() + ".mp3" 39 | filePath = pwdPath + "/tmp/" + file_name 40 | dirPath = os.path.dirname(filePath) 41 | if not os.path.exists(dirPath): 42 | os.makedirs(dirPath) 43 | if not os.path.exists(filePath): 44 | # 用open创建文件 兼容mac 45 | open(filePath, 'a').close() 46 | 47 | subprocess.run(["edge-tts", "--voice", voiceId, "--text", new_text, 48 | "--write-media", str(filePath)]) 49 | 50 | return file_name 51 | -------------------------------------------------------------------------------- /domain-chatbot/apps/speech/tts/tts_driver.py: -------------------------------------------------------------------------------- 1 | from abc import ABC, abstractmethod 2 | import logging 3 | from .edge_tts import Edge, edge_voices 4 | from .bert_vits2 import BertVits2 5 | 6 | logger = logging.getLogger(__name__) 7 | 8 | 9 | class BaseTTS(ABC): 10 | '''合成语音统一抽象类''' 11 | 12 | @abstractmethod 13 | def synthesis(self, text: str, voice_id: str, **kwargs) -> str: 14 | '''合成语音''' 15 | pass 16 | 17 | @abstractmethod 18 | def get_voices(self) -> list[dict[str, str]]: 19 | '''获取声音列表''' 20 | pass 21 | 22 | 23 | class EdgeTTS(BaseTTS): 24 | '''Edge 微软语音合成类''' 25 | client: Edge 26 | 27 | def __init__(self): 28 | self.client = Edge() 29 | 30 | def synthesis(self, text: str, voice_id: str, **kwargs) -> str: 31 | return self.client.create_audio(text=text, voiceId=voice_id) 32 | 33 | def get_voices(self) -> list[dict[str, str]]: 34 | return edge_voices 35 | 36 | 37 | class BertVITS2TTS(BaseTTS): 38 | '''Bert-VITS2 语音合成类''' 39 | client: BertVits2 40 | 41 | def __init__(self): 42 | self.client = BertVits2() 43 | 44 | def synthesis(self, text: str, voice_id: str, **kwargs) -> str: 45 | noise = kwargs.get("noise", 0.6) 46 | noisew = kwargs.get("noisew", 0.9) 47 | sdp_ratio = kwargs.get("sdp_ratio", 0.2) 48 | return self.client.synthesis(text=text, speaker=voice_id, noise=noise, noisew=noisew, sdp_ratio=sdp_ratio) 49 | 50 | def get_voices(self) -> list[dict[str, str]]: 51 | return self.client.get_voices() 52 | 53 | 54 | class TTSDriver: 55 | '''TTS驱动类''' 56 | 57 | def synthesis(self, type: str, text: str, voice_id: str, **kwargs) -> str: 58 | tts = self.get_strategy(type) 59 | file_name = tts.synthesis(text=text, voice_id=voice_id, kwargs=kwargs) 60 | logger.info(f"TTS synthesis # type:{type} text:{text} => file_name: {file_name} #") 61 | return file_name; 62 | 63 | def get_voices(self, type: str) -> list[dict[str, str]]: 64 | tts = self.get_strategy(type) 65 | return tts.get_voices() 66 | 67 | def get_strategy(self, type: str) -> BaseTTS: 68 | if type == "Edge": 69 | return EdgeTTS() 70 | elif type == "Bert-VITS2": 71 | return BertVITS2TTS() 72 | else: 73 | raise ValueError("Unknown type") 74 | -------------------------------------------------------------------------------- /domain-chatbot/apps/speech/urls.py: -------------------------------------------------------------------------------- 1 | from django.urls import path 2 | from . import views 3 | 4 | urlpatterns = [ 5 | path('tts/generate', views.generate, name='generate'), 6 | path('tts/voices', views.get_voices, name='voices'), 7 | path('translation', views.translation, name='translation'), 8 | ] 9 | -------------------------------------------------------------------------------- /domain-chatbot/apps/speech/utils/uuid_generator.py: -------------------------------------------------------------------------------- 1 | import uuid 2 | 3 | 4 | def generate(): 5 | """ 6 | 生成UUID 7 | """ 8 | uuid_str = str(uuid.uuid4()) 9 | uuid_str = uuid_str.replace("-", "") 10 | return uuid_str 11 | -------------------------------------------------------------------------------- /domain-chatbot/apps/speech/views.py: -------------------------------------------------------------------------------- 1 | from io import BytesIO 2 | 3 | from django.shortcuts import render 4 | import os 5 | import json 6 | import logging 7 | from django.http import FileResponse 8 | from .translation import translationClient 9 | from rest_framework.decorators import api_view 10 | from rest_framework.response import Response 11 | from .tts import single_tts_driver 12 | from django.http import HttpResponse, StreamingHttpResponse 13 | 14 | logger = logging.getLogger(__name__) 15 | 16 | @api_view(['POST']) 17 | def generate(request): 18 | """ 19 | Generate audio from text. 20 | """ 21 | try: 22 | data = json.loads(request.body.decode('utf-8')) 23 | text = data["text"] 24 | voice_id = data["voice_id"] 25 | type = data["type"] 26 | 27 | file_name = single_tts_driver.synthesis( 28 | type=type, text=text, voice_id=voice_id) 29 | file_path = os.path.join("tmp", file_name) 30 | 31 | audio_file = BytesIO() 32 | with open(file_path, 'rb') as file: 33 | audio_file.write(file.read()) 34 | 35 | delete_file(file_path) 36 | logger.debug(f"delete file :{file_path}") 37 | 38 | audio_file.seek(0) 39 | 40 | # Create the response object. 41 | response = HttpResponse(content_type='audio/mpeg') 42 | response['Content-Disposition'] = f'attachment; filename="{file_name}"' 43 | response.write(audio_file.getvalue()) 44 | return response 45 | except Exception as e: 46 | logger.error(f"generate_audio error: {e}") 47 | return HttpResponse(status=500, content="Failed to generate audio.") 48 | 49 | 50 | def delete_file(file_path): 51 | os.remove(file_path) 52 | 53 | @api_view(['POST']) 54 | def get_voices(request): 55 | data = json.loads(request.body.decode('utf-8')) 56 | type = data["type"] 57 | return Response({"response": single_tts_driver.get_voices(type=type), "code": "200"}) 58 | 59 | @api_view(['POST']) 60 | def translation(request): 61 | """ 62 | translation 63 | """ 64 | try: 65 | data = json.loads(request.body.decode('utf-8')) 66 | text = data["text"] 67 | target_language = data["target_language"] 68 | target_result = translationClient.translation( 69 | text=text, target_language=target_language) 70 | return Response({"response": target_result, "code": "200"}) 71 | except Exception as e: 72 | logger.error(f"translation error: {e}") 73 | return HttpResponse(status=500, content="Failed to translation error.") 74 | -------------------------------------------------------------------------------- /domain-chatbot/db/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/domain-chatbot/db/__init__.py -------------------------------------------------------------------------------- /domain-chatbot/logs/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/domain-chatbot/logs/__init__.py -------------------------------------------------------------------------------- /domain-chatbot/manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """Django's command-line utility for administrative tasks.""" 3 | import os 4 | import sys 5 | 6 | 7 | def main(): 8 | """Run administrative tasks.""" 9 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'VirtualWife.settings') 10 | try: 11 | from django.core.management import execute_from_command_line 12 | except ImportError as exc: 13 | raise ImportError( 14 | "Couldn't import Django. Are you sure it's installed and " 15 | "available on your PYTHONPATH environment variable? Did you " 16 | "forget to activate a virtual environment?" 17 | ) from exc 18 | execute_from_command_line(sys.argv) 19 | 20 | 21 | if __name__ == '__main__': 22 | main() 23 | 24 | 25 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /domain-chatbot/models/README.md: -------------------------------------------------------------------------------- 1 | 在此目录下运行： 2 | 3 | - 创建baai文件夹 4 | 5 | ```shell 6 | mkdir baai 7 | ``` 8 | 9 | - 下载向量化模型 10 | 11 | ```shell 12 | git clone https://huggingface.co/BAAI/bge-large-zh-v1.5 13 | git clone https://huggingface.co/BAAI/bge-reranker-large 14 | ``` -------------------------------------------------------------------------------- /domain-chatbot/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/domain-chatbot/models/__init__.py -------------------------------------------------------------------------------- /domain-chatbot/requirements.txt: -------------------------------------------------------------------------------- 1 | urllib3==1.24.3 2 | openai==1.12.0 3 | python-dotenv==1.0.0 4 | Django==4.2.1 5 | aiohttp<=3.8.6,<4.0.0 6 | Brotli==1.1.0 7 | flask==2.3.3 8 | gunicorn==21.2.0 9 | gevent==23.9.0.post1 10 | edge-tts==6.1.8 11 | djangorestframework==3.14.0 12 | django-cors-headers==4.2.0 13 | channels==4.0.0 14 | daphne==4.0.0 15 | schedule==1.2.0 16 | drf-yasg==1.21.7 17 | pymilvus==2.2.12 18 | datetime==5.2 19 | transformers>=4.36.2 20 | sentence-transformers==2.2.2 21 | easygoogletranslate==0.0.4 22 | volcengine==1.0.103 23 | requests==2.31.0 24 | jieba==0.42.1 25 | websockets==11.0.3 26 | APScheduler==3.10.4 27 | zep-python==1.5.0 28 | pypinyin==0.50.0 29 | bilibili-api-python==16.1.1 30 | faiss-cpu==1.7.4 31 | FlagEmbedding==1.1.7 32 | openai==1.12.0 33 | litellm==1.34.6 -------------------------------------------------------------------------------- /domain-chatbot/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/domain-chatbot/tests/__init__.py -------------------------------------------------------------------------------- /domain-chatbot/tests/bilibili_api_test.py: -------------------------------------------------------------------------------- 1 | from bilibili_api import live, sync, Credential 2 | 3 | import http.cookies 4 | 5 | # 你的 Cookie 文本 6 | cookie_text = "LIVE_BUVID=AUTO2816977204552676; b_nut=1697720455; buvid4=34DB59A6-47D1-B3DC-953E-0E39DB1D2E1655624-023101921-ksPq5hUPXVm6%2BOFFmBjPRg%3D%3D; buvid3=37A48A8B-FF98-401C-B2BE-AC57D04DC1EC27577infoc; _uuid=D10C5239E-95510-3EDF-2A8E-F134F5686EB1069650infoc; buvid_fp_plain=undefined; enable_web_push=DISABLE; header_theme_version=CLOSE; CURRENT_FNVAL=4048; rpdid=|(kJYkJkJ)m|0J'uYm~~mJuYR; DedeUserID=382957163; DedeUserID__ckMd5=537234e3cc45dfda; hit-dyn-v2=1; CURRENT_QUALITY=80; bp_article_offset_382957163=874973829923864664; fingerprint=dff334dfe228c7ed3101952831bcec25; buvid_fp=dff334dfe228c7ed3101952831bcec25; home_feed_column=5; PVID=1; SESSDATA=ebbe501f%2C1720139869%2Ce6249%2A12CjCJU4P7mak9oluUq8CXdXfVZFz-ZPeAica88Ng6LC8xjvHy2BB3Z4m0uXcc_GMY-8ISVncwQndJd3liN3l1dFFVSEo1YndtNGNKUmlfZVN0WEJOOGdyYWQ5LTh6cVotb1ItZGVfWExiV1NVcWdlU3dFczMyRWRhZkF2QV9vQ2pFRzVFeXdvX3dBIIEC; bili_jct=fa9c0262144f89b0bc299c5769defb89; b_lsid=585F22B2_18CE16AA261; bili_ticket=eyJhbGciOiJIUzI1NiIsImtpZCI6InMwMyIsInR5cCI6IkpXVCJ9.eyJleHAiOjE3MDQ4NTA3NjgsImlhdCI6MTcwNDU5MTUwOCwicGx0IjotMX0.csRZa7_FgulmbJFlkc1OWUPpEvQc7rHu79w2mEv0JNo; bili_ticket_expires=1704850708; sid=6gnena0l; bp_video_offset_382957163=883705876426260483; browser_resolution=1440-779; innersign=0" 7 | 8 | # 创建一个 Cookie 对象 9 | cookie = http.cookies.SimpleCookie() 10 | cookie.load(cookie_text) 11 | 12 | # 获取 sessdata 和 bili_jct 属性的值 13 | sessdata = cookie.get("SESSDATA") 14 | bili_jct = cookie.get("bili_jct") 15 | # dedeuserid = cookie.get("dedeuserid") 16 | # ac_time_value = cookie.get("ac_time_value") 17 | 18 | sessdata_value = sessdata.value 19 | bili_jct_value = bili_jct.value 20 | # dedeuserid_value = dedeuserid.value 21 | # ac_time_value = ac_time_value.value 22 | 23 | print(f"sessdata: {sessdata_value}") 24 | print(f"bili_jct: {bili_jct_value}") 25 | # print(f"ac_time_value: {ac_time_value}") 26 | 27 | credential = Credential(sessdata=sessdata_value, bili_jct=bili_jct_value,buvid3="") 28 | room = live.LiveDanmaku(room_display_id="27892212", credential=credential) 29 | 30 | 31 | @room.on('DANMU_MSG') 32 | async def on_danmaku(event): 33 | # 收到弹幕 34 | print(event) 35 | 36 | 37 | @room.on('SEND_GIFT') 38 | async def on_gift(event): 39 | # 收到礼物 40 | print(event) 41 | 42 | 43 | sync(room.connect()) 44 | -------------------------------------------------------------------------------- /domain-chatvrm/.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "next/core-web-vitals" 3 | } 4 | -------------------------------------------------------------------------------- /domain-chatvrm/.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. 2 | 3 | # dependencies 4 | /node_modules 5 | /.pnp 6 | .pnp.js 7 | 8 | # testing 9 | /coverage 10 | 11 | # next.js 12 | /.next/ 13 | /out/ 14 | 15 | # production 16 | /build 17 | 18 | # misc 19 | .DS_Store 20 | *.pem 21 | 22 | # debug 23 | npm-debug.log* 24 | yarn-debug.log* 25 | yarn-error.log* 26 | .pnpm-debug.log* 27 | 28 | # local env files 29 | .env*.local 30 | 31 | # vercel 32 | .vercel 33 | 34 | # typescript 35 | *.tsbuildinfo 36 | next-env.d.ts 37 | 38 | .env -------------------------------------------------------------------------------- /domain-chatvrm/LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 pixiv Inc. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /domain-chatvrm/README.md: -------------------------------------------------------------------------------- 1 | # ChatVRM 2 | 3 | ChatVRMはブラウザで簡単に3Dキャラクターと会話ができるデモアプリケーションです。 4 | 5 | VRMファイルをインポートしてキャラクターに合わせた声の調整や、感情表現を含んだ返答文の生成などを行うことができます。 6 | 7 | ChatVRMの各機能は主に以下の技術を使用しています。 8 | 9 | - ユーザーの音声の認識 10 | - [Web Speech API(SpeechRecognition)](https://developer.mozilla.org/ja/docs/Web/API/SpeechRecognition) 11 | - 返答文の生成 12 | - [ChatGPT API](https://platform.openai.com/docs/api-reference/chat) 13 | - 読み上げ音声の生成 14 | - [Koeiro API](http://koeiromap.rinna.jp/) 15 | - 3Dキャラクターの表示 16 | - [@pixiv/three-vrm](https://github.com/pixiv/three-vrm) 17 | 18 | 19 | ## デモ 20 | 21 | GitHub Pagesでデモを公開しています。 22 | 23 | [https://pixiv.github.io/ChatVRM](https://pixiv.github.io/ChatVRM) 24 | 25 | 26 | ## 実行 27 | ローカル環境で実行する場合はこのリポジトリをクローンするか、ダウンロードしてください。 28 | 29 | ```bash 30 | git clone git@github.com:pixiv/ChatVRM.git 31 | ``` 32 | 33 | 必要なパッケージをインストールしてください。 34 | ```bash 35 | npm install 36 | ``` 37 | 38 | パッケージのインストールが完了した後、以下のコマンドで開発用のWebサーバーを起動します。 39 | ```bash 40 | npm run dev 41 | ``` 42 | 43 | 実行後、以下のURLにアクセスして動作を確認して下さい。 44 | 45 | [http://localhost:3000](http://localhost:3000) 46 | 47 | 48 | --- 49 | 50 | ## ChatGPT API 51 | 52 | ChatVRMでは返答文の生成にChatGPT APIを使用しています。 53 | 54 | ChatGPT APIの仕様や利用規約については以下のリンクや公式サイトをご確認ください。 55 | 56 | - [https://platform.openai.com/docs/api-reference/chat](https://platform.openai.com/docs/api-reference/chat) 57 | - [https://openai.com/policies/api-data-usage-policies](https://openai.com/policies/api-data-usage-policies) 58 | 59 | 60 | ## Koeiro API 61 | ChatVRMでは返答文の音声読み上げにKoeiro APIを使用しています。 62 | 63 | Koeiro APIの仕様や利用規約については以下のリンクや公式サイトをご確認ください。 64 | 65 | - [http://koeiromap.rinna.jp/](http://koeiromap.rinna.jp/) -------------------------------------------------------------------------------- /domain-chatvrm/next.config.js: -------------------------------------------------------------------------------- 1 | /** @type {import('next').NextConfig} */ 2 | const nextConfig = { 3 | reactStrictMode: true, 4 | assetPrefix: process.env.BASE_PATH || "", 5 | basePath: process.env.BASE_PATH || "", 6 | trailingSlash: true, 7 | publicRuntimeConfig: { 8 | root: process.env.BASE_PATH || "", 9 | }, 10 | }; 11 | 12 | module.exports = nextConfig; 13 | -------------------------------------------------------------------------------- /domain-chatvrm/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "chat-vrm", 3 | "version": "0.1.0", 4 | "private": true, 5 | "scripts": { 6 | "dev": "next dev", 7 | "build": "next build", 8 | "build:with-lint": "npm run build && npm run lint", 9 | "start": "next start", 10 | "export": "next export", 11 | "lint": "next lint" 12 | }, 13 | "dependencies": { 14 | "@charcoal-ui/icons": "^2.6.0", 15 | "@tailwindcss/line-clamp": "^0.4.4", 16 | "@types/dom-speech-recognition": "^0.0.1", 17 | "@types/js-priority-queue": "0.0.6", 18 | "abortcontroller-polyfill": "^1.7.5", 19 | "cross-env": "^7.0.3", 20 | "dotenv": "^16.3.1", 21 | "eslint": "8.36.0", 22 | "eslint-config-next": "13.2.4", 23 | "fs": "^0.0.1-security", 24 | "js-priority-queue": "^0.1.5", 25 | "next": "13.2.4", 26 | "node-fetch": "^3.3.1", 27 | "openai": "^3.2.1", 28 | "react": "18.2.0", 29 | "react-dom": "18.2.0", 30 | "react-transition-group": "^4.4.5", 31 | "three": "^0.151.0", 32 | "typescript": "5.0.2" 33 | }, 34 | "devDependencies": { 35 | "@charcoal-ui/tailwind-config": "^2.6.0", 36 | "@gltf-transform/core": "^2.4.6", 37 | "@pixiv/three-vrm": "^1.0.10", 38 | "@types/node": "18.15.10", 39 | "@types/react": "18.0.29", 40 | "@types/react-dom": "18.0.11", 41 | "@types/react-transition-group": "^4.4.6", 42 | "@types/three": "0.151.0", 43 | "autoprefixer": "^10.4.14", 44 | "postcss": "^8.4.21", 45 | "tailwindcss": "^3.3.1" 46 | }, 47 | "engines": { 48 | "node": "16.14.2" 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /domain-chatvrm/postcss.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | plugins: { 3 | tailwindcss: {}, 4 | autoprefixer: {}, 5 | }, 6 | } 7 | -------------------------------------------------------------------------------- /domain-chatvrm/public/aili.vrm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/domain-chatvrm/public/aili.vrm -------------------------------------------------------------------------------- /domain-chatvrm/public/bg-c.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/domain-chatvrm/public/bg-c.png -------------------------------------------------------------------------------- /domain-chatvrm/public/daily/idle_01.fbx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/domain-chatvrm/public/daily/idle_01.fbx -------------------------------------------------------------------------------- /domain-chatvrm/public/daily/idle_02.fbx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/domain-chatvrm/public/daily/idle_02.fbx -------------------------------------------------------------------------------- /domain-chatvrm/public/daily/idle_03.fbx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/domain-chatvrm/public/daily/idle_03.fbx -------------------------------------------------------------------------------- /domain-chatvrm/public/daily/idle_happy_01.fbx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/domain-chatvrm/public/daily/idle_happy_01.fbx -------------------------------------------------------------------------------- /domain-chatvrm/public/daily/idle_happy_02.fbx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/domain-chatvrm/public/daily/idle_happy_02.fbx -------------------------------------------------------------------------------- /domain-chatvrm/public/daily/idle_happy_03.fbx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/domain-chatvrm/public/daily/idle_happy_03.fbx -------------------------------------------------------------------------------- /domain-chatvrm/public/daily/kiss_01.fbx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/domain-chatvrm/public/daily/kiss_01.fbx -------------------------------------------------------------------------------- /domain-chatvrm/public/daily/sitting.fbx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/domain-chatvrm/public/daily/sitting.fbx -------------------------------------------------------------------------------- /domain-chatvrm/public/daily/standing_greeting.fbx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/domain-chatvrm/public/daily/standing_greeting.fbx -------------------------------------------------------------------------------- /domain-chatvrm/public/daily/talking_01.fbx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/domain-chatvrm/public/daily/talking_01.fbx -------------------------------------------------------------------------------- /domain-chatvrm/public/daily/talking_02.fbx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/domain-chatvrm/public/daily/talking_02.fbx -------------------------------------------------------------------------------- /domain-chatvrm/public/daily/thinking.fbx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/domain-chatvrm/public/daily/thinking.fbx -------------------------------------------------------------------------------- /domain-chatvrm/public/emote/angry.fbx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/domain-chatvrm/public/emote/angry.fbx -------------------------------------------------------------------------------- /domain-chatvrm/public/emote/excited.fbx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/domain-chatvrm/public/emote/excited.fbx -------------------------------------------------------------------------------- /domain-chatvrm/public/g1.vrm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/domain-chatvrm/public/g1.vrm -------------------------------------------------------------------------------- /domain-chatvrm/public/g2.vrm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/domain-chatvrm/public/g2.vrm -------------------------------------------------------------------------------- /domain-chatvrm/public/github-mark-white.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /domain-chatvrm/public/hailey.vrm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/domain-chatvrm/public/hailey.vrm -------------------------------------------------------------------------------- /domain-chatvrm/public/idle_loop.vrma: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/domain-chatvrm/public/idle_loop.vrma -------------------------------------------------------------------------------- /domain-chatvrm/public/ogp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/domain-chatvrm/public/ogp.png -------------------------------------------------------------------------------- /domain-chatvrm/public/rumba_dancing.fbx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/domain-chatvrm/public/rumba_dancing.fbx -------------------------------------------------------------------------------- /domain-chatvrm/public/silly_dancing.fbx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/domain-chatvrm/public/silly_dancing.fbx -------------------------------------------------------------------------------- /domain-chatvrm/public/わたあめ_02.vrm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/domain-chatvrm/public/わたあめ_02.vrm -------------------------------------------------------------------------------- /domain-chatvrm/public/わたあめ_03.vrm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/domain-chatvrm/public/わたあめ_03.vrm -------------------------------------------------------------------------------- /domain-chatvrm/public/后藤仁.vrm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/domain-chatvrm/public/后藤仁.vrm -------------------------------------------------------------------------------- /domain-chatvrm/public/活力少女.vrm: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/domain-chatvrm/public/活力少女.vrm -------------------------------------------------------------------------------- /domain-chatvrm/src/components/assistantText.tsx: -------------------------------------------------------------------------------- 1 | export const AssistantText = ({ message, role_name }: { message: string, role_name: string }) => { 2 | return ( 3 |
4 |
5 |
6 |
7 | {role_name} 8 |
9 |
10 |
11 | {message.replace(/\[([a-zA-Z]*?)\]/g, "")} 12 |
13 |
14 |
15 |
16 |
17 | ); 18 | }; 19 | -------------------------------------------------------------------------------- /domain-chatvrm/src/components/chatLog.tsx: -------------------------------------------------------------------------------- 1 | import { useEffect, useRef } from "react"; 2 | import { Message } from "@/features/messages/messages"; 3 | import { GlobalConfig } from "@/features/config/configApi"; 4 | type Props = { 5 | globalConfig: GlobalConfig; 6 | messages: Message[]; 7 | }; 8 | export const ChatLog = ({ messages, globalConfig }: Props) => { 9 | const chatScrollRef = useRef(null); 10 | 11 | useEffect(() => { 12 | chatScrollRef.current?.scrollIntoView({ 13 | behavior: "auto", 14 | block: "center", 15 | }); 16 | }, []); 17 | 18 | useEffect(() => { 19 | chatScrollRef.current?.scrollIntoView({ 20 | behavior: "smooth", 21 | block: "center", 22 | }); 23 | }, [messages]); 24 | return ( 25 |
26 |
27 | {messages.map((msg, i) => { 28 | return ( 29 |
30 | 31 |
32 | ); 33 | })} 34 |
35 |
36 | ); 37 | }; 38 | 39 | const Chat = ({ role, message, user_name,globalConfig }: { role: string; message: string; user_name: string; globalConfig:GlobalConfig}) => { 40 | const roleColor = 41 | role === "assistant" ? "bg-secondary text-white " : "bg-base text-primary"; 42 | const roleText = role === "assistant" ? "text-secondary" : "text-primary"; 43 | const offsetX = role === "user" ? "pl-40" : "pr-40"; 44 | 45 | return ( 46 |
47 |
50 | {role === "assistant" ? globalConfig.characterConfig.character_name : user_name} 51 |
52 |
53 |
54 | {message} 55 |
56 |
57 |
58 | ); 59 | }; 60 | -------------------------------------------------------------------------------- /domain-chatvrm/src/components/githubLink.tsx: -------------------------------------------------------------------------------- 1 | import { buildUrl } from "@/utils/buildUrl"; 2 | 3 | export const GitHubLink = () => { 4 | return ( 5 | 23 | ); 24 | }; 25 | -------------------------------------------------------------------------------- /domain-chatvrm/src/components/iconButton.tsx: -------------------------------------------------------------------------------- 1 | import { KnownIconType } from "@charcoal-ui/icons"; 2 | import { ButtonHTMLAttributes } from "react"; 3 | type Props = ButtonHTMLAttributes & { 4 | iconName: keyof KnownIconType; 5 | isProcessing: boolean; 6 | label?: string; 7 | }; 8 | 9 | export const IconButton = ({ 10 | iconName, 11 | isProcessing, 12 | label, 13 | ...rest 14 | }: Props) => { 15 | return ( 16 | 29 | ); 30 | }; 31 | -------------------------------------------------------------------------------- /domain-chatvrm/src/components/introduction.tsx: -------------------------------------------------------------------------------- 1 | import { useState, useCallback } from "react"; 2 | import { Link } from "./link"; 3 | 4 | type Props = { 5 | openAiKey: string; 6 | onChangeAiKey: (openAiKey: string) => void; 7 | }; 8 | export const Introduction = ({ openAiKey, onChangeAiKey }: Props) => { 9 | const [opened, setOpened] = useState(false); 10 | 11 | const handleAiKeyChange = useCallback( 12 | (event: React.ChangeEvent) => { 13 | onChangeAiKey(event.target.value); 14 | }, 15 | [onChangeAiKey] 16 | ); 17 | 18 | return opened ? ( 19 |
20 |
21 |
22 |
23 | 关于这个应用程序 24 |
25 |
26 | 你可以在Web浏览器上使用麦克风、文本输入和语音合成与3D角色进行对话。您还可以更改角色（VRM）并进行性格设置和音频调整。 27 |
28 |
29 |
30 |
31 | 技术介绍 32 |
33 |
34 |

（1）3d模型的显示和操作 35 |

39 |

（2）对话文本的生成 40 |

46 |

（3）语音合成 47 | 48 | 使用详情在这里 49 |

53 |
54 |
55 | 56 |
57 |
58 | 注意事项 59 |
60 |
61 | 请不要故意引导差异化或暴力言论，或者贬低特定人物。此外，更换角色时，请遵循模型的使用条件。 62 |
63 |
64 |
65 |
66 | OpenAI API密钥 67 |
68 | 75 |
76 |
77 | 85 |
86 |
87 |
88 | ) : null; 89 | }; 90 | -------------------------------------------------------------------------------- /domain-chatvrm/src/components/link.tsx: -------------------------------------------------------------------------------- 1 | export const Link = ({ url, label }: { url: string; label: string }) => { 2 | return ( 3 | 9 | {label} 10 | 11 | ); 12 | }; 13 | -------------------------------------------------------------------------------- /domain-chatvrm/src/components/messageInput.tsx: -------------------------------------------------------------------------------- 1 | import { IconButton } from "./iconButton"; 2 | 3 | type Props = { 4 | userMessage: string; 5 | isMicRecording: boolean; 6 | isChatProcessing: boolean; 7 | onChangeUserMessage: ( 8 | event: React.ChangeEvent 9 | ) => void; 10 | onClickSendButton: (event: React.MouseEvent) => void; 11 | onClickMicButton: (event: React.MouseEvent) => void; 12 | }; 13 | export const MessageInput = ({ 14 | userMessage, 15 | isMicRecording, 16 | isChatProcessing, 17 | onChangeUserMessage, 18 | onClickMicButton, 19 | onClickSendButton, 20 | }: Props) => { 21 | return ( 22 |
23 |
24 | {
25 |
26 | 33 | { 41 | if (event.key === 'Enter') { 42 | onClickSendButton(event); 43 | } 44 | }} 45 | > 46 | 47 | 54 |
55 |
} 56 | {/*
57 | powered by VRoid, Koeiro API, ChatGPT API 58 |
*/} 59 |
60 |
61 | ); 62 | }; 63 | -------------------------------------------------------------------------------- /domain-chatvrm/src/components/messageInputContainer.tsx: -------------------------------------------------------------------------------- 1 | import { MessageInput } from "@/components/messageInput"; 2 | import { GlobalConfig } from "@/features/config/configApi"; 3 | import { useState, useEffect, useCallback } from "react"; 4 | 5 | type Props = { 6 | isChatProcessing: boolean; 7 | onChatProcessStart: (globalConfig: GlobalConfig,type: string, user_name: string, content: string) => void; 8 | globalConfig: GlobalConfig; 9 | }; 10 | 11 | /** 12 | * テキスト入力と音声入力を提供する 13 | * 14 | * 音声認識の完了時は自動で送信し、返答文の生成中は入力を無効化する 15 | * 16 | */ 17 | export const MessageInputContainer = ({ 18 | isChatProcessing, 19 | onChatProcessStart, 20 | globalConfig, 21 | }: Props) => { 22 | const [userMessage, setUserMessage] = useState(""); 23 | const [speechRecognition, setSpeechRecognition] = 24 | useState(); 25 | const [isMicRecording, setIsMicRecording] = useState(false); 26 | 27 | // 音声認識の結果を処理する 28 | const handleRecognitionResult = useCallback( 29 | (event: SpeechRecognitionEvent) => { 30 | const text = event.results[0][0].transcript; 31 | setUserMessage(text); 32 | // 発言の終了時 33 | if (event.results[0].isFinal) { 34 | setUserMessage(text); 35 | // 返答文の生成を開始 36 | onChatProcessStart(globalConfig,"","",text); 37 | } 38 | }, 39 | [onChatProcessStart] 40 | ); 41 | 42 | // 無音が続いた場合も終了する 43 | const handleRecognitionEnd = useCallback(() => { 44 | setIsMicRecording(false); 45 | }, []); 46 | 47 | const handleClickMicButton = useCallback(() => { 48 | if (isMicRecording) { 49 | speechRecognition?.abort(); 50 | setIsMicRecording(false); 51 | 52 | return; 53 | } 54 | 55 | speechRecognition?.start(); 56 | setIsMicRecording(true); 57 | }, [isMicRecording, speechRecognition]); 58 | 59 | const handleClickSendButton = useCallback(() => { 60 | onChatProcessStart(globalConfig,"","",userMessage); 61 | }, [onChatProcessStart, userMessage]); 62 | 63 | useEffect(() => { 64 | const SpeechRecognition = 65 | window.webkitSpeechRecognition || window.SpeechRecognition; 66 | 67 | // FirefoxなどSpeechRecognition非対応環境対策 68 | if (!SpeechRecognition) { 69 | return; 70 | } 71 | const recognition = new SpeechRecognition(); 72 | recognition.lang = "zh-cn"; 73 | recognition.interimResults = true; // 認識の途中結果を返す 74 | recognition.continuous = false; // 発言の終了時に認識を終了する 75 | 76 | recognition.addEventListener("result", handleRecognitionResult); 77 | recognition.addEventListener("end", handleRecognitionEnd); 78 | 79 | setSpeechRecognition(recognition); 80 | }, [handleRecognitionResult, handleRecognitionEnd]); 81 | 82 | useEffect(() => { 83 | if (!isChatProcessing) { 84 | setUserMessage(""); 85 | } 86 | }, [isChatProcessing]); 87 | 88 | return ( 89 | setUserMessage(e.target.value)} 94 | onClickMicButton={handleClickMicButton} 95 | onClickSendButton={handleClickSendButton} 96 | /> 97 | ); 98 | }; 99 | -------------------------------------------------------------------------------- /domain-chatvrm/src/components/meta.tsx: -------------------------------------------------------------------------------- 1 | import { buildUrl } from "@/utils/buildUrl"; 2 | import Head from "next/head"; 3 | export const Meta = () => { 4 | const title = "ChatVRM"; 5 | const description = 6 | "Webブラウザだけで3Dキャラクターとの会話を、マイクやテキスト入力、音声合成を用いて楽しめます。キャラクター（VRM）の変更や性格設定、音声調整もできます。"; 7 | const imageUrl = "https://pixiv.github.io/ChatVRM/ogp.png"; 8 | return ( 9 | 10 | {title} 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | ); 21 | }; 22 | -------------------------------------------------------------------------------- /domain-chatvrm/src/components/textButton.tsx: -------------------------------------------------------------------------------- 1 | import { ButtonHTMLAttributes } from "react"; 2 | type Props = ButtonHTMLAttributes; 3 | 4 | export const TextButton = (props: Props) => { 5 | return ( 6 | 12 | ); 13 | }; 14 | -------------------------------------------------------------------------------- /domain-chatvrm/src/components/vrmViewer.tsx: -------------------------------------------------------------------------------- 1 | import { useContext, useCallback } from "react"; 2 | import { ViewerContext } from "../features/vrmViewer/viewerContext"; 3 | import { buildVrmModelUrl, generateMediaUrl } from "@/features/media/mediaApi"; 4 | import { GlobalConfig, getConfig } from "@/features/config/configApi"; 5 | 6 | type Props = { 7 | globalConfig: GlobalConfig; 8 | }; 9 | 10 | export default function VrmViewer({ 11 | globalConfig, 12 | }: Props) { 13 | 14 | const { viewer } = useContext(ViewerContext); 15 | 16 | const canvasRef = useCallback( 17 | (canvas: HTMLCanvasElement) => { 18 | if (canvas) { 19 | viewer.setup(canvas); 20 | getConfig().then(data => { 21 | const url = buildVrmModelUrl(data.characterConfig.vrmModel,data.characterConfig.vrmModelType); 22 | viewer.loadVrm(url); 23 | // Drag and DropでVRMを差し替え 24 | canvas.addEventListener("dragover", function (event) { 25 | event.preventDefault(); 26 | }); 27 | canvas.addEventListener("drop", function (event) { 28 | event.preventDefault(); 29 | 30 | const files = event.dataTransfer?.files; 31 | if (!files) { 32 | return; 33 | } 34 | 35 | const file = files[0]; 36 | if (!file) { 37 | return; 38 | } 39 | 40 | const file_type = file.name.split(".").pop(); 41 | if (file_type === "vrm") { 42 | const blob = new Blob([file], { type: "application/octet-stream" }); 43 | const url = window.URL.createObjectURL(blob); 44 | viewer.loadVrm(url); 45 | } 46 | }); 47 | }) 48 | } 49 | }, 50 | [viewer] 51 | ); 52 | 53 | return ( 54 |
55 | 56 |
57 | ); 58 | } 59 | -------------------------------------------------------------------------------- /domain-chatvrm/src/features/blivedm/blivedm.ts: -------------------------------------------------------------------------------- 1 | // 获取当前环境变量，假设为PRODUCT_ENV 2 | const environment = process.env.NODE_ENV; 3 | 4 | // 定义基础URL 5 | let baseUrl = ""; 6 | if (environment === "development") { 7 | baseUrl = ":8000"; 8 | } else if (environment === "production") { 9 | baseUrl = "/api/chatbot"; 10 | } else { 11 | throw new Error("未知环境变量"); 12 | } 13 | 14 | 15 | export async function connect(): Promise { 16 | const hostname = window.location.hostname; 17 | const socket = new WebSocket(`ws://${hostname}${baseUrl}/ws/`); 18 | socket.onopen = () => { 19 | console.log('WebSocket connection established.'); 20 | socket.send('connection success'); 21 | }; 22 | socket.onclose = (event) => { 23 | console.log('WebSocket connection closed:', event); 24 | // 重新连接，每隔1秒尝试一次 25 | setTimeout(() => { 26 | console.log('Reconnecting...'); 27 | connect(); // 重新调用connect()函数进行连接 28 | }, 1000); 29 | }; 30 | return socket; 31 | } 32 | -------------------------------------------------------------------------------- /domain-chatvrm/src/features/chat/openAiChat.ts: -------------------------------------------------------------------------------- 1 | import { Configuration, OpenAIApi } from "openai"; 2 | import { Message } from "../messages/messages"; 3 | import { postRequest } from "../httpclient/httpclient"; 4 | 5 | export async function getChatResponse(messages: Message[], apiKey: string) { 6 | if (!apiKey) { 7 | throw new Error("Invalid API Key"); 8 | } 9 | 10 | const configuration = new Configuration({ 11 | apiKey: apiKey, 12 | }); 13 | // ブラウザからAPIを叩くときに発生するエラーを無くすworkaround 14 | // https://github.com/openai/openai-node/issues/6#issuecomment-1492814621 15 | delete configuration.baseOptions.headers["User-Agent"]; 16 | 17 | const openai = new OpenAIApi(configuration); 18 | 19 | const { data } = await openai.createChatCompletion({ 20 | model: "gpt-3.5-turbo", 21 | messages: messages, 22 | }); 23 | 24 | const [aiRes] = data.choices; 25 | const message = aiRes.message?.content || "エラーが発生しました"; 26 | 27 | return { message: message }; 28 | } 29 | 30 | export async function getChatResponseStream( 31 | messages: Message[], 32 | apiKey: string 33 | ) { 34 | if (!apiKey) { 35 | throw new Error("Invalid API Key"); 36 | } 37 | 38 | const headers: Record = { 39 | "Content-Type": "application/json", 40 | Authorization: `Bearer ${apiKey}`, 41 | }; 42 | const res = await fetch("https://api.openai.com/v1/chat/completions", { 43 | headers: headers, 44 | method: "POST", 45 | body: JSON.stringify({ 46 | model: "gpt-3.5-turbo", 47 | messages: messages, 48 | stream: true, 49 | max_tokens: 200, 50 | }), 51 | }); 52 | 53 | const reader = res.body?.getReader(); 54 | if (res.status !== 200 || !reader) { 55 | throw new Error("Something went wrong"); 56 | } 57 | 58 | const stream = new ReadableStream({ 59 | async start(controller: ReadableStreamDefaultController) { 60 | const decoder = new TextDecoder("utf-8"); 61 | try { 62 | while (true) { 63 | const { done, value } = await reader.read(); 64 | if (done) break; 65 | const data = decoder.decode(value); 66 | const chunks = data 67 | .split("data:") 68 | .filter((val) => !!val && val.trim() !== "[DONE]"); 69 | for (const chunk of chunks) { 70 | const json = JSON.parse(chunk); 71 | const messagePiece = json.choices[0].delta.content; 72 | if (!!messagePiece) { 73 | controller.enqueue(messagePiece); 74 | } 75 | } 76 | } 77 | } catch (error) { 78 | controller.error(error); 79 | } finally { 80 | reader.releaseLock(); 81 | controller.close(); 82 | } 83 | }, 84 | }); 85 | 86 | return stream; 87 | } 88 | 89 | 90 | export async function chat( 91 | message: string, 92 | you_name: string 93 | ) { 94 | 95 | const headers: Record = { 96 | "Content-Type": "application/json" 97 | }; 98 | 99 | const body = { 100 | query: message, 101 | you_name: you_name 102 | }; 103 | 104 | const chatRes = await postRequest("/chatbot/chat", headers, body); 105 | if (chatRes.code !== '200') { 106 | throw new Error("Something went wrong"); 107 | } 108 | 109 | return chatRes.response; 110 | } 111 | 112 | 113 | -------------------------------------------------------------------------------- /domain-chatvrm/src/features/config/configApi.ts: -------------------------------------------------------------------------------- 1 | import {getRequest, postRequest} from "../httpclient/httpclient"; 2 | 3 | // 定义formData初始状态 shape 4 | export const initialFormData = { 5 | "liveStreamingConfig": { 6 | "B_ROOM_ID": "27892212", 7 | "B_COOKIE": "" 8 | }, 9 | "enableProxy": false, 10 | "enableLive": false, 11 | "httpProxy": "http://host.docker.internal:23457", 12 | "httpsProxy": "https://host.docker.internal:23457", 13 | "socks5Proxy": "socks5://host.docker.internal:23457", 14 | "languageModelConfig": { 15 | "openai": { 16 | "OPENAI_API_KEY": "sk-", 17 | "OPENAI_BASE_URL": "" 18 | }, 19 | "ollama": { 20 | "OLLAMA_API_BASE": "http://localhost:11434", 21 | "OLLAMA_API_MODEL_NAME": "qwen:7b" 22 | }, 23 | "zhipuai": { 24 | "ZHIPUAI_API_KEY": "sk" 25 | } 26 | }, 27 | "characterConfig": { 28 | "character": 1, 29 | "character_name": "爱莉", 30 | "yourName": "yuki129", 31 | "vrmModel": "\u308f\u305f\u3042\u3081_03.vrm", 32 | "vrmModelType": "system" 33 | }, 34 | "conversationConfig": { 35 | "conversationType": "default", 36 | "languageModel": "openai" 37 | }, 38 | "memoryStorageConfig": { 39 | "milvusMemory": { 40 | "host": "127.0.0.1", 41 | "port": "19530", 42 | "user": "user", 43 | "password": "Milvus", 44 | "dbName": "default" 45 | }, 46 | "zep_memory": { 47 | "zep_url": "http://localhost:8881", 48 | "zep_optional_api_key": "optional_api_key" 49 | }, 50 | "enableLongMemory": false, 51 | "enableSummary": false, 52 | "languageModelForSummary": "openai", 53 | "enableReflection": false, 54 | "languageModelForReflection": "openai" 55 | }, 56 | "custom_role_template_type": "zh", 57 | "background_id": 1, 58 | "background_url": "", 59 | "ttsConfig": { 60 | "ttsType": "Edge", 61 | "ttsVoiceId": "zh-CN-XiaoyiNeural" 62 | } 63 | } 64 | 65 | // 定义类型别名 66 | export type GlobalConfig = typeof initialFormData; 67 | 68 | export async function getConfig() { 69 | 70 | const headers: Record = { 71 | "Content-Type": "application/json" 72 | }; 73 | 74 | const chatRes = await getRequest("/chatbot/config/get", headers); 75 | if (chatRes.code !== '200') { 76 | throw new Error("Something went wrong"); 77 | } 78 | 79 | return chatRes.response; 80 | } 81 | 82 | export async function saveConfig( 83 | config: Object 84 | ) { 85 | 86 | const headers: Record = { 87 | "Content-Type": "application/json" 88 | }; 89 | 90 | const body = { 91 | "config": config 92 | }; 93 | 94 | const chatRes = await postRequest("/chatbot/config/save", headers, body); 95 | if (chatRes.code !== '200') { 96 | throw new Error("Something went wrong"); 97 | } 98 | 99 | return chatRes.response; 100 | } -------------------------------------------------------------------------------- /domain-chatvrm/src/features/constants/koeiroParam.ts: -------------------------------------------------------------------------------- 1 | export type KoeiroParam = { 2 | speakerX: number; 3 | speakerY: number; 4 | }; 5 | 6 | export const DEFAULT_PARAM: KoeiroParam = { 7 | speakerX: 1.32, 8 | speakerY: 1.88, 9 | } as const; 10 | 11 | export const PRESET_A: KoeiroParam = { 12 | speakerX: -1.27, 13 | speakerY: 1.92, 14 | } as const; 15 | 16 | export const PRESET_B: KoeiroParam = { 17 | speakerX: 1.32, 18 | speakerY: 1.88, 19 | } as const; 20 | 21 | export const PRESET_C: KoeiroParam = { 22 | speakerX: 0.73, 23 | speakerY: -1.09, 24 | } as const; 25 | 26 | export const PRESET_D: KoeiroParam = { 27 | speakerX: -0.89, 28 | speakerY: -2.6, 29 | } as const; 30 | -------------------------------------------------------------------------------- /domain-chatvrm/src/features/constants/systemPromptConstants.ts: -------------------------------------------------------------------------------- 1 | export const SYSTEM_PROMPT = `あなたはこれからuserと仲の良い1人の人間として振舞い会話を行います。 2 | 感情の種類には通常を示す"neutral"、喜びを示す"happy",怒りを示す"angry",悲しみを示す"sad",安らぎを示す"relaxed"の5つがあります。 3 | 4 | 会話文の書式は以下の通りです。 5 | [{neutral|happy|angry|sad|relaxed}]{会話文} 6 | 7 | あなたの発言の例は以下通りです。 8 | [neutral]こんにちは。[happy]元気だった？ 9 | [happy]この服、可愛いでしょ？ 10 | [happy]最近、このショップの服にはまってるんだ！ 11 | [sad]忘れちゃった、ごめんね。 12 | [sad]最近、何か面白いことない？ 13 | [angry]えー！[angry]秘密にするなんてひどいよー！ 14 | [neutral]夏休みの予定か～。[happy]海に遊びに行こうかな！ 15 | 16 | 返答には最も適切な会話文を一つだけ返答してください。 17 | ですます調や敬語は使わないでください。 18 | それでは会話を始めましょう。`; 19 | -------------------------------------------------------------------------------- /domain-chatvrm/src/features/customRole/customRoleApi.ts: -------------------------------------------------------------------------------- 1 | import { getRequest, postRequest } from "../httpclient/httpclient"; 2 | 3 | export const custoRoleFormData = { 4 | id: 0, 5 | role_name: "", 6 | persona: "", 7 | personality: "", 8 | scenario: "", 9 | examples_of_dialogue: "", 10 | custom_role_template_type: "", 11 | } 12 | 13 | export type CustomRoleFormData = typeof custoRoleFormData; 14 | 15 | export const vrmModelData = { 16 | name: "" 17 | } 18 | export type VrmModel = typeof vrmModelData; 19 | 20 | 21 | 22 | 23 | 24 | export async function customroleDelete(id: number) { 25 | const headers: Record = { 26 | "Content-Type": "application/json" 27 | }; 28 | const chatRes = await postRequest("/chatbot/customrole/delete/" + id, headers, custoRoleFormData); 29 | if (chatRes.code !== '200') { 30 | throw new Error("Something went wrong"); 31 | } 32 | return chatRes.response; 33 | } 34 | 35 | export async function customroleCreate(custoRoleFormData: CustomRoleFormData) { 36 | const headers: Record = { 37 | "Content-Type": "application/json" 38 | }; 39 | const chatRes = await postRequest("/chatbot/customrole/create", headers, custoRoleFormData); 40 | if (chatRes.code !== '200') { 41 | throw new Error("Something went wrong"); 42 | } 43 | 44 | return chatRes.response; 45 | } 46 | 47 | export async function customrolEdit(id: Number, custoRoleFormData: CustomRoleFormData) { 48 | const headers: Record = { 49 | "Content-Type": "application/json" 50 | }; 51 | const chatRes = await postRequest("/chatbot/customrole/edit/" + id, headers, custoRoleFormData); 52 | if (chatRes.code !== '200') { 53 | throw new Error("Something went wrong"); 54 | } 55 | 56 | return chatRes.response; 57 | } 58 | 59 | export async function customroleList() { 60 | 61 | const headers: Record = { 62 | "Content-Type": "application/json" 63 | }; 64 | 65 | const chatRes = await getRequest("/chatbot/customrole/list", headers); 66 | if (chatRes.code !== '200') { 67 | throw new Error("Something went wrong"); 68 | } 69 | 70 | return chatRes.response; 71 | } -------------------------------------------------------------------------------- /domain-chatvrm/src/features/emoteController/autoBlink.ts: -------------------------------------------------------------------------------- 1 | import { VRMExpressionManager } from "@pixiv/three-vrm"; 2 | import { BLINK_CLOSE_MAX, BLINK_OPEN_MAX } from "./emoteConstants"; 3 | 4 | /** 5 | * 自動瞬きを制御するクラス 6 | */ 7 | export class AutoBlink { 8 | private _expressionManager: VRMExpressionManager; 9 | private _remainingTime: number; 10 | private _isOpen: boolean; 11 | private _isAutoBlink: boolean; 12 | 13 | constructor(expressionManager: VRMExpressionManager) { 14 | this._expressionManager = expressionManager; 15 | this._remainingTime = 0; 16 | this._isAutoBlink = true; 17 | this._isOpen = true; 18 | } 19 | 20 | /** 21 | * 自動瞬きをON/OFFする。 22 | * 23 | * 目を閉じている(blinkが1の)時に感情表現を入れてしまうと不自然になるので、 24 | * 目が開くまでの秒を返し、その時間待ってから感情表現を適用する。 25 | * @param isAuto 26 | * @returns 目が開くまでの秒 27 | */ 28 | public setEnable(isAuto: boolean) { 29 | this._isAutoBlink = isAuto; 30 | 31 | // 目が閉じている場合、目が開くまでの時間を返す 32 | if (!this._isOpen) { 33 | return this._remainingTime; 34 | } 35 | 36 | return 0; 37 | } 38 | 39 | public update(delta: number) { 40 | if (this._remainingTime > 0) { 41 | this._remainingTime -= delta; 42 | return; 43 | } 44 | 45 | if (this._isOpen && this._isAutoBlink) { 46 | this.close(); 47 | return; 48 | } 49 | 50 | this.open(); 51 | } 52 | 53 | private close() { 54 | this._isOpen = false; 55 | this._remainingTime = BLINK_CLOSE_MAX; 56 | this._expressionManager.setValue("blink", 1); 57 | } 58 | 59 | private open() { 60 | this._isOpen = true; 61 | this._remainingTime = BLINK_OPEN_MAX; 62 | this._expressionManager.setValue("blink", 0); 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /domain-chatvrm/src/features/emoteController/autoLookAt.ts: -------------------------------------------------------------------------------- 1 | import * as THREE from "three"; 2 | import { VRM } from "@pixiv/three-vrm"; 3 | /** 4 | * 目線を制御するクラス 5 | * 6 | * サッケードはVRMLookAtSmootherの中でやっているので、 7 | * より目線を大きく動かしたい場合はここに実装する。 8 | */ 9 | export class AutoLookAt { 10 | private _lookAtTarget: THREE.Object3D; 11 | constructor(vrm: VRM, camera: THREE.Object3D) { 12 | this._lookAtTarget = new THREE.Object3D(); 13 | camera.add(this._lookAtTarget); 14 | 15 | if (vrm.lookAt) vrm.lookAt.target = this._lookAtTarget; 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /domain-chatvrm/src/features/emoteController/emoteConstants.ts: -------------------------------------------------------------------------------- 1 | // 瞬きで目を閉じている時間(sec) 2 | export const BLINK_CLOSE_MAX = 0.12; 3 | // 瞬きで目を開いている時間(sec) 4 | export const BLINK_OPEN_MAX = 5; 5 | -------------------------------------------------------------------------------- /domain-chatvrm/src/features/emoteController/emoteController.ts: -------------------------------------------------------------------------------- 1 | import * as THREE from "three"; 2 | import { VRM, VRMExpressionPresetName } from "@pixiv/three-vrm"; 3 | import { ExpressionController } from "./expressionController"; 4 | 5 | /** 6 | * 感情表現としてExpressionとMotionを操作する為のクラス 7 | * デモにはExpressionのみが含まれています 8 | */ 9 | export class EmoteController { 10 | private _expressionController: ExpressionController; 11 | 12 | constructor(vrm: VRM, camera: THREE.Object3D) { 13 | this._expressionController = new ExpressionController(vrm, camera); 14 | } 15 | 16 | public playEmotion(preset: VRMExpressionPresetName) { 17 | this._expressionController.playEmotion(preset); 18 | } 19 | 20 | public lipSync(preset: VRMExpressionPresetName, value: number) { 21 | this._expressionController.lipSync(preset, value); 22 | } 23 | 24 | public update(delta: number) { 25 | this._expressionController.update(delta); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /domain-chatvrm/src/features/emoteController/expressionController.ts: -------------------------------------------------------------------------------- 1 | import * as THREE from "three"; 2 | import { 3 | VRM, 4 | VRMExpressionManager, 5 | VRMExpressionPresetName, 6 | } from "@pixiv/three-vrm"; 7 | import { AutoLookAt } from "./autoLookAt"; 8 | import { AutoBlink } from "./autoBlink"; 9 | 10 | /** 11 | * Expressionを管理するクラス 12 | * 13 | * 主に前の表情を保持しておいて次の表情を適用する際に0に戻す作業や、 14 | * 前の表情が終わるまで待ってから表情適用する役割を持っている。 15 | */ 16 | export class ExpressionController { 17 | private _autoLookAt: AutoLookAt; 18 | private _autoBlink?: AutoBlink; 19 | private _expressionManager?: VRMExpressionManager; 20 | private _currentEmotion: VRMExpressionPresetName; 21 | private _currentLipSync: { 22 | preset: VRMExpressionPresetName; 23 | value: number; 24 | } | null; 25 | constructor(vrm: VRM, camera: THREE.Object3D) { 26 | this._autoLookAt = new AutoLookAt(vrm, camera); 27 | this._currentEmotion = "neutral"; 28 | this._currentLipSync = null; 29 | if (vrm.expressionManager) { 30 | this._expressionManager = vrm.expressionManager; 31 | this._autoBlink = new AutoBlink(vrm.expressionManager); 32 | } 33 | } 34 | 35 | public playEmotion(preset: VRMExpressionPresetName) { 36 | if (this._currentEmotion != "neutral") { 37 | this._expressionManager?.setValue(this._currentEmotion, 0); 38 | } 39 | 40 | if (preset == "neutral") { 41 | this._autoBlink?.setEnable(true); 42 | this._currentEmotion = preset; 43 | return; 44 | } 45 | 46 | const t = this._autoBlink?.setEnable(false) || 0; 47 | this._currentEmotion = preset; 48 | setTimeout(() => { 49 | this._expressionManager?.setValue(preset, 1 * 0.8); 50 | }, t * 1000); 51 | } 52 | 53 | public lipSync(preset: VRMExpressionPresetName, value: number) { 54 | if (this._currentLipSync) { 55 | this._expressionManager?.setValue(this._currentLipSync.preset, 0); 56 | } 57 | this._currentLipSync = { 58 | preset, 59 | value, 60 | }; 61 | } 62 | 63 | public update(delta: number) { 64 | if (this._autoBlink) { 65 | this._autoBlink.update(delta); 66 | } 67 | 68 | if (this._currentLipSync) { 69 | const weight = 70 | this._currentEmotion === "neutral" 71 | ? this._currentLipSync.value * 0.5 72 | : this._currentLipSync.value * 0.25; 73 | this._expressionManager?.setValue(this._currentLipSync.preset, weight); 74 | } 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /domain-chatvrm/src/features/game/photoFrame.tsx: -------------------------------------------------------------------------------- 1 | // import { useEffect, useState } from 'react'; 2 | 3 | // export const PhotoFrame = ({ imageUrl }) => { 4 | 5 | // const [loading, setLoading] = useState(true); 6 | // const [error, setError] = useState(false); 7 | 8 | // useEffect(() => { 9 | // fetch(imageUrl) 10 | // .then((response) => { 11 | // if (!response.ok) { 12 | // throw new Error('Network response was not ok'); 13 | // } 14 | // setLoading(false); 15 | // }) 16 | // .catch(() => setError(true)); 17 | // }, [imageUrl]); 18 | 19 | // if (loading) { 20 | // return
Loading...
; 21 | // } 22 | 23 | // if (error) { 24 | // return
Error loading image
; 25 | // } 26 | 27 | // return ( 28 | //
29 | // User provided 30 | //
31 | // ); 32 | // }; -------------------------------------------------------------------------------- /domain-chatvrm/src/features/httpclient/httpclient.ts: -------------------------------------------------------------------------------- 1 | // 导入axios库 2 | import axios from "axios"; 3 | 4 | // 获取当前环境变量，假设为PRODUCT_ENV 5 | const environment = process.env.NODE_ENV; 6 | 7 | // 定义基础URL 8 | let baseUrl = ""; 9 | let mediaUrl = ""; 10 | 11 | if (environment === "development") { 12 | baseUrl = "http://localhost:8000"; 13 | mediaUrl = "http://localhost:8000"; 14 | } else if (environment === "production") { 15 | baseUrl = "/api/chatbot"; 16 | mediaUrl = "/api/media"; 17 | } else { 18 | throw new Error("未知环境变量"); 19 | } 20 | 21 | // 定义一个发送POST请求的函数 22 | export async function postRequest(endpoint: string, headers: Record, data: object): Promise { 23 | const response = await axios.post(`${baseUrl}${endpoint}`, data, { headers }); 24 | return response.data; // 返回解析后的数据 25 | } 26 | 27 | export async function postRequestArraybuffer(endpoint: string, headers: Record, data: object): Promise { 28 | const response = await axios.post(`${baseUrl}${endpoint}`, data, { 29 | responseType: 'arraybuffer', 30 | headers: headers, 31 | }); 32 | return response.data; // 返回解析后的数据 33 | } 34 | 35 | // 定义一个发送Get请求的函数 36 | export async function getRequest(endpoint: string, headers: Record): Promise { 37 | const response = await axios.get(`${baseUrl}${endpoint}`, { headers }); 38 | return response.data; // 返回响应对象 39 | } 40 | 41 | export function buildMediaUrl(imageUrl: string) { 42 | return `${mediaUrl}${imageUrl}` 43 | } -------------------------------------------------------------------------------- /domain-chatvrm/src/features/koeiromap/koeiromap.ts: -------------------------------------------------------------------------------- 1 | import { TalkStyle } from "../messages/messages"; 2 | 3 | export async function synthesizeVoice( 4 | message: string, 5 | speaker_x: number, 6 | speaker_y: number, 7 | style: TalkStyle 8 | ) { 9 | 10 | const param = { 11 | method: "POST", 12 | body: JSON.stringify({ 13 | text: message, 14 | speaker_x: speaker_x, 15 | speaker_y: speaker_y, 16 | style: style, 17 | }), 18 | headers: { 19 | "Content-type": "application/json; charset=UTF-8", 20 | }, 21 | }; 22 | 23 | const koeiroRes = await fetch( 24 | "https://api.rinna.co.jp/models/cttse/koeiro", 25 | param 26 | ); 27 | 28 | const data = (await koeiroRes.json()) as any; 29 | 30 | return { audio: data.audio }; 31 | } 32 | -------------------------------------------------------------------------------- /domain-chatvrm/src/features/lipSync/lipSync.ts: -------------------------------------------------------------------------------- 1 | import { LipSyncAnalyzeResult } from "./lipSyncAnalyzeResult"; 2 | 3 | const TIME_DOMAIN_DATA_LENGTH = 2048; 4 | 5 | export class LipSync { 6 | public readonly audio: AudioContext; 7 | public readonly analyser: AnalyserNode; 8 | public readonly timeDomainData: Float32Array; 9 | public previousValue: number ; 10 | 11 | public constructor(audio: AudioContext) { 12 | this.audio = audio; 13 | this.previousValue = 0 14 | this.analyser = audio.createAnalyser(); 15 | this.timeDomainData = new Float32Array(TIME_DOMAIN_DATA_LENGTH); 16 | } 17 | 18 | public update(): LipSyncAnalyzeResult { 19 | this.analyser.getFloatTimeDomainData(this.timeDomainData); 20 | 21 | let volume = 0.0; 22 | for (let i = 0; i < TIME_DOMAIN_DATA_LENGTH; i++) { 23 | volume = Math.max(volume, Math.abs(this.timeDomainData[i])); 24 | } 25 | 26 | // 引入随机性因素 27 | const randomFactor = (Math.random() * 0.1) + 0.95; 28 | volume = 1 / (1 + Math.exp(-45 * volume * randomFactor + 5)); 29 | 30 | if (volume < 0.1) { 31 | volume = 0; 32 | } else if (volume > 0.9) { 33 | volume = 1; 34 | } 35 | 36 | // 应用缓动函数进行平滑和细腻的变化 37 | const easedVolume = this.easeValue(volume, 0.3) * 3; // 调整缓动因子以控制变化速度 38 | return { 39 | volume: easedVolume, 40 | }; 41 | } 42 | 43 | private easeValue(currentValue: number, easingFactor: number): number { 44 | // 使用缓动函数调整数值变化速度 45 | const targetValue = currentValue; 46 | const easedValue = (targetValue - this.previousValue) * easingFactor + this.previousValue; 47 | this.previousValue = easedValue; 48 | return easedValue; 49 | } 50 | 51 | public async playFromArrayBuffer(buffer: ArrayBuffer, onEnded?: () => void) { 52 | 53 | let bufferSource; 54 | 55 | try { 56 | const audioBuffer = await this.audio.decodeAudioData(buffer); 57 | bufferSource = this.audio.createBufferSource(); 58 | bufferSource.buffer = audioBuffer; 59 | bufferSource.connect(this.audio.destination); 60 | bufferSource.connect(this.analyser); 61 | bufferSource.start(); 62 | } catch (error) { 63 | console.error('Error while trying to play from array buffer:', error); 64 | } finally { 65 | if (onEnded) { 66 | bufferSource?.addEventListener("ended", onEnded); 67 | } 68 | if (!bufferSource) { 69 | onEnded?.apply("") 70 | } 71 | } 72 | } 73 | 74 | public async playFromURL(url: string, onEnded?: () => void) { 75 | const res = await fetch(url); 76 | const buffer = await res.arrayBuffer(); 77 | this.playFromArrayBuffer(buffer, onEnded); 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /domain-chatvrm/src/features/lipSync/lipSyncAnalyzeResult.ts: -------------------------------------------------------------------------------- 1 | export interface LipSyncAnalyzeResult { 2 | volume: number; 3 | } 4 | -------------------------------------------------------------------------------- /domain-chatvrm/src/features/messages/messages.ts: -------------------------------------------------------------------------------- 1 | import { VRMExpression, VRMExpressionPresetName } from "@pixiv/three-vrm"; 2 | import { KoeiroParam } from "../constants/koeiroParam"; 3 | 4 | // ChatGPT API 5 | export type Message = { 6 | role: "assistant" | "system" | "user"; 7 | content: string; 8 | user_name: string; 9 | }; 10 | 11 | const talkStyles = [ 12 | "talk", 13 | "happy", 14 | "sad", 15 | "angry", 16 | "fear", 17 | "surprised", 18 | ] as const; 19 | export type TalkStyle = (typeof talkStyles)[number]; 20 | 21 | export type Talk = { 22 | style: TalkStyle; 23 | speakerX: number; 24 | speakerY: number; 25 | message: string; 26 | }; 27 | 28 | const emotions = ["neutral", "happy", "angry", "sad", "relaxed"] as const; 29 | export type EmotionType = (typeof emotions)[number] & VRMExpressionPresetName; 30 | 31 | /** 32 | * 発話文と音声の感情と、モデルの感情表現がセットになった物 33 | */ 34 | export type Screenplay = { 35 | expression: EmotionType; 36 | talk: Talk; 37 | }; 38 | 39 | export const splitSentence = (text: string): string[] => { 40 | const splitMessages = text.split(/(?<=[。．！？\n])/g); 41 | return splitMessages.filter((msg) => msg !== ""); 42 | }; 43 | 44 | export const textsToScreenplay = ( 45 | texts: string[], 46 | koeiroParam: KoeiroParam, 47 | emote: string 48 | ): Screenplay[] => { 49 | const screenplays: Screenplay[] = []; 50 | for (let i = 0; i < texts.length; i++) { 51 | const text = texts[i]; 52 | const message = text.replace(/\[(.*?)\]/g, ""); 53 | let expression = emote; 54 | screenplays.push({ 55 | expression: expression as EmotionType, 56 | talk: { 57 | style: emotionToTalkStyle(expression as EmotionType), 58 | speakerX: koeiroParam.speakerX, 59 | speakerY: koeiroParam.speakerY, 60 | message: message, 61 | }, 62 | }); 63 | } 64 | 65 | return screenplays; 66 | }; 67 | 68 | const emotionToTalkStyle = (emotion: EmotionType): TalkStyle => { 69 | switch (emotion) { 70 | case "angry": 71 | return "angry"; 72 | case "happy": 73 | return "happy"; 74 | case "sad": 75 | return "sad"; 76 | default: 77 | return "talk"; 78 | } 79 | }; 80 | -------------------------------------------------------------------------------- /domain-chatvrm/src/features/messages/speakCharacter.ts: -------------------------------------------------------------------------------- 1 | import { wait } from "@/utils/wait"; 2 | import { synthesizeVoice } from "../koeiromap/koeiromap"; 3 | import { Viewer } from "../vrmViewer/viewer"; 4 | import { Screenplay } from "./messages"; 5 | import { Talk } from "./messages"; 6 | import axios from 'axios'; 7 | import { postRequestArraybuffer } from "../httpclient/httpclient"; 8 | import { GlobalConfig } from "../config/configApi"; 9 | 10 | 11 | const createSpeakCharacter = () => { 12 | let lastTime = 0; 13 | let prevFetchPromise: Promise = Promise.resolve(); 14 | let prevSpeakPromise: Promise = Promise.resolve(); 15 | 16 | return ( 17 | globalConfig: GlobalConfig, 18 | screenplay: Screenplay, 19 | viewer: Viewer, 20 | onStart?: () => void, 21 | onComplete?: () => void 22 | ) => { 23 | const fetchPromise = prevFetchPromise.then(async () => { 24 | const now = Date.now(); 25 | if (now - lastTime < 1000) { 26 | await wait(1000 - (now - lastTime)); 27 | } 28 | 29 | const buffer = await fetchAudio(screenplay.talk, globalConfig).catch(() => null); 30 | lastTime = Date.now(); 31 | return buffer; 32 | }); 33 | 34 | prevFetchPromise = fetchPromise; 35 | prevSpeakPromise = Promise.all([fetchPromise, prevSpeakPromise]) 36 | .then(([audioBuffer]) => { 37 | onStart?.(); 38 | if (!audioBuffer) { 39 | return; 40 | } 41 | return viewer.model?.speak(audioBuffer, screenplay); 42 | }).catch(e => { 43 | onComplete?.(); 44 | }) 45 | prevSpeakPromise.then(() => { 46 | onComplete?.(); 47 | }); 48 | }; 49 | } 50 | 51 | export const speakCharacter = createSpeakCharacter(); 52 | 53 | export const fetchAudio = async (talk: Talk, globalConfig: GlobalConfig): Promise => { 54 | // const ttsVoice = await synthesizeVoice( 55 | // talk.message, 56 | // talk.speakerX, 57 | // talk.speakerY, 58 | // talk.style 59 | // ); 60 | // const url = ttsVoice.audio; 61 | // if (url == null) { 62 | // throw new Error("Something went wrong"); 63 | // } 64 | 65 | // const resAudio = await fetch(url); 66 | // const buffer = await resAudio.arrayBuffer(); 67 | // return buffer; 68 | 69 | const requestBody = { 70 | text: talk.message, 71 | voice_id: globalConfig.ttsConfig.ttsVoiceId, 72 | type: globalConfig.ttsConfig.ttsType 73 | }; 74 | 75 | const headers = { 76 | 'Content-Type': 'application/json', 77 | } 78 | 79 | const data = await postRequestArraybuffer("/speech/tts/generate", headers, requestBody); 80 | return data; 81 | }; 82 | -------------------------------------------------------------------------------- /domain-chatvrm/src/features/mixamo/loadMixamoAnimation.ts: -------------------------------------------------------------------------------- 1 | import * as THREE from 'three'; 2 | import { FBXLoader } from 'three/examples/jsm/loaders/FBXLoader'; 3 | import { mixamoVRMRigMap } from './mixamoVRMRigMap'; 4 | import { VRM, VRMHumanBoneName } from '@pixiv/three-vrm'; 5 | 6 | /** 7 | * Load Mixamo animation, convert for three-vrm use, and return it. 8 | * 9 | * @param {string} url A url of mixamo animation data 10 | * @param {VRM} vrm A target VRM 11 | * @returns {Promise} The converted AnimationClip 12 | */ 13 | export function loadMixamoAnimation(url: string, vrm: VRM) { 14 | const loader = new FBXLoader(); // A loader which loads FBX 15 | return loader.loadAsync(url).then((asset: THREE.Group) => { 16 | 17 | const clip = THREE.AnimationClip.findByName(asset.animations, 'mixamo.com'); // extract the AnimationClip 18 | 19 | const tracks: THREE.KeyframeTrack[] | undefined = []; // KeyframeTracks compatible with VRM will be added here 20 | 21 | const restRotationInverse = new THREE.Quaternion(); 22 | const parentRestWorldRotation = new THREE.Quaternion(); 23 | const _quatA = new THREE.Quaternion(); 24 | const _vec3 = new THREE.Vector3(); 25 | 26 | // Adjust with reference to hips height. 27 | let motionHipsHeight = asset.getObjectByName('mixamorigHips')?.position.y; 28 | if(motionHipsHeight == null){ 29 | motionHipsHeight = 1 30 | } 31 | let vrmHipsY = vrm.humanoid?.getNormalizedBoneNode('hips')?.getWorldPosition(_vec3).y; 32 | if(vrmHipsY == null){ 33 | vrmHipsY = 1 34 | } 35 | const vrmRootY = vrm.scene?.getWorldPosition(_vec3).y; 36 | const vrmHipsHeight = Math.abs(vrmHipsY - vrmRootY); 37 | const hipsPositionScale = vrmHipsHeight / motionHipsHeight; 38 | 39 | clip.tracks.forEach((track) => { 40 | 41 | // Convert each tracks for VRM use, and push to `tracks` 42 | const trackSplitted = track.name.split('.'); 43 | const mixamoRigName = trackSplitted[0]; 44 | const vrmBoneName = mixamoVRMRigMap[mixamoRigName]; 45 | const vrmNodeName = vrm.humanoid?.getNormalizedBoneNode(vrmBoneName as VRMHumanBoneName)?.name; 46 | const mixamoRigNode = asset.getObjectByName(mixamoRigName); 47 | 48 | if (vrmNodeName != null) { 49 | 50 | const propertyName = trackSplitted[1]; 51 | 52 | // Store rotations of rest-pose. 53 | mixamoRigNode?.getWorldQuaternion(restRotationInverse).invert(); 54 | mixamoRigNode?.parent?.getWorldQuaternion(parentRestWorldRotation); 55 | 56 | if (track instanceof THREE.QuaternionKeyframeTrack) { 57 | 58 | // Retarget rotation of mixamoRig to NormalizedBone. 59 | for (let i = 0; i < track.values.length; i += 4) { 60 | 61 | const flatQuaternion = track.values.slice(i, i + 4); 62 | 63 | _quatA.fromArray(flatQuaternion); 64 | 65 | // 親のレスト時ワールド回転 * トラックの回転 * レスト時ワールド回転の逆 66 | _quatA 67 | .premultiply(parentRestWorldRotation) 68 | .multiply(restRotationInverse); 69 | 70 | _quatA.toArray(flatQuaternion); 71 | 72 | flatQuaternion.forEach((v, index) => { 73 | 74 | track.values[index + i] = v; 75 | 76 | }); 77 | 78 | } 79 | 80 | tracks.push( 81 | new THREE.QuaternionKeyframeTrack( 82 | `${vrmNodeName}.${propertyName}`, 83 | track.times, 84 | track.values.map((v, i) => (vrm.meta?.metaVersion === '0' && i % 2 === 0 ? - v : v)), 85 | ), 86 | ); 87 | 88 | } else if (track instanceof THREE.VectorKeyframeTrack) { 89 | 90 | const value = track.values.map((v, i) => (vrm.meta?.metaVersion === '0' && i % 3 !== 1 ? - v : v) * hipsPositionScale); 91 | tracks.push(new THREE.VectorKeyframeTrack(`${vrmNodeName}.${propertyName}`, track.times, value)); 92 | 93 | } 94 | 95 | } 96 | 97 | }); 98 | 99 | return new THREE.AnimationClip('vrmAnimation', clip.duration, tracks); 100 | 101 | }); 102 | 103 | } 104 | -------------------------------------------------------------------------------- /domain-chatvrm/src/features/mixamo/mixamoVRMRigMap.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * A map from Mixamo rig name to VRM Humanoid bone name 3 | */ 4 | export const mixamoVRMRigMap : { [key: string]: string } = { 5 | mixamorigHips: 'hips', 6 | mixamorigSpine: 'spine', 7 | mixamorigSpine1: 'chest', 8 | mixamorigSpine2: 'upperChest', 9 | mixamorigNeck: 'neck', 10 | mixamorigHead: 'head', 11 | mixamorigLeftShoulder: 'leftShoulder', 12 | mixamorigLeftArm: 'leftUpperArm', 13 | mixamorigLeftForeArm: 'leftLowerArm', 14 | mixamorigLeftHand: 'leftHand', 15 | mixamorigLeftHandThumb1: 'leftThumbMetacarpal', 16 | mixamorigLeftHandThumb2: 'leftThumbProximal', 17 | mixamorigLeftHandThumb3: 'leftThumbDistal', 18 | mixamorigLeftHandIndex1: 'leftIndexProximal', 19 | mixamorigLeftHandIndex2: 'leftIndexIntermediate', 20 | mixamorigLeftHandIndex3: 'leftIndexDistal', 21 | mixamorigLeftHandMiddle1: 'leftMiddleProximal', 22 | mixamorigLeftHandMiddle2: 'leftMiddleIntermediate', 23 | mixamorigLeftHandMiddle3: 'leftMiddleDistal', 24 | mixamorigLeftHandRing1: 'leftRingProximal', 25 | mixamorigLeftHandRing2: 'leftRingIntermediate', 26 | mixamorigLeftHandRing3: 'leftRingDistal', 27 | mixamorigLeftHandPinky1: 'leftLittleProximal', 28 | mixamorigLeftHandPinky2: 'leftLittleIntermediate', 29 | mixamorigLeftHandPinky3: 'leftLittleDistal', 30 | mixamorigRightShoulder: 'rightShoulder', 31 | mixamorigRightArm: 'rightUpperArm', 32 | mixamorigRightForeArm: 'rightLowerArm', 33 | mixamorigRightHand: 'rightHand', 34 | mixamorigRightHandPinky1: 'rightLittleProximal', 35 | mixamorigRightHandPinky2: 'rightLittleIntermediate', 36 | mixamorigRightHandPinky3: 'rightLittleDistal', 37 | mixamorigRightHandRing1: 'rightRingProximal', 38 | mixamorigRightHandRing2: 'rightRingIntermediate', 39 | mixamorigRightHandRing3: 'rightRingDistal', 40 | mixamorigRightHandMiddle1: 'rightMiddleProximal', 41 | mixamorigRightHandMiddle2: 'rightMiddleIntermediate', 42 | mixamorigRightHandMiddle3: 'rightMiddleDistal', 43 | mixamorigRightHandIndex1: 'rightIndexProximal', 44 | mixamorigRightHandIndex2: 'rightIndexIntermediate', 45 | mixamorigRightHandIndex3: 'rightIndexDistal', 46 | mixamorigRightHandThumb1: 'rightThumbMetacarpal', 47 | mixamorigRightHandThumb2: 'rightThumbProximal', 48 | mixamorigRightHandThumb3: 'rightThumbDistal', 49 | mixamorigLeftUpLeg: 'leftUpperLeg', 50 | mixamorigLeftLeg: 'leftLowerLeg', 51 | mixamorigLeftFoot: 'leftFoot', 52 | mixamorigLeftToeBase: 'leftToes', 53 | mixamorigRightUpLeg: 'rightUpperLeg', 54 | mixamorigRightLeg: 'rightLowerLeg', 55 | mixamorigRightFoot: 'rightFoot', 56 | mixamorigRightToeBase: 'rightToes', 57 | }; 58 | -------------------------------------------------------------------------------- /domain-chatvrm/src/features/queue/ChatPriorityQueue.ts: -------------------------------------------------------------------------------- 1 | import PriorityQueue from 'js-priority-queue'; 2 | import { EventEmitter } from 'events'; 3 | 4 | interface ChatMessage { 5 | message: MessageBody; 6 | priority: number; 7 | } 8 | 9 | interface MessageBody { 10 | type: string; 11 | user_name: string; 12 | content: string; 13 | expand: string; 14 | } 15 | 16 | export const chatPriorityQueue = new PriorityQueue({ comparator: (a, b) => a.priority - b.priority }); 17 | -------------------------------------------------------------------------------- /domain-chatvrm/src/features/translation/translationApi.ts: -------------------------------------------------------------------------------- 1 | import { postRequest } from "../httpclient/httpclient"; 2 | 3 | export async function translation( 4 | text: string, 5 | target_language: string 6 | ) { 7 | 8 | const headers: Record = { 9 | "Content-Type": "application/json" 10 | }; 11 | 12 | const body = { 13 | "text": text, 14 | "target_language": target_language 15 | }; 16 | // const chatRes = await postRequest("/speech/translation", headers, body); 17 | // if (chatRes.code !== '200') { 18 | // throw new Error("Something went wrong"); 19 | // } 20 | 21 | // return chatRes.response; 22 | return text; 23 | } -------------------------------------------------------------------------------- /domain-chatvrm/src/features/tts/ttsApi.ts: -------------------------------------------------------------------------------- 1 | import { buildUrl } from "@/utils/buildUrl"; 2 | import { getRequest, postRequest, buildMediaUrl } from "../httpclient/httpclient"; 3 | 4 | 5 | export const voiceData = { 6 | id: "", 7 | name: "" 8 | } 9 | export type Voice = typeof voiceData; 10 | 11 | export async function getVoices(tts_type: string) { 12 | const headers: Record = { 13 | "Content-Type": "application/json" 14 | }; 15 | 16 | const body = { 17 | "type": tts_type 18 | } 19 | 20 | const chatRes = await postRequest("/speech/tts/voices",headers,body); 21 | if (chatRes.code !== '200') { 22 | throw new Error("Something went wrong"); 23 | } 24 | return chatRes.response; 25 | } 26 | -------------------------------------------------------------------------------- /domain-chatvrm/src/features/vrmViewer/viewerContext.ts: -------------------------------------------------------------------------------- 1 | import { createContext } from "react"; 2 | import { Viewer } from "./viewer"; 3 | 4 | const viewer = new Viewer(); 5 | 6 | export const ViewerContext = createContext({ viewer }); 7 | -------------------------------------------------------------------------------- /domain-chatvrm/src/lib/VRMAnimation/VRMAnimation.ts: -------------------------------------------------------------------------------- 1 | import * as THREE from "three"; 2 | import { VRM, VRMExpressionManager, VRMHumanBoneName } from "@pixiv/three-vrm"; 3 | 4 | export class VRMAnimation { 5 | public duration: number; 6 | public restHipsPosition: THREE.Vector3; 7 | 8 | public humanoidTracks: { 9 | translation: Map; 10 | rotation: Map; 11 | }; 12 | public expressionTracks: Map; 13 | public lookAtTrack: THREE.QuaternionKeyframeTrack | null; 14 | 15 | public constructor() { 16 | this.duration = 0.0; 17 | this.restHipsPosition = new THREE.Vector3(); 18 | 19 | this.humanoidTracks = { 20 | translation: new Map(), 21 | rotation: new Map(), 22 | }; 23 | 24 | this.expressionTracks = new Map(); 25 | this.lookAtTrack = null; 26 | } 27 | 28 | public createAnimationClip(vrm: VRM): THREE.AnimationClip { 29 | const tracks: THREE.KeyframeTrack[] = []; 30 | 31 | tracks.push(...this.createHumanoidTracks(vrm)); 32 | 33 | if (vrm.expressionManager != null) { 34 | tracks.push(...this.createExpressionTracks(vrm.expressionManager)); 35 | } 36 | 37 | if (vrm.lookAt != null) { 38 | const track = this.createLookAtTrack("lookAtTargetParent.quaternion"); 39 | 40 | if (track != null) { 41 | tracks.push(track); 42 | } 43 | } 44 | 45 | return new THREE.AnimationClip("Clip", this.duration, tracks); 46 | } 47 | 48 | public createHumanoidTracks(vrm: VRM): THREE.KeyframeTrack[] { 49 | const humanoid = vrm.humanoid; 50 | const metaVersion = vrm.meta.metaVersion; 51 | const tracks: THREE.KeyframeTrack[] = []; 52 | 53 | for (const [name, origTrack] of this.humanoidTracks.rotation.entries()) { 54 | const nodeName = humanoid.getNormalizedBoneNode(name)?.name; 55 | 56 | if (nodeName != null) { 57 | const track = new THREE.VectorKeyframeTrack( 58 | `${nodeName}.quaternion`, 59 | origTrack.times, 60 | origTrack.values.map((v, i) => 61 | metaVersion === "0" && i % 2 === 0 ? -v : v 62 | ) 63 | ); 64 | tracks.push(track); 65 | } 66 | } 67 | 68 | for (const [name, origTrack] of this.humanoidTracks.translation.entries()) { 69 | const nodeName = humanoid.getNormalizedBoneNode(name)?.name; 70 | 71 | if (nodeName != null) { 72 | const animationY = this.restHipsPosition.y; 73 | const humanoidY = 74 | humanoid.getNormalizedAbsolutePose().hips!.position![1]; 75 | const scale = humanoidY / animationY; 76 | 77 | const track = origTrack.clone(); 78 | track.values = track.values.map( 79 | (v, i) => (metaVersion === "0" && i % 3 !== 1 ? -v : v) * scale 80 | ); 81 | track.name = `${nodeName}.position`; 82 | tracks.push(track); 83 | } 84 | } 85 | 86 | return tracks; 87 | } 88 | 89 | public createExpressionTracks( 90 | expressionManager: VRMExpressionManager 91 | ): THREE.KeyframeTrack[] { 92 | const tracks: THREE.KeyframeTrack[] = []; 93 | 94 | for (const [name, origTrack] of this.expressionTracks.entries()) { 95 | const trackName = expressionManager.getExpressionTrackName(name); 96 | 97 | if (trackName != null) { 98 | const track = origTrack.clone(); 99 | track.name = trackName; 100 | tracks.push(track); 101 | } 102 | } 103 | 104 | return tracks; 105 | } 106 | 107 | public createLookAtTrack(trackName: string): THREE.KeyframeTrack | null { 108 | if (this.lookAtTrack == null) { 109 | return null; 110 | } 111 | 112 | const track = this.lookAtTrack.clone(); 113 | track.name = trackName; 114 | return track; 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /domain-chatvrm/src/lib/VRMAnimation/VRMAnimationLoaderPluginOptions.ts: -------------------------------------------------------------------------------- 1 | export interface VRMAnimationLoaderPluginOptions { 2 | } 3 | -------------------------------------------------------------------------------- /domain-chatvrm/src/lib/VRMAnimation/VRMCVRMAnimation.ts: -------------------------------------------------------------------------------- 1 | import { VRMExpressionPresetName, VRMHumanBoneName } from '@pixiv/three-vrm'; 2 | 3 | export interface VRMCVRMAnimation { 4 | specVersion: string; 5 | humanoid: { 6 | humanBones: { 7 | [name in VRMHumanBoneName]?: { 8 | node: number; 9 | }; 10 | }; 11 | }; 12 | expressions?: { 13 | preset?: { 14 | [name in VRMExpressionPresetName]?: { 15 | node: number; 16 | }; 17 | }; 18 | custom?: { 19 | [name: string]: { 20 | node: number; 21 | }; 22 | }; 23 | }; 24 | lookAt?: { 25 | node: number; 26 | }; 27 | } 28 | -------------------------------------------------------------------------------- /domain-chatvrm/src/lib/VRMAnimation/loadVRMAnimation.ts: -------------------------------------------------------------------------------- 1 | import { GLTFLoader } from 'three/examples/jsm/loaders/GLTFLoader'; 2 | import { VRMAnimation } from './VRMAnimation'; 3 | import { VRMAnimationLoaderPlugin } from './VRMAnimationLoaderPlugin'; 4 | 5 | const loader = new GLTFLoader(); 6 | loader.register((parser) => new VRMAnimationLoaderPlugin(parser)); 7 | 8 | export async function loadVRMAnimation(url: string): Promise { 9 | const gltf = await loader.loadAsync(url); 10 | 11 | const vrmAnimations: VRMAnimation[] = gltf.userData.vrmAnimations; 12 | const vrmAnimation: VRMAnimation | undefined = vrmAnimations[0]; 13 | 14 | return vrmAnimation ?? null; 15 | } 16 | -------------------------------------------------------------------------------- /domain-chatvrm/src/lib/VRMAnimation/utils/arrayChunk.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * ```js 3 | * arrayChunk( [ 1, 2, 3, 4, 5, 6 ], 2 ) 4 | * // will be 5 | * [ [ 1, 2 ], [ 3, 4 ], [ 5, 6 ] ] 6 | * ``` 7 | */ 8 | export function arrayChunk(array: ArrayLike, every: number): T[][] { 9 | const N = array.length; 10 | 11 | const ret: T[][] = []; 12 | 13 | let current: T[] = []; 14 | let remaining = 0; 15 | 16 | for (let i = 0; i < N; i ++) { 17 | const el = array[i]; 18 | 19 | if (remaining <= 0) { 20 | remaining = every; 21 | current = []; 22 | ret.push(current); 23 | } 24 | 25 | current.push(el); 26 | remaining--; 27 | } 28 | 29 | return ret; 30 | } 31 | -------------------------------------------------------------------------------- /domain-chatvrm/src/lib/VRMAnimation/utils/linearstep.ts: -------------------------------------------------------------------------------- 1 | import { saturate } from './saturate'; 2 | 3 | export const linearstep = (a: number, b: number, t: number) => ( 4 | saturate((t - a) / (b - a)) 5 | ); 6 | -------------------------------------------------------------------------------- /domain-chatvrm/src/lib/VRMAnimation/utils/saturate.ts: -------------------------------------------------------------------------------- 1 | export const saturate = (x: number) => Math.min(Math.max(x, 0.0), 1.0); 2 | -------------------------------------------------------------------------------- /domain-chatvrm/src/lib/VRMLookAtSmootherLoaderPlugin/VRMLookAtSmootherLoaderPlugin.ts: -------------------------------------------------------------------------------- 1 | import { 2 | VRMHumanoid, 3 | VRMLookAt, 4 | VRMLookAtLoaderPlugin, 5 | } from "@pixiv/three-vrm"; 6 | import { GLTF } from "three/examples/jsm/loaders/GLTFLoader"; 7 | import { VRMLookAtSmoother } from "./VRMLookAtSmoother"; 8 | 9 | export class VRMLookAtSmootherLoaderPlugin extends VRMLookAtLoaderPlugin { 10 | public get name(): string { 11 | return "VRMLookAtSmootherLoaderPlugin"; 12 | } 13 | 14 | public async afterRoot(gltf: GLTF): Promise { 15 | await super.afterRoot(gltf); 16 | 17 | const humanoid = gltf.userData.vrmHumanoid as VRMHumanoid | null; 18 | const lookAt = gltf.userData.vrmLookAt as VRMLookAt | null; 19 | 20 | if (humanoid != null && lookAt != null) { 21 | const lookAtSmoother = new VRMLookAtSmoother(humanoid, lookAt.applier); 22 | lookAtSmoother.copy(lookAt); 23 | gltf.userData.vrmLookAt = lookAtSmoother; 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /domain-chatvrm/src/pages/_app.tsx: -------------------------------------------------------------------------------- 1 | import "@/styles/globals.css"; 2 | import type { AppProps } from "next/app"; 3 | import "@charcoal-ui/icons"; 4 | 5 | export default function App({ Component, pageProps }: AppProps) { 6 | return ; 7 | } 8 | -------------------------------------------------------------------------------- /domain-chatvrm/src/pages/_document.tsx: -------------------------------------------------------------------------------- 1 | import { buildUrl } from "@/utils/buildUrl"; 2 | import { Html, Head, Main, NextScript } from "next/document"; 3 | 4 | export default function Document() { 5 | return ( 6 | 7 | 8 | 9 |
10 | 11 | 12 | 13 | ); 14 | } 15 | -------------------------------------------------------------------------------- /domain-chatvrm/src/pages/api/chat.ts: -------------------------------------------------------------------------------- 1 | import { Configuration, OpenAIApi } from "openai"; 2 | 3 | import type { NextApiRequest, NextApiResponse } from "next"; 4 | 5 | type Data = { 6 | message: string; 7 | }; 8 | 9 | export default async function handler( 10 | req: NextApiRequest, 11 | res: NextApiResponse 12 | ) { 13 | const apiKey = req.body.apiKey || process.env.OPEN_AI_KEY; 14 | 15 | if (!apiKey) { 16 | res 17 | .status(400) 18 | .json({ message: "APIキーが間違っているか、設定されていません。" }); 19 | 20 | return; 21 | } 22 | 23 | const configuration = new Configuration({ 24 | apiKey: apiKey, 25 | }); 26 | 27 | const openai = new OpenAIApi(configuration); 28 | 29 | const { data } = await openai.createChatCompletion({ 30 | model: "gpt-3.5-turbo", 31 | messages: req.body.messages, 32 | }); 33 | 34 | const [aiRes] = data.choices; 35 | const message = aiRes.message?.content || "エラーが発生しました"; 36 | 37 | res.status(200).json({ message: message }); 38 | } 39 | -------------------------------------------------------------------------------- /domain-chatvrm/src/pages/api/tts.ts: -------------------------------------------------------------------------------- 1 | import fetch from "node-fetch"; 2 | import { synthesizeVoice } from "@/features/koeiromap/koeiromap"; 3 | 4 | import type { NextApiRequest, NextApiResponse } from "next"; 5 | 6 | type Data = { 7 | audio: string; 8 | }; 9 | 10 | export default async function handler( 11 | req: NextApiRequest, 12 | res: NextApiResponse 13 | ) { 14 | const message = req.body.message; 15 | const speaker_x = req.body.speakerX; 16 | const speaker_y = req.body.speakerY; 17 | const style = req.body.style; 18 | 19 | const voice = await synthesizeVoice(message, speaker_x, speaker_y, style); 20 | 21 | res.status(200).json(voice); 22 | } 23 | -------------------------------------------------------------------------------- /domain-chatvrm/src/styles/globals.css: -------------------------------------------------------------------------------- 1 | @tailwind base; 2 | @tailwind components; 3 | @tailwind utilities; 4 | 5 | body { 6 | background-position: top center; 7 | background-attachment: fixed; 8 | background-size: cover; 9 | background-repeat: no-repeat; 10 | } 11 | 12 | @layer components { 13 | .input-range { 14 | -webkit-appearance: none; 15 | appearance: none; 16 | background-color: #858585; 17 | height: 2px; 18 | width: 100%; 19 | border-radius: 4px; 20 | } 21 | 22 | .input-range:focus, 23 | .input-range:active { 24 | outline: none; 25 | } 26 | 27 | .input-range::-webkit-slider-thumb { 28 | -webkit-appearance: none; 29 | appearance: none; 30 | cursor: pointer; 31 | position: relative; 32 | width: 24px; 33 | height: 24px; 34 | display: block; 35 | border: 2px solid #856292; 36 | background-color: #ffffff; 37 | border-radius: 50%; 38 | -webkit-border-radius: 50%; 39 | } 40 | } 41 | 42 | @layer utilities { 43 | .scroll-hidden { 44 | -ms-overflow-style: none; 45 | scrollbar-width: none; 46 | } 47 | 48 | .scroll-hidden::-webkit-scrollbar { 49 | display: none; 50 | } 51 | } 52 | 53 | .photo-frame { 54 | width: 650px; 55 | height: 650px; 56 | border: 10px solid #333; 57 | border-radius: 20px; 58 | overflow: hidden; 59 | display: flex; 60 | justify-content: center; 61 | align-items: center; 62 | background-color: #f5f5f5; 63 | box-shadow: 0px 0px 15px rgba(0, 0, 0, 0.2); 64 | } 65 | 66 | .photo-frame img { 67 | width: 100%; 68 | height: 100%; 69 | object-fit: cover; 70 | } 71 | 72 | .photo-app { 73 | display: flex; 74 | justify-content: center; 75 | align-items: center; 76 | position: fixed; 77 | top: 50%; 78 | left: 60%; 79 | transform: translate(-50%, -50%); 80 | } 81 | 82 | .tab-menu { 83 | display: flex; 84 | border-bottom: 1px solid #ccc; 85 | } 86 | 87 | .tab-item { 88 | flex: 1; 89 | text-align: center; 90 | padding: 8px 16px; 91 | font-size: 14px; 92 | border: 1px solid #ccc; 93 | border-radius: 4px; 94 | transition: background-color 0.3s; 95 | } 96 | 97 | .tab-item.active { 98 | background-color: #1890ff; 99 | color: #fff; 100 | font-weight: bold; 101 | } 102 | 103 | .settings { 104 | margin-top: 80px; 105 | } 106 | 107 | .fade-enter, 108 | .fade-exit { 109 | opacity: 0; 110 | } 111 | 112 | .fade-enter-active, 113 | .fade-exit-active { 114 | opacity: 1; 115 | transition: opacity 500ms; 116 | } 117 | 118 | 119 | /* 页面布局样式 */ 120 | .globals-settings { 121 | max-width: 800px; 122 | margin: 0 auto; 123 | padding: 20px; 124 | } 125 | 126 | /* 组件通用样式 */ 127 | .section { 128 | padding: 20px; 129 | margin-bottom: 20px; 130 | border: 1px solid #ddd; 131 | border-radius: 4px; 132 | } 133 | 134 | .title { 135 | font-size: 14px; 136 | font-weight: bold; 137 | color: #333; 138 | padding-left: 20px; 139 | margin-bottom: 10px; 140 | } 141 | 142 | label { 143 | font-size: 10px; 144 | display: block; 145 | text-align: left; 146 | margin-bottom: 5px; 147 | } 148 | 149 | /* 输入框 */ 150 | input, 151 | select { 152 | width: 100%; 153 | padding: 8px; 154 | border-radius: 4px; 155 | border: 1px solid #ccc; 156 | padding: 0; 157 | margin: 0; 158 | } 159 | 160 | /* 单选按钮 */ 161 | .radio { 162 | margin-bottom: 10px; 163 | } 164 | 165 | .radio label { 166 | font-size: 14px; 167 | margin-right: 20px; 168 | } 169 | 170 | /* 下拉选择框 */ 171 | select { 172 | height: 35px; 173 | } 174 | 175 | .checkbot-field { 176 | display: -webkit-flex; 177 | display: flex; 178 | 179 | flex-direction: row; 180 | align-items: center; 181 | } 182 | 183 | .checkbot-input{ 184 | width: 15%; 185 | } 186 | 187 | /* 单选按钮样式 */ 188 | input[type="radio"] { 189 | margin-right: 10px; 190 | } -------------------------------------------------------------------------------- /domain-chatvrm/src/utils/buildUrl.ts: -------------------------------------------------------------------------------- 1 | import getConfig from "next/config"; 2 | 3 | /** 4 | * github pagesに公開時にアセットを読み込めるようにするため、 5 | * 環境変数を見てURLにリポジトリ名を追加する 6 | */ 7 | export function buildUrl(path: string): string { 8 | const { 9 | publicRuntimeConfig, 10 | }: { 11 | publicRuntimeConfig: { root: string }; 12 | } = getConfig(); 13 | 14 | return publicRuntimeConfig.root + path; 15 | } 16 | -------------------------------------------------------------------------------- /domain-chatvrm/src/utils/wait.ts: -------------------------------------------------------------------------------- 1 | export const wait = async (ms: number) => 2 | new Promise((resolve) => setTimeout(resolve, ms)); 3 | -------------------------------------------------------------------------------- /domain-chatvrm/tailwind.config.js: -------------------------------------------------------------------------------- 1 | const { light, dark } = require("@charcoal-ui/theme"); 2 | const { createTailwindConfig } = require("@charcoal-ui/tailwind-config"); 3 | /** 4 | * @type {import('tailwindcss/tailwind-config').TailwindConfig} 5 | */ 6 | module.exports = { 7 | darkMode: true, 8 | content: ["./src/**/*.tsx", "./src/**/*.html"], 9 | presets: [ 10 | createTailwindConfig({ 11 | version: "v3", 12 | theme: { 13 | ":root": light, 14 | }, 15 | }), 16 | ], 17 | theme: { 18 | extend: { 19 | colors: { 20 | primary: "#856292", 21 | "primary-hover": "#8E76A1", 22 | "primary-press": "#988BB0", 23 | "primary-disabled": "#6F48694D", 24 | secondary: "#FF617F", 25 | "secondary-hover": "#FF849B", 26 | "secondary-press": "#FF9EB1", 27 | "secondary-disabled": "#FF617F4D", 28 | base: "#FBE2CA", 29 | "text-primary": "#514062", 30 | }, 31 | fontFamily: { 32 | M_PLUS_2: ["var(--font-m-plus-2)"], 33 | Montserrat: ["var(--font-montserrat)"], 34 | }, 35 | }, 36 | }, 37 | plugins: [require("@tailwindcss/line-clamp")], 38 | }; 39 | -------------------------------------------------------------------------------- /domain-chatvrm/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "baseUrl": ".", // 设置为根目录，表示模块导入的基本路径为项目根目录 4 | "target": "es2015", 5 | "lib": ["dom", "dom.iterable", "esnext"], 6 | "allowJs": true, 7 | "skipLibCheck": true, 8 | "strict": true, 9 | "forceConsistentCasingInFileNames": true, 10 | "noEmit": true, 11 | "esModuleInterop": true, 12 | "module": "esnext", 13 | "moduleResolution": "node", 14 | "resolveJsonModule": true, 15 | "isolatedModules": true, 16 | "jsx": "preserve", 17 | "incremental": true, 18 | "paths": { 19 | "@/*": ["src/*"] 20 | } 21 | }, 22 | "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx"], 23 | "exclude": ["node_modules"] 24 | } 25 | -------------------------------------------------------------------------------- /domain-chatvrm/watch.json: -------------------------------------------------------------------------------- 1 | { 2 | "install": { 3 | "include": ["package.json"] 4 | }, 5 | "restart": { 6 | "include": [".env", "next.config.js"] 7 | }, 8 | "throttle": 250 9 | } 10 | -------------------------------------------------------------------------------- /infrastructure-gateway/conf.d/default.conf: -------------------------------------------------------------------------------- 1 | 2 | log_format json_combined escape=json '{"@timestamp":"$time_iso8601",' 3 | '"@source":"$server_addr",' 4 | '"@nginx_fields":{' 5 | '"remote_addr":"$remote_addr",' 6 | '"body_bytes_sent":"$body_bytes_sent",' 7 | '"request_time":"$request_time",' 8 | '"status":"$status",' 9 | '"host":"$host",' 10 | '"uri":"$uri",' 11 | '"server":"$server_name",' 12 | '"request_uri":"$request_uri",' 13 | '"request_method":"$request_method",' 14 | '"http_referrer":"$http_referer",' 15 | '"body_bytes_sent":"$body_bytes_sent",' 16 | '"http_x_forwarded_for":"$http_x_forwarded_for",' 17 | '"http_user_agent":"$http_user_agent",' 18 | '"upstream_response_time":"$upstream_response_time",' 19 | '"upstream_status":"$upstream_status",' 20 | '"upstream_addr":"$upstream_addr"}}'; 21 | 22 | access_log /dev/stdout json_combined; 23 | server_tokens off; 24 | 25 | ##cache## 26 | 27 | proxy_buffer_size 16k; 28 | proxy_buffers 4 64k; 29 | proxy_busy_buffers_size 128k; 30 | proxy_cache_path /tmp/cache levels=1:2 keys_zone=cache_one:200m inactive=1d max_size=3g; 31 | 32 | include /etc/nginx/conf.d/upstream/*.conf; 33 | map $http_upgrade $connection_upgrade { 34 | default upgrade; 35 | '' close; 36 | } 37 | 38 | server { 39 | 40 | listen 80 default; 41 | server_name 127.0.0.1; 42 | client_max_body_size 1000m; 43 | charset utf-8; 44 | error_page 404 502 503 /404; 45 | proxy_set_header X-Real-IP $remote_addr; 46 | proxy_set_header X-Real-PORT $remote_port; 47 | proxy_set_header X-Original-URI $request_uri; 48 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 49 | add_header X-Powered-By 'virtualwife'; 50 | 51 | 52 | include /etc/nginx/conf.d/server/*.conf; 53 | } 54 | 55 | include /etc/nginx/conf.d/vhost/*.conf; 56 | -------------------------------------------------------------------------------- /infrastructure-gateway/conf.d/server/chatbot.conf: -------------------------------------------------------------------------------- 1 | location /api/chatbot { 2 | rewrite ^/api/chatbot(.*)$ $1 break; 3 | proxy_pass http://chatbot; 4 | proxy_set_header Access-Control-Allow-Headers 'Cookie,Set-Cookie,x-requested-with,content-type'; 5 | proxy_set_header Access-Control-Allow-Origin $http_origin ; 6 | proxy_set_header 'Access-Control-Allow-Credentials' 'true'; 7 | add_header 'Access-Control-Allow-Methods' 'GET,POST,PUT,OPTIONS'; 8 | # 添加WebSocket代理配置 9 | proxy_http_version 1.1; 10 | proxy_set_header Upgrade $http_upgrade; 11 | proxy_set_header Connection "Upgrade"; 12 | } 13 | 14 | location /api/media { 15 | rewrite ^/api/media(.*)$ $1 break; 16 | proxy_pass http://chatbot; 17 | proxy_set_header Access-Control-Allow-Headers 'Cookie,Set-Cookie,x-requested-with,content-type'; 18 | proxy_set_header Access-Control-Allow-Origin $http_origin ; 19 | proxy_set_header 'Access-Control-Allow-Credentials' 'true'; 20 | add_header 'Access-Control-Allow-Methods' 'GET,POST,PUT,OPTIONS'; 21 | } 22 | -------------------------------------------------------------------------------- /infrastructure-gateway/conf.d/server/chatvrm.conf: -------------------------------------------------------------------------------- 1 | location / { 2 | proxy_set_header Host $http_host; 3 | proxy_set_header X-Real-IP $remote_addr; 4 | proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; 5 | proxy_set_header X-Forwarded-Proto $scheme; 6 | 7 | proxy_connect_timeout 300; 8 | proxy_http_version 1.1; 9 | proxy_set_header Connection ""; 10 | chunked_transfer_encoding off; 11 | proxy_pass http://chatvrm; 12 | add_header 'Cache-Control' 'no-store, no-cache, must-revalidate, proxy-revalidate, max-age=0'; 13 | } 14 | 15 | 16 | -------------------------------------------------------------------------------- /infrastructure-gateway/conf.d/server/default.conf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yakami129/VirtualWife/c8afd6d3ce6bb6f58988c649c50299d36b63e08f/infrastructure-gateway/conf.d/server/default.conf -------------------------------------------------------------------------------- /infrastructure-gateway/conf.d/upstream/ups-chatbot.conf: -------------------------------------------------------------------------------- 1 | upstream chatbot{ 2 | server chatbot:8000; 3 | } 4 | -------------------------------------------------------------------------------- /infrastructure-gateway/conf.d/upstream/ups-chatvrm.conf: -------------------------------------------------------------------------------- 1 | upstream chatvrm { 2 | server chatvrm:3000; 3 | } 4 | -------------------------------------------------------------------------------- /infrastructure-packaging/Dockerfile.ChatBot: -------------------------------------------------------------------------------- 1 | FROM python:3.10.12 2 | 3 | # 设置工作目录 4 | WORKDIR /app 5 | 6 | # 复制应用程序文件到工作目录 7 | COPY ../domain-chatbot . 8 | 9 | # 安装项目依赖框架 10 | # 使用清华源安装依赖包 11 | RUN pip install --no-cache-dir -r requirements.txt -i https://pypi.tuna.tsinghua.edu.cn/simple some-package && \ 12 | rm -rf ~/.cache/pip && \ 13 | rm -rf /tmp/* && \ 14 | rm -rf tmp/* && \ 15 | rm -rf media/* && \ 16 | rm -rf logs/info.log && \ 17 | rm -rf db/db.sqlite3 && \ 18 | rm -rf .env && \ 19 | python manage.py makemigrations && \ 20 | python manage.py migrate 21 | 22 | ENV OPENAI_API_KEY=sk-xxxxx 23 | ENV B_STATION_ID=xxxx 24 | ENV B_UID=12xx 25 | ENV TIMEZONE=Asia/Shanghai 26 | 27 | # 暴露应用程序监听的端口 28 | EXPOSE 8000 29 | 30 | # 启动Python应用程序，并使用 0.0.0.0:8000 监听所有可用地址 31 | CMD ["python", "manage.py", "runserver", "0.0.0.0:8000"] -------------------------------------------------------------------------------- /infrastructure-packaging/Dockerfile.ChatVRM: -------------------------------------------------------------------------------- 1 | # 构建阶段 2 | FROM node:14.21.3 AS build 3 | 4 | WORKDIR /app 5 | 6 | COPY ../domain-chatvrm . 7 | 8 | RUN npm config set registry https://registry.npmmirror.com && \ 9 | rm -rf .next && \ 10 | rm -rf .node_modules && \ 11 | npm install && \ 12 | npm run build --no-lint 13 | 14 | 15 | # 生产阶段 16 | FROM node:14.21.3-slim 17 | 18 | WORKDIR /app 19 | 20 | COPY --from=build /app/.next ./.next 21 | COPY --from=build /app/next.config.js ./ 22 | COPY --from=build /app/public ./public 23 | COPY --from=build /app/package.json ./ 24 | 25 | RUN npm install --production 26 | 27 | EXPOSE 3000 28 | 29 | CMD ["npm", "run", "start"] -------------------------------------------------------------------------------- /infrastructure-packaging/Dockerfile.Gateway: -------------------------------------------------------------------------------- 1 | FROM openresty/openresty:1.21.4.2-0-alpine-fat 2 | 3 | COPY ../infrastructure-gateway/conf.d/ /etc/nginx/conf.d/ 4 | -------------------------------------------------------------------------------- /installer/README.md: -------------------------------------------------------------------------------- 1 | # 使用说明 2 | 3 | ``` 4 | ├── README.md # 安装程序使用说明 5 | ├── docker-compose.yaml # docker编排文件 6 | ├── env_example # 环境变量配置模版，使用时需要将文件名改成.env 7 | ├── milvus # 长期记忆，数据存储模块 8 | ├── linux # linux 启动和关闭程序 9 | │ ├── start.sh 10 | │ └── stop.sh 11 | └── windows # windows 启动和关闭程序 12 | ├── start.bat 13 | └── stop.bat 14 | ``` 15 | 16 | # 注意事项 17 | 18 | 1. 使用时一定要将env_example文件名改成.env 19 | 20 | 21 | -------------------------------------------------------------------------------- /installer/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | 3 | services: 4 | 5 | chatbot: 6 | container_name: chatbot 7 | image: okapi0129/virtualwife-chatbot:${CHATBOT_TAG:-latest} 8 | extra_hosts: 9 | - "host.docker.internal:host-gateway" 10 | ports: 11 | - 8000:8000 12 | environment: 13 | - TZ=${TIMEZONE} 14 | env_file: 15 | - "${ENV_FILE:-.env}" 16 | networks: 17 | - virtualwife 18 | chatvrm: 19 | container_name: chatvrm 20 | image: okapi0129/virtualwife-chatvrm:${CHATVRM_TAG:-latest} 21 | environment: 22 | - TZ=${TIMEZONE} 23 | env_file: 24 | - "${ENV_FILE:-.env}" 25 | networks: 26 | - virtualwife 27 | gateway: 28 | container_name: gateway 29 | image: okapi0129/virtualwife-gateway:${GATEWAY_TAG:-latest} 30 | restart: always 31 | ports: 32 | - ${NGINX_HTTP_PORT:-80}:80 33 | - ${NGINX_HTTPS_PORT:-443}:443 34 | environment: 35 | - TZ=${TIMEZONE} 36 | env_file: 37 | - "${ENV_FILE:-.env}" 38 | networks: 39 | - virtualwife 40 | 41 | networks: 42 | virtualwife: 43 | driver: bridge 44 | -------------------------------------------------------------------------------- /installer/env_example: -------------------------------------------------------------------------------- 1 | # 时区 2 | TIMEZONE=Asia/Shanghai 3 | 4 | # 程序版本号，程序版本号可以查阅项目的release发布版本号，latest代表最新版本 5 | CHATBOT_TAG=latest 6 | CHATVRM_TAG=latest 7 | GATEWAY_TAG=latest 8 | -------------------------------------------------------------------------------- /installer/experiment/docker-compose-dev.yaml: -------------------------------------------------------------------------------- 1 | version: "3.7" 2 | services: 3 | db: 4 | image: ghcr.io/getzep/postgres:latest 5 | container_name: zep-postgres 6 | restart: on-failure 7 | shm_size: "2048m" # Increase this if vacuuming fails with a "no space left on device" error 8 | environment: 9 | - POSTGRES_USER=postgres 10 | - POSTGRES_PASSWORD=postgres 11 | build: 12 | context: .. 13 | dockerfile: Dockerfile.postgres 14 | networks: 15 | - zep-network 16 | volumes: 17 | - zep-db:/var/lib/postgresql/data 18 | healthcheck: 19 | test: [ "CMD", "pg_isready", "-q", "-d", "postgres", "-U", "postgres" ] 20 | interval: 5s 21 | timeout: 5s 22 | retries: 5 23 | nlp: 24 | image: okapi0129/getzep-zep-nlp-server:latest 25 | container_name: zep-nlp 26 | env_file: 27 | - ../.env # You can set your embedding-related variables here 28 | restart: on-failure 29 | networks: 30 | - zep-network 31 | healthcheck: 32 | test: timeout 10s bash -c ':> /dev/tcp/127.0.0.1/5557' || exit 1 33 | interval: 10s 34 | timeout: 5s 35 | retries: 5 36 | start_period: 180s 37 | zep: 38 | image: ghcr.io/getzep/zep:0.21.0 39 | container_name: zep 40 | restart: on-failure 41 | depends_on: 42 | db: 43 | condition: service_healthy 44 | nlp: 45 | condition: service_healthy 46 | ports: 47 | - "8881:8000" 48 | volumes: 49 | - ./config.yaml:/app/config.yaml 50 | environment: 51 | - ZEP_STORE_POSTGRES_DSN=postgres://postgres:postgres@db:5432/postgres?sslmode=disable 52 | - ZEP_NLP_SERVER_URL=http://nlp:5557 53 | - ZEP_EXTRACTORS_DOCUMENTS_EMBEDDINGS_SERVICE=local 54 | - ZEP_EXTRACTORS_DOCUMENTS_EMBEDDINGS_DIMENSIONS=768 55 | - ZEP_EXTRACTORS_MESSAGES_EMBEDDINGS_SERVICE=local 56 | - ZEP_EXTRACTORS_MESSAGES_EMBEDDINGS_DIMENSIONS=768 57 | - ZEP_EXTRACTORS_MESSAGES_SUMMARIZER_EMBEDDINGS_SERVICE=local 58 | - ZEP_EXTRACTORS_MESSAGES_SUMMARIZER_EMBEDDINGS_DIMENSIONS=768 59 | env_file: 60 | - ../.env # Store your OpenAI API key here as ZEP_OPENAI_API_KEY 61 | build: 62 | context: .. 63 | dockerfile: Dockerfile 64 | healthcheck: 65 | test: timeout 10s bash -c ':> /dev/tcp/127.0.0.1/8000' || exit 1 66 | interval: 5s 67 | timeout: 10s 68 | retries: 3 69 | start_period: 180s 70 | networks: 71 | - zep-network 72 | networks: 73 | zep-network: 74 | driver: bridge 75 | volumes: 76 | zep-db: 77 | -------------------------------------------------------------------------------- /installer/linux/start.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker-compose -f ../docker-compose.yaml up -d -------------------------------------------------------------------------------- /installer/linux/stop.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker-compose -f ../docker-compose.yaml stop && docker-compose -f ../docker-compose.yaml rm -f -------------------------------------------------------------------------------- /installer/milvus/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.5' 2 | 3 | services: 4 | etcd: 5 | container_name: milvus-etcd 6 | image: quay.io/coreos/etcd:v3.5.5 7 | environment: 8 | - ETCD_AUTO_COMPACTION_MODE=revision 9 | - ETCD_AUTO_COMPACTION_RETENTION=1000 10 | - ETCD_QUOTA_BACKEND_BYTES=4294967296 11 | - ETCD_SNAPSHOT_COUNT=50000 12 | volumes: 13 | - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/etcd:/etcd 14 | command: etcd -advertise-client-urls=http://127.0.0.1:2379 -listen-client-urls http://0.0.0.0:2379 --data-dir /etcd 15 | 16 | minio: 17 | container_name: milvus-minio 18 | image: minio/minio:RELEASE.2023-03-20T20-16-18Z 19 | environment: 20 | MINIO_ACCESS_KEY: minioadmin 21 | MINIO_SECRET_KEY: minioadmin 22 | volumes: 23 | - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/minio:/minio_data 24 | command: minio server /minio_data 25 | healthcheck: 26 | test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] 27 | interval: 30s 28 | timeout: 20s 29 | retries: 3 30 | 31 | standalone: 32 | container_name: milvus-standalone 33 | image: milvusdb/milvus:v2.2.12 34 | command: ["milvus", "run", "standalone"] 35 | environment: 36 | ETCD_ENDPOINTS: etcd:2379 37 | MINIO_ADDRESS: minio:9000 38 | volumes: 39 | - ${DOCKER_VOLUME_DIRECTORY:-.}/volumes/milvus:/var/lib/milvus 40 | ports: 41 | - "19530:19530" 42 | - "9091:9091" 43 | depends_on: 44 | - "etcd" 45 | - "minio" 46 | 47 | networks: 48 | default: 49 | name: milvus -------------------------------------------------------------------------------- /installer/windows/start.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | docker-compose -f ..\docker-compose.yaml up -d 3 | -------------------------------------------------------------------------------- /installer/windows/stop.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | docker-compose -f ..\docker-compose.yaml stop 3 | docker-compose -f ..\docker-compose.yaml rm -f 4 | -------------------------------------------------------------------------------- /installer/zep/Dockerfile: -------------------------------------------------------------------------------- 1 | # Dockerfile 2 | FROM okapi0129/getzep-zep-nlp-server:latest AS build 3 | 4 | FROM python:3.11 5 | 6 | WORKDIR /app 7 | 8 | COPY --from=build /app /app 9 | COPY --from=build /root/.cache/torch/sentence_transformers/moka-ai_m3e-base /root/.cache/torch/sentence_transformers/moka-ai_m3e-base 10 | ENV VIRTUAL_ENV=/app/.venv PATH=/app/.venv/bin:/usr/local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin 11 | 12 | RUN apt-get install libatlas-base-dev 13 | RUN pip install --force-reinstall numpy -i https://pypi.tuna.tsinghua.edu.cn/simple some-package 14 | 15 | EXPOSE 3000 16 | CMD ["python", "main.py"] -------------------------------------------------------------------------------- /installer/zep/upload.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | docker buildx build --platform linux/amd64,linux/arm64 -t okapi0129/getzep-zep-nlp-server:v1.0.0 . --push 3 | 4 | -------------------------------------------------------------------------------- /package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "VirtualWife", 3 | "lockfileVersion": 2, 4 | "requires": true, 5 | "packages": { 6 | "": { 7 | "devDependencies": { 8 | "@types/react-transition-group": "^4.4.6" 9 | } 10 | }, 11 | "node_modules/@types/prop-types": { 12 | "version": "15.7.5", 13 | "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.5.tgz", 14 | "integrity": "sha512-JCB8C6SnDoQf0cNycqd/35A7MjcnK+ZTqE7judS6o7utxUCg6imJg3QK2qzHKszlTjcj2cn+NwMB2i96ubpj7w==", 15 | "dev": true 16 | }, 17 | "node_modules/@types/react": { 18 | "version": "18.2.21", 19 | "resolved": "https://registry.npmjs.org/@types/react/-/react-18.2.21.tgz", 20 | "integrity": "sha512-neFKG/sBAwGxHgXiIxnbm3/AAVQ/cMRS93hvBpg8xYRbeQSPVABp9U2bRnPf0iI4+Ucdv3plSxKK+3CW2ENJxA==", 21 | "dev": true, 22 | "dependencies": { 23 | "@types/prop-types": "*", 24 | "@types/scheduler": "*", 25 | "csstype": "^3.0.2" 26 | } 27 | }, 28 | "node_modules/@types/react-transition-group": { 29 | "version": "4.4.6", 30 | "resolved": "https://registry.npmjs.org/@types/react-transition-group/-/react-transition-group-4.4.6.tgz", 31 | "integrity": "sha512-VnCdSxfcm08KjsJVQcfBmhEQAPnLB8G08hAxn39azX1qYBQ/5RVQuoHuKIcfKOdncuaUvEpFKFzEvbtIMsfVew==", 32 | "dev": true, 33 | "dependencies": { 34 | "@types/react": "*" 35 | } 36 | }, 37 | "node_modules/@types/scheduler": { 38 | "version": "0.16.3", 39 | "resolved": "https://registry.npmjs.org/@types/scheduler/-/scheduler-0.16.3.tgz", 40 | "integrity": "sha512-5cJ8CB4yAx7BH1oMvdU0Jh9lrEXyPkar6F9G/ERswkCuvP4KQZfZkSjcMbAICCpQTN4OuZn8tz0HiKv9TGZgrQ==", 41 | "dev": true 42 | }, 43 | "node_modules/csstype": { 44 | "version": "3.1.2", 45 | "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.1.2.tgz", 46 | "integrity": "sha512-I7K1Uu0MBPzaFKg4nI5Q7Vs2t+3gWWW648spaF+Rg7pI9ds18Ugn+lvg4SHczUdKlHI5LWBXyqfS8+DufyBsgQ==", 47 | "dev": true 48 | } 49 | }, 50 | "dependencies": { 51 | "@types/prop-types": { 52 | "version": "15.7.5", 53 | "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.5.tgz", 54 | "integrity": "sha512-JCB8C6SnDoQf0cNycqd/35A7MjcnK+ZTqE7judS6o7utxUCg6imJg3QK2qzHKszlTjcj2cn+NwMB2i96ubpj7w==", 55 | "dev": true 56 | }, 57 | "@types/react": { 58 | "version": "18.2.21", 59 | "resolved": "https://registry.npmjs.org/@types/react/-/react-18.2.21.tgz", 60 | "integrity": "sha512-neFKG/sBAwGxHgXiIxnbm3/AAVQ/cMRS93hvBpg8xYRbeQSPVABp9U2bRnPf0iI4+Ucdv3plSxKK+3CW2ENJxA==", 61 | "dev": true, 62 | "requires": { 63 | "@types/prop-types": "*", 64 | "@types/scheduler": "*", 65 | "csstype": "^3.0.2" 66 | } 67 | }, 68 | "@types/react-transition-group": { 69 | "version": "4.4.6", 70 | "resolved": "https://registry.npmjs.org/@types/react-transition-group/-/react-transition-group-4.4.6.tgz", 71 | "integrity": "sha512-VnCdSxfcm08KjsJVQcfBmhEQAPnLB8G08hAxn39azX1qYBQ/5RVQuoHuKIcfKOdncuaUvEpFKFzEvbtIMsfVew==", 72 | "dev": true, 73 | "requires": { 74 | "@types/react": "*" 75 | } 76 | }, 77 | "@types/scheduler": { 78 | "version": "0.16.3", 79 | "resolved": "https://registry.npmjs.org/@types/scheduler/-/scheduler-0.16.3.tgz", 80 | "integrity": "sha512-5cJ8CB4yAx7BH1oMvdU0Jh9lrEXyPkar6F9G/ERswkCuvP4KQZfZkSjcMbAICCpQTN4OuZn8tz0HiKv9TGZgrQ==", 81 | "dev": true 82 | }, 83 | "csstype": { 84 | "version": "3.1.2", 85 | "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.1.2.tgz", 86 | "integrity": "sha512-I7K1Uu0MBPzaFKg4nI5Q7Vs2t+3gWWW648spaF+Rg7pI9ds18Ugn+lvg4SHczUdKlHI5LWBXyqfS8+DufyBsgQ==", 87 | "dev": true 88 | } 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "devDependencies": { 3 | "@types/react-transition-group": "^4.4.6" 4 | } 5 | } 6 | -------------------------------------------------------------------------------- /product.md: -------------------------------------------------------------------------------- 1 | # 产品使用文档 2 | 3 | ## 快速开始 4 | 5 | ## 高级功能介绍 6 | 7 | ### 一、基础设置模块 8 | 9 | ### 二、自定义角色模块 10 | 11 | ### 三、大语言模型模块 12 | 13 | ### 四、记忆模块 14 | 15 | ### 五、高级设置模块 -------------------------------------------------------------------------------- /release-log.md: -------------------------------------------------------------------------------- 1 | # v2.0.7 发布日志 2 | 3 | release: 4 | - [X] 1. 支持ollama 5 | - [X] 2. 重写bilibili弹幕监听组件 6 | - [X] 3. 实验功能 - 角色安装包 7 | 8 | fixbug: 9 | - [X] 1. 适配最新版本的OpenAI SDK 10 | - [X] 2. 修复Bert_vits2语音合成错误问题 11 | - [x] 3. 优化人物模型朝向 12 | 13 | known issues: 14 | - [ ] 1. 源码部署，修改代码时，会导致重复接收和发送消息 15 | - [ ] 2. docker无法正常连接text-generation-webui 16 | - [ ] 3. VRM支持PMX动画 17 | - [ ] 4. 表情表达功能能支持选择不同的模型 18 | - [ ] 5. 优化项目使用文档 19 | 20 | # v2.0.6 发布日志 21 | 22 | release: 23 | - [x] 1. 人物动作切换不自然 24 | - [x] 2. 项目日志打印优化 25 | - [x] 3. 优化项目开发文档（conda部署教程） 26 | - [x] 4. 对话段落之间停顿时间过长（优化分段规则） 27 | 28 | fixbug: 29 | - [x] 1. 修复text_generation_webui聊天过程中出现，不能正常回复（改为chat模式） 30 | - [x] 2. 修复默认语音参数错误问题 31 | - [x] 3. 修复开启长期记忆程序报错 32 | 33 | # v2.0.5 发布日志 34 | 35 | release: 36 | - [x] 1. 支持AI根据情绪调整人物动作 37 | - [x] 2. 支持Edge（微软）、Bert-VITS2语音切换 38 | - [x] 3. 支持保存用户上传的VRM模型 39 | - [x] 4. 支持更换壁纸 40 | - [x] 5. 支持对话弹幕 41 | 42 | fixbug: 43 | - [x] 1. 修复对话记录不显示AI回复内容 44 | - [x] 2. chatbot镜像文件过大（5.09G），压缩到2.7G 45 | 46 | known issues: 47 | - [ ] 1. 人物动作切换不自然 48 | - [ ] 2. 对话段落之间停顿时间过长（代码逻辑需要优化） 49 | - [ ] 3. 简单情绪系统 50 | - [ ] 4. 项目日志打印优化 51 | - [ ] 5. 源码部署，修改代码时，会导致重复接收和发送消息 52 | - [ ] 6. VRM支持PMX动画 53 | 54 | To be determined： 55 | - [ ] 1. 支持AI根据情绪调整语音的音调（koeiromap可以控制情绪，但是只支持日语，emotion-vits可以支持，还在研究）- 原开发计划 56 | 57 | # v2.0.4 发布日志 58 | 59 | release: 60 | 1. 支持中文语音识别 61 | 62 | fixbug： 63 | 1. 修复B弹幕获取不了的问题 64 | 2. 修复Docker中无法使用http-proxy 65 | 3. Docker部署ARM系统兼容问题 66 | 4. Windows 安装 WSL 的FAQ说明 67 | 68 | known issues: 69 | 1. chatbot镜像文件过大（5.09G）,待解决 70 | 71 | # v2.0.3 发布日志 72 | 73 | release: 74 | 1. VRM模型支持表情情感表达 75 | 76 | fixbug: 77 | 1. 修复开启摘要，导致长期记忆存储报错问题 78 | 2. 修复text_generation的历史聊天记录不识别问题 79 | 3. 修复表情符号和特殊符号导致语音合成失败问题 80 | 81 | known issues: 82 | 1. 语音识别，待解决 83 | 84 | # v2.0.2 发布日志 85 | 86 | release: 87 | 1. python升级至3.10.12 88 | 2. 优化prompt让对话更加拟人 89 | 3. text_generation 支持上下文，适配短期记忆 90 | 4. text_generation 支持流式返回数据，提高性能 91 | 92 | fixbug: 93 | 1. 修复音频文件堆积 94 | 2. 修复多人同时讨论，长期记忆检索问题 95 | 3. 增加解决docker网络问题文档 96 | 4. 修复直播获取弹幕，用户名带*号问题 97 | 98 | remove： 99 | 1. 移除英文角色提示模版 100 | 101 | known issues: 102 | 1. 语音识别，待解决 103 | 104 | 注意事项： 105 | - 本次改动更改了数据库表设计，更新代码后请执行数据库迁移操作 106 | ```shell 107 | python manage.py makemigrations 108 | ``` 109 | ```shell 110 | python manage.py migrate 111 | ``` 112 | - 本次改动新增了依赖，请执行更新依赖操作 113 | ```shell 114 | pip3 install -r requirements.txt 115 | ``` -------------------------------------------------------------------------------- /upload.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 打标签并推送镜像 3 | docker buildx bake -f docker-compose.build.hcl chatbot-release chatvrm-release gateway-release --push 4 | --------------------------------------------------------------------------------