├── .gitignore ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── docker-compose.yml ├── fe ├── .gitignore ├── index.html ├── package.json ├── src │ ├── App.vue │ ├── assets │ │ └── main.css │ ├── components │ │ ├── KnowledgeNameSetting.vue │ │ └── KnowledgeSelector.vue │ ├── main.js │ ├── router │ │ └── index.js │ ├── utils │ │ ├── knowledgeIdStore.js │ │ └── knowledgeStore.js │ └── views │ │ ├── Chat.vue │ │ ├── Indexer.vue │ │ ├── KnowledgeBase.vue │ │ └── Retriever.vue └── vite.config.js └── server ├── README.md ├── api └── rag │ ├── rag.go │ └── v1 │ ├── chat.go │ ├── indexer.go │ ├── knowledge_base.go │ └── retriever.go ├── core ├── common │ ├── chat_model.go │ ├── consts.go │ ├── embedding.go │ ├── es.go │ ├── helper.go │ └── stream.go ├── config │ └── config.go ├── grader │ ├── grader.go │ └── message.go ├── indexer │ ├── indexer.go │ ├── lambda_func.go │ ├── loader.go │ ├── orchestration.go │ ├── parser.go │ └── transformer.go ├── message.go ├── rag.go ├── rag_test.go ├── readme.md ├── retriever │ ├── orchestration.go │ ├── retriever.go │ └── transformer.go └── test_file │ ├── readme.html │ ├── readme.md │ ├── readme2.md │ └── test.pdf ├── go.mod ├── go.sum ├── hack └── config.yaml ├── internal ├── cmd │ └── cmd.go ├── controller │ └── rag │ │ ├── rag.go │ │ ├── rag_new.go │ │ ├── rag_v1_chat.go │ │ ├── rag_v1_chat_stream.go │ │ ├── rag_v1_indexer.go │ │ ├── rag_v1_kb.go │ │ └── rag_v1_retriever.go ├── dao │ ├── dao.go │ ├── internal │ │ └── knowledge_base.go │ └── knowledge_base.go ├── logic │ ├── chat │ │ ├── chat.go │ │ └── message.go │ └── rag │ │ └── retriever.go ├── mcp │ ├── indexer.go │ ├── knowledgebase.go │ ├── mcp.go │ └── retriever.go └── model │ ├── do │ └── knowledge_base.go │ ├── entity │ └── knowledge_base.go │ └── knowledge_base.go ├── main.go ├── manifest └── config │ └── config_demo.yaml └── static ├── chat.png ├── indexer.png ├── kb-select.png ├── kb.png ├── mcp-cfg.png ├── mcp-use.png └── retriever.png /.gitignore: -------------------------------------------------------------------------------- 1 | # If you prefer the allow list template instead of the deny list, see community template: 2 | # https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore 3 | # 4 | # Binaries for programs and plugins 5 | *.exe 6 | *.exe~ 7 | *.dll 8 | *.so 9 | *.dylib 10 | 11 | # Test binary, built with `go test -c` 12 | *.test 13 | 14 | # Output of the go coverage tool, specifically when used with LiteIDE 15 | *.out 16 | 17 | # Dependency directories (remove the comment below to include it) 18 | # vendor/ 19 | 20 | # Go workspace file 21 | go.work 22 | go.work.sum 23 | 24 | # env file 25 | .env 26 | .idea 27 | 28 | server/manifest/config/config.yaml 29 | /server/uploads/ 30 | /node_modules/ 31 | /fe/package-lock.json 32 | /server/static/fe/ 33 | /fe/.vite 34 | /server/go-rag-server 35 | /server/logs/ 36 | /data/ -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # 多阶段构建 2 | # 阶段2: 构建后端 3 | FROM golang:1.23.9-alpine AS server-builder 4 | WORKDIR /app 5 | # 复制整个项目代码 6 | COPY . . 7 | # 复制前端构建产物到server目录 8 | # 构建后端 9 | RUN cd server && go mod tidy && go build -o go-rag-server main.go 10 | 11 | # 阶段3: 最终镜像 12 | FROM alpine:latest 13 | WORKDIR /app 14 | # 安装运行时依赖 15 | RUN apk --no-cache add ca-certificates tzdata 16 | # 设置时区 17 | ENV TZ=Asia/Shanghai 18 | # 复制后端构建产物 19 | COPY --from=server-builder /app/server/go-rag-server /app/ 20 | COPY --from=server-builder /app/server/static/ /app/static/ 21 | COPY --from=server-builder /app/server/manifest/ /app/manifest/ 22 | 23 | # 暴露端口 24 | EXPOSE 8000 25 | 26 | # 启动命令 27 | CMD ["/app/go-rag-server"] -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Makefile for go-rag project 2 | 3 | .PHONY: build-fe build-server build docker-build clean 4 | 5 | # 默认目标 6 | all: build 7 | 8 | # 构建前端 9 | build-fe: 10 | cd fe && npm install && npm run build 11 | mkdir -p server/static/fe 12 | cp -r fe/dist/* server/static/fe/ 13 | 14 | # 构建后端 15 | build-server: 16 | cd server && go mod tidy && go build -o go-rag-server main.go 17 | 18 | # 构建整个项目 19 | build: build-fe build-server 20 | 21 | # 运行 22 | run: 23 | cd server && ./go-rag-server 24 | 25 | # 清理构建产物 26 | clean: 27 | rm -rf fe/dist 28 | rm -rf server/static/fe 29 | rm -f server/go-rag-server 30 | 31 | # 构建Docker镜像 32 | docker-build: build 33 | docker build -t go-rag:latest -f Dockerfile . 34 | 35 | run-local: 36 | cd server && go mod tidy && go run . 37 | 38 | build-linux: 39 | cd server && go mod tidy && GOOS=linux GOARCH=amd64 go build -o go-rag-server 40 | 41 | run-by-docker: 42 | docker compose -f docker-compose.yml up -d 43 | 44 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # go-rag 2 | 基于eino+gf+vue实现知识库的rag 3 | 1. 创建知识库 4 | ![](./server/static/kb.png) 5 | 2. 选择需要使用的知识库，上传文档 6 | ![](./server/static/kb-select.png) 7 | ![](./server/static/indexer.png) 8 | 3. 检索 9 | ![](./server/static/retriever.png) 10 | 4. 对话 11 | ![](./server/static/chat.png) 12 | 5. mcp （以集成到deepchat为例） 13 | ![](./server/static/mcp-cfg.png) 14 | ![](./server/static/mcp-use.png) 15 | 16 | 17 | ## 存储层 18 | - [x] es8存储向量相关数据 19 | 20 | ## 功能列表 21 | - [x] md、pdf、html 文档解析 22 | - [x] 网页解析 23 | - [x] 文档检索 24 | - [x] 长文档自动切割(chunk) 25 | - [x] 提供http接口 [rag-api](./server/README.md) 26 | - [x] 提供 index、retrieve、chat 的前端界面 27 | - [x] 多知识库支持 28 | 29 | 30 | ## 未来计划 31 | - [ ] 使用mysql存储chunk和文档的映射关系，目前放在es的ext字段 32 | 33 | ## 使用 34 | ### clone项目 35 | ```bash 36 | git clone https://github.com/wangle201210/go-rag.git 37 | ``` 38 | 39 | ### 使用 Docker Compose 快速启动（推荐） 40 | ```bash 41 | cp server/manifest/config/config.example.yaml server/manifest/config/config.yaml 42 | # 修改配置文件中的embedding、chat的配置 43 | docker compose up -d 44 | # 浏览器打开 http://localhost:8000 45 | ``` 46 | 47 | ### 使用源码启动 48 | 如果有可用的es8和mysql,可以直接快速启动项目，否则需要先安装es8和mysql 49 | 需要修改`config.yaml`文件的相关配置 50 | ```bash 51 | cp server/manifest/config/config.example.yaml server/manifest/config/config.yaml 52 | # 修改配置文件中的embedding、chat、mysql、es的配置 53 | make build 54 | make run 55 | # 浏览器打开 http://localhost:8000 56 | ```` 57 | 58 | ### 安装依赖 59 | *如果有可用的es8和mysql,可以不用安装* 60 | 安装es8 61 | ```bash 62 | docker run -d --name elasticsearch \ 63 | -e "discovery.type=single-node" \ 64 | -e "ES_JAVA_OPTS=-Xms512m -Xmx512m" \ 65 | -e "xpack.security.enabled=false" \ 66 | -p 9200:9200 \ 67 | -p 9300:9300 \ 68 | elasticsearch:8.18.0 69 | ``` 70 | 安装mysql 71 | ```bash 72 | docker run -p 3306:3306 --name mysql \ 73 | -v /Users/wanna/docker/mysql/log:/var/log/mysql \ 74 | -v /Users/wanna/docker/mysql/data:/var/lib/mysql \ 75 | --restart=always \ 76 | -e MYSQL_ROOT_PASSWORD=123456 \ 77 | -e MYSQL_DATABASE=go-rag \ 78 | -d mysql:8.0 79 | ``` 80 | 81 | ### 运行 api 项目 82 | 83 | ```bash 84 | cd server 85 | go mod tidy 86 | go run main.go 87 | ``` 88 | 89 | ### 运行前端项目 90 | 91 | ```bash 92 | cd fe 93 | npm install 94 | npm run dev 95 | ``` 96 | 97 | ## 使用Makefile构建 98 | 99 | - 构建前端并将产物复制到server/static/fe目录 `make build-fe` 100 | 101 | - 构建后端 `make build-server` 102 | 103 | - 构建整个项目（前端+后端）`make build` 104 | 105 | - 清理构建产物 `make clean` 106 | 107 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | 3 | services: 4 | go-rag: 5 | image: iwangle/go-rag:v0.0.1 6 | # build: 7 | # context: . 8 | # dockerfile: Dockerfile 9 | ports: 10 | - "8000:8000" 11 | restart: unless-stopped 12 | environment: 13 | - TZ=Asia/Shanghai 14 | - ES_HOST=elasticsearch 15 | - ES_PORT=9200 16 | - MYSQL_HOST=mysql 17 | - MYSQL_PORT=3306 18 | - MYSQL_USER=root 19 | - MYSQL_PASSWORD=123456 20 | - MYSQL_DATABASE=go-rag 21 | volumes: 22 | # 如果需要持久化配置或数据，可以添加相应的卷挂载，目前是在构建镜像时就copy过去的 23 | - ./server/manifest:/app/manifest 24 | depends_on: 25 | - elasticsearch 26 | - mysql 27 | 28 | elasticsearch: 29 | image: elasticsearch:8.11.3 30 | environment: 31 | - discovery.type=single-node 32 | - xpack.security.enabled=false 33 | - "ES_JAVA_OPTS=-Xms512m -Xmx512m" 34 | ports: 35 | - "9200:9200" 36 | volumes: 37 | - ./data/es_data:/usr/share/elasticsearch/data 38 | restart: unless-stopped 39 | 40 | mysql: 41 | image: mysql:8.0 42 | environment: 43 | - MYSQL_ROOT_PASSWORD=123456 44 | - MYSQL_DATABASE=go-rag 45 | - MYSQL_ROOT_HOST=% # 允许root从任意主机连接 46 | ports: 47 | - "3306:3306" 48 | volumes: 49 | - ./data/mysql_data:/var/lib/mysql 50 | restart: unless-stopped 51 | -------------------------------------------------------------------------------- /fe/.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | pnpm-debug.log* 8 | lerna-debug.log* 9 | 10 | node_modules 11 | dist 12 | dist-ssr 13 | *.local 14 | 15 | # Editor directories and files 16 | .vscode/* 17 | !.vscode/extensions.json 18 | .idea 19 | .DS_Store 20 | *.suo 21 | *.ntvs* 22 | *.njsproj 23 | *.sln 24 | *.sw? -------------------------------------------------------------------------------- /fe/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | Go-RAG - 基于检索增强生成的智能问答系统 8 | 9 | 10 |
11 | 12 | 13 | -------------------------------------------------------------------------------- /fe/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "go-rag-fe", 3 | "version": "0.1.0", 4 | "private": true, 5 | "scripts": { 6 | "dev": "vite", 7 | "build": "vite build", 8 | "preview": "vite preview" 9 | }, 10 | "dependencies": { 11 | "vue": "^3.3.4", 12 | "element-plus": "^2.9.11", 13 | "axios": "^1.4.0", 14 | "vue-router": "^4.2.4", 15 | "@element-plus/icons-vue": "^2.1.0", 16 | "uuid": "^9.0.0", 17 | "marked": "^4.3.0", 18 | "highlight.js": "^11.8.0", 19 | "dompurify": "^3.0.5" 20 | }, 21 | "devDependencies": { 22 | "@vitejs/plugin-vue": "^4.2.3", 23 | "vite": "^4.4.6", 24 | "sass": "^1.64.1" 25 | } 26 | } -------------------------------------------------------------------------------- /fe/src/App.vue: -------------------------------------------------------------------------------- 1 | 53 | 54 | 58 | 59 | -------------------------------------------------------------------------------- /fe/src/assets/main.css: -------------------------------------------------------------------------------- 1 | /* 全局样式 */ 2 | * { 3 | margin: 0; 4 | padding: 0; 5 | box-sizing: border-box; 6 | } 7 | 8 | html, body { 9 | height: 100%; 10 | font-family: 'Helvetica Neue', Helvetica, 'PingFang SC', 'Hiragino Sans GB', 'Microsoft YaHei', '微软雅黑', Arial, sans-serif; 11 | -webkit-font-smoothing: antialiased; 12 | -moz-osx-font-smoothing: grayscale; 13 | color: #2c3e50; 14 | background-color: #f5f7fa; 15 | } 16 | 17 | #app { 18 | height: 100%; 19 | } 20 | 21 | /* 滚动条样式 */ 22 | ::-webkit-scrollbar { 23 | width: 6px; 24 | height: 6px; 25 | } 26 | 27 | ::-webkit-scrollbar-track { 28 | background: #f1f1f1; 29 | border-radius: 3px; 30 | } 31 | 32 | ::-webkit-scrollbar-thumb { 33 | background: #c0c4cc; 34 | border-radius: 3px; 35 | } 36 | 37 | ::-webkit-scrollbar-thumb:hover { 38 | background: #909399; 39 | } 40 | 41 | /* 常用工具类 */ 42 | .text-center { 43 | text-align: center; 44 | } 45 | 46 | .text-right { 47 | text-align: right; 48 | } 49 | 50 | .flex-center { 51 | display: flex; 52 | justify-content: center; 53 | align-items: center; 54 | } 55 | 56 | .mb-10 { 57 | margin-bottom: 10px; 58 | } 59 | 60 | .mt-10 { 61 | margin-top: 10px; 62 | } 63 | 64 | .ml-10 { 65 | margin-left: 10px; 66 | } 67 | 68 | .mr-10 { 69 | margin-right: 10px; 70 | } 71 | 72 | .p-10 { 73 | padding: 10px; 74 | } 75 | 76 | /* 动画效果 */ 77 | .fade-enter-active, 78 | .fade-leave-active { 79 | transition: opacity 0.3s ease; 80 | } 81 | 82 | .fade-enter-from, 83 | .fade-leave-to { 84 | opacity: 0; 85 | } -------------------------------------------------------------------------------- /fe/src/components/KnowledgeNameSetting.vue: -------------------------------------------------------------------------------- 1 | 41 | 42 | 74 | 75 | -------------------------------------------------------------------------------- /fe/src/components/KnowledgeSelector.vue: -------------------------------------------------------------------------------- 1 | 64 | 65 | 166 | 167 | -------------------------------------------------------------------------------- /fe/src/main.js: -------------------------------------------------------------------------------- 1 | import { createApp } from 'vue' 2 | import ElementPlus from 'element-plus' 3 | import 'element-plus/dist/index.css' 4 | import * as ElementPlusIconsVue from '@element-plus/icons-vue' 5 | import App from './App.vue' 6 | import router from './router' 7 | 8 | // 导入全局样式 9 | import './assets/main.css' 10 | 11 | const app = createApp(App) 12 | 13 | // 注册所有图标 14 | for (const [key, component] of Object.entries(ElementPlusIconsVue)) { 15 | app.component(key, component) 16 | } 17 | 18 | app.use(ElementPlus) 19 | app.use(router) 20 | app.mount('#app') -------------------------------------------------------------------------------- /fe/src/router/index.js: -------------------------------------------------------------------------------- 1 | import { createRouter, createWebHashHistory } from 'vue-router' 2 | 3 | const routes = [ 4 | { 5 | path: '/', 6 | redirect: '/indexer' 7 | }, 8 | { 9 | path: '/indexer', 10 | name: 'Indexer', 11 | component: () => import('../views/Indexer.vue') 12 | }, 13 | { 14 | path: '/retriever', 15 | name: 'Retriever', 16 | component: () => import('../views/Retriever.vue') 17 | }, 18 | { 19 | path: '/chat', 20 | name: 'Chat', 21 | component: () => import('../views/Chat.vue') 22 | }, 23 | { 24 | path: '/knowledge-base', 25 | name: 'KnowledgeBase', 26 | component: () => import('../views/KnowledgeBase.vue') 27 | } 28 | ] 29 | 30 | const router = createRouter({ 31 | history: createWebHashHistory(), 32 | routes 33 | }) 34 | 35 | export default router -------------------------------------------------------------------------------- /fe/src/utils/knowledgeIdStore.js: -------------------------------------------------------------------------------- 1 | /** 2 | * 知识库ID管理服务 3 | * 用于在不同页面之间共享选中的知识库ID 4 | * 使用localStorage存储，确保页面刷新后仍然可用 5 | */ 6 | 7 | const STORAGE_KEY = 'go_rag_selected_knowledge_id' 8 | 9 | /** 10 | * 获取当前选中的知识库ID 11 | * 如果不存在，则返回空字符串 12 | */ 13 | export function getSelectedKnowledgeId() { 14 | return localStorage.getItem(STORAGE_KEY) || '' 15 | } 16 | 17 | /** 18 | * 设置选中的知识库ID 19 | */ 20 | export function setSelectedKnowledgeId(id) { 21 | localStorage.setItem(STORAGE_KEY, id) 22 | return id 23 | } 24 | 25 | /** 26 | * 清除选中的知识库ID 27 | */ 28 | export function clearSelectedKnowledgeId() { 29 | localStorage.removeItem(STORAGE_KEY) 30 | } -------------------------------------------------------------------------------- /fe/src/utils/knowledgeStore.js: -------------------------------------------------------------------------------- 1 | /** 2 | * 知识库名称管理服务 3 | * 用于在不同页面之间共享knowledge_name参数 4 | * 使用localStorage存储，确保页面刷新后仍然可用 5 | */ 6 | 7 | import { v4 as uuidv4 } from 'uuid' 8 | 9 | const STORAGE_KEY = 'go_rag_knowledge_name' 10 | 11 | /** 12 | * 获取当前知识库名称 13 | * 如果不存在，则自动生成一个新的 14 | */ 15 | export function getKnowledgeName() { 16 | let knowledgeName = localStorage.getItem(STORAGE_KEY) 17 | 18 | // 如果不存在，则生成一个新的知识库名称 19 | if (!knowledgeName) { 20 | knowledgeName = generateKnowledgeName() 21 | localStorage.setItem(STORAGE_KEY, knowledgeName) 22 | } 23 | 24 | return knowledgeName 25 | } 26 | 27 | /** 28 | * 设置知识库名称 29 | */ 30 | export function setKnowledgeName(name) { 31 | localStorage.setItem(STORAGE_KEY, name) 32 | return name 33 | } 34 | 35 | /** 36 | * 生成一个新的知识库名称 37 | * 使用UUID确保唯一性 38 | */ 39 | export function generateKnowledgeName() { 40 | return `knowledge_${uuidv4().substring(0, 8)}` 41 | } -------------------------------------------------------------------------------- /fe/src/views/Chat.vue: -------------------------------------------------------------------------------- 1 | 172 | 173 | 431 | 432 | -------------------------------------------------------------------------------- /fe/src/views/Indexer.vue: -------------------------------------------------------------------------------- 1 | 65 | 66 | 132 | 133 | -------------------------------------------------------------------------------- /fe/src/views/KnowledgeBase.vue: -------------------------------------------------------------------------------- 1 | 109 | 110 | 266 | 267 | -------------------------------------------------------------------------------- /fe/src/views/Retriever.vue: -------------------------------------------------------------------------------- 1 | 92 | 93 | 169 | 170 | -------------------------------------------------------------------------------- /fe/vite.config.js: -------------------------------------------------------------------------------- 1 | import { defineConfig } from 'vite' 2 | import vue from '@vitejs/plugin-vue' 3 | import path from 'path' 4 | 5 | // https://vitejs.dev/config/ 6 | export default defineConfig({ 7 | plugins: [vue()], 8 | resolve: { 9 | alias: { 10 | '@': path.resolve(__dirname, 'src'), 11 | }, 12 | }, 13 | server: { 14 | proxy: { 15 | '/v1': { 16 | target: 'http://localhost:8000', 17 | changeOrigin: true 18 | } 19 | } 20 | } 21 | }) -------------------------------------------------------------------------------- /server/README.md: -------------------------------------------------------------------------------- 1 | # go-rag api 2 | rag api 项目 3 | -------------------------------------------------------------------------------- /server/api/rag/rag.go: -------------------------------------------------------------------------------- 1 | // ================================================================================= 2 | // Code generated and maintained by GoFrame CLI tool. DO NOT EDIT. 3 | // ================================================================================= 4 | 5 | package rag 6 | 7 | import ( 8 | "context" 9 | 10 | "github.com/wangle201210/go-rag/server/api/rag/v1" 11 | ) 12 | 13 | type IRagV1 interface { 14 | Chat(ctx context.Context, req *v1.ChatReq) (res *v1.ChatRes, err error) 15 | ChatStream(ctx context.Context, req *v1.ChatStreamReq) (res *v1.ChatStreamRes, err error) 16 | Indexer(ctx context.Context, req *v1.IndexerReq) (res *v1.IndexerRes, err error) 17 | KBCreate(ctx context.Context, req *v1.KBCreateReq) (res *v1.KBCreateRes, err error) 18 | KBUpdate(ctx context.Context, req *v1.KBUpdateReq) (res *v1.KBUpdateRes, err error) 19 | KBDelete(ctx context.Context, req *v1.KBDeleteReq) (res *v1.KBDeleteRes, err error) 20 | KBGetOne(ctx context.Context, req *v1.KBGetOneReq) (res *v1.KBGetOneRes, err error) 21 | KBGetList(ctx context.Context, req *v1.KBGetListReq) (res *v1.KBGetListRes, err error) 22 | Retriever(ctx context.Context, req *v1.RetrieverReq) (res *v1.RetrieverRes, err error) 23 | } 24 | -------------------------------------------------------------------------------- /server/api/rag/v1/chat.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | "github.com/cloudwego/eino/schema" 5 | "github.com/gogf/gf/v2/frame/g" 6 | ) 7 | 8 | type ChatReq struct { 9 | g.Meta `path:"/v1/chat" method:"post" tags:"rag"` 10 | ConvID string `json:"conv_id" v:"required"` // 会话id 11 | Question string `json:"question" v:"required"` 12 | KnowledgeName string `json:"knowledge_name" v:"required"` 13 | TopK int `json:"top_k"` // 默认为5 14 | Score float64 `json:"score"` // 默认为0.2 15 | } 16 | 17 | type ChatRes struct { 18 | g.Meta `mime:"application/json"` 19 | Answer string `json:"answer"` 20 | References []*schema.Document `json:"references"` 21 | } 22 | 23 | // ChatStreamReq 流式输出请求 24 | type ChatStreamReq struct { 25 | g.Meta `path:"/v1/chat/stream" method:"post" tags:"rag"` 26 | ConvID string `json:"conv_id" v:"required"` // 会话id 27 | Question string `json:"question" v:"required"` 28 | KnowledgeName string `json:"knowledge_name" v:"required"` 29 | TopK int `json:"top_k"` // 默认为5 30 | Score float64 `json:"score"` // 默认为0.2 31 | } 32 | 33 | // ChatStreamRes 流式输出响应 34 | type ChatStreamRes struct { 35 | g.Meta `mime:"text/event-stream"` 36 | // 流式输出不需要返回具体内容，内容通过HTTP响应流返回 37 | } 38 | -------------------------------------------------------------------------------- /server/api/rag/v1/indexer.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | "github.com/gogf/gf/v2/frame/g" 5 | "github.com/gogf/gf/v2/net/ghttp" 6 | ) 7 | 8 | type IndexerReq struct { 9 | g.Meta `path:"/v1/indexer" method:"post" mime:"multipart/form-data" tags:"rag"` 10 | File *ghttp.UploadFile `p:"file" type:"file" dc:"如果是本地文件，怎上传文件"` 11 | URL string `p:"url" dc:"如果是网络文件则直接输入url即可"` 12 | KnowledgeName string `p:"knowledge_name" dc:"知识库名称" v:"required"` 13 | } 14 | 15 | type IndexerRes struct { 16 | g.Meta `mime:"application/json"` 17 | DocIDs []string `json:"doc_ids"` 18 | } 19 | -------------------------------------------------------------------------------- /server/api/rag/v1/knowledge_base.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | "github.com/gogf/gf/v2/frame/g" 5 | "github.com/wangle201210/go-rag/server/internal/model/entity" 6 | ) 7 | 8 | // Status marks kb status. 9 | type Status int 10 | 11 | const ( 12 | StatusOK Status = 1 13 | StatusDisabled Status = 2 14 | ) 15 | 16 | type KBCreateReq struct { 17 | g.Meta `path:"/v1/kb" method:"post" tags:"kb" summary:"Create kb"` 18 | Name string `v:"required|length:3,20" dc:"kb name"` 19 | Description string `v:"required|length:3,200" dc:"kb description"` 20 | Category string `v:"length:3,10" dc:"kb category"` 21 | } 22 | 23 | type KBCreateRes struct { 24 | Id int64 `json:"id" dc:"kb id"` 25 | } 26 | 27 | type KBUpdateReq struct { 28 | g.Meta `path:"/v1/kb/{id}" method:"put" tags:"kb" summary:"Update kb"` 29 | Id int64 `v:"required" dc:"kb id"` 30 | Name *string `v:"length:3,10" dc:"kb name"` 31 | Description *string `v:"length:3,200" dc:"kb description"` 32 | Category *string `v:"length:3,10" dc:"kb category"` 33 | Status *Status `v:"in:1,2" dc:"kb status"` 34 | } 35 | type KBUpdateRes struct{} 36 | 37 | type KBDeleteReq struct { 38 | g.Meta `path:"/v1/kb/{id}" method:"delete" tags:"kb" summary:"Delete kb"` 39 | Id int64 `v:"required" dc:"kb id"` 40 | } 41 | type KBDeleteRes struct{} 42 | 43 | type KBGetOneReq struct { 44 | g.Meta `path:"/v1/kb/{id}" method:"get" tags:"kb" summary:"Get one kb"` 45 | Id int64 `v:"required" dc:"kb id"` 46 | } 47 | type KBGetOneRes struct { 48 | *entity.KnowledgeBase `dc:"kb"` 49 | } 50 | 51 | type KBGetListReq struct { 52 | g.Meta `path:"/v1/kb" method:"get" tags:"kb" summary:"Get kbs"` 53 | Name *string `v:"length:3,10" dc:"kb name"` 54 | Status *Status `v:"in:1,2" dc:"kb age"` 55 | Category *string `v:"length:3,10" dc:"kb category"` 56 | } 57 | 58 | type KBGetListRes struct { 59 | List []*entity.KnowledgeBase `json:"list" dc:"kb list"` 60 | } 61 | -------------------------------------------------------------------------------- /server/api/rag/v1/retriever.go: -------------------------------------------------------------------------------- 1 | package v1 2 | 3 | import ( 4 | "github.com/cloudwego/eino/schema" 5 | "github.com/gogf/gf/v2/frame/g" 6 | ) 7 | 8 | type RetrieverReq struct { 9 | g.Meta `path:"/v1/retriever" method:"post" tags:"rag"` 10 | Question string `json:"question" v:"required"` 11 | TopK int `json:"top_k"` // 默认为5 12 | Score float64 `json:"score"` // 默认为0.2 13 | KnowledgeName string `json:"knowledge_name" v:"required"` 14 | } 15 | 16 | type RetrieverRes struct { 17 | g.Meta `mime:"application/json"` 18 | Document []*schema.Document `json:"document"` 19 | } 20 | -------------------------------------------------------------------------------- /server/core/common/chat_model.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/cloudwego/eino-ext/components/model/openai" 7 | "github.com/cloudwego/eino/components/model" 8 | ) 9 | 10 | var chatModel model.BaseChatModel 11 | 12 | func GetChatModel(ctx context.Context, cfg *openai.ChatModelConfig) (model.BaseChatModel, error) { 13 | if chatModel != nil { 14 | return chatModel, nil 15 | } 16 | cm, err := openai.NewChatModel(ctx, cfg) 17 | if err != nil { 18 | return nil, err 19 | } 20 | chatModel = cm 21 | return cm, nil 22 | } 23 | -------------------------------------------------------------------------------- /server/core/common/consts.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | const ( 4 | FieldContent = "content" 5 | FieldContentVector = "content_vector" 6 | FieldExtra = "ext" 7 | DocExtra = "ext" 8 | KnowledgeName = "_knowledge_name" 9 | 10 | Title1 = "h1" 11 | Title2 = "h2" 12 | Title3 = "h3" 13 | ) 14 | -------------------------------------------------------------------------------- /server/core/common/embedding.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "context" 5 | "os" 6 | 7 | "github.com/cloudwego/eino-ext/components/embedding/openai" 8 | "github.com/cloudwego/eino/components/embedding" 9 | "github.com/wangle201210/go-rag/server/core/config" 10 | ) 11 | 12 | func NewEmbedding(ctx context.Context, conf *config.Config) (eb embedding.Embedder, err error) { 13 | econf := &openai.EmbeddingConfig{ 14 | APIKey: conf.APIKey, 15 | Model: conf.EmbeddingModel, 16 | Dimensions: Of(1024), 17 | Timeout: 0, 18 | BaseURL: conf.BaseURL, 19 | } 20 | if econf.APIKey == "" { 21 | econf.APIKey = os.Getenv("OPENAI_API_KEY") 22 | } 23 | if econf.BaseURL == "" { 24 | econf.BaseURL = os.Getenv("OPENAI_BASE_URL") 25 | } 26 | if econf.Model == "" { 27 | econf.Model = "text-embedding-3-large" 28 | } 29 | eb, err = openai.NewEmbedder(ctx, econf) 30 | if err != nil { 31 | return nil, err 32 | } 33 | return eb, nil 34 | } 35 | -------------------------------------------------------------------------------- /server/core/common/es.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/elastic/go-elasticsearch/v8" 7 | "github.com/elastic/go-elasticsearch/v8/typedapi/indices/create" 8 | "github.com/elastic/go-elasticsearch/v8/typedapi/indices/exists" 9 | "github.com/elastic/go-elasticsearch/v8/typedapi/types" 10 | ) 11 | 12 | // createIndex create index for example in add_documents.go. 13 | func createIndex(ctx context.Context, client *elasticsearch.Client, indexName string) error { 14 | _, err := create.NewCreateFunc(client)(indexName).Request(&create.Request{ 15 | Mappings: &types.TypeMapping{ 16 | Properties: map[string]types.Property{ 17 | FieldContent: types.NewTextProperty(), 18 | FieldExtra: types.NewTextProperty(), 19 | KnowledgeName: types.NewKeywordProperty(), 20 | FieldContentVector: &types.DenseVectorProperty{ 21 | Dims: Of(1024), // same as embedding dimensions 22 | Index: Of(true), 23 | Similarity: Of("cosine"), 24 | }, 25 | }, 26 | }, 27 | }).Do(ctx) 28 | 29 | return err 30 | } 31 | 32 | func CreateIndexIfNotExists(ctx context.Context, client *elasticsearch.Client, indexName string) error { 33 | indexExists, err := exists.NewExistsFunc(client)(indexName).Do(ctx) 34 | if err != nil { 35 | return err 36 | } 37 | if indexExists { 38 | return nil 39 | } 40 | err = createIndex(ctx, client, indexName) 41 | return err 42 | } 43 | -------------------------------------------------------------------------------- /server/core/common/helper.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import "net/url" 4 | 5 | func Of[T any](v T) *T { 6 | return &v 7 | } 8 | 9 | func IsURL(str string) bool { 10 | u, err := url.Parse(str) 11 | if err != nil { 12 | return false 13 | } 14 | return u.Scheme != "" && u.Host != "" 15 | } 16 | -------------------------------------------------------------------------------- /server/core/common/stream.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import "github.com/cloudwego/eino/schema" 4 | 5 | type StreamData struct { 6 | Id string `json:"id"` // 同一个消息里面的id是相同的 7 | Created int64 `json:"created"` // 消息初始生成时间 8 | Content string `json:"content"` // 消息具体内容 9 | Document []*schema.Document `json:"document"` 10 | } 11 | -------------------------------------------------------------------------------- /server/core/config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "github.com/cloudwego/eino-ext/components/model/openai" 5 | "github.com/elastic/go-elasticsearch/v8" 6 | ) 7 | 8 | type Config struct { 9 | Client *elasticsearch.Client 10 | IndexName string // es index name 11 | // embedding 时使用 12 | APIKey string 13 | BaseURL string 14 | EmbeddingModel string 15 | ChatModel string 16 | } 17 | 18 | func (x *Config) GetChatModelConfig() *openai.ChatModelConfig { 19 | return &openai.ChatModelConfig{ 20 | APIKey: x.APIKey, 21 | BaseURL: x.BaseURL, 22 | Model: x.ChatModel, 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /server/core/grader/grader.go: -------------------------------------------------------------------------------- 1 | package grader 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "strings" 7 | 8 | "github.com/cloudwego/eino/components/model" 9 | "github.com/cloudwego/eino/schema" 10 | "github.com/gogf/gf/v2/frame/g" 11 | ) 12 | 13 | type Grader struct { 14 | cm model.BaseChatModel 15 | } 16 | 17 | func NewGrader(cm model.BaseChatModel) *Grader { 18 | return &Grader{ 19 | cm: cm, 20 | } 21 | } 22 | 23 | // Retriever 检查下检索到的结果是否能够回答当前问题 24 | func (x *Grader) Retriever(ctx context.Context, docs []*schema.Document, question string) (pass bool, err error) { 25 | messages, err := retrieverMessages(docs, question) 26 | if err != nil { 27 | return 28 | } 29 | result, err := x.cm.Generate(ctx, messages) 30 | if err != nil { 31 | return false, fmt.Errorf("检查下检索到的结果是否能够回答当前问题失败: %v", err) 32 | } 33 | pass = isPass(result.Content) 34 | return 35 | } 36 | 37 | func (x *Grader) Related(ctx context.Context, doc *schema.Document, question string) (pass bool, err error) { 38 | messages, err := docRelatedMessages(doc, question) 39 | if err != nil { 40 | return 41 | } 42 | result, err := x.cm.Generate(ctx, messages) 43 | if err != nil { 44 | return false, fmt.Errorf("检查下检索到的结果是否和用户问题相关失败: %v", err) 45 | } 46 | pass = isPass(result.Content) 47 | return 48 | } 49 | 50 | func isPass(msg string) bool { 51 | g.Log().Infof(context.Background(), "isPass: %s", msg) 52 | msg = strings.ToLower(msg) 53 | return strings.Contains(msg, "yes") 54 | } 55 | -------------------------------------------------------------------------------- /server/core/grader/message.go: -------------------------------------------------------------------------------- 1 | package grader 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/cloudwego/eino/components/prompt" 8 | "github.com/cloudwego/eino/schema" 9 | ) 10 | 11 | // createRetrieverTemplate 判断检索到的文档是否足够回答用户问题 12 | func createRetrieverTemplate() prompt.ChatTemplate { 13 | // 创建模板，使用 FString 格式 14 | return prompt.FromMessages(schema.FString, 15 | // 系统消息模板 16 | schema.SystemMessage( 17 | "您是一名评估检索到的文档是否足够回答用户问题的专家。"+ 18 | "请先仔细理解用户问题"+ 19 | "如果检索到的文档足够回答用户问题，请给出 'yes'，"+ 20 | "如果检索到的文档不足以回答用户问题，请给出 'no'。"+ 21 | "不要给出任何其他解释。", 22 | ), 23 | // 用户消息模板 24 | schema.UserMessage( 25 | "这是检索到的文档: \n"+ 26 | "{document} \n\n"+ 27 | "这是用户的问题: {question}"), 28 | ) 29 | } 30 | 31 | // createDocRelatedTemplate 判断检索到的文档是否和用户问题相关 32 | func createDocRelatedTemplate() prompt.ChatTemplate { 33 | // 创建模板，使用 FString 格式 34 | return prompt.FromMessages(schema.FString, 35 | // 系统消息模板 36 | schema.SystemMessage( 37 | "您是一名评估检索到的文档是否和用户问题相关的专家。"+ 38 | "这里不需要是一个严格的测试,目标是过滤掉错误的检索。"+ 39 | "如果检索到的文档和用户问题相关，请给出 'yes'，"+ 40 | "如果检索到的文档和用户问题不相关，请给出 'no'。"+ 41 | "不要给出任何其他解释。", 42 | ), 43 | // 用户消息模板 44 | schema.UserMessage( 45 | "<|start_documents|> \n"+ 46 | "{document} <|end_documents|>\n"+ 47 | "<|start_query|>{question}<|end_query|>"), 48 | ) 49 | } 50 | 51 | // formatMessages 格式化消息并处理错误 52 | func formatMessages(template prompt.ChatTemplate, data map[string]any) ([]*schema.Message, error) { 53 | messages, err := template.Format(context.Background(), data) 54 | if err != nil { 55 | return nil, fmt.Errorf("格式化模板失败: %w", err) 56 | } 57 | return messages, nil 58 | } 59 | 60 | func retrieverMessages(docs []*schema.Document, question string) ([]*schema.Message, error) { 61 | document := "" 62 | for i, doc := range docs { 63 | document += fmt.Sprintf("docs[%d]: %s", i, doc.Content) 64 | } 65 | template := createRetrieverTemplate() 66 | data := map[string]any{ 67 | "question": question, 68 | "document": document, 69 | } 70 | messages, err := formatMessages(template, data) 71 | if err != nil { 72 | return nil, err 73 | } 74 | return messages, nil 75 | } 76 | 77 | func docRelatedMessages(doc *schema.Document, question string) ([]*schema.Message, error) { 78 | template := createDocRelatedTemplate() 79 | data := map[string]any{ 80 | "question": question, 81 | "document": doc, 82 | } 83 | messages, err := formatMessages(template, data) 84 | if err != nil { 85 | return nil, err 86 | } 87 | return messages, nil 88 | } 89 | -------------------------------------------------------------------------------- /server/core/indexer/indexer.go: -------------------------------------------------------------------------------- 1 | package indexer 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/bytedance/sonic" 8 | "github.com/cloudwego/eino-ext/components/indexer/es8" 9 | "github.com/cloudwego/eino/components/indexer" 10 | "github.com/cloudwego/eino/schema" 11 | "github.com/google/uuid" 12 | "github.com/wangle201210/go-rag/server/core/common" 13 | "github.com/wangle201210/go-rag/server/core/config" 14 | ) 15 | 16 | // newIndexer component initialization function of node 'Indexer2' in graph 'rag' 17 | func newIndexer(ctx context.Context, conf *config.Config) (idr indexer.Indexer, err error) { 18 | indexerConfig := &es8.IndexerConfig{ 19 | Client: conf.Client, 20 | Index: conf.IndexName, 21 | BatchSize: 10, 22 | DocumentToFields: func(ctx context.Context, doc *schema.Document) (field2Value map[string]es8.FieldValue, err error) { 23 | var knowledgeName string 24 | if value, ok := ctx.Value(common.KnowledgeName).(string); ok { 25 | knowledgeName = value 26 | } else { 27 | err = fmt.Errorf("必须提供知识库名称") 28 | return 29 | } 30 | doc.ID = uuid.New().String() 31 | if doc.MetaData != nil { 32 | marshal, _ := sonic.Marshal(doc.MetaData) 33 | doc.MetaData[common.DocExtra] = string(marshal) 34 | } 35 | return map[string]es8.FieldValue{ 36 | common.FieldContent: { 37 | Value: getMdContentWithTitle(doc), 38 | EmbedKey: common.FieldContentVector, // vectorize doc content and save vector to field "content_vector" 39 | }, 40 | common.FieldExtra: { 41 | Value: doc.MetaData[common.DocExtra], 42 | }, 43 | common.KnowledgeName: { 44 | Value: knowledgeName, 45 | }, 46 | }, nil 47 | }, 48 | } 49 | embeddingIns11, err := common.NewEmbedding(ctx, conf) 50 | if err != nil { 51 | return nil, err 52 | } 53 | indexerConfig.Embedding = embeddingIns11 54 | idr, err = es8.NewIndexer(ctx, indexerConfig) 55 | if err != nil { 56 | return nil, err 57 | } 58 | return idr, nil 59 | } 60 | 61 | func getMdContentWithTitle(doc *schema.Document) string { 62 | if doc.MetaData == nil { 63 | return doc.Content 64 | } 65 | title := "" 66 | list := []string{"h1", "h2", "h3", "h4", "h5", "h6"} 67 | for _, v := range list { 68 | if d, e := doc.MetaData[v].(string); e && len(d) > 0 { 69 | title += fmt.Sprintf("%s:%s ", v, d) 70 | } 71 | } 72 | if len(title) == 0 { 73 | return doc.Content 74 | } 75 | return title + "\n" + doc.Content 76 | } 77 | -------------------------------------------------------------------------------- /server/core/indexer/lambda_func.go: -------------------------------------------------------------------------------- 1 | package indexer 2 | 3 | import ( 4 | "context" 5 | "strings" 6 | 7 | "github.com/cloudwego/eino-ext/components/document/loader/file" 8 | "github.com/cloudwego/eino/schema" 9 | "github.com/wangle201210/go-rag/server/core/common" 10 | ) 11 | 12 | // newLambda component initialization function of node 'Lambda1' in graph 't' 13 | func newLambda(ctx context.Context, docs []*schema.Document) (output []*schema.Document, err error) { 14 | // 不是md文档不处理 15 | if len(docs) == 0 || docs[0].MetaData[file.MetaKeyExtension] != ".md" { 16 | return docs, nil 17 | } 18 | ndocs := make([]*schema.Document, 0, len(docs)) 19 | var nd *schema.Document 20 | maxLen := 512 21 | for _, doc := range docs { 22 | // 不是同一个文件的就不要放一起了 23 | if nd != nil && doc.MetaData[file.MetaKeySource] != nd.MetaData[file.MetaKeySource] { 24 | ndocs = append(ndocs, nd) 25 | nd = nil 26 | } 27 | // 两个文档长度之和大于maxLen就不要放一起了 28 | if nd != nil && len(nd.Content)+len(doc.Content) > maxLen { 29 | ndocs = append(ndocs, nd) 30 | nd = nil 31 | } 32 | // 不是同一个一级标题的就不要放一起了 33 | if nd != nil && doc.MetaData[common.Title1] != nd.MetaData[common.Title1] { 34 | ndocs = append(ndocs, nd) 35 | nd = nil 36 | } 37 | // 不是同一个二级标题的就不要放一起了 38 | // 如果nd的h2是nil，证明之前只有h1,且两个的h1相等，则直接合并 39 | if nd != nil && nd.MetaData[common.Title2] != nil && doc.MetaData[common.Title2] != nd.MetaData[common.Title2] { 40 | ndocs = append(ndocs, nd) 41 | nd = nil 42 | } 43 | if nd == nil { 44 | nd = doc 45 | } else { 46 | mergeTitle(nd, doc, common.Title2) 47 | mergeTitle(nd, doc, common.Title3) 48 | nd.Content += doc.Content 49 | } 50 | } 51 | if nd != nil { 52 | ndocs = append(ndocs, nd) 53 | } 54 | return ndocs, nil 55 | } 56 | 57 | func mergeTitle(orgDoc, addDoc *schema.Document, key string) { 58 | // 相等就不管了 59 | if orgDoc.MetaData[key] == addDoc.MetaData[key] { 60 | return 61 | } 62 | var title []string 63 | if orgDoc.MetaData[key] != nil { 64 | title = append(title, orgDoc.MetaData[key].(string)) 65 | } 66 | if addDoc.MetaData[key] != nil { 67 | title = append(title, addDoc.MetaData[key].(string)) 68 | } 69 | if len(title) > 0 { 70 | orgDoc.MetaData[key] = strings.Join(title, ",") 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /server/core/indexer/loader.go: -------------------------------------------------------------------------------- 1 | package indexer 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/cloudwego/eino-ext/components/document/loader/file" 7 | "github.com/cloudwego/eino-ext/components/document/loader/url" 8 | "github.com/cloudwego/eino/components/document" 9 | "github.com/cloudwego/eino/schema" 10 | "github.com/wangle201210/go-rag/server/core/common" 11 | ) 12 | 13 | // newLoader component initialization function of node 'Loader1' in graph 'rag' 14 | func newLoader(ctx context.Context) (ldr document.Loader, err error) { 15 | mldr := &multiLoader{} 16 | parser, err := newParser(ctx) 17 | if err != nil { 18 | return nil, err 19 | } 20 | fldr, err := file.NewFileLoader(ctx, &file.FileLoaderConfig{ 21 | UseNameAsID: true, 22 | Parser: parser, 23 | }) 24 | if err != nil { 25 | return nil, err 26 | } 27 | mldr.fileLoader = fldr 28 | uldr, err := url.NewLoader(ctx, &url.LoaderConfig{}) 29 | if err != nil { 30 | return nil, err 31 | } 32 | mldr.urlLoader = uldr 33 | return mldr, nil 34 | } 35 | 36 | type multiLoader struct { 37 | fileLoader document.Loader 38 | urlLoader document.Loader 39 | } 40 | 41 | func (x *multiLoader) Load(ctx context.Context, src document.Source, opts ...document.LoaderOption) ([]*schema.Document, error) { 42 | if common.IsURL(src.URI) { 43 | return x.urlLoader.Load(ctx, src, opts...) 44 | } 45 | return x.fileLoader.Load(ctx, src, opts...) 46 | } 47 | -------------------------------------------------------------------------------- /server/core/indexer/orchestration.go: -------------------------------------------------------------------------------- 1 | package indexer 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/cloudwego/eino/compose" 7 | "github.com/wangle201210/go-rag/server/core/config" 8 | ) 9 | 10 | func BuildIndexer(ctx context.Context, conf *config.Config) (r compose.Runnable[any, []string], err error) { 11 | const ( 12 | Loader1 = "Loader1" 13 | Indexer2 = "Indexer2" 14 | DocumentTransformer3 = "DocumentTransformer3" 15 | Lambda1 = "Lambda1" 16 | ) 17 | 18 | g := compose.NewGraph[any, []string]() 19 | loader1KeyOfLoader, err := newLoader(ctx) 20 | if err != nil { 21 | return nil, err 22 | } 23 | _ = g.AddLoaderNode(Loader1, loader1KeyOfLoader) 24 | indexer2KeyOfIndexer, err := newIndexer(ctx, conf) 25 | if err != nil { 26 | return nil, err 27 | } 28 | _ = g.AddIndexerNode(Indexer2, indexer2KeyOfIndexer) 29 | documentTransformer2KeyOfDocumentTransformer, err := newDocumentTransformer(ctx) 30 | if err != nil { 31 | return nil, err 32 | } 33 | _ = g.AddLambdaNode(Lambda1, compose.InvokableLambda(newLambda)) 34 | 35 | _ = g.AddDocumentTransformerNode(DocumentTransformer3, documentTransformer2KeyOfDocumentTransformer) 36 | _ = g.AddEdge(compose.START, Loader1) 37 | _ = g.AddEdge(Loader1, DocumentTransformer3) 38 | _ = g.AddEdge(DocumentTransformer3, Lambda1) 39 | _ = g.AddEdge(Lambda1, Indexer2) 40 | _ = g.AddEdge(Indexer2, compose.END) 41 | r, err = g.Compile(ctx, compose.WithGraphName("indexer")) 42 | if err != nil { 43 | return nil, err 44 | } 45 | return r, err 46 | } 47 | -------------------------------------------------------------------------------- /server/core/indexer/parser.go: -------------------------------------------------------------------------------- 1 | package indexer 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/cloudwego/eino-ext/components/document/parser/html" 7 | "github.com/cloudwego/eino-ext/components/document/parser/pdf" 8 | "github.com/cloudwego/eino/components/document/parser" 9 | "github.com/wangle201210/go-rag/server/core/common" 10 | ) 11 | 12 | func newParser(ctx context.Context) (p parser.Parser, err error) { 13 | textParser := parser.TextParser{} 14 | 15 | htmlParser, err := html.NewParser(ctx, &html.Config{ 16 | Selector: common.Of("body"), 17 | }) 18 | if err != nil { 19 | return nil, err 20 | } 21 | 22 | pdfParser, err := pdf.NewPDFParser(ctx, &pdf.Config{}) 23 | if err != nil { 24 | return 25 | } 26 | 27 | // 创建扩展解析器 28 | p, err = parser.NewExtParser(ctx, &parser.ExtParserConfig{ 29 | // 注册特定扩展名的解析器 30 | Parsers: map[string]parser.Parser{ 31 | ".html": htmlParser, 32 | ".pdf": pdfParser, 33 | }, 34 | // 设置默认解析器，用于处理未知格式 35 | FallbackParser: textParser, 36 | }) 37 | if err != nil { 38 | return nil, err 39 | } 40 | return 41 | } 42 | -------------------------------------------------------------------------------- /server/core/indexer/transformer.go: -------------------------------------------------------------------------------- 1 | package indexer 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/cloudwego/eino-ext/components/document/transformer/splitter/markdown" 7 | "github.com/cloudwego/eino-ext/components/document/transformer/splitter/recursive" 8 | "github.com/cloudwego/eino/components/document" 9 | "github.com/cloudwego/eino/schema" 10 | "github.com/wangle201210/go-rag/server/core/common" 11 | ) 12 | 13 | // newDocumentTransformer component initialization function of node 'DocumentTransformer3' in graph 'rag' 14 | func newDocumentTransformer(ctx context.Context) (tfr document.Transformer, err error) { 15 | trans := &transformer{} 16 | // 递归分割 17 | config := &recursive.Config{ 18 | ChunkSize: 1000, // 每段内容1000字 19 | OverlapSize: 100, // 有10%的重叠 20 | Separators: []string{"\n", "。", "?", "？", "!", "！"}, 21 | } 22 | recTrans, err := recursive.NewSplitter(ctx, config) 23 | if err != nil { 24 | return nil, err 25 | } 26 | // md 文档特殊处理 27 | mdTrans, err := markdown.NewHeaderSplitter(ctx, &markdown.HeaderConfig{ 28 | Headers: map[string]string{"#": common.Title1, "##": common.Title2, "###": common.Title3}, 29 | TrimHeaders: false, 30 | }) 31 | if err != nil { 32 | return nil, err 33 | } 34 | trans.recursive = recTrans 35 | trans.markdown = mdTrans 36 | return trans, nil 37 | } 38 | 39 | type transformer struct { 40 | markdown document.Transformer 41 | recursive document.Transformer 42 | } 43 | 44 | func (x *transformer) Transform(ctx context.Context, docs []*schema.Document, opts ...document.TransformerOption) ([]*schema.Document, error) { 45 | isMd := false 46 | for _, doc := range docs { 47 | // 只需要判断第一个是不是.md 48 | if doc.MetaData["_extension"] == ".md" { 49 | isMd = true 50 | break 51 | } 52 | } 53 | if isMd { 54 | return x.markdown.Transform(ctx, docs, opts...) 55 | } 56 | return x.recursive.Transform(ctx, docs, opts...) 57 | } 58 | -------------------------------------------------------------------------------- /server/core/message.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "time" 7 | 8 | "github.com/cloudwego/eino/components/prompt" 9 | "github.com/cloudwego/eino/schema" 10 | ) 11 | 12 | var system = "你非常擅长于使用rag进行数据检索，" + 13 | "你的目标是在充分理解用户的问题后进行向量化检索" + 14 | "现在时间{time_now}" + 15 | "你要提取并优化搜索的查询内容。" + 16 | "请遵循以下规则重写查询内容：\n " + 17 | "- 根据用户的问题和上下文，重写应该进行搜索的关键词\n" + 18 | "- 如果需要使用时间，则根据当前时间给出需要查询的具体时间日期信息\n" + 19 | // "- 生成的查询关键词要选择合适的语言，考虑用户的问题类型使用最适合的语言进行搜索，例如某些问题应该保持用户的问题语言，而有一些则更适合翻译成英语或其他语言\n" + 20 | "- 保持查询简洁，查询内容通常不超过3个关键词, 最多不要超过5个关键词\n" + 21 | "- 参考当前搜索引擎的查询习惯重写关键字，直接返回优化后的搜索词，不要有任何额外说明。\n" + 22 | "- 尽量不要使用下面这些已使用过的关键词，因为之前使用这些关键词搜索到的结果不符合预期，\n" + 23 | "- 已使用过的关键词：{used}\n" 24 | 25 | // createTemplate 创建并返回一个配置好的聊天模板 26 | func createTemplate() prompt.ChatTemplate { 27 | return prompt.FromMessages(schema.FString, 28 | // 系统消息模板 29 | schema.SystemMessage("{system}"), 30 | // 用户消息模板 31 | schema.UserMessage( 32 | "如下是用户的问题: {question}"), 33 | ) 34 | } 35 | 36 | // formatMessages 格式化消息并处理错误 37 | func formatMessages(template prompt.ChatTemplate, data map[string]any) ([]*schema.Message, error) { 38 | messages, err := template.Format(context.Background(), data) 39 | if err != nil { 40 | return nil, fmt.Errorf("格式化模板失败: %w", err) 41 | } 42 | return messages, nil 43 | } 44 | 45 | func getOptimizedQueryMessages(used string, question string) ([]*schema.Message, error) { 46 | template := createTemplate() 47 | data := map[string]any{ 48 | "system": system, 49 | "time_now": time.Now().Format(time.RFC3339), 50 | "question": question, 51 | "used": used, 52 | } 53 | messages, err := formatMessages(template, data) 54 | if err != nil { 55 | return nil, err 56 | } 57 | return messages, nil 58 | } 59 | -------------------------------------------------------------------------------- /server/core/rag.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "sync" 8 | 9 | "github.com/cloudwego/eino-ext/components/retriever/es8" 10 | "github.com/cloudwego/eino/components/document" 11 | "github.com/cloudwego/eino/components/model" 12 | er "github.com/cloudwego/eino/components/retriever" 13 | "github.com/cloudwego/eino/compose" 14 | "github.com/cloudwego/eino/schema" 15 | "github.com/elastic/go-elasticsearch/v8" 16 | "github.com/elastic/go-elasticsearch/v8/typedapi/core/search" 17 | "github.com/elastic/go-elasticsearch/v8/typedapi/types" 18 | "github.com/gogf/gf/v2/frame/g" 19 | "github.com/wangle201210/go-rag/server/core/common" 20 | "github.com/wangle201210/go-rag/server/core/config" 21 | "github.com/wangle201210/go-rag/server/core/grader" 22 | "github.com/wangle201210/go-rag/server/core/indexer" 23 | "github.com/wangle201210/go-rag/server/core/retriever" 24 | ) 25 | 26 | type Rag struct { 27 | idxer compose.Runnable[any, []string] 28 | rtrvr compose.Runnable[string, []*schema.Document] 29 | client *elasticsearch.Client 30 | cm model.BaseChatModel 31 | grader *grader.Grader 32 | } 33 | 34 | func New(ctx context.Context, conf *config.Config) (*Rag, error) { 35 | if len(conf.IndexName) == 0 { 36 | return nil, fmt.Errorf("indexName is empty") 37 | } 38 | // 确保es index存在 39 | err := common.CreateIndexIfNotExists(ctx, conf.Client, conf.IndexName) 40 | if err != nil { 41 | return nil, err 42 | } 43 | buildIndex, err := indexer.BuildIndexer(ctx, conf) 44 | if err != nil { 45 | return nil, err 46 | } 47 | buildRetriever, err := retriever.BuildRetriever(ctx, conf) 48 | if err != nil { 49 | return nil, err 50 | } 51 | cm, err := common.GetChatModel(ctx, conf.GetChatModelConfig()) 52 | if err != nil { 53 | g.Log().Error(ctx, "GetChatModel failed, err=%v", err) 54 | return nil, err 55 | } 56 | return &Rag{ 57 | idxer: buildIndex, 58 | rtrvr: buildRetriever, 59 | client: conf.Client, 60 | cm: cm, 61 | grader: grader.NewGrader(cm), 62 | }, nil 63 | } 64 | 65 | type IndexReq struct { 66 | URI string // 文档地址，可以是文件路径（pdf，html，md等），也可以是网址 67 | KnowledgeName string // 知识库名称 68 | } 69 | 70 | // Index 71 | // uri: 72 | // ids: 文档id 73 | func (x *Rag) Index(ctx context.Context, req *IndexReq) (ids []string, err error) { 74 | s := document.Source{ 75 | URI: req.URI, 76 | } 77 | ctx = context.WithValue(ctx, common.KnowledgeName, req.KnowledgeName) 78 | ids, err = x.idxer.Invoke(ctx, s) 79 | if err != nil { 80 | return 81 | } 82 | return 83 | } 84 | 85 | type RetrieveReq struct { 86 | Query string // 检索关键词 87 | TopK int // 检索结果数量 88 | Score float64 // 分数阀值(0-2, 0 完全相反，1 毫不相干，2 完全相同,一般需要传入一个大于1的数字，如1.5) 89 | KnowledgeName string // 知识库名字 90 | optQuery string // 优化后的检索关键词 91 | excludeIDs []string // 要排除的 _id 列表 92 | } 93 | 94 | // Retrieve 检索 95 | func (x *Rag) Retrieve(ctx context.Context, req *RetrieveReq) (msg []*schema.Document, err error) { 96 | used := "" 97 | relatedDocs := &sync.Map{} 98 | docNum := 0 99 | // 最多尝试N次,后续做成可配置 100 | for i := 0; i < 3; i++ { 101 | question := req.Query 102 | var ( 103 | messages []*schema.Message 104 | generate *schema.Message 105 | docs []*schema.Document 106 | pass bool 107 | ) 108 | messages, err = getOptimizedQueryMessages(used, question) 109 | if err != nil { 110 | return 111 | } 112 | generate, err = x.cm.Generate(ctx, messages) 113 | if err != nil { 114 | return 115 | } 116 | optimizedQuery := generate.Content 117 | used += optimizedQuery + " " 118 | req.optQuery = optimizedQuery 119 | docs, err = x.retrieve(ctx, req) 120 | if err != nil { 121 | return 122 | } 123 | wg := &sync.WaitGroup{} 124 | for _, doc := range docs { 125 | wg.Add(1) 126 | go func() { 127 | defer wg.Done() 128 | pass, err = x.grader.Related(ctx, doc, req.Query) 129 | if err != nil { 130 | return 131 | } 132 | req.excludeIDs = append(req.excludeIDs, doc.ID) // 后续不要检索这个_id对应的数据 133 | if pass { 134 | relatedDocs.Store(doc.ID, doc) 135 | docNum++ 136 | } else { 137 | g.Log().Infof(ctx, "not doc score: %v, related: %v", doc.Score(), doc.Content) 138 | } 139 | }() 140 | } 141 | wg.Wait() 142 | // 数量不够就再次检索 143 | if docNum < req.TopK { 144 | continue 145 | } 146 | // 数量够了，就直接返回 147 | rDocs := make([]*schema.Document, 0, req.TopK) 148 | relatedDocs.Range(func(key, value any) bool { 149 | rDocs = append(rDocs, value.(*schema.Document)) 150 | return true 151 | }) 152 | pass, err = x.grader.Retriever(ctx, rDocs, req.Query) 153 | if err != nil { 154 | return 155 | } 156 | if pass { 157 | return rDocs, nil 158 | } 159 | } 160 | // 最后数量不够，就返回所有数据 161 | relatedDocs.Range(func(key, value any) bool { 162 | msg = append(msg, value.(*schema.Document)) 163 | return true 164 | }) 165 | return 166 | } 167 | 168 | func (x *Rag) retrieve(ctx context.Context, req *RetrieveReq) (msg []*schema.Document, err error) { 169 | g.Log().Infof(ctx, "query: %v", req.optQuery) 170 | esQuery := []types.Query{ 171 | { 172 | Bool: &types.BoolQuery{ 173 | Must: []types.Query{{Match: map[string]types.MatchQuery{common.KnowledgeName: {Query: req.KnowledgeName}}}}, 174 | }, 175 | }, 176 | } 177 | if len(req.excludeIDs) > 0 { 178 | esQuery[0].Bool.MustNot = []types.Query{ 179 | { 180 | Terms: &types.TermsQuery{ 181 | TermsQuery: map[string]types.TermsQueryField{ 182 | "_id": req.excludeIDs, 183 | }, 184 | }, 185 | }, 186 | } 187 | } 188 | msg, err = x.rtrvr.Invoke(ctx, req.optQuery, 189 | compose.WithRetrieverOption( 190 | er.WithScoreThreshold(req.Score), 191 | er.WithTopK(req.TopK), 192 | es8.WithFilters(esQuery), 193 | ), 194 | ) 195 | if err != nil { 196 | return 197 | } 198 | return 199 | } 200 | 201 | // GetKnowledgeBaseList 获取知识库列表 202 | func (x *Rag) GetKnowledgeBaseList(ctx context.Context) (list []string, err error) { 203 | names := "distinct_knowledge_names" 204 | query := search.NewRequest() 205 | query.Size = common.Of(0) // 不返回原始文档 206 | query.Aggregations = map[string]types.Aggregations{ 207 | names: { 208 | Terms: &types.TermsAggregation{ 209 | Field: common.Of(common.KnowledgeName), 210 | Size: common.Of(10000), 211 | }, 212 | }, 213 | } 214 | res, err := search.NewSearchFunc(x.client)(). 215 | Request(query). 216 | Do(ctx) 217 | if err != nil { 218 | return 219 | } 220 | if res.Aggregations == nil { 221 | g.Log().Infof(ctx, "No aggregations found") 222 | return 223 | } 224 | termsAgg, ok := res.Aggregations[names].(*types.StringTermsAggregate) 225 | if !ok || termsAgg == nil { 226 | err = errors.New("failed to parse terms aggregation") 227 | return 228 | } 229 | for _, bucket := range termsAgg.Buckets.([]types.StringTermsBucket) { 230 | list = append(list, bucket.Key.(string)) 231 | } 232 | return 233 | } 234 | -------------------------------------------------------------------------------- /server/core/rag_test.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "context" 5 | "log" 6 | "testing" 7 | 8 | "github.com/elastic/go-elasticsearch/v8" 9 | "github.com/gogf/gf/v2/frame/g" 10 | "github.com/wangle201210/go-rag/server/core/config" 11 | ) 12 | 13 | var ragSvr = &Rag{} 14 | 15 | func _init() { 16 | ctx := context.Background() 17 | client, err := elasticsearch.NewClient(elasticsearch.Config{ 18 | Addresses: []string{"http://localhost:9200"}, 19 | }) 20 | if err != nil { 21 | log.Printf("NewClient of es8 failed, err=%v", err) 22 | return 23 | } 24 | ragSvr, err = New(context.Background(), &config.Config{ 25 | Client: client, 26 | IndexName: "rag-test", 27 | APIKey: g.Cfg().MustGet(ctx, "embedding.apiKey").String(), 28 | BaseURL: g.Cfg().MustGet(ctx, "embedding.baseURL").String(), 29 | EmbeddingModel: g.Cfg().MustGet(ctx, "embedding.model").String(), 30 | ChatModel: g.Cfg().MustGet(ctx, "chat.model").String(), 31 | }) 32 | if err != nil { 33 | log.Printf("New of rag failed, err=%v", err) 34 | return 35 | } 36 | } 37 | func TestIndex(t *testing.T) { 38 | ctx := context.Background() 39 | uriList := []string{ 40 | "./test_file/readme.md", 41 | "./test_file/readme2.md", 42 | "./test_file/readme.html", 43 | "./test_file/test.pdf", 44 | "https://deepchat.thinkinai.xyz/docs/guide/advanced-features/shortcuts.html", 45 | } 46 | for _, s := range uriList { 47 | req := &IndexReq{ 48 | URI: s, 49 | KnowledgeName: "wanna", 50 | } 51 | ids, err := ragSvr.Index(ctx, req) 52 | if err != nil { 53 | t.Fatal(err) 54 | } 55 | for _, id := range ids { 56 | t.Log(id) 57 | } 58 | } 59 | } 60 | 61 | func TestRetriever(t *testing.T) { 62 | _init() 63 | ctx := context.Background() 64 | req := &RetrieveReq{ 65 | Query: "deepchat支持哪些国家的语言", 66 | TopK: 5, 67 | Score: 1.2, 68 | KnowledgeName: "deepchat", 69 | } 70 | msg, err := ragSvr.Retrieve(ctx, req) 71 | if err != nil { 72 | t.Fatal(err) 73 | } 74 | for _, m := range msg { 75 | t.Logf("content: %v, score: %v", m.Content, m.Score()) 76 | } 77 | } 78 | 79 | func TestRag_GetKnowledgeList(t *testing.T) { 80 | ctx := context.Background() 81 | list, err := ragSvr.GetKnowledgeBaseList(ctx) 82 | if err != nil { 83 | t.Fatal(err) 84 | return 85 | } 86 | t.Logf("list: %v", list) 87 | } 88 | -------------------------------------------------------------------------------- /server/core/readme.md: -------------------------------------------------------------------------------- 1 | # go-rag 核心逻辑 2 | 此目录本应放在internal目录下，但为了便于被其他项目直接引用，将其放在core目录下 3 | 4 | 详情可以参照 [test文件](../core/rag_test.go) 5 | -------------------------------------------------------------------------------- /server/core/retriever/orchestration.go: -------------------------------------------------------------------------------- 1 | package retriever 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/cloudwego/eino/compose" 7 | "github.com/cloudwego/eino/schema" 8 | "github.com/wangle201210/go-rag/server/core/config" 9 | ) 10 | 11 | func BuildRetriever(ctx context.Context, conf *config.Config) (r compose.Runnable[string, []*schema.Document], err error) { 12 | const ( 13 | Retriever1 = "Retriever1" 14 | DocumentTransformer1 = "DocumentTransformer1" 15 | ) 16 | g := compose.NewGraph[string, []*schema.Document]() 17 | retriever1KeyOfRetriever, err := newRetriever(ctx, conf) 18 | if err != nil { 19 | return nil, err 20 | } 21 | _ = g.AddRetrieverNode(Retriever1, retriever1KeyOfRetriever) 22 | documentTransformer1KeyOfDocumentTransformer, err := newDocumentTransformer(ctx) 23 | if err != nil { 24 | return nil, err 25 | } 26 | _ = g.AddDocumentTransformerNode(DocumentTransformer1, documentTransformer1KeyOfDocumentTransformer) 27 | _ = g.AddEdge(compose.START, Retriever1) 28 | _ = g.AddEdge(Retriever1, DocumentTransformer1) 29 | _ = g.AddEdge(DocumentTransformer1, compose.END) 30 | r, err = g.Compile(ctx, compose.WithGraphName("retriever")) 31 | if err != nil { 32 | return nil, err 33 | } 34 | return r, err 35 | } 36 | -------------------------------------------------------------------------------- /server/core/retriever/retriever.go: -------------------------------------------------------------------------------- 1 | package retriever 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | 8 | "github.com/cloudwego/eino-ext/components/retriever/es8" 9 | "github.com/cloudwego/eino-ext/components/retriever/es8/search_mode" 10 | "github.com/cloudwego/eino/components/retriever" 11 | "github.com/cloudwego/eino/schema" 12 | "github.com/elastic/go-elasticsearch/v8/typedapi/types" 13 | "github.com/wangle201210/go-rag/server/core/common" 14 | "github.com/wangle201210/go-rag/server/core/config" 15 | ) 16 | 17 | // newRetriever component initialization function of node 'Retriever1' in graph 'retriever' 18 | func newRetriever(ctx context.Context, conf *config.Config) (rtr retriever.Retriever, err error) { 19 | retrieverConfig := &es8.RetrieverConfig{ 20 | Client: conf.Client, 21 | Index: conf.IndexName, 22 | TopK: 5, 23 | SearchMode: search_mode.SearchModeDenseVectorSimilarity( 24 | search_mode.DenseVectorSimilarityTypeCosineSimilarity, 25 | common.FieldContentVector, 26 | ), 27 | ResultParser: func(ctx context.Context, hit types.Hit) (doc *schema.Document, err error) { 28 | doc = &schema.Document{ 29 | ID: *hit.Id_, 30 | Content: "", 31 | MetaData: map[string]any{}, 32 | } 33 | 34 | var src map[string]any 35 | if err = json.Unmarshal(hit.Source_, &src); err != nil { 36 | return nil, err 37 | } 38 | 39 | for field, val := range src { 40 | switch field { 41 | case common.FieldContent: 42 | doc.Content = val.(string) 43 | case common.FieldContentVector: 44 | var v []float64 45 | for _, item := range val.([]interface{}) { 46 | v = append(v, item.(float64)) 47 | } 48 | doc.WithDenseVector(v) 49 | 50 | case common.FieldExtra: 51 | if val == nil { 52 | continue 53 | } 54 | doc.MetaData[common.DocExtra] = val.(string) 55 | case common.KnowledgeName: 56 | doc.MetaData[common.KnowledgeName] = val.(string) 57 | default: 58 | return nil, fmt.Errorf("unexpected field=%s, val=%v", field, val) 59 | } 60 | } 61 | 62 | if hit.Score_ != nil { 63 | doc.WithScore(float64(*hit.Score_)) 64 | } 65 | 66 | return doc, nil 67 | }, 68 | } 69 | embeddingIns11, err := common.NewEmbedding(ctx, conf) 70 | if err != nil { 71 | return nil, err 72 | } 73 | retrieverConfig.Embedding = embeddingIns11 74 | rtr, err = es8.NewRetriever(ctx, retrieverConfig) 75 | if err != nil { 76 | return nil, err 77 | } 78 | return rtr, nil 79 | } 80 | -------------------------------------------------------------------------------- /server/core/retriever/transformer.go: -------------------------------------------------------------------------------- 1 | package retriever 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/cloudwego/eino-ext/components/document/transformer/reranker/score" 7 | "github.com/cloudwego/eino/components/document" 8 | ) 9 | 10 | // newDocumentTransformer component initialization function of node 'DocumentTransformer1' in graph 'retriever' 11 | func newDocumentTransformer(ctx context.Context) (tfr document.Transformer, err error) { 12 | config := &score.Config{} 13 | tfr, err = score.NewReranker(ctx, config) 14 | if err != nil { 15 | return nil, err 16 | } 17 | return tfr, nil 18 | } 19 | -------------------------------------------------------------------------------- /server/core/test_file/readme.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | golang项目搭建 4 | 5 | 6 |

1. 安装环境

7 |

2. 写代码

8 |

3. 执行

9 | 10 | -------------------------------------------------------------------------------- /server/core/test_file/readme.md: -------------------------------------------------------------------------------- 1 | # 这是一个readme文件，这里有很多内容 -------------------------------------------------------------------------------- /server/core/test_file/readme2.md: -------------------------------------------------------------------------------- 1 | # 开饭了 -------------------------------------------------------------------------------- /server/core/test_file/test.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangle201210/go-rag/29d54bcfff76f50e5f5f8532653a2091d16cbca5/server/core/test_file/test.pdf -------------------------------------------------------------------------------- /server/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/wangle201210/go-rag/server 2 | 3 | go 1.23.1 4 | 5 | require ( 6 | github.com/ThinkInAIXYZ/go-mcp v0.2.14 7 | github.com/bytedance/sonic v1.13.2 8 | github.com/cloudwego/eino v0.3.31 9 | github.com/cloudwego/eino-ext/components/document/loader/file v0.0.0-20250424061409-ccd60fbc7c1c 10 | github.com/cloudwego/eino-ext/components/document/loader/url v0.0.0-20250605072634-0f875e04269d 11 | github.com/cloudwego/eino-ext/components/document/parser/html v0.0.0-20250424061409-ccd60fbc7c1c 12 | github.com/cloudwego/eino-ext/components/document/parser/pdf v0.0.0-20250424061409-ccd60fbc7c1c 13 | github.com/cloudwego/eino-ext/components/document/transformer/reranker/score v0.0.0-20250513023651-7b19c6ffbf4a 14 | github.com/cloudwego/eino-ext/components/document/transformer/splitter/markdown v0.0.0-20250605072634-0f875e04269d 15 | github.com/cloudwego/eino-ext/components/document/transformer/splitter/recursive v0.0.0-20250424061409-ccd60fbc7c1c 16 | github.com/cloudwego/eino-ext/components/embedding/openai v0.0.0-20250424061409-ccd60fbc7c1c 17 | github.com/cloudwego/eino-ext/components/indexer/es8 v0.0.0-20250605072634-0f875e04269d 18 | github.com/cloudwego/eino-ext/components/model/openai v0.0.0-20250512035704-1e06fdfda207 19 | github.com/cloudwego/eino-ext/components/retriever/es8 v0.0.0-20250424061409-ccd60fbc7c1c 20 | github.com/elastic/go-elasticsearch/v8 v8.16.0 21 | github.com/gogf/gf/contrib/drivers/mysql/v2 v2.9.0 22 | github.com/gogf/gf/v2 v2.9.0 23 | github.com/google/uuid v1.6.0 24 | github.com/wangle201210/chat-history v0.0.0-20250402104704-5eec15d5419e 25 | gorm.io/driver/mysql v1.5.7 26 | gorm.io/gorm v1.25.12 27 | ) 28 | 29 | require ( 30 | github.com/BurntSushi/toml v1.4.0 // indirect 31 | github.com/PuerkitoBio/goquery v1.8.1 // indirect 32 | github.com/andybalholm/cascadia v1.3.1 // indirect 33 | github.com/aymerick/douceur v0.2.0 // indirect 34 | github.com/bytedance/sonic/loader v0.2.4 // indirect 35 | github.com/clbanning/mxj/v2 v2.7.0 // indirect 36 | github.com/cloudwego/base64x v0.1.5 // indirect 37 | github.com/cloudwego/eino-ext/libs/acl/openai v0.0.0-20250422092704-54e372e1fa3d // indirect 38 | github.com/dslipak/pdf v0.0.2 // indirect 39 | github.com/dustin/go-humanize v1.0.1 // indirect 40 | github.com/elastic/elastic-transport-go/v8 v8.7.0 // indirect 41 | github.com/emirpasic/gods v1.18.1 // indirect 42 | github.com/fatih/color v1.18.0 // indirect 43 | github.com/fsnotify/fsnotify v1.7.0 // indirect 44 | github.com/getkin/kin-openapi v0.118.0 // indirect 45 | github.com/go-logr/logr v1.4.2 // indirect 46 | github.com/go-logr/stdr v1.2.2 // indirect 47 | github.com/go-openapi/jsonpointer v0.21.1 // indirect 48 | github.com/go-openapi/swag v0.23.1 // indirect 49 | github.com/go-sql-driver/mysql v1.7.1 // indirect 50 | github.com/goph/emperror v0.17.2 // indirect 51 | github.com/gorilla/css v1.0.1 // indirect 52 | github.com/gorilla/websocket v1.5.3 // indirect 53 | github.com/grokify/html-strip-tags-go v0.1.0 // indirect 54 | github.com/invopop/yaml v0.1.0 // indirect 55 | github.com/jinzhu/inflection v1.0.0 // indirect 56 | github.com/jinzhu/now v1.1.5 // indirect 57 | github.com/josharian/intern v1.0.0 // indirect 58 | github.com/json-iterator/go v1.1.12 // indirect 59 | github.com/klauspost/cpuid/v2 v2.2.10 // indirect 60 | github.com/magiconair/properties v1.8.9 // indirect 61 | github.com/mailru/easyjson v0.9.0 // indirect 62 | github.com/mattn/go-colorable v0.1.13 // indirect 63 | github.com/mattn/go-isatty v0.0.20 // indirect 64 | github.com/mattn/go-runewidth v0.0.16 // indirect 65 | github.com/meguminnnnnnnnn/go-openai v0.0.0-20250408071642-761325becfd6 // indirect 66 | github.com/microcosm-cc/bluemonday v1.0.27 // indirect 67 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 68 | github.com/modern-go/reflect2 v1.0.2 // indirect 69 | github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect 70 | github.com/nikolalohinski/gonja v1.5.3 // indirect 71 | github.com/olekukonko/tablewriter v0.0.5 // indirect 72 | github.com/orcaman/concurrent-map/v2 v2.0.1 // indirect 73 | github.com/pelletier/go-toml/v2 v2.2.3 // indirect 74 | github.com/perimeterx/marshmallow v1.1.5 // indirect 75 | github.com/pkg/errors v0.9.1 // indirect 76 | github.com/rivo/uniseg v0.4.7 // indirect 77 | github.com/sirupsen/logrus v1.9.3 // indirect 78 | github.com/slongfield/pyfmt v0.0.0-20220222012616-ea85ff4c361f // indirect 79 | github.com/tidwall/gjson v1.18.0 // indirect 80 | github.com/tidwall/match v1.1.1 // indirect 81 | github.com/tidwall/pretty v1.2.0 // indirect 82 | github.com/twitchyliquid64/golang-asm v0.15.1 // indirect 83 | github.com/yargevad/filepathx v1.0.0 // indirect 84 | github.com/yosida95/uritemplate/v3 v3.0.2 // indirect 85 | go.opentelemetry.io/otel v1.32.0 // indirect 86 | go.opentelemetry.io/otel/metric v1.32.0 // indirect 87 | go.opentelemetry.io/otel/sdk v1.32.0 // indirect 88 | go.opentelemetry.io/otel/trace v1.32.0 // indirect 89 | golang.org/x/arch v0.15.0 // indirect 90 | golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 // indirect 91 | golang.org/x/net v0.33.0 // indirect 92 | golang.org/x/sys v0.31.0 // indirect 93 | golang.org/x/text v0.21.0 // indirect 94 | gopkg.in/yaml.v3 v3.0.1 // indirect 95 | ) 96 | -------------------------------------------------------------------------------- /server/go.sum: -------------------------------------------------------------------------------- 1 | github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0= 2 | github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= 3 | github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM= 4 | github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ= 5 | github.com/ThinkInAIXYZ/go-mcp v0.2.14 h1:gyZ4Dv47Ozr4k4h329Qk8TOSDr4SsyBJn0o21oAs2Ec= 6 | github.com/ThinkInAIXYZ/go-mcp v0.2.14/go.mod h1:KnUWUymko7rmOgzvIjxwX0uB9oiJeLF/Q3W9cRt8fVg= 7 | github.com/airbrake/gobrake v3.6.1+incompatible/go.mod h1:wM4gu3Cn0W0K7GUuVWnlXZU11AGBXMILnrdOU8Kn00o= 8 | github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= 9 | github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= 10 | github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= 11 | github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= 12 | github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= 13 | github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= 14 | github.com/bugsnag/bugsnag-go v1.4.0/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= 15 | github.com/bugsnag/panicwrap v1.2.0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= 16 | github.com/bytedance/mockey v1.2.14 h1:KZaFgPdiUwW+jOWFieo3Lr7INM1P+6adO3hxZhDswY8= 17 | github.com/bytedance/mockey v1.2.14/go.mod h1:1BPHF9sol5R1ud/+0VEHGQq/+i2lN+GTsr3O2Q9IENY= 18 | github.com/bytedance/sonic v1.13.2 h1:8/H1FempDZqC4VqjptGo14QQlJx8VdZJegxs6wwfqpQ= 19 | github.com/bytedance/sonic v1.13.2/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1+KgkJhz4= 20 | github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= 21 | github.com/bytedance/sonic/loader v0.2.4 h1:ZWCw4stuXUsn1/+zQDqeE7JKP+QO47tz7QCNan80NzY= 22 | github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= 23 | github.com/certifi/gocertifi v0.0.0-20190105021004-abcd57078448/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4= 24 | github.com/clbanning/mxj/v2 v2.7.0 h1:WA/La7UGCanFe5NpHF0Q3DNtnCsVoxbPKuyBNHWRyME= 25 | github.com/clbanning/mxj/v2 v2.7.0/go.mod h1:hNiWqW14h+kc+MdF9C6/YoRfjEJoR3ou6tn/Qo+ve2s= 26 | github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4= 27 | github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= 28 | github.com/cloudwego/eino v0.3.31 h1:CByfu37j7YjYp+T/aS2oCffPCOY986F3Jbb6a6BfO4Y= 29 | github.com/cloudwego/eino v0.3.31/go.mod h1:wUjz990apdsaOraOXdh6CdhVXq8DJsOvLsVlxNTcNfY= 30 | github.com/cloudwego/eino-ext/components/document/loader/file v0.0.0-20250424061409-ccd60fbc7c1c h1:mMJtZczl2Cs8Cou+xzsrbT+I49seDhigISPeB6cQtX8= 31 | github.com/cloudwego/eino-ext/components/document/loader/file v0.0.0-20250424061409-ccd60fbc7c1c/go.mod h1:I4vbBCIMMKeF436Lc+L3DSPQ3f1nmiHD0JS+LhMYCdQ= 32 | github.com/cloudwego/eino-ext/components/document/loader/url v0.0.0-20250605072634-0f875e04269d h1:wUaqvsLQfSVEZka0CPjB+7scZ1tBbHf6eEt4ahmlRDo= 33 | github.com/cloudwego/eino-ext/components/document/loader/url v0.0.0-20250605072634-0f875e04269d/go.mod h1:qnTfPHmDIMJSF6/tbmcKUmVgEfuBrFoTfVgfFHxNZ84= 34 | github.com/cloudwego/eino-ext/components/document/parser/html v0.0.0-20250424061409-ccd60fbc7c1c h1:0Fn0GR1jM9WO4xaY8cwjj25F+T5tS4Jpo2bO1s9aDf8= 35 | github.com/cloudwego/eino-ext/components/document/parser/html v0.0.0-20250424061409-ccd60fbc7c1c/go.mod h1:w6f/1eEScd32tenRKwcYIJKqZ+3cArFOia1QxFsQmVE= 36 | github.com/cloudwego/eino-ext/components/document/parser/pdf v0.0.0-20250424061409-ccd60fbc7c1c h1:BBdsRDuIf7aSkUSUbrh2ojzIxCBxStPC05ofqjenR2E= 37 | github.com/cloudwego/eino-ext/components/document/parser/pdf v0.0.0-20250424061409-ccd60fbc7c1c/go.mod h1:Vpoaj8exHtu8EbRaAZTFRT7UaKslXd5nx7Z0EEVDIvY= 38 | github.com/cloudwego/eino-ext/components/document/transformer/reranker/score v0.0.0-20250513023651-7b19c6ffbf4a h1:Qce5GskHlDgAxyecVY8ur/5PFUmDQydusnHhRj4SCCg= 39 | github.com/cloudwego/eino-ext/components/document/transformer/reranker/score v0.0.0-20250513023651-7b19c6ffbf4a/go.mod h1:jdx8WFGep99tjWYAPp36pM8Gnw4pzPy+li2UZZZFjDE= 40 | github.com/cloudwego/eino-ext/components/document/transformer/splitter/markdown v0.0.0-20250605072634-0f875e04269d h1:x2NyMDCTXcJOdkKIVoOPXUacwkjznvEuls/epMVILrs= 41 | github.com/cloudwego/eino-ext/components/document/transformer/splitter/markdown v0.0.0-20250605072634-0f875e04269d/go.mod h1:qZxYTU/Snj6bvoyOe4ZKhb1ZgLUiuYcKbClTl15rPRI= 42 | github.com/cloudwego/eino-ext/components/document/transformer/splitter/recursive v0.0.0-20250424061409-ccd60fbc7c1c h1:u6iPDaZkgSrlHm0cDm2zTIEF8ODS/2inRRmfeI2Sebs= 43 | github.com/cloudwego/eino-ext/components/document/transformer/splitter/recursive v0.0.0-20250424061409-ccd60fbc7c1c/go.mod h1:a9JdTcj+WJkWjnm94RyhdWVNoiR2ancjsy01S5e/uPo= 44 | github.com/cloudwego/eino-ext/components/embedding/openai v0.0.0-20250424061409-ccd60fbc7c1c h1:HiryaRjOTeTmqT6NH/VQbZSKy03hXdSsDIu0B35g/p8= 45 | github.com/cloudwego/eino-ext/components/embedding/openai v0.0.0-20250424061409-ccd60fbc7c1c/go.mod h1:2vwPuBqNFHMSJUSngiuisoEmS8dGP7rujNqDQuUCFLg= 46 | github.com/cloudwego/eino-ext/components/indexer/es8 v0.0.0-20250605072634-0f875e04269d h1:5wFaXeXKWOO/JxqwSAdAaFUvUbVj5etwwrI6CxPxr/E= 47 | github.com/cloudwego/eino-ext/components/indexer/es8 v0.0.0-20250605072634-0f875e04269d/go.mod h1:gcTplPr8fzw00XS+twNkrQXORfRXL7UYFbAe1Z8v/Ck= 48 | github.com/cloudwego/eino-ext/components/model/openai v0.0.0-20250512035704-1e06fdfda207 h1:5/ZodDj/W1Iiw7voJP1ScA+M62ctE5WAA93D6/9rws0= 49 | github.com/cloudwego/eino-ext/components/model/openai v0.0.0-20250512035704-1e06fdfda207/go.mod h1:uXIWTFbaAbZ1128EIXjFc4S+tDqmz1idMZd5qt5kkwU= 50 | github.com/cloudwego/eino-ext/components/retriever/es8 v0.0.0-20250424061409-ccd60fbc7c1c h1:dB4WP0krLiz8URDs7//kPGr1wBU+BVKM3wp+/rbbbNY= 51 | github.com/cloudwego/eino-ext/components/retriever/es8 v0.0.0-20250424061409-ccd60fbc7c1c/go.mod h1:GhhtNox5sRTgU9yYfmp0MOyF0SCvO7c21VqIzg78boE= 52 | github.com/cloudwego/eino-ext/libs/acl/openai v0.0.0-20250422092704-54e372e1fa3d h1:unNqPz9vuJmJCZAw5YKFcszRX9e3CdVEjh0lR6QArxk= 53 | github.com/cloudwego/eino-ext/libs/acl/openai v0.0.0-20250422092704-54e372e1fa3d/go.mod h1:Ye0YAqpESCxMlnALNrjeNJjhS9q2PIdxVdJbtFeni8o= 54 | github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= 55 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 56 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 57 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 58 | github.com/dslipak/pdf v0.0.2 h1:djAvcM5neg9Ush+zR6QXB+VMJzR6TdnX766HPIg1JmI= 59 | github.com/dslipak/pdf v0.0.2/go.mod h1:2L3SnkI9cQwnAS9gfPz2iUoLC0rUZwbucpbKi5R1mUo= 60 | github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= 61 | github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= 62 | github.com/elastic/elastic-transport-go/v8 v8.7.0 h1:OgTneVuXP2uip4BA658Xi6Hfw+PeIOod2rY3GVMGoVE= 63 | github.com/elastic/elastic-transport-go/v8 v8.7.0/go.mod h1:YLHer5cj0csTzNFXoNQ8qhtGY1GTvSqPnKWKaqQE3Hk= 64 | github.com/elastic/go-elasticsearch/v8 v8.16.0 h1:f7bR+iBz8GTAVhwyFO3hm4ixsz2eMaEy0QroYnXV3jE= 65 | github.com/elastic/go-elasticsearch/v8 v8.16.0/go.mod h1:lGMlgKIbYoRvay3xWBeKahAiJOgmFDsjZC39nmO3H64= 66 | github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= 67 | github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= 68 | github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= 69 | github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= 70 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 71 | github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= 72 | github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= 73 | github.com/getkin/kin-openapi v0.118.0 h1:z43njxPmJ7TaPpMSCQb7PN0dEYno4tyBPQcrFdHoLuM= 74 | github.com/getkin/kin-openapi v0.118.0/go.mod h1:l5e9PaFUo9fyLJCPGQeXI2ML8c3P8BHOEV2VaAVf/pc= 75 | github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= 76 | github.com/go-check/check v0.0.0-20180628173108-788fd7840127 h1:0gkP6mzaMqkmpcJYCFOLkIBwI7xFExG03bbkOkCvUPI= 77 | github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= 78 | github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= 79 | github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= 80 | github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= 81 | github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= 82 | github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= 83 | github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= 84 | github.com/go-openapi/jsonpointer v0.21.1 h1:whnzv/pNXtK2FbX/W9yJfRmE2gsmkfahjMKB0fZvcic= 85 | github.com/go-openapi/jsonpointer v0.21.1/go.mod h1:50I1STOfbY1ycR8jGz8DaMeLCdXiI6aDteEdRNNzpdk= 86 | github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= 87 | github.com/go-openapi/swag v0.23.1 h1:lpsStH0n2ittzTnbaSloVZLuB5+fvSY/+hnagBjSNZU= 88 | github.com/go-openapi/swag v0.23.1/go.mod h1:STZs8TbRvEQQKUA+JZNAm3EWlgaOBGpyFDqQnDHMef0= 89 | github.com/go-sql-driver/mysql v1.7.0/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= 90 | github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI= 91 | github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= 92 | github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= 93 | github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= 94 | github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= 95 | github.com/gogf/gf/contrib/drivers/mysql/v2 v2.9.0 h1:1f7EeD0lfPHoXfaJDSL7cxRcSRelbsAKgF3MGXY+Uyo= 96 | github.com/gogf/gf/contrib/drivers/mysql/v2 v2.9.0/go.mod h1:tToO1PjGkLIR+9DbJ0wrKicYma0H/EUHXOpwel6Dw+0= 97 | github.com/gogf/gf/v2 v2.9.0 h1:semN5Q5qGjDQEv4620VzxcJzJlSD07gmyJ9Sy9zfbHk= 98 | github.com/gogf/gf/v2 v2.9.0/go.mod h1:sWGQw+pLILtuHmbOxoe0D+0DdaXxbleT57axOLH2vKI= 99 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 100 | github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= 101 | github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 102 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 103 | github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= 104 | github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 105 | github.com/goph/emperror v0.17.2 h1:yLapQcmEsO0ipe9p5TaN22djm3OFV/TfM/fcYP0/J18= 106 | github.com/goph/emperror v0.17.2/go.mod h1:+ZbQ+fUNO/6FNiUo0ujtMjhgad9Xa6fQL9KhH4LNHic= 107 | github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= 108 | github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k= 109 | github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= 110 | github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= 111 | github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= 112 | github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= 113 | github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= 114 | github.com/grokify/html-strip-tags-go v0.1.0 h1:03UrQLjAny8xci+R+qjCce/MYnpNXCtgzltlQbOBae4= 115 | github.com/grokify/html-strip-tags-go v0.1.0/go.mod h1:ZdzgfHEzAfz9X6Xe5eBLVblWIxXfYSQ40S/VKrAOGpc= 116 | github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= 117 | github.com/invopop/yaml v0.1.0 h1:YW3WGUoJEXYfzWBjn00zIlrw7brGVD0fUKRYDPAPhrc= 118 | github.com/invopop/yaml v0.1.0/go.mod h1:2XuRLgs/ouIrW3XNzuNj7J3Nvu/Dig5MXvbCEdiBN3Q= 119 | github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= 120 | github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= 121 | github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= 122 | github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= 123 | github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= 124 | github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= 125 | github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= 126 | github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= 127 | github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= 128 | github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= 129 | github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= 130 | github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= 131 | github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= 132 | github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= 133 | github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= 134 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 135 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 136 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 137 | github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= 138 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 139 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 140 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 141 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 142 | github.com/magiconair/properties v1.8.9 h1:nWcCbLq1N2v/cpNsy5WvQ37Fb+YElfq20WJ/a8RkpQM= 143 | github.com/magiconair/properties v1.8.9/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= 144 | github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= 145 | github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= 146 | github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= 147 | github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= 148 | github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= 149 | github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= 150 | github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= 151 | github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= 152 | github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= 153 | github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= 154 | github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= 155 | github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= 156 | github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= 157 | github.com/meguminnnnnnnnn/go-openai v0.0.0-20250408071642-761325becfd6 h1:nmdXxiUX48DZ2ELC/jSYzyGUVgxVEF2QJRGhLJ933zA= 158 | github.com/meguminnnnnnnnn/go-openai v0.0.0-20250408071642-761325becfd6/go.mod h1:kyz7fcXqXtccmRAIARn1Q+cKLNXJHC3AoqqJGeCqNI0= 159 | github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4= 160 | github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= 161 | github.com/microcosm-cc/bluemonday v1.0.27 h1:MpEUotklkwCSLeH+Qdx1VJgNqLlpY2KXwXFM08ygZfk= 162 | github.com/microcosm-cc/bluemonday v1.0.27/go.mod h1:jFi9vgW+H7c3V0lb6nR74Ib/DIB5OBs92Dimizgw2cA= 163 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 164 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= 165 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 166 | github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= 167 | github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= 168 | github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= 169 | github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= 170 | github.com/nikolalohinski/gonja v1.5.3 h1:GsA+EEaZDZPGJ8JtpeGN78jidhOlxeJROpqMT9fTj9c= 171 | github.com/nikolalohinski/gonja v1.5.3/go.mod h1:RmjwxNiXAEqcq1HeK5SSMmqFJvKOfTfXhkJv6YBtPa4= 172 | github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= 173 | github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= 174 | github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 175 | github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= 176 | github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= 177 | github.com/orcaman/concurrent-map/v2 v2.0.1 h1:jOJ5Pg2w1oeB6PeDurIYf6k9PQ+aTITr/6lP/L/zp6c= 178 | github.com/orcaman/concurrent-map/v2 v2.0.1/go.mod h1:9Eq3TG2oBe5FirmYWQfYO5iH1q0Jv47PLaNK++uCdOM= 179 | github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= 180 | github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= 181 | github.com/perimeterx/marshmallow v1.1.4/go.mod h1:dsXbUu8CRzfYP5a87xpp0xq9S3u0Vchtcl8we9tYaXw= 182 | github.com/perimeterx/marshmallow v1.1.5 h1:a2LALqQ1BlHM8PZblsDdidgv1mWi1DgC2UmX50IvK2s= 183 | github.com/perimeterx/marshmallow v1.1.5/go.mod h1:dsXbUu8CRzfYP5a87xpp0xq9S3u0Vchtcl8we9tYaXw= 184 | github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 185 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 186 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 187 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 188 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 189 | github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= 190 | github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= 191 | github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= 192 | github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= 193 | github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= 194 | github.com/rollbar/rollbar-go v1.0.2/go.mod h1:AcFs5f0I+c71bpHlXNNDbOWJiKwjFDtISeXco0L5PKQ= 195 | github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= 196 | github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= 197 | github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= 198 | github.com/slongfield/pyfmt v0.0.0-20220222012616-ea85ff4c361f h1:Z2cODYsUxQPofhpYRMQVwWz4yUVpHF+vPi+eUdruUYI= 199 | github.com/slongfield/pyfmt v0.0.0-20220222012616-ea85ff4c361f/go.mod h1:JqzWyvTuI2X4+9wOHmKSQCYxybB/8j6Ko43qVmXDuZg= 200 | github.com/smarty/assertions v1.16.0 h1:EvHNkdRA4QHMrn75NZSoUQ/mAUXAYWfatfB01yTCzfY= 201 | github.com/smarty/assertions v1.16.0/go.mod h1:duaaFdCS0K9dnoM50iyek/eYINOZ64gbh1Xlf6LG7AI= 202 | github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sSznIX1xY= 203 | github.com/smartystreets/goconvey v1.8.1/go.mod h1:+/u4qLyY6x1jReYOp7GOM2FSt8aP9CzCZL03bI28W60= 204 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 205 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 206 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 207 | github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= 208 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 209 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 210 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 211 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 212 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 213 | github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= 214 | github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= 215 | github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 216 | github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= 217 | github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= 218 | github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= 219 | github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= 220 | github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= 221 | github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= 222 | github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= 223 | github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= 224 | github.com/ugorji/go v1.2.7 h1:qYhyWUUd6WbiM+C6JZAUkIJt/1WrjzNHY9+KCIjVqTo= 225 | github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6M= 226 | github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0= 227 | github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= 228 | github.com/wangle201210/chat-history v0.0.0-20250402104704-5eec15d5419e h1:pdG2LdTh2toVNke05pGmFtS4ZkuA6xt0rqT4pWE9HZk= 229 | github.com/wangle201210/chat-history v0.0.0-20250402104704-5eec15d5419e/go.mod h1:MzB/53ZNftLNedu73UwySUHUFFCYZNJ3WmBrukalTrU= 230 | github.com/x-cray/logrus-prefixed-formatter v0.5.2 h1:00txxvfBM9muc0jiLIEAkAcIMJzfthRT6usrui8uGmg= 231 | github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= 232 | github.com/yargevad/filepathx v1.0.0 h1:SYcT+N3tYGi+NvazubCNlvgIPbzAk7i7y2dwg3I5FYc= 233 | github.com/yargevad/filepathx v1.0.0/go.mod h1:BprfX/gpYNJHJfc35GjRRpVcwWXS89gGulUIU5tK3tA= 234 | github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= 235 | github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= 236 | github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= 237 | go.opentelemetry.io/otel v1.32.0 h1:WnBN+Xjcteh0zdk01SVqV55d/m62NJLJdIyb4y/WO5U= 238 | go.opentelemetry.io/otel v1.32.0/go.mod h1:00DCVSB0RQcnzlwyTfqtxSm+DRr9hpYrHjNGiBHVQIg= 239 | go.opentelemetry.io/otel/metric v1.32.0 h1:xV2umtmNcThh2/a/aCP+h64Xx5wsj8qqnkYZktzNa0M= 240 | go.opentelemetry.io/otel/metric v1.32.0/go.mod h1:jH7CIbbK6SH2V2wE16W05BHCtIDzauciCRLoc/SyMv8= 241 | go.opentelemetry.io/otel/sdk v1.32.0 h1:RNxepc9vK59A8XsgZQouW8ue8Gkb4jpWtJm9ge5lEG4= 242 | go.opentelemetry.io/otel/sdk v1.32.0/go.mod h1:LqgegDBjKMmb2GC6/PrTnteJG39I8/vJCAP9LlJXEjU= 243 | go.opentelemetry.io/otel/trace v1.32.0 h1:WIC9mYrXf8TmY/EXuULKc8hR17vE+Hjv2cssQDe03fM= 244 | go.opentelemetry.io/otel/trace v1.32.0/go.mod h1:+i4rkvCraA+tG6AzwloGaCtkx53Fa+L+V8e9a7YvhT8= 245 | go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU= 246 | go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= 247 | golang.org/x/arch v0.15.0 h1:QtOrQd0bTUnhNVNndMpLHNWrDmYzZ2KDqSrEymqInZw= 248 | golang.org/x/arch v0.15.0/go.mod h1:JmwW7aLIoRUKgaTzhkiEFxvcEiQGyOg9BMonBJUS7EE= 249 | golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 250 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 251 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= 252 | golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= 253 | golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= 254 | golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 h1:nDVHiLt8aIbd/VzvPWN6kSOPE7+F/fNFDSXLVYkE/Iw= 255 | golang.org/x/exp v0.0.0-20250305212735-054e65f0b394/go.mod h1:sIifuuw/Yco/y6yb6+bDNfyeQ/MdPUy/hKEMYQV17cM= 256 | golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= 257 | golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 258 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 259 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 260 | golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= 261 | golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= 262 | golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= 263 | golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= 264 | golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= 265 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 266 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 267 | golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 268 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 269 | golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 270 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 271 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 272 | golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 273 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 274 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 275 | golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 276 | golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 277 | golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 278 | golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 279 | golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 280 | golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= 281 | golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= 282 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 283 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 284 | golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= 285 | golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= 286 | golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= 287 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 288 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 289 | golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 290 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 291 | golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= 292 | golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= 293 | golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= 294 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 295 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 296 | golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= 297 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 298 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 299 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 300 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 301 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= 302 | gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= 303 | gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= 304 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 305 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 306 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 307 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 308 | gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 309 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 310 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 311 | gorm.io/driver/mysql v1.5.7 h1:MndhOPYOfEp2rHKgkZIhJ16eVUIRf2HmzgoPmh7FCWo= 312 | gorm.io/driver/mysql v1.5.7/go.mod h1:sEtPWMiqiN1N1cMXoXmBbd8C6/l+TESwriotuRRpkDM= 313 | gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8= 314 | gorm.io/gorm v1.25.12 h1:I0u8i2hWQItBq1WfE0o2+WuL9+8L21K9e2HHSTE/0f8= 315 | gorm.io/gorm v1.25.12/go.mod h1:xh7N7RHfYlNc5EmcI/El95gXusucDrQnHXe0+CgWcLQ= 316 | nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= 317 | -------------------------------------------------------------------------------- /server/hack/config.yaml: -------------------------------------------------------------------------------- 1 | gfcli: 2 | gen: 3 | dao: 4 | - link: "mysql:root:123456@tcp(127.0.0.1:3306)/go-rag" 5 | descriptionTag: true 6 | tables: "knowledge_base" 7 | -------------------------------------------------------------------------------- /server/internal/cmd/cmd.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/ThinkInAIXYZ/go-mcp/server" 7 | "github.com/ThinkInAIXYZ/go-mcp/transport" 8 | "github.com/gogf/gf/v2/frame/g" 9 | "github.com/gogf/gf/v2/net/ghttp" 10 | "github.com/gogf/gf/v2/os/gcmd" 11 | "github.com/wangle201210/go-rag/server/internal/controller/rag" 12 | "github.com/wangle201210/go-rag/server/internal/mcp" 13 | ) 14 | 15 | var ( 16 | Main = gcmd.Command{ 17 | Name: "main", 18 | Usage: "main", 19 | Brief: "start http server", 20 | Func: func(ctx context.Context, parser *gcmd.Parser) (err error) { 21 | s := g.Server() 22 | Mcp(ctx, s) 23 | s.Group("/", func(group *ghttp.RouterGroup) { 24 | s.AddStaticPath("", "./static/fe/") 25 | s.SetIndexFiles([]string{"index.html"}) 26 | group.Middleware(ghttp.MiddlewareHandlerResponse, ghttp.MiddlewareCORS) 27 | group.Bind( 28 | rag.NewV1(), 29 | ) 30 | }) 31 | s.Run() 32 | return nil 33 | }, 34 | } 35 | ) 36 | 37 | func Mcp(ctx context.Context, s *ghttp.Server) { 38 | trans, handler, err := transport.NewStreamableHTTPServerTransportAndHandler() 39 | if err != nil { 40 | g.Log().Panicf(ctx, "new sse transport and hander with error: %v", err) 41 | } 42 | // new mcp server 43 | mcpServer, _ := server.NewServer(trans) 44 | mcpServer.RegisterTool(mcp.GetRetrieverTool(), mcp.HandleRetriever) 45 | mcpServer.RegisterTool(mcp.GetKnowledgeBaseTool(), mcp.HandleKnowledgeBase) 46 | // start mcp Server 47 | go func() { 48 | mcpServer.Run() 49 | }() 50 | // mcpServer.Shutdown(context.Background()) 51 | s.Group("/", func(r *ghttp.RouterGroup) { 52 | r.ALL("/mcp", func(r *ghttp.Request) { 53 | handler.HandleMCP().ServeHTTP(r.Response.Writer, r.Request) 54 | }) 55 | }) 56 | } 57 | -------------------------------------------------------------------------------- /server/internal/controller/rag/rag.go: -------------------------------------------------------------------------------- 1 | // ================================================================================= 2 | // This is auto-generated by GoFrame CLI tool only once. Fill this file as you wish. 3 | // ================================================================================= 4 | 5 | package rag 6 | -------------------------------------------------------------------------------- /server/internal/controller/rag/rag_new.go: -------------------------------------------------------------------------------- 1 | // ================================================================================= 2 | // This is auto-generated by GoFrame CLI tool only once. Fill this file as you wish. 3 | // ================================================================================= 4 | 5 | package rag 6 | 7 | import ( 8 | "github.com/wangle201210/go-rag/server/api/rag" 9 | ) 10 | 11 | type ControllerV1 struct{} 12 | 13 | func NewV1() rag.IRagV1 { 14 | return &ControllerV1{} 15 | } 16 | -------------------------------------------------------------------------------- /server/internal/controller/rag/rag_v1_chat.go: -------------------------------------------------------------------------------- 1 | package rag 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/wangle201210/go-rag/server/api/rag/v1" 7 | "github.com/wangle201210/go-rag/server/internal/logic/chat" 8 | ) 9 | 10 | func (c *ControllerV1) Chat(ctx context.Context, req *v1.ChatReq) (res *v1.ChatRes, err error) { 11 | retriever, err := c.Retriever(ctx, &v1.RetrieverReq{ 12 | Question: req.Question, 13 | TopK: req.TopK, 14 | Score: req.Score, 15 | KnowledgeName: req.KnowledgeName, 16 | }) 17 | if err != nil { 18 | return 19 | } 20 | chatI := chat.GetChat() 21 | answer, err := chatI.GetAnswer(ctx, req.ConvID, retriever.Document, req.Question) 22 | if err != nil { 23 | return 24 | } 25 | res = &v1.ChatRes{ 26 | Answer: answer, 27 | References: retriever.Document, 28 | } 29 | return 30 | } 31 | -------------------------------------------------------------------------------- /server/internal/controller/rag/rag_v1_chat_stream.go: -------------------------------------------------------------------------------- 1 | package rag 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "time" 8 | 9 | "github.com/bytedance/sonic" 10 | "github.com/gogf/gf/v2/frame/g" 11 | "github.com/gogf/gf/v2/net/ghttp" 12 | "github.com/google/uuid" 13 | v1 "github.com/wangle201210/go-rag/server/api/rag/v1" 14 | "github.com/wangle201210/go-rag/server/core/common" 15 | "github.com/wangle201210/go-rag/server/internal/logic/chat" 16 | ) 17 | 18 | // ChatStream 流式输出接口 19 | func (c *ControllerV1) ChatStream(ctx context.Context, req *v1.ChatStreamReq) (res *v1.ChatStreamRes, err error) { 20 | // 获取HTTP响应对象 21 | httpReq := ghttp.RequestFromCtx(ctx) 22 | httpResp := httpReq.Response 23 | // 设置响应头 24 | httpResp.Header().Set("Content-Type", "text/event-stream") 25 | httpResp.Header().Set("Cache-Control", "no-cache") 26 | httpResp.Header().Set("Connection", "keep-alive") 27 | httpResp.Header().Set("X-Accel-Buffering", "no") // 禁用Nginx缓冲 28 | httpResp.Header().Set("Access-Control-Allow-Origin", "*") 29 | 30 | // 获取检索结果 31 | retriever, err := c.Retriever(ctx, &v1.RetrieverReq{ 32 | Question: req.Question, 33 | TopK: req.TopK, 34 | Score: req.Score, 35 | KnowledgeName: req.KnowledgeName, 36 | }) 37 | if err != nil { 38 | writeSSEError(httpResp, err) 39 | return &v1.ChatStreamRes{}, nil 40 | } 41 | sd := &common.StreamData{ 42 | Id: uuid.NewString(), 43 | Created: time.Now().Unix(), 44 | Document: retriever.Document, 45 | } 46 | marshal, _ := sonic.Marshal(sd) 47 | writeSSEDocuments(httpResp, string(marshal)) 48 | // 获取Chat实例 49 | chatI := chat.GetChat() 50 | // 获取流式响应 51 | streamReader, err := chatI.GetAnswerStream(ctx, req.ConvID, retriever.Document, req.Question) 52 | if err != nil { 53 | writeSSEError(httpResp, err) 54 | return &v1.ChatStreamRes{}, nil 55 | } 56 | defer streamReader.Close() 57 | 58 | // 处理流式响应 59 | for { 60 | chunk, err := streamReader.Recv() 61 | if err == io.EOF { 62 | break 63 | } 64 | if err != nil { 65 | writeSSEError(httpResp, err) 66 | break 67 | } 68 | if len(chunk.Content) == 0 { 69 | continue 70 | } 71 | 72 | sd.Content = chunk.Content 73 | marshal, _ := sonic.Marshal(sd) 74 | // 发送数据事件 75 | writeSSEData(httpResp, string(marshal)) 76 | } 77 | // 发送结束事件 78 | writeSSEDone(httpResp) 79 | return &v1.ChatStreamRes{}, nil 80 | } 81 | 82 | // writeSSEData 写入SSE事件 83 | func writeSSEData(resp *ghttp.Response, data string) { 84 | if len(data) == 0 { 85 | return 86 | } 87 | // g.Log().Infof(context.Background(), "data: %s", data) 88 | resp.Writeln(fmt.Sprintf("data:%s\n", data)) 89 | resp.Flush() 90 | } 91 | 92 | func writeSSEDone(resp *ghttp.Response) { 93 | resp.Writeln(fmt.Sprintf("data:%s\n", "[DONE]")) 94 | resp.Flush() 95 | } 96 | 97 | func writeSSEDocuments(resp *ghttp.Response, data string) { 98 | resp.Writeln(fmt.Sprintf("documents:%s\n", data)) 99 | resp.Flush() 100 | } 101 | 102 | // writeSSEError 写入SSE错误 103 | func writeSSEError(resp *ghttp.Response, err error) { 104 | g.Log().Error(context.Background(), err) 105 | resp.Writeln(fmt.Sprintf("event: error\ndata: %s\n\n", err.Error())) 106 | resp.Flush() 107 | } 108 | -------------------------------------------------------------------------------- /server/internal/controller/rag/rag_v1_indexer.go: -------------------------------------------------------------------------------- 1 | package rag 2 | 3 | import ( 4 | "context" 5 | 6 | gorag "github.com/wangle201210/go-rag/server/core" 7 | "github.com/wangle201210/go-rag/server/internal/logic/rag" 8 | 9 | "github.com/wangle201210/go-rag/server/api/rag/v1" 10 | ) 11 | 12 | func (c *ControllerV1) Indexer(ctx context.Context, req *v1.IndexerReq) (res *v1.IndexerRes, err error) { 13 | svr := rag.GetRagSvr() 14 | uri := req.URL 15 | if req.File != nil { 16 | filename, e := req.File.Save("./uploads/") 17 | if e != nil { 18 | err = e 19 | return 20 | } 21 | uri = "./uploads/" + filename 22 | } 23 | indexReq := &gorag.IndexReq{ 24 | URI: uri, 25 | KnowledgeName: req.KnowledgeName, 26 | } 27 | ids, err := svr.Index(ctx, indexReq) 28 | if err != nil { 29 | return 30 | } 31 | res = &v1.IndexerRes{ 32 | DocIDs: ids, 33 | } 34 | return 35 | } 36 | -------------------------------------------------------------------------------- /server/internal/controller/rag/rag_v1_kb.go: -------------------------------------------------------------------------------- 1 | package rag 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/wangle201210/go-rag/server/internal/dao" 7 | "github.com/wangle201210/go-rag/server/internal/model/do" 8 | 9 | "github.com/wangle201210/go-rag/server/api/rag/v1" 10 | ) 11 | 12 | func (c *ControllerV1) KBCreate(ctx context.Context, req *v1.KBCreateReq) (res *v1.KBCreateRes, err error) { 13 | insertId, err := dao.KnowledgeBase.Ctx(ctx).Data(do.KnowledgeBase{ 14 | Name: req.Name, 15 | Status: v1.StatusOK, 16 | Description: req.Description, 17 | Category: req.Category, 18 | }).InsertAndGetId() 19 | if err != nil { 20 | return nil, err 21 | } 22 | res = &v1.KBCreateRes{ 23 | Id: insertId, 24 | } 25 | return 26 | } 27 | 28 | func (c *ControllerV1) KBDelete(ctx context.Context, req *v1.KBDeleteReq) (res *v1.KBDeleteRes, err error) { 29 | _, err = dao.KnowledgeBase.Ctx(ctx).WherePri(req.Id).Delete() 30 | return 31 | } 32 | 33 | func (c *ControllerV1) KBGetList(ctx context.Context, req *v1.KBGetListReq) (res *v1.KBGetListRes, err error) { 34 | res = &v1.KBGetListRes{} 35 | err = dao.KnowledgeBase.Ctx(ctx).Where(do.KnowledgeBase{ 36 | Status: req.Status, 37 | Name: req.Name, 38 | Category: req.Category, 39 | }).Scan(&res.List) 40 | return 41 | } 42 | 43 | func (c *ControllerV1) KBGetOne(ctx context.Context, req *v1.KBGetOneReq) (res *v1.KBGetOneRes, err error) { 44 | res = &v1.KBGetOneRes{} 45 | err = dao.KnowledgeBase.Ctx(ctx).WherePri(req.Id).Scan(&res.KnowledgeBase) 46 | return 47 | } 48 | 49 | func (c *ControllerV1) KBUpdate(ctx context.Context, req *v1.KBUpdateReq) (res *v1.KBUpdateRes, err error) { 50 | _, err = dao.KnowledgeBase.Ctx(ctx).Data(do.KnowledgeBase{ 51 | Name: req.Name, 52 | Status: req.Status, 53 | Description: req.Description, 54 | Category: req.Category, 55 | }).WherePri(req.Id).Update() 56 | return 57 | } 58 | -------------------------------------------------------------------------------- /server/internal/controller/rag/rag_v1_retriever.go: -------------------------------------------------------------------------------- 1 | package rag 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "sort" 7 | 8 | "github.com/gogf/gf/v2/frame/g" 9 | gorag "github.com/wangle201210/go-rag/server/core" 10 | "github.com/wangle201210/go-rag/server/internal/logic/rag" 11 | 12 | "github.com/wangle201210/go-rag/server/api/rag/v1" 13 | ) 14 | 15 | func (c *ControllerV1) Retriever(ctx context.Context, req *v1.RetrieverReq) (res *v1.RetrieverRes, err error) { 16 | ragSvr := rag.GetRagSvr() 17 | if req.TopK == 0 { 18 | req.TopK = 5 19 | } 20 | if req.Score == 0 { 21 | req.Score = 0.2 22 | } 23 | if req.Score < 1.0 { 24 | req.Score += 1 25 | } 26 | ragReq := &gorag.RetrieveReq{ 27 | Query: req.Question, 28 | TopK: req.TopK, 29 | Score: req.Score, 30 | KnowledgeName: req.KnowledgeName, 31 | } 32 | g.Log().Infof(ctx, "ragReq: %v", ragReq) 33 | msg, err := ragSvr.Retrieve(ctx, ragReq) 34 | if err != nil { 35 | return 36 | } 37 | for _, document := range msg { 38 | if document.MetaData != nil { 39 | delete(document.MetaData, "_dense_vector") 40 | if v, e := document.MetaData["_score"]; e { 41 | vf := v.(float64) 42 | document.MetaData["_score"] = vf - 1 43 | } 44 | m := make(map[string]interface{}) 45 | if err = json.Unmarshal([]byte(document.MetaData["ext"].(string)), &m); err != nil { 46 | return 47 | } 48 | document.MetaData["ext"] = m 49 | } 50 | } 51 | // eino 默认是把分高的排在两边，这里我xiu gai 52 | sort.Slice(msg, func(i, j int) bool { 53 | return msg[i].Score() > msg[j].Score() 54 | }) 55 | res = &v1.RetrieverRes{ 56 | Document: msg, 57 | } 58 | return 59 | } 60 | -------------------------------------------------------------------------------- /server/internal/dao/dao.go: -------------------------------------------------------------------------------- 1 | package dao 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "time" 7 | 8 | "github.com/gogf/gf/v2/frame/g" 9 | "github.com/wangle201210/go-rag/server/internal/model" 10 | "gorm.io/driver/mysql" 11 | "gorm.io/gorm" 12 | "gorm.io/gorm/logger" 13 | ) 14 | 15 | var db *gorm.DB 16 | 17 | func init() { 18 | err := InitDB() 19 | if err != nil { 20 | g.Log().Fatal(context.Background(), "database connection not initialized") 21 | } 22 | } 23 | 24 | // InitDB 初始化数据库连接 25 | func InitDB() error { 26 | config := &gorm.Config{ 27 | Logger: logger.Default.LogMode(logger.Info), 28 | } 29 | dsn := GetDsn() 30 | 31 | println(dsn) 32 | var err error 33 | db, err = gorm.Open(mysql.Open(dsn), config) 34 | if err != nil { 35 | return fmt.Errorf("failed to connect database: %v", err) 36 | } 37 | 38 | sqlDB, err := db.DB() 39 | if err != nil { 40 | return fmt.Errorf("failed to get database instance: %v", err) 41 | } 42 | 43 | // 设置连接池 44 | sqlDB.SetMaxIdleConns(10) 45 | sqlDB.SetMaxOpenConns(100) 46 | sqlDB.SetConnMaxLifetime(time.Hour) 47 | 48 | // 自动迁移数据库表结构 49 | if err = autoMigrateTables(); err != nil { 50 | return fmt.Errorf("failed to migrate database tables: %v", err) 51 | } 52 | 53 | return nil 54 | } 55 | 56 | func GetDsn() string { 57 | cfg := g.DB().GetConfig() 58 | return fmt.Sprintf("%s:%s@tcp(%s:%s)/%s", cfg.User, cfg.Pass, cfg.Host, cfg.Port, cfg.Name) 59 | } 60 | 61 | // GetDB 获取数据库实例 62 | func GetDB() *gorm.DB { 63 | if db == nil { 64 | g.Log().Fatal(context.Background(), "database connection not initialized") 65 | } 66 | return db 67 | } 68 | 69 | // autoMigrateTables 自动迁移数据库表结构 70 | func autoMigrateTables() error { 71 | // 自动迁移会创建表、缺失的外键、约束、列和索引 72 | return db.AutoMigrate(&model.KnowledgeBase{}) 73 | } 74 | -------------------------------------------------------------------------------- /server/internal/dao/internal/knowledge_base.go: -------------------------------------------------------------------------------- 1 | // ========================================================================== 2 | // Code generated and maintained by GoFrame CLI tool. DO NOT EDIT. 3 | // ========================================================================== 4 | 5 | package internal 6 | 7 | import ( 8 | "context" 9 | 10 | "github.com/gogf/gf/v2/database/gdb" 11 | "github.com/gogf/gf/v2/frame/g" 12 | ) 13 | 14 | // KnowledgeBaseDao is the data access object for the table knowledge_base. 15 | type KnowledgeBaseDao struct { 16 | table string // table is the underlying table name of the DAO. 17 | group string // group is the database configuration group name of the current DAO. 18 | columns KnowledgeBaseColumns // columns contains all the column names of Table for convenient usage. 19 | } 20 | 21 | // KnowledgeBaseColumns defines and stores column names for the table knowledge_base. 22 | type KnowledgeBaseColumns struct { 23 | Id string // 主键ID 24 | Name string // 知识库名称 25 | Description string // 知识库描述 26 | Category string // 知识库分类 27 | Status string // 状态：0-禁用，1-启用 28 | CreateTime string // 创建时间 29 | UpdateTime string // 更新时间 30 | } 31 | 32 | // knowledgeBaseColumns holds the columns for the table knowledge_base. 33 | var knowledgeBaseColumns = KnowledgeBaseColumns{ 34 | Id: "id", 35 | Name: "name", 36 | Description: "description", 37 | Category: "category", 38 | Status: "status", 39 | CreateTime: "create_time", 40 | UpdateTime: "update_time", 41 | } 42 | 43 | // NewKnowledgeBaseDao creates and returns a new DAO object for table data access. 44 | func NewKnowledgeBaseDao() *KnowledgeBaseDao { 45 | return &KnowledgeBaseDao{ 46 | group: "default", 47 | table: "knowledge_base", 48 | columns: knowledgeBaseColumns, 49 | } 50 | } 51 | 52 | // DB retrieves and returns the underlying raw database management object of the current DAO. 53 | func (dao *KnowledgeBaseDao) DB() gdb.DB { 54 | return g.DB(dao.group) 55 | } 56 | 57 | // Table returns the table name of the current DAO. 58 | func (dao *KnowledgeBaseDao) Table() string { 59 | return dao.table 60 | } 61 | 62 | // Columns returns all column names of the current DAO. 63 | func (dao *KnowledgeBaseDao) Columns() KnowledgeBaseColumns { 64 | return dao.columns 65 | } 66 | 67 | // Group returns the database configuration group name of the current DAO. 68 | func (dao *KnowledgeBaseDao) Group() string { 69 | return dao.group 70 | } 71 | 72 | // Ctx creates and returns a Model for the current DAO. It automatically sets the context for the current operation. 73 | func (dao *KnowledgeBaseDao) Ctx(ctx context.Context) *gdb.Model { 74 | return dao.DB().Model(dao.table).Safe().Ctx(ctx) 75 | } 76 | 77 | // Transaction wraps the transaction logic using function f. 78 | // It rolls back the transaction and returns the error if function f returns a non-nil error. 79 | // It commits the transaction and returns nil if function f returns nil. 80 | // 81 | // Note: Do not commit or roll back the transaction in function f, 82 | // as it is automatically handled by this function. 83 | func (dao *KnowledgeBaseDao) Transaction(ctx context.Context, f func(ctx context.Context, tx gdb.TX) error) (err error) { 84 | return dao.Ctx(ctx).Transaction(ctx, f) 85 | } 86 | -------------------------------------------------------------------------------- /server/internal/dao/knowledge_base.go: -------------------------------------------------------------------------------- 1 | // ================================================================================= 2 | // This file is auto-generated by the GoFrame CLI tool. You may modify it as needed. 3 | // ================================================================================= 4 | 5 | package dao 6 | 7 | import ( 8 | "github.com/wangle201210/go-rag/server/internal/dao/internal" 9 | ) 10 | 11 | // internalKnowledgeBaseDao is an internal type for wrapping the internal DAO implementation. 12 | type internalKnowledgeBaseDao = *internal.KnowledgeBaseDao 13 | 14 | // knowledgeBaseDao is the data access object for the table knowledge_base. 15 | // You can define custom methods on it to extend its functionality as needed. 16 | type knowledgeBaseDao struct { 17 | internalKnowledgeBaseDao 18 | } 19 | 20 | var ( 21 | // KnowledgeBase is a globally accessible object for table knowledge_base operations. 22 | KnowledgeBase = knowledgeBaseDao{ 23 | internal.NewKnowledgeBaseDao(), 24 | } 25 | ) 26 | 27 | // Add your custom methods and functionality below. 28 | -------------------------------------------------------------------------------- /server/internal/logic/chat/chat.go: -------------------------------------------------------------------------------- 1 | package chat 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "io" 8 | 9 | "github.com/cloudwego/eino-ext/components/model/openai" 10 | "github.com/cloudwego/eino/components/model" 11 | "github.com/cloudwego/eino/schema" 12 | "github.com/gogf/gf/v2/frame/g" 13 | "github.com/gogf/gf/v2/os/gctx" 14 | "github.com/wangle201210/chat-history/eino" 15 | "github.com/wangle201210/go-rag/server/internal/dao" 16 | ) 17 | 18 | var chat *Chat 19 | 20 | type Chat struct { 21 | cm model.BaseChatModel 22 | eh *eino.History 23 | } 24 | 25 | func GetChat() *Chat { 26 | return chat 27 | } 28 | 29 | // 暂时用不上chat功能，先不init 30 | func init() { 31 | ctx := gctx.New() 32 | c, err := newChat(&openai.ChatModelConfig{ 33 | APIKey: g.Cfg().MustGet(ctx, "chat.apiKey").String(), 34 | BaseURL: g.Cfg().MustGet(ctx, "chat.baseURL").String(), 35 | Model: g.Cfg().MustGet(ctx, "chat.model").String(), 36 | }) 37 | if err != nil { 38 | g.Log().Fatalf(ctx, "newChat failed, err=%v", err) 39 | return 40 | } 41 | c.eh = eino.NewEinoHistory(dao.GetDsn()) 42 | chat = c 43 | } 44 | 45 | func newChat(cfg *openai.ChatModelConfig) (res *Chat, err error) { 46 | chatModel, err := openai.NewChatModel(context.Background(), cfg) 47 | if err != nil { 48 | return nil, err 49 | } 50 | return &Chat{cm: chatModel}, nil 51 | } 52 | 53 | func (x *Chat) GetAnswer(ctx context.Context, convID string, docs []*schema.Document, question string) (answer string, err error) { 54 | messages, err := x.docsMessages(ctx, convID, docs, question) 55 | if err != nil { 56 | return "", err 57 | } 58 | result, err := generate(ctx, x.cm, messages) 59 | if err != nil { 60 | return "", fmt.Errorf("生成答案失败: %w", err) 61 | } 62 | err = x.eh.SaveMessage(result, convID) 63 | if err != nil { 64 | g.Log().Error(ctx, "save assistant message err: %v", err) 65 | return 66 | } 67 | return result.Content, nil 68 | } 69 | 70 | // GetAnswerStream 流式生成答案 71 | func (x *Chat) GetAnswerStream(ctx context.Context, convID string, docs []*schema.Document, question string) (answer *schema.StreamReader[*schema.Message], err error) { 72 | messages, err := x.docsMessages(ctx, convID, docs, question) 73 | if err != nil { 74 | return 75 | } 76 | ctx = context.Background() 77 | streamData, err := stream(ctx, x.cm, messages) 78 | if err != nil { 79 | err = fmt.Errorf("生成答案失败: %w", err) 80 | } 81 | srs := streamData.Copy(2) 82 | go func() { 83 | // for save 84 | fullMsgs := make([]*schema.Message, 0) 85 | defer func() { 86 | srs[1].Close() 87 | fullMsg, err := schema.ConcatMessages(fullMsgs) 88 | if err != nil { 89 | g.Log().Error(ctx, "error concatenating messages: %v", err) 90 | return 91 | } 92 | err = x.eh.SaveMessage(fullMsg, convID) 93 | if err != nil { 94 | g.Log().Error(ctx, "save assistant message err: %v", err) 95 | return 96 | } 97 | }() 98 | outer: 99 | for { 100 | select { 101 | case <-ctx.Done(): 102 | fmt.Println("context done", ctx.Err()) 103 | return 104 | default: 105 | chunk, err := srs[1].Recv() 106 | if err != nil { 107 | if errors.Is(err, io.EOF) { 108 | break outer 109 | } 110 | } 111 | fullMsgs = append(fullMsgs, chunk) 112 | } 113 | } 114 | }() 115 | 116 | return srs[0], nil 117 | 118 | } 119 | 120 | func generate(ctx context.Context, llm model.BaseChatModel, in []*schema.Message) (message *schema.Message, err error) { 121 | message, err = llm.Generate(ctx, in) 122 | if err != nil { 123 | err = fmt.Errorf("llm generate failed: %v", err) 124 | return 125 | } 126 | return 127 | } 128 | 129 | func stream(ctx context.Context, llm model.BaseChatModel, in []*schema.Message) (res *schema.StreamReader[*schema.Message], err error) { 130 | res, err = llm.Stream(ctx, in) 131 | if err != nil { 132 | err = fmt.Errorf("llm generate failed: %v", err) 133 | return 134 | } 135 | return 136 | } 137 | -------------------------------------------------------------------------------- /server/internal/logic/chat/message.go: -------------------------------------------------------------------------------- 1 | package chat 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/cloudwego/eino/components/prompt" 8 | "github.com/cloudwego/eino/schema" 9 | "github.com/gogf/gf/v2/frame/g" 10 | ) 11 | 12 | const ( 13 | role = "你是一个专业的AI助手，能够根据提供的参考信息准确回答用户问题。" 14 | ) 15 | 16 | // createTemplate 创建并返回一个配置好的聊天模板 17 | func createTemplate() prompt.ChatTemplate { 18 | // 创建模板，使用 FString 格式 19 | return prompt.FromMessages(schema.FString, 20 | // 系统消息模板 21 | schema.SystemMessage("{role}"+ 22 | "请严格遵守以下规则：\n"+ 23 | "1. 回答必须基于提供的参考内容，不要依赖外部知识\n"+ 24 | "2. 如果参考内容中有明确答案，直接使用参考内容回答\n"+ 25 | "3. 如果参考内容不完整或模糊，可以合理推断但需说明\n"+ 26 | "4. 如果参考内容完全不相关或不存在，如实告知用户'根据现有资料无法回答'\n"+ 27 | "5. 保持回答专业、简洁、准确\n"+ 28 | "6. 必要时可引用参考内容中的具体数据或原文\n\n"+ 29 | "当前提供的参考内容：\n"+ 30 | "{docs}\n\n"+ 31 | ""), 32 | schema.MessagesPlaceholder("chat_history", true), 33 | // 用户消息模板 34 | schema.UserMessage("Question: {question}"), 35 | ) 36 | } 37 | 38 | // formatMessages 格式化消息并处理错误 39 | func formatMessages(template prompt.ChatTemplate, data map[string]any) ([]*schema.Message, error) { 40 | messages, err := template.Format(context.Background(), data) 41 | if err != nil { 42 | return nil, fmt.Errorf("格式化模板失败: %w", err) 43 | } 44 | return messages, nil 45 | } 46 | 47 | // docsMessages 将检索到的上下文和问题转换为消息列表 48 | func (x *Chat) docsMessages(ctx context.Context, convID string, docs []*schema.Document, question string) (messages []*schema.Message, err error) { 49 | chatHistory, err := x.eh.GetHistory(convID, 100) 50 | if err != nil { 51 | return 52 | } 53 | // 插入一条用户数据 54 | err = x.eh.SaveMessage(&schema.Message{ 55 | Role: schema.User, 56 | Content: question, 57 | }, convID) 58 | if err != nil { 59 | return 60 | } 61 | template := createTemplate() 62 | for i, doc := range docs { 63 | g.Log().Debugf(context.Background(), "docs[%d]: %s", i, doc.Content) 64 | } 65 | data := map[string]any{ 66 | "role": role, 67 | "question": question, 68 | "docs": docs, 69 | "chat_history": chatHistory, 70 | } 71 | messages, err = formatMessages(template, data) 72 | if err != nil { 73 | return 74 | } 75 | return 76 | } 77 | -------------------------------------------------------------------------------- /server/internal/logic/rag/retriever.go: -------------------------------------------------------------------------------- 1 | package rag 2 | 3 | import ( 4 | "context" 5 | "log" 6 | 7 | "github.com/elastic/go-elasticsearch/v8" 8 | "github.com/gogf/gf/v2/frame/g" 9 | "github.com/wangle201210/go-rag/server/core" 10 | "github.com/wangle201210/go-rag/server/core/config" 11 | ) 12 | 13 | var ragSvr = &core.Rag{} 14 | 15 | func init() { 16 | ctx := context.Background() 17 | client, err := elasticsearch.NewClient(elasticsearch.Config{ 18 | Addresses: []string{g.Cfg().MustGet(ctx, "es.address").String()}, 19 | Username: g.Cfg().MustGet(ctx, "es.username").String(), 20 | Password: g.Cfg().MustGet(ctx, "es.password").String(), 21 | }) 22 | if err != nil { 23 | log.Printf("NewClient of es8 failed, err=%v", err) 24 | return 25 | } 26 | ragSvr, err = core.New(ctx, &config.Config{ 27 | Client: client, 28 | IndexName: g.Cfg().MustGet(ctx, "es.indexName").String(), 29 | APIKey: g.Cfg().MustGet(ctx, "embedding.apiKey").String(), 30 | BaseURL: g.Cfg().MustGet(ctx, "embedding.baseURL").String(), 31 | EmbeddingModel: g.Cfg().MustGet(ctx, "embedding.model").String(), 32 | ChatModel: g.Cfg().MustGet(ctx, "chat.model").String(), 33 | }) 34 | if err != nil { 35 | log.Printf("New of rag failed, err=%v", err) 36 | return 37 | } 38 | } 39 | 40 | func GetRagSvr() *core.Rag { 41 | return ragSvr 42 | } 43 | -------------------------------------------------------------------------------- /server/internal/mcp/indexer.go: -------------------------------------------------------------------------------- 1 | package mcp 2 | 3 | import ( 4 | "context" 5 | "encoding/base64" 6 | "fmt" 7 | 8 | "github.com/ThinkInAIXYZ/go-mcp/protocol" 9 | "github.com/gogf/gf/v2/frame/g" 10 | "github.com/gogf/gf/v2/os/gctx" 11 | gorag "github.com/wangle201210/go-rag/server/core" 12 | "github.com/wangle201210/go-rag/server/internal/logic/rag" 13 | ) 14 | 15 | type IndexParam struct { 16 | URI string `json:"uri" description:"文件路径" required:"true"` // 可以是文件路径（pdf，html，md等），也可以是网址 17 | KnowledgeName string `json:"knowledge_name" description:"知识库名字,请先通过getKnowledgeBaseList获取列表后判断是否有符合的知识库，如果没有则根据用户提示词自己生成" required:"true"` 18 | } 19 | 20 | func GetIndexerByFilePathTool() *protocol.Tool { 21 | tool, err := protocol.NewTool("Indexer_by_filepath", "通过文件路径进行文本嵌入", IndexParam{}) 22 | if err != nil { 23 | g.Log().Errorf(gctx.New(), "Failed to create tool: %v", err) 24 | return nil 25 | } 26 | return tool 27 | } 28 | 29 | func HandleIndexerByFilePath(ctx context.Context, req *protocol.CallToolRequest) (*protocol.CallToolResult, error) { 30 | var reqData IndexParam 31 | if err := protocol.VerifyAndUnmarshal(req.RawArguments, &reqData); err != nil { 32 | return nil, err 33 | } 34 | svr := rag.GetRagSvr() 35 | uri := reqData.URI 36 | indexReq := &gorag.IndexReq{ 37 | URI: uri, 38 | KnowledgeName: reqData.KnowledgeName, 39 | } 40 | ids, err := svr.Index(ctx, indexReq) 41 | if err != nil { 42 | return nil, err 43 | } 44 | msg := fmt.Sprintf("index file %s successfully, knowledge_name: %s, doc_ids: %v", uri, reqData.KnowledgeName, ids) 45 | return &protocol.CallToolResult{ 46 | Content: []protocol.Content{ 47 | &protocol.TextContent{ 48 | Type: "text", 49 | Text: msg, 50 | }, 51 | }, 52 | }, nil 53 | } 54 | 55 | type IndexFileParam struct { 56 | Filename string `json:"filename" description:"文件名字" required:"true"` 57 | Content string `json:"content" description:"被base64编码后的文件内容，先调用工具获取base64信息" required:"true"` // 可以是文件路径（pdf，html，md等），也可以是网址文件" required:"true"` // 可以是文件路径（pdf，html，md等），也可以是网址 58 | KnowledgeName string `json:"knowledge_name" description:"知识库名字,请先通过getKnowledgeBaseList获取列表后判断是否有符合的知识库，如果没有则根据用户提示词自己生成" required:"true"` 59 | } 60 | 61 | func GetIndexerByFileBase64ContentTool() *protocol.Tool { 62 | tool, err := protocol.NewTool("Indexer_by_base64_file_content", "获取文件base64信息后上传，然后对内容进行文本嵌入", IndexFileParam{}) 63 | if err != nil { 64 | g.Log().Errorf(gctx.New(), "Failed to create tool: %v", err) 65 | return nil 66 | } 67 | return tool 68 | } 69 | 70 | func HandleIndexerByFileBase64Content(ctx context.Context, req *protocol.CallToolRequest) (*protocol.CallToolResult, error) { 71 | var reqData IndexFileParam 72 | if err := protocol.VerifyAndUnmarshal(req.RawArguments, &reqData); err != nil { 73 | return nil, err 74 | } 75 | // svr := rag.GetRagSvr() 76 | decoded, err := base64.StdEncoding.DecodeString(reqData.Content) 77 | if err != nil { 78 | return nil, err 79 | } 80 | fmt.Println(decoded) 81 | // indexReq := &gorag.IndexReq{ 82 | // URI: uri, 83 | // KnowledgeName: reqData.KnowledgeName, 84 | // } 85 | // ids, err := svr.Index(ctx, indexReq) 86 | // if err != nil { 87 | // return nil, err 88 | // } 89 | // msg := fmt.Sprintf("index file %s successfully, knowledge_name: %s, doc_ids: %v", uri, reqData.KnowledgeName, ids) 90 | return &protocol.CallToolResult{ 91 | Content: []protocol.Content{ 92 | &protocol.TextContent{ 93 | Type: "text", 94 | Text: string(decoded), 95 | }, 96 | }, 97 | }, nil 98 | } 99 | -------------------------------------------------------------------------------- /server/internal/mcp/knowledgebase.go: -------------------------------------------------------------------------------- 1 | package mcp 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/ThinkInAIXYZ/go-mcp/protocol" 8 | "github.com/gogf/gf/v2/frame/g" 9 | "github.com/gogf/gf/v2/os/gctx" 10 | v1 "github.com/wangle201210/go-rag/server/api/rag/v1" 11 | ) 12 | 13 | type KnowledgeBaseParam struct { 14 | } 15 | 16 | func GetKnowledgeBaseTool() *protocol.Tool { 17 | tool, err := protocol.NewTool("getKnowledgeBaseList", "获取知识库列表", KnowledgeBaseParam{}) 18 | if err != nil { 19 | g.Log().Errorf(gctx.New(), "Failed to create tool: %v", err) 20 | return nil 21 | } 22 | return tool 23 | } 24 | 25 | func HandleKnowledgeBase(ctx context.Context, toolReq *protocol.CallToolRequest) (res *protocol.CallToolResult, err error) { 26 | statusOK := v1.StatusOK 27 | getList, err := c.KBGetList(ctx, &v1.KBGetListReq{ 28 | Status: &statusOK, 29 | }) 30 | if err != nil { 31 | return nil, err 32 | } 33 | list := getList.List 34 | msg := fmt.Sprintf("get %d knowledgeBase", len(list)) 35 | for _, l := range list { 36 | msg += fmt.Sprintf("\n - name: %s, description: %s", l.Name, l.Description) 37 | } 38 | return &protocol.CallToolResult{ 39 | Content: []protocol.Content{ 40 | &protocol.TextContent{ 41 | Type: "text", 42 | Text: msg, 43 | }, 44 | }, 45 | }, nil 46 | } 47 | -------------------------------------------------------------------------------- /server/internal/mcp/mcp.go: -------------------------------------------------------------------------------- 1 | package mcp 2 | 3 | import "github.com/wangle201210/go-rag/server/internal/controller/rag" 4 | 5 | var c = rag.NewV1() 6 | -------------------------------------------------------------------------------- /server/internal/mcp/retriever.go: -------------------------------------------------------------------------------- 1 | package mcp 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | "github.com/ThinkInAIXYZ/go-mcp/protocol" 8 | "github.com/gogf/gf/v2/frame/g" 9 | "github.com/gogf/gf/v2/os/gctx" 10 | v1 "github.com/wangle201210/go-rag/server/api/rag/v1" 11 | ) 12 | 13 | type RetrieverParam struct { 14 | Question string `json:"question" description:"用户提问的问题" required:"true"` 15 | KnowledgeName string `json:"knowledge_name" description:"知识库名称，请先通过getKnowledgeBaseList获取列表后判断是否有符合用户提示词的知识库" required:"true"` 16 | TopK int `json:"top_k" description:"检索结果的数量，默认为5" required:"false"` // 默认为5 17 | Score float64 `json:"score" description:"检索结果的分数阀值，默认为0.2" required:"false"` // 默认为0.2 18 | } 19 | 20 | func GetRetrieverTool() *protocol.Tool { 21 | tool, err := protocol.NewTool("retriever", "检索知识库文档", RetrieverParam{}) 22 | if err != nil { 23 | g.Log().Errorf(gctx.New(), "Failed to create tool: %v", err) 24 | return nil 25 | } 26 | return tool 27 | } 28 | 29 | func HandleRetriever(ctx context.Context, toolReq *protocol.CallToolRequest) (res *protocol.CallToolResult, err error) { 30 | var req RetrieverParam 31 | if err := protocol.VerifyAndUnmarshal(toolReq.RawArguments, &req); err != nil { 32 | return nil, err 33 | } 34 | retriever, err := c.Retriever(ctx, &v1.RetrieverReq{ 35 | Question: req.Question, 36 | TopK: req.TopK, 37 | Score: req.Score, 38 | KnowledgeName: req.KnowledgeName, 39 | }) 40 | if err != nil { 41 | return nil, err 42 | } 43 | docs := retriever.Document 44 | msg := fmt.Sprintf("retrieve %d documents", len(docs)) 45 | for i, doc := range docs { 46 | msg += fmt.Sprintf("\n%d. score: %.2f, content: %s", i+1, doc.Score(), doc.Content) 47 | } 48 | return &protocol.CallToolResult{ 49 | Content: []protocol.Content{ 50 | &protocol.TextContent{ 51 | Type: "text", 52 | Text: msg, 53 | }, 54 | }, 55 | }, nil 56 | } 57 | -------------------------------------------------------------------------------- /server/internal/model/do/knowledge_base.go: -------------------------------------------------------------------------------- 1 | // ================================================================================= 2 | // Code generated and maintained by GoFrame CLI tool. DO NOT EDIT. 3 | // ================================================================================= 4 | 5 | package do 6 | 7 | import ( 8 | "github.com/gogf/gf/v2/frame/g" 9 | "github.com/gogf/gf/v2/os/gtime" 10 | ) 11 | 12 | // KnowledgeBase is the golang structure of table knowledge_base for DAO operations like Where/Data. 13 | type KnowledgeBase struct { 14 | g.Meta `orm:"table:knowledge_base, do:true"` 15 | Id interface{} // 主键ID 16 | Name interface{} // 知识库名称 17 | Description interface{} // 知识库描述 18 | Category interface{} // 知识库分类 19 | Status interface{} // 状态：0-禁用，1-启用 20 | CreateTime *gtime.Time // 创建时间 21 | UpdateTime *gtime.Time // 更新时间 22 | } 23 | -------------------------------------------------------------------------------- /server/internal/model/entity/knowledge_base.go: -------------------------------------------------------------------------------- 1 | // ================================================================================= 2 | // Code generated and maintained by GoFrame CLI tool. DO NOT EDIT. 3 | // ================================================================================= 4 | 5 | package entity 6 | 7 | import ( 8 | "github.com/gogf/gf/v2/os/gtime" 9 | ) 10 | 11 | // KnowledgeBase is the golang structure for table knowledge_base. 12 | type KnowledgeBase struct { 13 | Id int64 `json:"id" orm:"id" description:"主键ID"` // 主键ID 14 | Name string `json:"name" orm:"name" description:"知识库名称"` // 知识库名称 15 | Description string `json:"description" orm:"description" description:"知识库描述"` // 知识库描述 16 | Category string `json:"category" orm:"category" description:"知识库分类"` // 知识库分类 17 | Status int `json:"status" orm:"status" description:"状态：0-禁用，1-启用"` // 状态：0-禁用，1-启用 18 | CreateTime *gtime.Time `json:"createTime" orm:"create_time" description:"创建时间"` // 创建时间 19 | UpdateTime *gtime.Time `json:"updateTime" orm:"update_time" description:"更新时间"` // 更新时间 20 | } 21 | -------------------------------------------------------------------------------- /server/internal/model/knowledge_base.go: -------------------------------------------------------------------------------- 1 | package model 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/wangle201210/chat-history/models" 7 | "gorm.io/gorm" 8 | ) 9 | 10 | // KnowledgeBase 11 | // 仅供gorm自动创建表使用 12 | type KnowledgeBase struct { 13 | Id int64 `gorm:"primaryKey;column:id"` 14 | Name string `gorm:"column:name;type:varchar(255)"` 15 | Description string `gorm:"column:description;type:varchar(255)"` 16 | Category string `gorm:"column:category;type:varchar(255)"` 17 | Status int `gorm:"column:status;default:1"` 18 | CreateTime time.Time `gorm:"column:created_at"` 19 | UpdateTime time.Time `gorm:"column:updated_at"` 20 | } 21 | 22 | // TableName 设置表名 23 | func (KnowledgeBase) TableName() string { 24 | return "knowledge_base" 25 | } 26 | 27 | func autoMigrateTables(db *gorm.DB) error { 28 | // 自动迁移会创建表、缺失的外键、约束、列和索引 29 | return db.AutoMigrate( 30 | &models.Conversation{}, 31 | &models.Message{}, 32 | &models.Attachment{}, 33 | &models.MessageAttachment{}, 34 | ) 35 | } 36 | -------------------------------------------------------------------------------- /server/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/gogf/gf/v2/os/gctx" 5 | 6 | _ "github.com/gogf/gf/contrib/drivers/mysql/v2" 7 | "github.com/wangle201210/go-rag/server/internal/cmd" 8 | ) 9 | 10 | func main() { 11 | cmd.Main.Run(gctx.GetInitCtx()) 12 | } 13 | -------------------------------------------------------------------------------- /server/manifest/config/config_demo.yaml: -------------------------------------------------------------------------------- 1 | server: 2 | address: ":8000" 3 | openapiPath: "/api.json" 4 | swaggerPath: "/swagger" 5 | 6 | logger: 7 | level : "all" 8 | stdout: true 9 | 10 | database: 11 | default: 12 | host: "mysql" # 地址 13 | port: "3306" # 端口 14 | user: "root" # 账号 15 | pass: "123456" # 密码 16 | name: "go-rag" # 数据库名称 17 | type: "mysql" # 数据库类型 18 | 19 | es: 20 | address: "http://elasticsearch:9200" 21 | indexName: "rag-test" 22 | # username: "elastic" 23 | # password: "123456" 24 | 25 | embedding: 26 | model: "text-embedding-3-large" 27 | apiKey: "sk-***" 28 | baseURL: "https://api.openai.com/v1" 29 | 30 | chat: 31 | apiKey: "sk-***" 32 | baseURL: "https://api.openai.com/v1" 33 | model: "gpt-4o" 34 | -------------------------------------------------------------------------------- /server/static/chat.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangle201210/go-rag/29d54bcfff76f50e5f5f8532653a2091d16cbca5/server/static/chat.png -------------------------------------------------------------------------------- /server/static/indexer.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangle201210/go-rag/29d54bcfff76f50e5f5f8532653a2091d16cbca5/server/static/indexer.png -------------------------------------------------------------------------------- /server/static/kb-select.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangle201210/go-rag/29d54bcfff76f50e5f5f8532653a2091d16cbca5/server/static/kb-select.png -------------------------------------------------------------------------------- /server/static/kb.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangle201210/go-rag/29d54bcfff76f50e5f5f8532653a2091d16cbca5/server/static/kb.png -------------------------------------------------------------------------------- /server/static/mcp-cfg.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangle201210/go-rag/29d54bcfff76f50e5f5f8532653a2091d16cbca5/server/static/mcp-cfg.png -------------------------------------------------------------------------------- /server/static/mcp-use.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangle201210/go-rag/29d54bcfff76f50e5f5f8532653a2091d16cbca5/server/static/mcp-use.png -------------------------------------------------------------------------------- /server/static/retriever.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangle201210/go-rag/29d54bcfff76f50e5f5f8532653a2091d16cbca5/server/static/retriever.png --------------------------------------------------------------------------------