├── .dumi ├── theme │ ├── slots │ │ ├── PublicContent │ │ │ ├── index.less │ │ │ └── index.tsx │ │ ├── Content │ │ │ ├── heti.scss │ │ │ └── index.tsx │ │ ├── NavbarExtra │ │ │ └── index.tsx │ │ ├── HeroTitle │ │ │ ├── index.tsx │ │ │ └── index.less │ │ ├── ContentText │ │ │ ├── index.tsx │ │ │ └── index.less │ │ ├── HomeTitle │ │ │ ├── index.tsx │ │ │ └── index.less │ │ ├── Footer │ │ │ ├── index.tsx │ │ │ └── index.less │ │ ├── Banner │ │ │ ├── index.tsx │ │ │ └── index.less │ │ ├── Logo │ │ │ ├── index.tsx │ │ │ └── index.less │ │ ├── AboutDocs │ │ │ ├── index.tsx │ │ │ └── index.less │ │ ├── LearnMore │ │ │ ├── index.tsx │ │ │ └── index.less │ │ ├── Foot │ │ │ └── index.tsx │ │ ├── SearchBar │ │ │ ├── Mask.tsx │ │ │ └── input.tsx │ │ ├── Contribution │ │ │ ├── index.tsx │ │ │ └── index.less │ │ ├── BlogDetails │ │ │ └── index.tsx │ │ ├── AutomatedTesting │ │ │ ├── index.tsx │ │ │ └── index.less │ │ ├── CodeAnalysis │ │ │ ├── index.tsx │ │ │ └── index.less │ │ ├── LangSwitch │ │ │ ├── index.less │ │ │ └── index.tsx │ │ ├── DevOps │ │ │ ├── index.tsx │ │ │ └── index.less │ │ ├── IntelligentInference │ │ │ ├── index.tsx │ │ │ └── index.less │ │ ├── ColorSwitch │ │ │ ├── index.less │ │ │ └── index.tsx │ │ ├── Hero │ │ │ └── index.tsx │ │ ├── PerformanceEvaluation │ │ │ ├── index.tsx │ │ │ └── index.less │ │ ├── Publication │ │ │ ├── index.tsx │ │ │ └── index.less │ │ ├── Navbar │ │ │ └── menu.tsx │ │ ├── Toc │ │ │ └── index.less │ │ ├── Sidebar │ │ │ └── index.less │ │ ├── Header │ │ │ └── index.tsx │ │ └── CodeGeneration │ │ │ └── index.tsx │ ├── styles │ │ └── variables.less │ ├── locales │ │ ├── zh-CN.json │ │ └── en-US.json │ └── constants │ │ └── index.ts ├── tsconfig.json └── hooks │ └── useLocale.ts ├── .prettierignore ├── .vscode └── settings.json ├── .husky ├── pre-commit └── commit-msg ├── static ├── LOGO.png ├── wechat.webp ├── codefuse_web.webp ├── slick.25572f22.eot ├── slick.653a4cbb.woff ├── slick.6aa1ee46.ttf ├── baseagent.37d27565.png ├── agent-flow.c2afcd91.png ├── reactagent.a2c7af88.webp ├── executoragent.3131e5b1.png ├── selectoragent.b3dcf0af.webp ├── muagent_framework.56842181.png └── slick.d4bc62a2.svg ├── .gitignore ├── docs ├── blogDetails │ ├── 001.en-US.md │ ├── blogDeatils.zh-CN.md │ ├── blogDetails.en-US.md │ ├── 20231101.en-US.md │ ├── 20231211.en-US.md │ ├── 20231220.en-US.md │ ├── 20240119.en-US.md │ ├── 20240123.en-US.md │ ├── 20240614.en-US.md │ ├── 20240703.en-US.md │ ├── 20240705.en-US.md │ ├── 20240706.en-US.md │ ├── 20240805.en-US.md │ ├── 20240807.en-US.md │ ├── 20240820.en-US.md │ ├── 20240914.en-US.md │ ├── 20231101.zh-CN.md │ ├── 20231211.zh-CN.md │ ├── 001.zh-CN.md │ ├── 20241111.en-US.md │ └── 20241111.zh-CN.md ├── static │ └── api-docs │ │ └── muAgent │ │ ├── baseagent.png │ │ ├── agent-flow.png │ │ ├── reactagent.webp │ │ ├── executoragent.png │ │ ├── selectoragent.webp │ │ └── muagent_framework.png ├── contribution │ ├── acknowledgements.zh-CN.md │ ├── acknowledgements.en-US.md │ ├── contribution.zh-CN.md │ ├── issue.zh-CN.md │ ├── pr.zh-CN.md │ ├── pr.en-US.md │ └── issue.en-US.md ├── docs │ ├── api-docs │ │ └── MuAgent │ │ │ ├── connector │ │ │ ├── connector_memory.zh-CN.md │ │ │ └── connector_memory.en-US.md │ │ │ ├── llm_models │ │ │ ├── llm_config.zh-CN.md │ │ │ ├── llm_config.en-US.md │ │ │ ├── embedding_config.zh-CN.md │ │ │ └── embedding_config.en-US.md │ │ │ └── overview │ │ │ ├── agent-flow.zh-CN.md │ │ │ └── agent-flow.en-US.md │ ├── developer-docs │ │ ├── CodeFuse-ModelCache │ │ │ └── main │ │ │ │ ├── config.zh-CN.md │ │ │ │ ├── config.en-US.md │ │ │ │ ├── CodeFuseModelCache.zh-CN.md │ │ │ │ ├── release_note.zh-CN.md │ │ │ │ ├── CodeFuseModelCache.en-US.md │ │ │ │ ├── quickstart.zh-CN.md │ │ │ │ └── quickstart.en-US.md │ │ ├── CodeFuse-evalution │ │ │ └── main │ │ │ │ ├── codefuse-evalution.zh-CN.md │ │ │ │ └── codefuse-evalution.en-US.md │ │ ├── CodeFuse-ChatBot │ │ │ └── master │ │ │ │ ├── quickstart.zh-CN.md │ │ │ │ ├── quickstart.en-US.md │ │ │ │ ├── roadmap.zh-CN.md │ │ │ │ ├── codefusechatbot.zh-CN.md │ │ │ │ └── roadmap.en-US.md │ │ ├── MFTCoder │ │ │ └── main │ │ │ │ ├── introduction.zh-CN.md │ │ │ │ └── introduction.en-US.md │ │ ├── CodeFuse-DevOps-Model │ │ │ └── main │ │ │ │ ├── traindetail.zh-CN.md │ │ │ │ ├── quickstart.zh-CN.md │ │ │ │ ├── codefuseDevopsModel.zh-CN.md │ │ │ │ └── quickstart.en-US.md │ │ ├── CodeFuse-Query │ │ │ └── main │ │ │ │ ├── CodeFuseQuery.zh-CN.md │ │ │ │ └── user_case.zh-CN.md │ │ ├── CodeFuse-DevOps-Eval │ │ │ └── master │ │ │ │ ├── evaluate.zh-CN.md │ │ │ │ └── evaluate.en-US.md │ │ ├── Test-Agent │ │ │ └── main │ │ │ │ └── quickstart.zh-CN.md │ │ └── CodeFuse-MFT-VLM │ │ │ └── main │ │ │ ├── quickstart.zh-CN.md │ │ │ ├── mftvlm.en-US.md │ │ │ └── quickstart.en-US.md │ └── about │ │ ├── overview.zh-CN.md │ │ └── overview.en-US.md ├── aboutDocs │ ├── aboutdocs.zh-CN.md │ └── aboutdocs.en-US.md ├── publication │ ├── publication.zh-CN.md │ └── publication.en-US.md └── blogs │ └── blogs.en-US.md ├── .editorconfig ├── .prettierrc.js ├── tsconfig.json ├── LEGAL.md ├── .dumirc.ts ├── package.json └── README_ZH.md /.dumi/theme/slots/PublicContent/index.less: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.dumi/theme/slots/PublicContent/index.tsx: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.prettierignore: -------------------------------------------------------------------------------- 1 | .dumi/tmp 2 | .dumi/tmp-production 3 | *.yaml 4 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "typescript.tsdk": "node_modules/typescript/lib" 3 | } 4 | -------------------------------------------------------------------------------- /.husky/pre-commit: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | . "$(dirname -- "$0")/_/husky.sh" 3 | 4 | npx lint-staged 5 | -------------------------------------------------------------------------------- /static/LOGO.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/codefuse-ai/codefuse-ai.github.io/main/static/LOGO.png -------------------------------------------------------------------------------- /static/wechat.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/codefuse-ai/codefuse-ai.github.io/main/static/wechat.webp -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | .dumi/tmp 3 | .dumi/tmp-production 4 | .DS_Store 5 | .node 6 | package-lock.json 7 | *log -------------------------------------------------------------------------------- /.husky/commit-msg: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | . "$(dirname -- "$0")/_/husky.sh" 3 | 4 | npx commitlint --edit "${1}" 5 | -------------------------------------------------------------------------------- /static/codefuse_web.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/codefuse-ai/codefuse-ai.github.io/main/static/codefuse_web.webp -------------------------------------------------------------------------------- /static/slick.25572f22.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/codefuse-ai/codefuse-ai.github.io/main/static/slick.25572f22.eot -------------------------------------------------------------------------------- /static/slick.653a4cbb.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/codefuse-ai/codefuse-ai.github.io/main/static/slick.653a4cbb.woff -------------------------------------------------------------------------------- /static/slick.6aa1ee46.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/codefuse-ai/codefuse-ai.github.io/main/static/slick.6aa1ee46.ttf -------------------------------------------------------------------------------- /static/baseagent.37d27565.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/codefuse-ai/codefuse-ai.github.io/main/static/baseagent.37d27565.png -------------------------------------------------------------------------------- /static/agent-flow.c2afcd91.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/codefuse-ai/codefuse-ai.github.io/main/static/agent-flow.c2afcd91.png -------------------------------------------------------------------------------- /static/reactagent.a2c7af88.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/codefuse-ai/codefuse-ai.github.io/main/static/reactagent.a2c7af88.webp -------------------------------------------------------------------------------- /static/executoragent.3131e5b1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/codefuse-ai/codefuse-ai.github.io/main/static/executoragent.3131e5b1.png -------------------------------------------------------------------------------- /static/selectoragent.b3dcf0af.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/codefuse-ai/codefuse-ai.github.io/main/static/selectoragent.b3dcf0af.webp -------------------------------------------------------------------------------- /docs/blogDetails/001.en-US.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: 'Not updated yet' 3 | time: '2024-06-05' 4 | toc: content 5 | --- 6 | 7 | Not updated yet 8 | -------------------------------------------------------------------------------- /docs/blogDetails/blogDeatils.zh-CN.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: blogDetails 3 | nav: 4 | title: blogDetails 5 | order: 11 6 | toc: content 7 | --- 8 | -------------------------------------------------------------------------------- /docs/blogDetails/blogDetails.en-US.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: blogDetails 3 | nav: 4 | title: blogDetails 5 | order: 11 6 | toc: content 7 | --- 8 | -------------------------------------------------------------------------------- /static/muagent_framework.56842181.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/codefuse-ai/codefuse-ai.github.io/main/static/muagent_framework.56842181.png -------------------------------------------------------------------------------- /docs/blogDetails/20231101.en-US.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: 'Not updated yet' 3 | time: '2023-12-11' 4 | toc: content 5 | --- 6 | 7 | Not updated yet 8 | -------------------------------------------------------------------------------- /docs/blogDetails/20231211.en-US.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: 'Not updated yet' 3 | time: '2023-12-11' 4 | toc: content 5 | --- 6 | 7 | Not updated yet 8 | -------------------------------------------------------------------------------- /docs/blogDetails/20231220.en-US.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: 'Not updated yet' 3 | time: '2023-12-20 ' 4 | toc: content 5 | --- 6 | 7 | Not updated yet 8 | -------------------------------------------------------------------------------- /docs/blogDetails/20240119.en-US.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: 'Not updated yet' 3 | time: '2024-01-19' 4 | toc: content 5 | --- 6 | 7 | Not updated yet 8 | -------------------------------------------------------------------------------- /docs/blogDetails/20240123.en-US.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: 'Not updated yet' 3 | time: '2024-01-23' 4 | toc: content 5 | --- 6 | 7 | Not updated yet 8 | -------------------------------------------------------------------------------- /docs/blogDetails/20240614.en-US.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: 'Not updated yet' 3 | time: '2024-06-14' 4 | toc: content 5 | --- 6 | 7 | Not updated yet 8 | -------------------------------------------------------------------------------- /docs/blogDetails/20240703.en-US.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: 'Not updated yet' 3 | time: '2024-07-03' 4 | toc: content 5 | --- 6 | 7 | Not updated yet 8 | -------------------------------------------------------------------------------- /docs/blogDetails/20240705.en-US.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: 'Not updated yet' 3 | time: '2024-07-05' 4 | toc: content 5 | --- 6 | 7 | Not updated yet 8 | -------------------------------------------------------------------------------- /docs/blogDetails/20240706.en-US.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: 'Not updated yet' 3 | time: '2024-07-06' 4 | toc: content 5 | --- 6 | 7 | Not updated yet 8 | -------------------------------------------------------------------------------- /docs/blogDetails/20240805.en-US.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: 'Not updated yet' 3 | time: '2024-08-05' 4 | toc: content 5 | --- 6 | 7 | Not updated yet 8 | -------------------------------------------------------------------------------- /docs/blogDetails/20240807.en-US.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: 'Not updated yet' 3 | time: '2024-08-07' 4 | toc: content 5 | --- 6 | 7 | Not updated yet 8 | -------------------------------------------------------------------------------- /docs/blogDetails/20240820.en-US.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: 'Not updated yet' 3 | time: '2024-08-20' 4 | toc: content 5 | --- 6 | 7 | Not updated yet 8 | -------------------------------------------------------------------------------- /docs/blogDetails/20240914.en-US.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: 'Not updated yet' 3 | time: '2024-09-14' 4 | toc: content 5 | --- 6 | 7 | Not updated yet 8 | -------------------------------------------------------------------------------- /docs/static/api-docs/muAgent/baseagent.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/codefuse-ai/codefuse-ai.github.io/main/docs/static/api-docs/muAgent/baseagent.png -------------------------------------------------------------------------------- /docs/static/api-docs/muAgent/agent-flow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/codefuse-ai/codefuse-ai.github.io/main/docs/static/api-docs/muAgent/agent-flow.png -------------------------------------------------------------------------------- /docs/static/api-docs/muAgent/reactagent.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/codefuse-ai/codefuse-ai.github.io/main/docs/static/api-docs/muAgent/reactagent.webp -------------------------------------------------------------------------------- /.dumi/theme/slots/Content/heti.scss: -------------------------------------------------------------------------------- 1 | // override .heti root-selector 2 | $root-selector: '.markdown'; 3 | $line-length: 100%; 4 | @import 'heti/lib/heti.scss'; 5 | -------------------------------------------------------------------------------- /docs/static/api-docs/muAgent/executoragent.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/codefuse-ai/codefuse-ai.github.io/main/docs/static/api-docs/muAgent/executoragent.png -------------------------------------------------------------------------------- /docs/static/api-docs/muAgent/selectoragent.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/codefuse-ai/codefuse-ai.github.io/main/docs/static/api-docs/muAgent/selectoragent.webp -------------------------------------------------------------------------------- /.dumi/theme/slots/NavbarExtra/index.tsx: -------------------------------------------------------------------------------- 1 | import React, { type FC } from 'react'; 2 | 3 | const NavbarExtra: FC = () => <>; 4 | 5 | export default NavbarExtra; 6 | -------------------------------------------------------------------------------- /docs/static/api-docs/muAgent/muagent_framework.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/codefuse-ai/codefuse-ai.github.io/main/docs/static/api-docs/muAgent/muagent_framework.png -------------------------------------------------------------------------------- /docs/blogDetails/20231101.zh-CN.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: '在 Visual Studio Code 中使用 CodeFuse' 3 | time: '2023-11-01' 4 | toc: content 5 | --- 6 | 7 | https://codefuse.yuque.com/eoxx1u/codefuse/vscode-extension 8 | -------------------------------------------------------------------------------- /.dumi/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "jsx": "react" 4 | }, 5 | "extends": "../tsconfig.json", 6 | "include": ["**/*"], 7 | "paths": { 8 | "@/*": ["./src/*"], 9 | "@@/*": ["./src/.umi/*"] 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # http://editorconfig.org 2 | root = true 3 | 4 | [*] 5 | indent_style = space 6 | indent_size = 2 7 | end_of_line = lf 8 | charset = utf-8 9 | trim_trailing_whitespace = true 10 | insert_final_newline = true 11 | 12 | [*.md] 13 | trim_trailing_whitespace = false 14 | -------------------------------------------------------------------------------- /.prettierrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | printWidth: 80, 3 | proseWrap: 'never', 4 | singleQuote: true, 5 | trailingComma: 'all', 6 | overrides: [ 7 | { 8 | files: '*.md', 9 | options: { 10 | proseWrap: 'preserve', 11 | }, 12 | }, 13 | ], 14 | }; 15 | -------------------------------------------------------------------------------- /.dumi/theme/slots/HeroTitle/index.tsx: -------------------------------------------------------------------------------- 1 | import React, { type FC, type string } from 'react'; 2 | import './index.less'; 3 | 4 | const HeroTitle: FC<{ children: string }> = (props) => ( 5 |
6 | 7 |
8 | ); 9 | 10 | export default HeroTitle; 11 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "strict": true, 4 | "skipLibCheck": true, 5 | "esModuleInterop": true, 6 | "jsx": "react", 7 | "baseUrl": "./", 8 | "paths": { 9 | "@@/*": [ 10 | ".dumi/tmp/*" 11 | ] 12 | } 13 | }, 14 | "include": [ 15 | ".dumirc.ts" 16 | ] 17 | } 18 | -------------------------------------------------------------------------------- /docs/contribution/acknowledgements.zh-CN.md: -------------------------------------------------------------------------------- 1 | --- 2 | nav: 3 | title: 社区 4 | order: 2 5 | group: 6 | title: ❤️ 致谢 7 | index: true 8 | order: 2 9 | title: 致谢 10 | toc: content 11 | --- 12 | 13 | ChatBot 项目基于[langchain-chatchat](https://github.com/chatchat-space/Langchain-Chatchat)和[codebox-api](https://github.com/shroominic/codebox-api)! 14 | 15 | ...... 16 | 17 | 在此深深感谢他们的开源贡献！ 18 | -------------------------------------------------------------------------------- /.dumi/theme/slots/ContentText/index.tsx: -------------------------------------------------------------------------------- 1 | import React, { type FC, type string } from 'react'; 2 | import './index.less'; 3 | 4 | const ContentText: FC<{ title: string , desc:any }> = (props) => ( 5 |
6 |
{props.title}
7 |
{props.desc}
8 |
9 | ); 10 | 11 | export default ContentText; 12 | -------------------------------------------------------------------------------- /.dumi/hooks/useLocale.ts: -------------------------------------------------------------------------------- 1 | import { useIntl, useSiteData } from 'dumi'; 2 | import { useState } from 'react'; 3 | import type { ILocale } from './types'; 4 | 5 | export const useLocale = (): ILocale => { 6 | const intl = useIntl(); 7 | const { locales } = useSiteData(); 8 | const [locale] = useState( 9 | () => locales.find(({ id }) => id === intl.locale)!, 10 | ); 11 | return locale; 12 | }; 13 | -------------------------------------------------------------------------------- /.dumi/theme/slots/HomeTitle/index.tsx: -------------------------------------------------------------------------------- 1 | import './index.less'; 2 | import React, { type FC } from 'react'; 3 | const HomeTitle: FC<{ title: string, textAlign?: string, line?: string }> = (props) => { 4 | return
5 | {props.title} 6 |
7 |
8 | }; 9 | 10 | export default HomeTitle; 11 | -------------------------------------------------------------------------------- /LEGAL.md: -------------------------------------------------------------------------------- 1 | Legal Disclaimer 2 | 3 | Within this source code, the comments in Chinese shall be the original, governing version. Any comment in other languages are for reference only. In the event of any conflict between the Chinese language version comments and other language version comments, the Chinese language version shall prevail. 4 | 5 | 法律免责声明 6 | 7 | 关于代码注释部分，中文注释为官方版本，其它语言注释仅做参考。中文注释可能与其它语言注释存在不一致，当中文注释与其它语言注释存在不一致时，请以中文注释为准。 8 | -------------------------------------------------------------------------------- /docs/contribution/acknowledgements.en-US.md: -------------------------------------------------------------------------------- 1 | --- 2 | nav: 3 | title: Community 4 | order: 2 5 | group: 6 | title: ❤️ Acknowledgements 7 | index: true 8 | order: 2 9 | toc: content 10 | --- 11 | 12 | The ChatBot project is based on [langchain-chatchat](https://github.com/chatchat-space/Langchain-Chatchat) and [codebox-api](https://github.com/shroominic/codebox-api). 13 | 14 | ...... 15 | 16 | Deep gratitude is extended for their open-source contributions! 17 | -------------------------------------------------------------------------------- /.dumi/theme/slots/Footer/index.tsx: -------------------------------------------------------------------------------- 1 | import { useSiteData } from 'dumi'; 2 | import React, { type FC } from 'react'; 3 | import './index.less'; 4 | 5 | const Footer: FC = () => { 6 | const { themeConfig } = useSiteData(); 7 | 8 | if (!themeConfig.footer) return null; 9 | 10 | return ( 11 |
15 | ); 16 | }; 17 | 18 | export default Footer; 19 | -------------------------------------------------------------------------------- /docs/docs/api-docs/MuAgent/connector/connector_memory.zh-CN.md: -------------------------------------------------------------------------------- 1 | --- 2 | group: 3 | title: Connector 4 | order: 0 5 | title: Memory 6 | order: 3 7 | toc: content 8 | --- 9 | 10 | ## Memory Manager 11 | 12 | - 将 chat history 在数据库进行读写管理，包括 user input、 llm output、doc retrieval、code retrieval、search retrieval 13 | - 对 chat history 进行关键信息总结 summary context，作为 prompt context 14 | - 提供检索功能，检索 chat history 或者 summary context 中与问题相关信息，辅助问答 15 | 16 | ## 使用示例 17 | 18 | 完整示例见 ~/tests/connector/memory_manager_test.py 19 | -------------------------------------------------------------------------------- /.dumi/theme/slots/Banner/index.tsx: -------------------------------------------------------------------------------- 1 | import React, { type FC } from 'react'; 2 | import './index.less'; 3 | const Banner: FC <{ bannerBg?: string; bannerTitle?: string; }> = (props) => { 4 | return ( 5 |
6 |
7 |
8 | 11 |
12 |
13 |
14 | ); 15 | }; 16 | export default Banner; 17 | -------------------------------------------------------------------------------- /.dumi/theme/slots/Logo/index.tsx: -------------------------------------------------------------------------------- 1 | import { Link, useLocale, useSiteData } from 'dumi'; 2 | import { type FC } from 'react'; 3 | import './index.less'; 4 | import React from 'react'; 5 | 6 | const Logo: FC = () => { 7 | const { themeConfig } = useSiteData(); 8 | const locale = useLocale(); 9 | return ( 10 | 14 | {themeConfig.logo !== false && ( 15 |
16 | )} 17 | {themeConfig.name} 18 | 19 | ); 20 | }; 21 | 22 | export default Logo; 23 | -------------------------------------------------------------------------------- /.dumi/theme/slots/HeroTitle/index.less: -------------------------------------------------------------------------------- 1 | @import (reference) '.dumi/theme/styles/variables.less'; 2 | 3 | @font-face { 4 | font-family: AlibabaPuHuiTi_2_85_Bold; 5 | } 6 | 7 | .@{prefix}-hero-title { 8 | margin: 0px; 9 | display: inline-block; 10 | font-family: Alibaba-PuHuiTi, 'Gill Sans', 'Gill Sans MT', Calibri, 11 | 'Trebuchet MS', sans-serif; 12 | color: lighten(desaturate(spin(@c-primary, -13), 10.5), 20); 13 | line-height: 1; 14 | 15 | img { 16 | margin: 0 auto; 17 | width: 1100px; 18 | } 19 | 20 | @media @mobile { 21 | font-size: 60px; 22 | } 23 | 24 | // @{dark-selector} & { 25 | // opacity: 0.7; 26 | // } 27 | } 28 | -------------------------------------------------------------------------------- /.dumi/theme/slots/AboutDocs/index.tsx: -------------------------------------------------------------------------------- 1 | import { useRouteMeta } from 'dumi'; 2 | import React, { type FC } from 'react'; 3 | import './index.less'; 4 | import ContentText from '../ContentText'; 5 | 6 | const AboutDocs: FC = () => { 7 | const { frontmatter } = useRouteMeta(); 8 | return ( 9 |
10 |
11 |
12 | 15 |
16 |
17 | 18 |
19 | ); 20 | }; 21 | export default AboutDocs; 22 | -------------------------------------------------------------------------------- /docs/aboutDocs/aboutdocs.zh-CN.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: About 3 | nav: 4 | title: 关于 5 | order: 4 6 | bannerTitle: https://mdn.alipayobjects.com/huamei_bvbxju/afts/img/A*10l4RpTepNIAAAAAAAAAAAAADlHYAQ/original 7 | contentTitle: CodeFuse 团队 8 | 9 | # group: 10 | # title: ❤️ Contribution Guide 11 | # index: true 12 | # order: -1 13 | toc: content 14 | --- 15 | 16 | ## CodeFuse 团队 17 | 18 | CodeFuse 团队由一群充满热情的成员组成，我们的目标是构建大型编码语言模型（Code LLMs），以支持和提升在整个软件开发生命周期中的 AI 原生开发工作。我们的工作覆盖了从设计需求、编写代码、测试、构建、部署、运维到洞察分析等关键环节。我们积极推广开源精神，目前已经推出了 15 个代码相关的模型，并开源了一系列技术工具，例如 MFTCoder、CodeFuse-VLM、CodeFuse-DevOps、CodeFuse-Query、TestGPU 和 CodeFuse-muAgent。此外，为了让用户在日常编码工作中能够直接受益，我们还开发了 CodeFuse-IDE 插件。随时欢迎和我们交流，如果您对这项激动人心的工作感兴趣，欢迎加入我们！ 19 | -------------------------------------------------------------------------------- /docs/docs/api-docs/MuAgent/connector/connector_memory.en-US.md: -------------------------------------------------------------------------------- 1 | --- 2 | group: 3 | title: Connector 4 | order: 0 5 | title: Memory 6 | order: 3 7 | toc: content 8 | --- 9 | 10 | ## Memory Manager 11 | 12 | Primarily used for managing chat history, not yet completed 13 | 14 | - Read and write chat history in the database, including user input, llm output, doc retrieval, code retrieval, search retrieval. 15 | - Summarize key information from the chat history into a summary context, serving as a prompt context. 16 | - Provide a search function to retrieve information related to the question from chat history or summary context, aiding in Q&A. 17 | 18 | ## Usage Example 19 | 20 | Examples see ~/tests/connector/memory_manager_test.py 21 | -------------------------------------------------------------------------------- /.dumi/theme/slots/Banner/index.less: -------------------------------------------------------------------------------- 1 | .banner { 2 | position: relative; 3 | padding-top: 30px; 4 | height: 380px; 5 | width: 100%; 6 | box-sizing: border-box; 7 | // background: linear-gradient(to top, #30cfd0 0%, #330867 100%); 8 | background-repeat: no-repeat; 9 | background-size: cover; 10 | background-position: center; 11 | display: flex; 12 | align-items: center; 13 | justify-content: center; 14 | flex-direction: column; 15 | 16 | .bannerContent { 17 | padding: 0px 24px; 18 | margin-top: 30px; 19 | width: 900px; 20 | color: #f8f9fa; 21 | font-size: 2.7rem; 22 | display: flex; 23 | justify-content: center; 24 | 25 | img { 26 | width: 30%; 27 | } 28 | 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /.dumi/theme/slots/Footer/index.less: -------------------------------------------------------------------------------- 1 | @import (reference) '.dumi/theme/styles/variables.less'; 2 | 3 | .@{prefix}-footer { 4 | margin-top: @s-content-padding; 5 | border-top: 1px solid @c-border-light; 6 | color: @c-text-note; 7 | font-size: 15px; 8 | line-height: 26px; 9 | text-align: center; 10 | padding: @s-content-padding * 0.6 0; 11 | 12 | @{dark-selector} & { 13 | border-top-color: @c-border-less-dark; 14 | color: @c-text-note-dark; 15 | } 16 | 17 | @media @mobile { 18 | padding: @s-content-padding * 0.3 0; 19 | font-size: 13px; 20 | } 21 | 22 | a { 23 | color: @c-primary; 24 | 25 | @{dark-selector} & { 26 | color: @c-primary-dark; 27 | } 28 | 29 | &:not(:hover) { 30 | text-decoration: none; 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /.dumi/theme/slots/LearnMore/index.tsx: -------------------------------------------------------------------------------- 1 | import { useLocale, history } from 'dumi'; 2 | import './index.less'; 3 | import React, { type FC } from 'react'; 4 | import { SwapRightOutlined } from '@ant-design/icons'; 5 | const LearnMore: FC<{ children: string, link: string }> = (props) => { 6 | const locale = useLocale(); 7 | const handleClick = (link: string) => { 8 | if (link.indexOf('http') === -1) { 9 | history.push(link); 10 | window.scrollTo({ top: 0 }); 11 | } else { 12 | window.open(link, '_blank') 13 | } 14 | }; 15 | return ( 16 |
handleClick(props.link)}> 17 | {locale.id === 'zh-CN' ? props.children : 'Learn more'} 18 | 19 |
20 | ); 21 | }; 22 | export default LearnMore; 23 | 24 | -------------------------------------------------------------------------------- /.dumirc.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig } from 'dumi'; 2 | 3 | export default defineConfig({ 4 | favicons: [ 5 | 'https://mdn.alipayobjects.com/huamei_v98cj4/afts/img/A*EfwQTpYQfq4AAAAAAAAAAAAADo6VAQ/original', 6 | ], 7 | locales: [ 8 | { id: 'en-US', name: 'EN' }, 9 | { id: 'zh-CN', name: '中文' }, 10 | ], 11 | themeConfig: { 12 | logo: 'https://mdn.alipayobjects.com/huamei_bvbxju/afts/img/A*_pzERpyma84AAAAAAAAAAAAADlHYAQ/original', 13 | footer: false, 14 | // 'Copyright © 支付宝（中国）网络技术有限公司 ｜ 备案号：沪ICP备15027489号', 15 | socialLinks: { 16 | github: 'https://github.com/codefuse-ai', 17 | }, 18 | editLink: true, 19 | }, 20 | 21 | mfsu: false, 22 | resolve: { 23 | forceKebabCaseRouting: false, 24 | }, 25 | extraRemarkPlugins: ['remark-math'], 26 | extraRehypePlugins: ['rehype-katex'] 27 | }); 28 | -------------------------------------------------------------------------------- /.dumi/theme/slots/ContentText/index.less: -------------------------------------------------------------------------------- 1 | @import (reference) '.dumi/theme/styles/variables.less'; 2 | 3 | @font-face { 4 | font-family: AlibabaPuHuiTi_2_85_Bold; 5 | } 6 | 7 | .dumi-content-text { 8 | width: 900px; 9 | margin: 0 auto; 10 | color: #e4e9ec; 11 | padding: 24px; 12 | min-height: 700px; 13 | 14 | .title { 15 | margin: 40px 0 32px; 16 | text-align: left; 17 | color: #e4e9ec; 18 | font-weight: 600; 19 | font-size: 32px; 20 | 21 | @{dark-selector} & { 22 | font-size: 32px; 23 | color: #5c6cf7; 24 | font-weight: 600; 25 | } 26 | } 27 | 28 | .desc { 29 | margin: 0 0 20px; 30 | line-height: 30px; 31 | font-size: 16px; 32 | letter-spacing: 0.61px; 33 | 34 | @{dark-selector} & { 35 | font-size: 16px; 36 | color: #1b1b1b; 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /docs/docs/developer-docs/CodeFuse-ModelCache/main/config.zh-CN.md: -------------------------------------------------------------------------------- 1 | --- 2 | store: 3 | title: CodeFuse-ModelCache 4 | version: main 5 | group: 6 | title: 🌱 CodeFuse-ModelCache 7 | order: -1 8 | title: 最佳配置 9 | order: 2 10 | toc: content 11 | github: https://github.com/codefuse-ai/CodeFuse-ModelCache 12 | --- 13 | 14 | ## 环境依赖 15 | 16 | - python 版本: 3.8 及以上 17 | - 依赖包安装： 18 | `pip install requirements.txt ` 19 | 20 | ## 服务启动 21 | 22 | - 在启动服务前，应该进行如下环境配置： 23 | - 安装关系数据库 mysql， 导入 sql 创建数据表，sql 文件: reference_doc/create_table.sql 24 | - 安装向量数据库 milvus 25 | - 在配置文件中添加数据库访问信息，配置文件为： 26 | - modelcache/config/milvus_config.ini 27 | - modelcache/config/mysql_config.ini 28 | - 离线模型 bin 文件下载， 参考地址：https://huggingface.co/shibing624/text2vec-base-chinese/tree/main，并将下载的bin文件，放到 model/text2vec-base-chinese 文件夹中 29 | - 通过 flask4modelcache.py 脚本启动后端服务。 30 | -------------------------------------------------------------------------------- /.dumi/theme/slots/HomeTitle/index.less: -------------------------------------------------------------------------------- 1 | .homeTitle { 2 | margin: 0 auto; 3 | text-align: center; 4 | min-width: 343px; 5 | font-size: 40px; 6 | color: #ffffff; 7 | letter-spacing: 1.35px; 8 | font-weight: 600; 9 | background-image: linear-gradient(90deg, #d8d8d8 0%, #545eff 100%); 10 | display: inline-block; 11 | background-clip: text; 12 | -webkit-background-clip: text; 13 | -webkit-text-fill-color: transparent; 14 | 15 | .line { 16 | margin-top: 19px; 17 | background-image: linear-gradient(90deg, #e5b2ca 0%, #7546f3 100%); 18 | height: 4px; 19 | width: 323px; 20 | 21 | @{dark-selector} & { 22 | background-image: linear-gradient(90deg, #545eff 0%, #9466ff 100%); 23 | } 24 | } 25 | 26 | @{dark-selector} & { 27 | background-image: linear-gradient(90deg, #545eff 0%, #9466ff 100%); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /.dumi/theme/slots/Foot/index.tsx: -------------------------------------------------------------------------------- 1 | import { useRouteMeta, useSidebarData, useSiteData } from 'dumi'; 2 | import React, { type FC, type ReactNode } from 'react'; 3 | import { Row, Col, Collapse } from "antd"; 4 | import classNames from "classnames"; 5 | import './index.less'; 6 | 7 | const Foot: FC = () => { 8 | return ( 9 |
10 |
11 |
12 | Copyright © 支付宝（中国）网络技术有限公司 13 | {/* ｜ */} 14 | {/* 备案号：沪ICP备15027489号 */} 15 |
16 |
17 |
18 |
19 | Copyright © 支付宝（中国）网络技术有限公司 20 | {/* ｜ */} 21 | {/* 备案号：沪ICP备15027489号 */} 22 |
23 |
24 |
25 | ); 26 | } 27 | export default Foot; 28 | -------------------------------------------------------------------------------- /.dumi/theme/slots/SearchBar/Mask.tsx: -------------------------------------------------------------------------------- 1 | import React, { useEffect, type FC, type ReactNode } from 'react'; 2 | 3 | type MaskProps = { 4 | visible: boolean; 5 | children: ReactNode; 6 | onMaskClick?: () => void; 7 | onClose?: () => void; 8 | }; 9 | 10 | export const Mask: FC = (props) => { 11 | useEffect(() => { 12 | if (props.visible) { 13 | document.body.style.overflow = 'hidden'; 14 | } else if (document.body.style.overflow) { 15 | document.body.style.overflow = ''; 16 | props.onClose?.(); 17 | } 18 | }, [props.visible]); 19 | 20 | return props.visible ? ( 21 |
22 |
26 |
{props.children}
27 |
28 | ) : null; 29 | }; 30 | -------------------------------------------------------------------------------- /.dumi/theme/slots/Contribution/index.tsx: -------------------------------------------------------------------------------- 1 | import { useRouteMeta, useOutlet, } from 'dumi'; 2 | import React, { type FC } from 'react'; 3 | import './index.less'; 4 | import ContentText from '../ContentText'; 5 | 6 | const Contribution: FC = () => { 7 | const { frontmatter } = useRouteMeta(); 8 | console.log('frontmatter,,,', frontmatter); 9 | 10 | return ( 11 |
12 |
13 |
14 | 15 |
16 |
17 |
18 |
{frontmatter.contentTitle}
19 |
    20 | { 21 | frontmatter.list.map((item: string) => { 22 | return
  • {item}
    • 23 | }) 24 | } 25 |
26 |
27 |
28 | ); 29 | }; 30 | export default Contribution; 31 | -------------------------------------------------------------------------------- /docs/docs/developer-docs/CodeFuse-evalution/main/codefuse-evalution.zh-CN.md: -------------------------------------------------------------------------------- 1 | --- 2 | nav: 3 | title: Docs 4 | order: -1 5 | second: 6 | title: Developer-Docs 7 | order: -1 8 | store: 9 | title: CodeFuse-evalution 10 | version: main 11 | group: 12 | title: 🌱 CodeFuse-evalution 13 | index: true 14 | order: -1 15 | title: 代码大语言模型的多任务评估基准 16 | order: -1 17 | toc: content 18 | github: https://github.com/codefuse-ai/codefuse-evaluation 19 | --- 20 | 21 | 27 | 28 | CodeFuseEval 在 HumanEval-x、MBPP 的基准上，结合 CodeFuse 大模型多任务场景，开发的编程领域多任务的评测基准， 可用于评估模型在代码补全，自然语言生成代码，测试用例生成、跨语言代码翻译，中文指令生成代码等多类任务的性能。持续开放中，敬请期待！ 29 | 30 | ![img](https://mdn.alipayobjects.com/huamei_bvbxju/afts/img/A*7CZtSpgj3K4AAAAAAAAAAAAADlHYAQ/original) 31 | -------------------------------------------------------------------------------- /docs/aboutDocs/aboutdocs.en-US.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: About 3 | nav: 4 | title: About 5 | order: 4 6 | bannerTitle: https://mdn.alipayobjects.com/huamei_bvbxju/afts/img/A*i0atTYYpUEMAAAAAAAAAAAAADlHYAQ/original 7 | contentTitle: CodeFuse Team 8 | # group: 9 | # title: ❤️ Contribution Guide 10 | # index: true 11 | # order: -1 12 | toc: content 13 | --- 14 | 15 | ## CodeFuse Team 16 | 17 | The CodeFuse Team is a group of enthusiastic people that aims to build Code Large Language Models (Code LLMs) to support and enhance AI-native software development throughout the full life cycle, covering crucial stages such as design requirements, coding, testing, building, deployment, operations, and insight analysis.We embrace open source and have released 15 code-related models, along with a series of open-sourced techniques, such as MFTCoder, CodeFuse-VLM, CodeFuse-DevOps, CodeFuse-Query, TestGPU, CodeFuse-muAgent.We also built CodeFuse-IDE plugins for users to benefit from the assistance of CodeFuse in their daily coding work. Feel free to chat with us, and you are welcome to join us for this amazing work! 18 | -------------------------------------------------------------------------------- /.dumi/theme/slots/LearnMore/index.less: -------------------------------------------------------------------------------- 1 | .buttom { 2 | z-index: 1111 !important; 3 | cursor: pointer; 4 | width: 150px; 5 | height: 37px; 6 | background-image: url("https://mdn.alipayobjects.com/huamei_bvbxju/afts/img/A*eMQHT670srQAAAAAAAAAAAAADlHYAQ/original"); 7 | background-repeat: no-repeat; 8 | background-size: contain; 9 | background-position: center; 10 | border-radius: 7px; 11 | color: #a8abff; 12 | display: flex; 13 | align-items: center; 14 | justify-content: center; 15 | position: relative; 16 | 17 | .anticon { 18 | svg { 19 | width: 30px; 20 | height: 23px; 21 | position: absolute; 22 | bottom: 5px; 23 | } 24 | 25 | margin-left: 8px; 26 | width: 25px; 27 | height: 25px; 28 | } 29 | 30 | &:hover { 31 | background-image: url("https://mdn.alipayobjects.com/huamei_bvbxju/afts/img/A*Pm85SKQpQ88AAAAAAAAAAAAADlHYAQ/original"); 32 | background-repeat: no-repeat; 33 | background-size: contain; 34 | background-position: center; 35 | color: #fff; 36 | 37 | @{dark-selector} & { 38 | color: #7031FD; 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /docs/docs/developer-docs/CodeFuse-ModelCache/main/config.en-US.md: -------------------------------------------------------------------------------- 1 | --- 2 | store: 3 | title: CodeFuse-ModelCache 4 | version: main 5 | group: 6 | title: 🌱 CodeFuse-ModelCache 7 | order: -1 8 | title: How to better configure your cache 9 | order: 2 10 | toc: content 11 | github: https://github.com/codefuse-ai/CodeFuse-ModelCache 12 | --- 13 | 14 | ## Environment Dependencies 15 | 16 | - Python version: 3.8 or higher 17 | - To install dependencies: pip install requirements.txt 18 | 19 | ## Service Startup 20 | 21 | - Before starting the service, the following environment configurations should be performed: 22 | - Install relational database MySQL, import SQL to create tables, SQL file: reference_doc/create_table.sql 23 | - Install vector database Milvus 24 | - Add database access information to the configuration files, which are: 25 | - modelcache/config/milvus_config.ini 26 | - modelcache/config/mysql_config.ini 27 | - Download offline model bin files, refer to: https://huggingface.co/shibing624/text2vec-base-chinese/tree/main, and place the downloaded bin files into the model/text2vec-base-chinese folder 28 | - Start the backend service using the flask4modelcache.py script. 29 | -------------------------------------------------------------------------------- /.dumi/theme/slots/Content/index.tsx: -------------------------------------------------------------------------------- 1 | import { useRouteMeta, useSidebarData, useSiteData } from 'dumi'; 2 | import React, { type FC, type ReactNode } from 'react'; 3 | import './heti.scss'; 4 | import './index.less'; 5 | 6 | const Content: FC<{ children: ReactNode }> = (props) => { 7 | const sidebar = useSidebarData(); 8 | const { themeConfig } = useSiteData(); 9 | const { frontmatter } = useRouteMeta(); 10 | return ( 11 |
12 |
17 | {props.children} 18 |
19 | {/*
20 |
21 | Copyright © 支付宝（中国）网络技术有限公司 ｜ 22 | 备案号：沪ICP备15027489号 23 |
24 |
25 |
26 |
27 | Copyright © 支付宝（中国）网络技术有限公司 ｜ 28 | 备案号：沪ICP备15027489号 29 |
30 |
*/} 31 |
32 | ); 33 | }; 34 | 35 | export default Content; 36 | -------------------------------------------------------------------------------- /.dumi/theme/slots/BlogDetails/index.tsx: -------------------------------------------------------------------------------- 1 | import { useRouteMeta, useOutlet } from 'dumi'; 2 | import React, { type FC } from 'react'; 3 | import Content from 'dumi/theme/slots/Content'; 4 | import './index.less'; 5 | import Toc from 'dumi/theme/slots/Toc'; 6 | 7 | const BlogDetails: FC = () => { 8 | const outlet = useOutlet(); 9 | const { frontmatter: fm } = useRouteMeta(); 10 | const { frontmatter } = useRouteMeta(); 11 | return ( 12 |
13 |
14 |
15 |
{frontmatter.time}
16 |
{frontmatter.title}
17 |
18 |
19 |
20 | {fm.toc === 'content' && ( 21 |
22 |
23 | 24 |
25 |
26 | )} 27 | 28 | {
{outlet}
} 29 | 30 |
31 |
32 | ); 33 | }; 34 | export default BlogDetails; 35 | -------------------------------------------------------------------------------- /docs/docs/about/overview.zh-CN.md: -------------------------------------------------------------------------------- 1 | --- 2 | nav: 3 | title: 文档 4 | order: -1 5 | second: 6 | title: 关于 CodeFuse 7 | order: 1 8 | group: 9 | title: 📖 CodeFuse-AI 整体介绍 10 | index: true 11 | order: 0 12 | title: 概览 13 | toc: content 14 | description: Learn more about the team maintaining Docura, how and why the project started, and how to get involved. 15 | --- 16 | 17 |

18 | 19 |

20 |
21 | HuggingFace | 22 | 魔搭社区 | 23 | 产品主页 24 |
25 | 26 | Hello World! This is CodeFuse! 27 | 28 | **CodeFuse 的使命是开发专门设计用于支持整个软件开发生命周期的大型代码语言模型（Code LLMs），涵盖设计、需求、编码、测试、部署、运维等关键阶段。我们致力于打造创新的解决方案，让软件开发者们在研发的过程中如丝般顺滑。** 29 | 30 |

31 | 32 |

33 | 34 | 我们非常有激情去构建创新的解决方案来支持全生命周期 AI 驱动的软件开发，如上图所示。同时，我们也诚邀志同道合的工程师和研究人员加入这个社区，共同构建和增强 CodeFuse。 35 | -------------------------------------------------------------------------------- /.dumi/theme/slots/AutomatedTesting/index.tsx: -------------------------------------------------------------------------------- 1 | import { useRouteMeta, useLocale, usePrefersColor } from 'dumi'; 2 | import './index.less'; 3 | import React, { type FC } from 'react'; 4 | import { SwapRightOutlined } from '@ant-design/icons'; 5 | import HomeTitle from '../HomeTitle'; 6 | 7 | const AutomatedTesting: FC = () => { 8 | const { frontmatter } = useRouteMeta(); 9 | const locale = useLocale(); 10 | const [color] = usePrefersColor(); 11 | 12 | if (!('AutomatedTesting' in frontmatter)) return null; 13 | return
14 |
15 | 16 |
17 | < HomeTitle title={frontmatter.AutomatedTesting.title} /> 18 |
19 | {frontmatter.AutomatedTesting.description} 20 |
21 |
{ window.open(frontmatter.AutomatedTesting.link) }}> 22 | {locale.id === 'zh-CN' ? '了解更多' : 'Learn more'} 23 | 24 |
25 |
26 |
27 |
28 | }; 29 | 30 | export default AutomatedTesting; 31 | -------------------------------------------------------------------------------- /.dumi/theme/slots/CodeAnalysis/index.tsx: -------------------------------------------------------------------------------- 1 | import { useLocale, usePrefersColor, useRouteMeta } from 'dumi'; 2 | import './index.less'; 3 | import React, { type FC } from 'react'; 4 | import { SwapRightOutlined } from '@ant-design/icons'; 5 | import HomeTitle from '../HomeTitle'; 6 | 7 | const CodeAnalysis: FC = () => { 8 | const { frontmatter } = useRouteMeta(); 9 | const [color] = usePrefersColor(); 10 | const locale = useLocale(); 11 | if (!('CodeAnalysis' in frontmatter)) return null; 12 | return
13 |
14 | 18 |
19 | 20 |
21 | {frontmatter.CodeAnalysis.description} 22 |
23 |
{ window.open(frontmatter.CodeAnalysis.link) }}> 24 | {locale.id === 'zh-CN' ? '了解更多' : 'Learn more'} 25 | 26 |
27 |
28 |
29 |
30 | }; 31 | 32 | export default CodeAnalysis; 33 | -------------------------------------------------------------------------------- /.dumi/theme/slots/LangSwitch/index.less: -------------------------------------------------------------------------------- 1 | @import (reference) '../../styles/variables.less'; 2 | 3 | .@{prefix}-lang-switch { 4 | color: @c-text-secondary; 5 | font-size: 14px; 6 | line-height: 16px; 7 | text-decoration: none; 8 | transition: all 0.3s; 9 | cursor: pointer; 10 | 11 | @{dark-selector} & { 12 | color: @c-text-secondary-dark; 13 | } 14 | 15 | &:hover { 16 | color: @c-primary; 17 | 18 | @{dark-selector} & { 19 | color: @c-primary-dark; 20 | } 21 | } 22 | } 23 | 24 | .@{prefix}-lang-select { 25 | display: inline-flex; 26 | align-items: center; 27 | 28 | > select { 29 | appearance: none; 30 | padding: 6px 0; 31 | padding-inline-start: 10px; 32 | padding-inline-end: 18px; 33 | color: @c-text-secondary; 34 | text-align: right; 35 | font-size: 14px; 36 | line-height: 1; 37 | border: 0; 38 | background-color: transparent; 39 | cursor: pointer; 40 | 41 | @{dark-selector} & { 42 | color: @c-text-secondary-dark; 43 | } 44 | } 45 | 46 | > svg { 47 | margin-inline-start: -16px; 48 | width: 12px; 49 | fill: darken(@c-border, 10%); 50 | pointer-events: none; 51 | 52 | @{dark-selector} & { 53 | fill: lighten(@c-border-dark, 10%); 54 | } 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /.dumi/theme/slots/Logo/index.less: -------------------------------------------------------------------------------- 1 | @import (reference) '.dumi/theme/styles/variables.less'; 2 | 3 | .@{prefix}-logo { 4 | display: inline-flex; 5 | align-items: center; 6 | color: @c-text; 7 | font-size: 22px; 8 | line-height: 1; 9 | font-weight: bold; 10 | text-decoration: none; 11 | margin-right: 40px; 12 | 13 | @{dark-selector} & { 14 | color: @c-text-dark; 15 | } 16 | .logoImg { 17 | width: 148px; 18 | height: 30px; 19 | background-image: url('https://mdn.alipayobjects.com/huamei_bvbxju/afts/img/A*_pzERpyma84AAAAAAAAAAAAADlHYAQ/original'); 20 | background-repeat: no-repeat; 21 | background-size: contain; 22 | background-position: center; 23 | 24 | @{dark-selector} & { 25 | width: 168px; 26 | height: 30px; 27 | background-image: url('https://mdn.alipayobjects.com/huamei_bvbxju/afts/img/A*9K9dSYx7NaMAAAAAAAAAAAAADlHYAQ/original'); 28 | background-repeat: no-repeat; 29 | background-size: contain; 30 | background-position: left; 31 | } 32 | } 33 | 34 | @media @mobile { 35 | font-size: 18px; 36 | 37 | img { 38 | height: 32px; 39 | } 40 | } 41 | 42 | img { 43 | margin-inline-end: 10px; 44 | width: 168px; 45 | height: 30px; 46 | 47 | @media @mobile { 48 | height: 32px; 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "CodeFuse-Docs", 3 | "version": "0.0.1", 4 | "description": "A static-site powered by dumi", 5 | "repository": { 6 | "type": "git", 7 | "url": "https://code.alipay.com/LinkE/CodeFuse-Docs" 8 | }, 9 | "license": "MIT", 10 | "author": "zhenjinsong.szj", 11 | "scripts": { 12 | "build": "dumi build", 13 | "dev": "dumi dev", 14 | "prepare": "husky install && dumi setup", 15 | "start": "npm run dev" 16 | }, 17 | "commitlint": { 18 | "extends": [ 19 | "@commitlint/config-conventional" 20 | ] 21 | }, 22 | "lint-staged": { 23 | "*.{md,json}": [ 24 | "prettier --write --no-error-on-unmatched-pattern" 25 | ] 26 | }, 27 | "dependencies": { 28 | "@ant-design/icons": "^5.3.7", 29 | "antd": "^5.16.5", 30 | "react-slick": "^0.30.2", 31 | "rehype-katex": "^7.0.0", 32 | "remark-math": "^6.0.0", 33 | "slick-carousel": "^1.8.1", 34 | "styled-components": "^6.1.8" 35 | }, 36 | "devDependencies": { 37 | "@commitlint/cli": "^17.1.2", 38 | "@commitlint/config-conventional": "^17.1.0", 39 | "@types/lodash": "^4.17.0", 40 | "dumi": "^2.2.0", 41 | "husky": "^8.0.1", 42 | "lint-staged": "^13.0.3", 43 | "prettier": "^2.7.1" 44 | }, 45 | "tnpm": { 46 | "mode": "npm" 47 | }, 48 | "yuyanId": "180020010001264005" 49 | } 50 | -------------------------------------------------------------------------------- /.dumi/theme/slots/DevOps/index.tsx: -------------------------------------------------------------------------------- 1 | import { useLocale, useRouteMeta, usePrefersColor } from 'dumi'; 2 | import './index.less'; 3 | import React, { type FC } from 'react'; 4 | import { SwapRightOutlined } from '@ant-design/icons'; 5 | import HomeTitle from '../HomeTitle'; 6 | 7 | 8 | const DevOps: FC = () => { 9 | const { frontmatter } = useRouteMeta(); 10 | const [color] = usePrefersColor(); 11 | const locale = useLocale(); 12 | if (!('DevOps' in frontmatter)) return null; 13 | return
14 |
15 | 16 |
17 |
    18 | {frontmatter.DevOps.map((item: any) => { 19 | return
  • window.open(item.link)}> 20 | 21 |
    {item.cardTitle}
    22 |
    {item.description}
    23 |
    24 | {locale.id === 'zh-CN' ? '了解更多' : 'Learn more'} 25 | 26 |
    27 |
    • 28 | }) 29 | } 30 |
31 |
32 |
33 |
34 | }; 35 | 36 | export default DevOps; 37 | -------------------------------------------------------------------------------- /.dumi/theme/slots/IntelligentInference/index.tsx: -------------------------------------------------------------------------------- 1 | import { useLocale, useRouteMeta, usePrefersColor } from 'dumi'; 2 | import './index.less'; 3 | import React, { type FC } from 'react'; 4 | import { SwapRightOutlined } from '@ant-design/icons'; 5 | import HomeTitle from '../HomeTitle'; 6 | 7 | const IntelligentInference: FC = () => { 8 | const { frontmatter } = useRouteMeta(); 9 | const locale = useLocale(); 10 | const [color] = usePrefersColor(); 11 | if (!('IntelligentInference' in frontmatter)) return null; 12 | return
13 |
14 |
15 | 16 |
17 | {frontmatter.IntelligentInference.description} 18 |
19 |
{ window.open(frontmatter.IntelligentInference.link) }}> 20 | {locale.id === 'zh-CN' ? '了解更多' : 'Learn more'} 21 | 22 |
23 |
24 | 30 |
31 |
32 | }; 33 | 34 | export default IntelligentInference; 35 | -------------------------------------------------------------------------------- /.dumi/theme/slots/ColorSwitch/index.less: -------------------------------------------------------------------------------- 1 | @import (reference) '../../styles/variables.less'; 2 | 3 | .@{prefix}-color-switch { 4 | position: relative; 5 | font-size: 0; 6 | line-height: 0; 7 | 8 | @media screen and (max-width: 1430px) { 9 | &::before { 10 | left: auto; 11 | right: auto; 12 | inset-inline-end: -15px; 13 | transform: none; 14 | 15 | [class*='-switch']+&, 16 | [class*='-select']+& { 17 | inset-inline-end: 0; 18 | } 19 | } 20 | } 21 | 22 | [class*='-switch']+&, 23 | [class*='-select']+& { 24 | margin-inline-start: 35px; 25 | margin-inline-start: 15px; 26 | margin-inline-end: -15px; 27 | padding-right: 15px; 28 | // padding-inline: 15px; 29 | // border-inline-start: 1px solid @c-border; 30 | 31 | @{dark-selector} & { 32 | border-inline-start-color: @c-border-dark; 33 | } 34 | } 35 | 36 | svg { 37 | width: 16px; 38 | fill: @c-text-secondary; 39 | 40 | @{dark-selector} & { 41 | fill: @c-text-secondary-dark; 42 | } 43 | } 44 | 45 | &:hover svg { 46 | fill: @c-primary; 47 | 48 | @{dark-selector} & { 49 | fill: @c-primary-dark; 50 | } 51 | } 52 | 53 | select { 54 | position: absolute; 55 | inset: 0 15%; 56 | opacity: 0; 57 | width: 100%; 58 | min-width: 16px; 59 | max-width: 70%; 60 | height: 16px; 61 | cursor: pointer; 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /docs/docs/api-docs/MuAgent/llm_models/llm_config.zh-CN.md: -------------------------------------------------------------------------------- 1 | --- 2 | group: 3 | title: llm_models 4 | order: 1 5 | title: LLM 配置 6 | order: -1 7 | toc: content 8 | --- 9 | 10 | ## 准备相关参数 11 | 12 | 首先增加 openai 配置，也可以是其它类似于 openai 接口的模型（通过 fastchat 启动） 13 | 14 | ``` 15 | import os, sys 16 | 17 | api_key = "sk-xxx" 18 | api_base_url= "https://api.openai.com/v1" 19 | model_name = "gpt-3.5-turbo" 20 | ``` 21 | 22 | ## 构建 LLM Config 23 | 24 | - 通过调用 类 openai 传入 25 | 26 | ``` 27 | from muagent.llm_models.llm_config import EmbedConfig, LLMConfig 28 | llm_config = LLMConfig( 29 | model_name=model_name, api_key=api_key, api_base_url=api_base_url, temperature=0.3, 30 | stop="**Observation:**" 31 | ) 32 | ``` 33 | 34 | - 自定义 langchain LLM 传入 35 | 36 | ``` 37 | from muagent.llm_models.llm_config import EmbedConfig, LLMConfig 38 | from langchain.llms.base import BaseLLM, LLM 39 | 40 | class CustomizedModel(LLM): 41 | repetition_penalty = 1.1 42 | temperature = 0.2 43 | top_k = 40 44 | top_p = 0.9 45 | 46 | def predict(self, prompt: str, stop: Optional[List[str]] = None) -> str: 47 | return self._call(prompt, stop) 48 | 49 | def _call(self, prompt: str, 50 | stop: Optional[List[str]] = None) -> str: 51 | """_call 52 | """ 53 | return "" 54 | 55 | llm = CustomizedModel() 56 | llm_config = LLMConfig( 57 | llm=llm 58 | ) 59 | ``` 60 | -------------------------------------------------------------------------------- /.dumi/theme/slots/Hero/index.tsx: -------------------------------------------------------------------------------- 1 | import { Link, useRouteMeta, usePrefersColor } from 'dumi'; 2 | import HeroTitle from 'dumi/theme/slots/HeroTitle'; 3 | import CodeGeneration from '../CodeGeneration'; 4 | import React, { type FC } from 'react'; 5 | import './index.less'; 6 | import DevOps from '../DevOps'; 7 | import CodeAnalysis from '../CodeAnalysis'; 8 | import IntelligentInference from '../IntelligentInference'; 9 | import AutomatedTesting from '../AutomatedTesting'; 10 | import PerformanceEvaluation from '../PerformanceEvaluation'; 11 | 12 | const Hero: FC = () => { 13 | const { frontmatter } = useRouteMeta(); 14 | const [color] = usePrefersColor(); 15 | 16 | if (!('hero' in frontmatter)) return null; 17 | return ( 18 |
19 |
20 | {frontmatter.hero!.title && ( 21 | {color === 'dark' ? frontmatter.hero!.titleLight : frontmatter.hero!.title} 22 | )} 23 | {frontmatter.hero!.description && ( 24 |

28 | )} 29 |

30 | 31 | 32 | 33 | 34 | 35 | 36 |
37 | ); 38 | }; 39 | export default Hero; 40 | -------------------------------------------------------------------------------- /.dumi/theme/slots/PerformanceEvaluation/index.tsx: -------------------------------------------------------------------------------- 1 | import { Link, useLocale, usePrefersColor, useRouteMeta } from 'dumi'; 2 | import './index.less'; 3 | import React, { type FC } from 'react'; 4 | import { SwapRightOutlined } from '@ant-design/icons'; 5 | import HomeTitle from '../HomeTitle'; 6 | 7 | const PerformanceEvaluation: FC = () => { 8 | const { frontmatter } = useRouteMeta(); 9 | const locale = useLocale(); 10 | const [color] = usePrefersColor(); 11 | if (!('PerformanceEvaluation' in frontmatter)) return null; 12 | return
13 |
14 |
15 | 16 |
window.open(frontmatter.PerformanceEvaluation.link)}> 17 | {locale.id === 'zh-CN' ? '了解更多' : 'Learn more'} 18 | 19 |
20 |
21 |
22 |
23 | {frontmatter.PerformanceEvaluation.description} 24 |
25 | 31 |
32 |
33 |
34 | }; 35 | 36 | export default PerformanceEvaluation; 37 | -------------------------------------------------------------------------------- /.dumi/theme/styles/variables.less: -------------------------------------------------------------------------------- 1 | @prefix: dumi-default; 2 | @s-content-width: 1000px; 3 | @s-content-padding: 48px; 4 | @s-sidebar-width: 248px; 5 | @s-header-height: 100px; 6 | @s-header-height-m: 52px; 7 | 8 | // default theme colors 9 | @c-primary: #5c6cf7; 10 | @c-warning: #d59200; 11 | @c-success: #208a41; 12 | @c-error: #ce1f31; 13 | @c-text: #30363f; 14 | @c-text-secondary: #4f5866; 15 | @c-text-note: #8a9099; 16 | @c-border: #d0d5d8; 17 | @c-border-light: #e4e9ec; 18 | @c-site-bg: #04040e; 19 | 20 | // dark theme colors 21 | // @dark-selector be injected by less-loader in feature/theme/index.ts 22 | @dark-solid-amount: 15%; 23 | @dark-light-amount: 22%; 24 | @dark-border-amount: 71%; 25 | @c-primary-dark: darken(@c-primary, @dark-solid-amount); 26 | @c-warning-dark: darken(@c-warning, @dark-solid-amount); 27 | @c-success-dark: darken(@c-success, @dark-solid-amount); 28 | @c-error-dark: darken(@c-error, @dark-solid-amount); 29 | @c-text-dark: lighten(@c-text-note, @dark-light-amount); 30 | @c-text-secondary-dark: lighten(@c-text-secondary, @dark-light-amount); 31 | @c-text-note-dark: lighten(@c-text, @dark-light-amount); 32 | @c-border-dark: darken(@c-border, @dark-border-amount); 33 | @c-border-less-dark: darken(@c-border-light, @dark-border-amount); 34 | @c-site-bg-dark: darken(@c-site-bg, 95%); 35 | 36 | @mobile: ~'only screen and (max-width: 767px)'; 37 | @tablet: ~'only screen and (min-width: 768px) and (max-width: 1024px)'; 38 | @desktop: ~'only screen and (min-width: 1025px)'; 39 | -------------------------------------------------------------------------------- /docs/docs/developer-docs/CodeFuse-evalution/main/codefuse-evalution.en-US.md: -------------------------------------------------------------------------------- 1 | --- 2 | nav: 3 | title: Docs 4 | order: -1 5 | second: 6 | title: Developer-Docs 7 | order: -1 8 | store: 9 | title: CodeFuse-evalution 10 | version: main 11 | group: 12 | title: 🌱 CodeFuse-evalution 13 | index: true 14 | order: -1 15 | title: CodeFuse-evalution 16 | order: -1 17 | toc: content 18 | github: https://github.com/codefuse-ai/codefuse-evaluation 19 | --- 20 | 21 | # CodeFuseEval: Multi-tasking Evaluation Benchmark for Code Large Language Model 22 | 23 |
24 |

25 | CodeFuseEval on ModelScope｜ 26 | CodeFuseEval on Hugging Face 27 |

28 | 29 |
30 | 31 | CodeFuseEval is a Code Generation benchmark that combines the multi-tasking scenarios of CodeFuse Model with the benchmarks of HumanEval-x and MBPP. This benchmark is designed to evaluate the performance of models in various multi-tasking tasks, including code completion, code generation from natural language, test case generation, cross-language code translation, and code generation from Chinese commands, among others.Continuously open, stay tuned ! 32 | 33 |

34 | English Introduction 35 |

36 | -------------------------------------------------------------------------------- /docs/docs/developer-docs/CodeFuse-ChatBot/master/quickstart.zh-CN.md: -------------------------------------------------------------------------------- 1 | --- 2 | nav: 3 | title: 文档 4 | order: -1 5 | second: 6 | title: 开发者文档 7 | order: -1 8 | store: 9 | title: CodeFuse-ChatBot 10 | version: master 11 | group: 12 | title: 🌱 CodeFuse-ChatBot 13 | order: -1 14 | title: 快速开始 15 | order: -1 16 | toc: content 17 | github: https://github.com/codefuse-ai/codefuse-chatbot 18 | --- 19 | 20 | ## 🚀 快速使用 21 | 22 | 如需使用私有化模型部署，请自行安装 nvidia 驱动程序，本项目已在 Python 3.9.18，CUDA 11.7 环境下，Windows、X86 架构的 macOS 系统中完成测试。 23 | 24 | Docker 安装、私有化 LLM 接入及相关启动问题见：[快速使用明细](/zh-CN/docs/developer-docs/CodeFuse-ChatBot/master/start-detail) 25 | 26 | 1、python 环境准备 27 | 28 | - 推荐采用 conda 对 python 环境进行管理（可选） 29 | 30 | ```bash 31 | # 准备 conda 环境 32 | conda create --name devopsgpt python=3.9 33 | conda activate devopsgpt 34 | ``` 35 | 36 | - 安装相关依赖 37 | 38 | ```bash 39 | cd codefuse-chatbot 40 | # python=3.9，notebook用最新即可，python=3.8用notebook=6.5.6 41 | pip install -r requirements.txt 42 | ``` 43 | 44 | 2、启动服务 45 | 46 | ```bash 47 | # 完成server_config.py配置后，可一键启动 48 | cd examples 49 | bash start.sh 50 | # 开始在页面进行相关配置，然后打开`启动对话服务`即可 51 | ``` 52 | 53 |
54 | 图片 55 |
56 | 57 | 或者通过`start.py`进行启动[老版启动方式](/zh-CN/docs/developer-docs/CodeFuse-ChatBot/master/start-detail) 58 | 更多 LLM 接入方法见[更多细节...](/zh-CN/docs/developer-docs/CodeFuse-ChatBot/master/fastchat) 59 |
60 | -------------------------------------------------------------------------------- /docs/docs/developer-docs/CodeFuse-ModelCache/main/CodeFuseModelCache.zh-CN.md: -------------------------------------------------------------------------------- 1 | --- 2 | nav: 3 | title: 文档 4 | order: -1 5 | second: 6 | title: 开发者文档 7 | order: -1 8 | store: 9 | title: CodeFuse-ModelCache 10 | version: main 11 | group: 12 | title: 🌱 CodeFuse-ModelCache 13 | index: true 14 | order: -1 15 | title: CodeFuse-ModelCache 16 | order: -1 17 | toc: content 18 | github: https://github.com/codefuse-ai/CodeFuse-ModelCache 19 | --- 20 | 21 | ## Contents 22 | 23 | - [新闻](#新闻) 24 | - [项目简介](#项目简介) 25 | - [架构大图](#架构大图) 26 | - [致谢](#致谢) 27 | - [Contributing](#Contributing) 28 | 29 | ## 新闻 30 | 31 | - 🔥🔥[2023.12.10] 增加 llmEmb、onnx、paddlenlp、fasttext 等 LLM embedding 框架，并增加 timm 图片 embedding 框架，用于提供更丰富的 embedding 能力。 32 | - 🔥🔥[2023.11.20] codefuse-ModelCache 增加本地存储能力, 适配了嵌入式数据库 sqlite、faiss，方便用户快速启动测试。 33 | - [2023.10.31] codefuse-ModelCache... 34 | 35 | ## 项目简介 36 | 37 | Codefuse-ModelCache 是一个开源的大模型语义缓存系统，通过缓存已生成的模型结果，降低类似请求的响应时间，提升用户体验。该项目从服务优化角度出发，引入缓存机制，在资源有限和对实时性要求较高的场景下，帮助企业和研究机构降低推理部署成本、提升模型性能和效率、提供规模化大模型服务。我们希望通过开源，分享交流大模型语义 Cache 的相关技术。 38 | 39 | ## 架构大图 40 | 41 | ![modelcache modules](https://mdn.alipayobjects.com/huamei_bvbxju/afts/img/A*Z-6cSr6udKAAAAAAAAAAAAAADlHYAQ/original) 42 | 43 | ## 致谢 44 | 45 | 本项目参考了以下开源项目，在此对相关项目和研究开发人员表示感谢。
[GPTCache](https://github.com/zilliztech/GPTCache) 46 | 47 | ## Contributing 48 | 49 | ModelCache 是一个非常有趣且有用的项目，我们相信这个项目有很大的潜力，无论你是经验丰富的开发者，还是刚刚入门的新手，都欢迎你为这个项目做出一些贡献，包括但不限于：提交问题和建议，参与代码编写，完善文档和示例。你的参与将会使这个项目变得更好，同时也会为开源社区做出贡献。 50 | -------------------------------------------------------------------------------- /docs/docs/about/overview.en-US.md: -------------------------------------------------------------------------------- 1 | --- 2 | nav: 3 | title: Docs 4 | order: -1 5 | second: 6 | title: About CodeFuse 7 | order: 0 8 | group: 9 | title: 📖 CodeFuse-AI Introduce 10 | index: true 11 | order: 0 12 | title: Overview 13 | toc: content 14 | description: Learn more about the team maintaining Docura, how and why the project started, and how to get involved. 15 | --- 16 | 17 |

18 | 19 |

20 | 21 |
22 | HuggingFace | 23 | ModelScope 24 |
25 | 26 | Hello World! This is CodeFuse! 27 |
28 | 29 | **CodeFuse aims to develop Code Large Language Models (Code LLMs) to support and enhance full-lifecycle AI native sotware developing, covering crucial stages such as design requirements, coding, testing, building, deployment, operations, and insight analysis.** 30 | 31 |

32 | 33 |

34 | We are passionating about creating innovative open-source solutions that empower developers throughout the software development process as shown above. We also encourage engineers and researchers within this community to join us in co-constructing/improving CodeFuse. 35 | -------------------------------------------------------------------------------- /docs/docs/api-docs/MuAgent/llm_models/llm_config.en-US.md: -------------------------------------------------------------------------------- 1 | --- 2 | group: 3 | title: llm_models 4 | order: 1 5 | title: LLM Config 6 | order: -1 7 | toc: content 8 | --- 9 | 10 | ## Prepare Relevant Parameters 11 | 12 | First, add the OpenAI configuration, or you can use another model similar to the OpenAI interface (launched through fastchat). 13 | 14 | ``` 15 | import os, sys 16 | 17 | api_key = "sk-xxx" 18 | api_base_url= "https://api.openai.com/v1" 19 | model_name = "gpt-3.5-turbo" 20 | ``` 21 | 22 | ## Build LLM Config 23 | 24 | - By passing the class `openai` 25 | 26 | ``` 27 | from muagent.llm_models.llm_config import EmbedConfig, LLMConfig 28 | 29 | llm_config = LLMConfig( 30 | model_name=model_name, api_key=api_key, api_base_url=api_base_url, temperature=0.3, 31 | stop="**Observation:**" 32 | ) 33 | ``` 34 | 35 | - Customizing and inputting langchain LLM 36 | 37 | ``` 38 | from muagent.llm_models.llm_config import EmbedConfig, LLMConfig 39 | from langchain.llms.base import BaseLLM, LLM 40 | 41 | 42 | class CustomizedModel(LLM): 43 | repetition_penalty = 1.1 44 | temperature = 0.2 45 | top_k = 40 46 | top_p = 0.9 47 | 48 | def predict(self, prompt: str, stop: Optional[List[str]] = None) -> str: 49 | return self._call(prompt, stop) 50 | 51 | def _call(self, prompt: str, 52 | stop: Optional[List[str]] = None) -> str: 53 | """_call""" 54 | return "" 55 | 56 | 57 | llm = CustomizedModel() 58 | llm_config = LLMConfig( 59 | llm=llm 60 | ) 61 | ``` 62 | -------------------------------------------------------------------------------- /docs/docs/api-docs/MuAgent/overview/agent-flow.zh-CN.md: -------------------------------------------------------------------------------- 1 | --- 2 | group: 3 | title: ❤️ Codefuse-muAgent 4 | order: -1 5 | title: Agent 编排 6 | order: 0 7 | toc: content 8 | --- 9 | 10 | ## 核心 Connector 介绍 11 | 12 | 为了便于大家理解整个 CoAgent 的链路，我们采取 Flow 的形式来详细介绍如何通过配置构建 13 | 14 |
15 | 图片 16 |
17 | 18 |
下面，我们先介绍相关的核心组件
19 | 20 | ### Agent 21 | 22 | 在 Agent 设计层面，我们提供了四种基本的 Agent 类型，对这些 Agent 进行 Role 的基础设定，可满足多种通用场景的交互和使用 23 | 24 | 1. BaseAgent：提供基础问答、工具使用、代码执行的功能，根据 Prompt 格式实现 输入 => 输出 25 | 26 | 2. ReactAgent：提供标准 React 的功能，根据问题实现当前任务 27 | 28 | 3. ExecutorAgent：对任务清单进行顺序执行，根据 User 或 上一个 Agent 编排的计划，完成相关任务 29 | 30 | 4. SelectorAgent：提供选择 Agent 的功能，根据 User 或 上一个 Agent 的问题选择合适的 Agent 来进行回答. 31 | 32 | 输出后将 message push 到 memory pool 之中，后续通过 Memory Manager 进行管理 33 | 34 | ### Chain 35 | 36 | 基础链路：BaseChain，串联 agent 的交互，完成相关 message 和 memory 的管理 37 | 38 | ### Phase 39 | 40 | 基础场景：BasePhase，串联 chain 的交互，完成相关 message 和 memory 的管理 41 | 42 | ### Prompt Manager 43 | 44 | Mutli-Agent 链路中每一个 agent 的 prompt 创建 45 | 46 | - 通过对 promtp_input_keys 和 promtp_output_keys 对的简单设定，可以沿用预设 Prompt Context 创建逻辑，从而实现 agent prompt 快速配置 47 | - 也可以对 prompt manager 模块进行新的 key-context 设计，实现个性化的 Agent Prompt 48 | 49 | ### Memory Manager 50 | 51 | 主要用于 chat history 的管理 52 | 53 | - 将 chat history 在数据库进行读写管理，包括 user input、 llm output、doc retrieval、code retrieval、search retrieval 54 | - 对 chat history 进行关键信息总结 summary context，作为 prompt context 55 | - 提供检索功能，检索 chat history 或者 summary context 中与问题相关信息，辅助问答 56 | -------------------------------------------------------------------------------- /docs/contribution/contribution.zh-CN.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Community 3 | nav: 4 | title: 社区 5 | order: 1 6 | group: 7 | title: ❤️ 贡献指南 8 | index: true 9 | order: -1 10 | toc: content 11 | --- 12 | 13 | 非常感谢您对 Codefuse 项目感兴趣，我们非常欢迎您对 Codefuse 项目的各种建议、意见（包括批评）、评论和贡献。 14 | 15 | 您对 Codefuse 的各种建议、意见、评论可以直接通过 GitHub 的 Issues 提出。 16 | 17 | 参与 Codefuse 项目并为其作出贡献的方法有很多：代码实现、测试编写、流程工具改进、文档完善等等。任何贡献我们都会非常欢迎，并将您加入贡献者列表. 18 | 19 | 进一步，有了足够的贡献后，您还可以有机会成为 Codefuse 的 Committer。 20 | 21 | 任何问题，您都可以联系我们得到及时解答，联系方式包括微信、Gitter（GitHub 提供的即时聊天工具）、邮件等等。 22 | 23 | ## 初次接触 24 | 25 | 初次来到 Codefuse 社区，您可以： 26 | 27 | - 关注 Codefuse Github 代码库 28 | - 加入 Codefuse 相关的微信群 随时提问； 29 | 通过以上方式及时了解 Codefuse 项目的开发动态并为您关注的话题发表意见。 30 | 31 | ## 贡献方式 32 | 33 | 这份贡献指南并不仅仅关于编写代码。我们重视并感激在各个领域的帮助。以下是一些您可以贡献的方式 34 | 35 | - 文档 36 | - Issue 37 | - PR 38 | 39 | ### 改进文档 40 | 41 | 文档是您了解 Codefuse 的最主要的方式，也是我们最需要帮助的地方！ 42 | 43 | 浏览文档，可以加深您对 Codefuse 的了解，也可以帮助您理解 Codefuse 的功能和技术细节，如果您发现文档有问题，请及时联系我们； 44 | 45 | 如果您对改进文档的质量感兴趣，不论是修订一个页面的地址、更正一个链接、以及写一篇更优秀的入门文档，我们都非常欢迎！ 46 | 47 | 我们的文档大多数是使用 markdown 格式编写的，您可以直接通过在 GitHub 中的 docs/ 中修改并提交文档变更。如果提交代码变更，可以参阅 Pull Request。 48 | 49 | ### 如果发现了一个 Bug 或问题 50 | 51 | 如果发现了一个 Bug 或问题，您可以直接通过 GitHub 的 Issues 提一个新的 Issue，我们会有人定期处理。详情见[Issue 模板](/contribution/issue) 52 | 53 | 您也可以通过阅读分析代码自己修复（当然在这之前最好能和我们交流下，或许已经有人在修复同样的问题了），然后提交一个 Pull Request。 54 | 55 | ### 修改代码和提交 PR（Pull Request） 56 | 57 | 您可以下载代码，编译安装，部署运行试一试（可以参考编译文档，看看是否与您预想的一样工作。如果有问题，您可以直接联系我们，提 Issue 或者通过阅读和分析源代码自己修复。详情见[如何提交 pr](/contribution/pr) 58 | 59 | 无论是修复 Bug 还是增加 Feature，我们都非常欢迎。如果您希望给 Doris 提交代码，您需要从 GitHub 上 fork 代码库至您的项目空间下，为您提交的代码创建一个新的分支，添加源项目为 upstream，并提交 PR。 提交 PR 的方式可以参考文档 Pull Request。 60 | -------------------------------------------------------------------------------- /docs/contribution/issue.zh-CN.md: -------------------------------------------------------------------------------- 1 | --- 2 | group: 3 | title: ❤️ 贡献指南 4 | order: 0 5 | title: 如何提交Issue 6 | order: 0 7 | toc: content 8 | --- 9 | 10 | ## Issue Type 11 | 12 | Issue 分为三种类型 13 | 14 | - Bug: 代码或者执行示例存在 bug 或缺少依赖导致无法正确执行 15 | - Documentation：文档表述存在争议、文档内容与代码不一致等 16 | - Feature：在当前代码基础继续演进的新功能 17 | 18 | ## Issue Template 19 | 20 | ### Issue: Bug Template 21 | 22 | **提交 Issue 前的确认清单** 23 |
要先确认是否查看 document、issue、discussion(github 功能) 等公开的文档信息 24 | 25 | - 我搜索了 Codefuse 相关的所有文档。 26 | - 我使用 GitHub 搜索寻找了一个类似的问题，但没有找到。 27 | - 我为这个问题添加了一个非常描述性的标题。 28 | 29 | **系统信息** 30 |
确认系统，如 mac -xx 、windwos-xx、linux-xx 31 | 32 | **代码版本** 33 |
确认代码版本或者分支，master、release 等 34 | 35 | **问题描述** 36 |
描述您碰到的问题，想要实现的事情、或代码执行 Bug 37 | 38 | **代码示例** 39 |
附上你的执行代码和相关配置，以便能够快速介入进行复现 40 | 41 | **报错信息、日志** 42 |
执行上述代码示例后的报错日志和相关信息 43 | 44 | **相关依赖的模块** 45 |
以 chatbot 项目为例 46 | 47 | - connector 48 | - codechat 49 | - sandbox 50 | - ... 51 | 52 | ### Issue: Documentation Template 53 | 54 | **Issue with current documentation:** 55 |
请帮忙指出当前文档中的问题、错别字或者令人困惑的地方 56 | 57 | **Idea or request for content** 58 |
您觉得合理的文档表述方式应该是什么样的 59 | 60 | ### Issue: Feature Template 61 | 62 | **提交 Issue 前的确认清单** 63 |
要先确认是否查看 document、issue、discussion(github 功能) 等公开的文档信息 64 | 65 | - 我搜索了 Codefuse 相关的所有文档。 66 | - 我使用 GitHub Issue 搜索寻找了一个类似的问题，但没有找到。 67 | - 我为这个问题添加了一个非常描述性的标题。 68 | 69 | **功能描述** 70 |
描述这个功能作何用途 71 | 72 | **相关示例** 73 |
提供参考的文档、仓库等信息，Please provide links to any relevant GitHub repos, papers, or other resources if relevant. 74 | 75 | **动机** 76 |
描述下这个 feature 的动机，为什么需要这个功能，提供足够的上下文信息帮助理解这个 feature 的诉求 77 | 78 | **Contribution** 79 |
你如何参与到这个 feature 的构建（如果参与的话） 80 | -------------------------------------------------------------------------------- /docs/contribution/pr.zh-CN.md: -------------------------------------------------------------------------------- 1 | --- 2 | group: 3 | title: ❤️ 贡献指南 4 | order: -1 5 | title: 如何提交PR 6 | order: 1 7 | toc: content 8 | --- 9 | 10 | ## Contribution 11 | 12 | ### Pre-Checklist 13 | 14 | - 要先确认是否查看 document、issue、discussion(github 功能) 等公开的文档信息 15 | - 找到你想处理的 GitHub 问题。如果不存在，创建一个问题或草案 PR，并请求维护者进行检查。 16 | - 检查相关的、相似的或重复的拉取请求。 17 | - 创建一个草案拉取请求。 18 | - 完成 PR 模板中的描述。 19 | - 链接任何被你的 PR 解决的 GitHub 问题。 20 | 21 | ### Description 22 | 23 | PR 的描述信息，用简洁的语言表达 PR 完成的事情，具体规范见[Commit 格式规范](#commit-格式规范) 24 | 25 | ### Related Issue 26 | 27 | `#xx` if has 28 | 29 | ### Test Code with Result 30 | 31 | 请提供相关的测试代码如果有必要的话 32 | 33 | ## Commit 格式规范 34 | 35 | Commit 分为“标题”和“内容”。原则上标题全部小写。内容首字母大写。 36 | 37 | ### 标题 38 | 39 | commit message 的标题：`[]() (#pr)` 40 | 41 | ### type 可选值 42 | 43 | 本次提交的类型，限定在以下类型（全小写） 44 | 45 | - fix：bug 修复 46 | - feature：新增功能 47 | - feature-wip：开发中的功能，比如某功能的部分代码。 48 | - improvement：原有功能的优化和改进 49 | - style：代码风格调整 50 | - typo：代码或文档勘误 51 | - refactor：代码重构（不涉及功能变动） 52 | - performance/optimize：性能优化 53 | - test：单元测试的添加或修复 54 | - deps：第三方依赖库的修改 55 | - community：社区相关的修改，如修改 Github Issue 模板等。 56 | 57 | 几点说明： 58 | 59 | 如在一次提交中出现多种类型，需增加多个类型。 60 | 如代码重构带来了性能提升，可以同时添加 [refactor][optimize] 61 | 不得出现如上所列类型之外的其他类型。如有必要，需要将新增类型添加到这个文档中。 62 | 63 | ### scope 可选值 64 | 65 | 本次提交涉及的模块范围。因为功能模块繁多，在此仅罗列部分，后续根据需求不断完善。 66 |
以 chatbot 的框架为例 67 | 68 | - connector 69 | - codechat 70 | - sandbox 71 | - ... 72 | 73 | 几点说明： 74 | 75 | 尽量使用列表中已存在的选项。如需添加，请及时更新本文档。 76 | 77 | ### subject 内容 78 | 79 | 标题需尽量清晰表明本次提交的主要内容。 80 | 81 | ## 示例 82 | 83 | comming soon 84 | 85 | ## Reference 86 | 87 | [doris-commit-format](https://doris.apache.org/zh-CN/community/how-to-contribute/commit-format-specification) 88 | -------------------------------------------------------------------------------- /static/slick.d4bc62a2.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/docs/developer-docs/MFTCoder/main/introduction.zh-CN.md: -------------------------------------------------------------------------------- 1 | --- 2 | store: 3 | title: MFTCoder 4 | version: main 5 | group: 6 | title: 🌱 MFTCoder 7 | order: -1 8 | title: 基本介绍 9 | order: 0 10 | toc: content 11 | github: https://github.com/codefuse-ai/MFTCoder 12 | --- 13 | 14 | ## 项目简介 15 | 16 | **国际首个高精度、高效率、多任务、多模型支持、多训练算法，大模型代码能力微调框架；** 17 | 18 | **Codefuse-MFTCoder** 是一个开源的多任务代码大语言模型项目，包含代码大模型的模型、数据、训练等。我们希望通过开源，分享交流大语言模型在代码领域的进步。 19 | 20 | ### 项目框架 21 | 22 | ![img_1.jpg](https://mdn.alipayobjects.com/huamei_bvbxju/afts/img/A*zc9pRJ-hdZMAAAAAAAAAAAAADlHYAQ/original) 23 | 24 | ### 项目优势 25 | 26 | :white_check_mark: **多任务**：一个模型同时支持多个任务，会保证多个任务之间的平衡，甚至可以泛化到新的没有见过的任务上去； 27 | 28 | :white_check_mark: **多模型**：支持最新的多个开源模型，包括 gpt-neox，llama，llama-2，baichuan，Qwen，chatglm2 等； 29 | 30 | :white_check_mark: **多框架**：既支持主流开源的 Accelerate+DeepSpeed/FSDP，也支持新开源的[ATorch 框架](https://github.com/intelligent-machine-learning/dlrover)； 31 | 32 | :white_check_mark: **高效微调**：支持 LoRA 和 QLoRA，可以用很少的资源去微调很大的模型，且训练速度能满足几乎所有微调场景； 33 | 34 | 本项目主要内容如下： 35 | 36 | - 同时支持单任务 SFT(Supervised FineTuning)和 MFT(Multi-task FineTuning), 当前开源支持数据均衡，未来将持续开源难易均衡， 收敛均衡等 37 | - 支持 QLoRA 低成本高效指令微调、LoRA 高效指令微调、全量参数高精度微调。 38 | - 支持绝大部分主流的开源大模型，重点关注代码能力优秀的开源大模型，如 DeepSeek-coder, Mistral, Mistral(MoE), Chatglm3, Qwen, GPT-Neox, Starcoder, Codegeex2, Code-LLaMA 等。 39 | - 支持 lora 与 base model 进行权重合并，推理更便捷。 40 | - 整理并开源 2 个指令微调数据集：[Evol-instruction-66k](https://huggingface.co/datasets/codefuse-ai/Evol-instruction-66k)和[CodeExercise-Python-27k](https://huggingface.co/datasets/codefuse-ai/CodeExercise-Python-27k)。 41 | - 开源多个[Codefuse 系列指令微调模型权重]，具体参见我们的 huggingface 组织和 modelscope 组织下的模型：[codefuse-ai huggingface](https://huggingface.co/codefuse-ai) or [codefuse-ai 魔搭](https://modelscope.cn/organization/codefuse-ai)。 42 | -------------------------------------------------------------------------------- /docs/docs/api-docs/MuAgent/llm_models/embedding_config.zh-CN.md: -------------------------------------------------------------------------------- 1 | --- 2 | group: 3 | title: llm_models 4 | order: 1 5 | title: Embedding 配置 6 | order: 0 7 | toc: content 8 | --- 9 | 10 | ## 准备相关参数 11 | 12 | 首先增加 openai 配置，也可以是其它类似于 openai 接口的模型（通过 fastchat 启动） 13 | 14 | ``` 15 | import os, sys 16 | 17 | api_key = "sk-xxx" 18 | api_base_url= "https://api.openai.com/v1" 19 | embed_model = "{{embed_model_name}}" 20 | embed_model_path = "{{embed_model_path}}" 21 | ``` 22 | 23 | ## 构建 LLM Config 24 | 25 | - 通过本地模型文件构建 26 | 27 | ``` 28 | from muagent.llm_models.llm_config import EmbedConfig, LLMConfig 29 | 30 | embed_config = EmbedConfig( 31 | embed_engine="model", embed_model=embed_model, embed_model_path=embed_model_path 32 | ) 33 | ``` 34 | 35 | - 通过 openai 构建 36 | 37 | ``` 38 | from muagent.llm_models.llm_config import EmbedConfig, LLMConfig 39 | 40 | embed_config = EmbedConfig( 41 | embed_engine="openai", api_key=api_key, api_base_url=api_base_url, 42 | ) 43 | ``` 44 | 45 | - 自定义 langchain embeddings 传入 46 | 47 | ``` 48 | from muagent.llm_models.llm_config import EmbedConfig, LLMConfig 49 | 50 | 51 | class CustomizedEmbeddings(Embeddings): 52 | 53 | def embed_documents(self, texts: List[str]) -> List[List[float]]: 54 | embeddings = [] 55 | # add your embedding code 56 | return embeddings 57 | 58 | def embed_query(self, text: str) -> List[float]: 59 | """Compute query embeddings using a HuggingFace transformer model. 60 | 61 | Args: 62 | text: The text to embed. 63 | 64 | Returns: 65 | Embeddings for the text. 66 | """ 67 | # add your embedding code 68 | return embedding 69 | 70 | embeddings = CustomizedEmbeddings() 71 | embed_config = EmbedConfig( 72 | embed_model="default", 73 | langchain_embeddings=embeddings 74 | ) 75 | ``` 76 | -------------------------------------------------------------------------------- /.dumi/theme/slots/SearchBar/input.tsx: -------------------------------------------------------------------------------- 1 | import { useIntl } from 'dumi'; 2 | import React, { forwardRef, useImperativeHandle, useRef } from 'react'; 3 | 4 | type NativeInputProps = React.DetailedHTMLProps< 5 | React.InputHTMLAttributes, 6 | HTMLInputElement 7 | >; 8 | 9 | type InputProps = { 10 | onChange: (keywords: string) => void; 11 | } & Pick; 12 | 13 | export const Input = forwardRef((props, ref) => { 14 | const intl = useIntl(); 15 | 16 | const imeWaiting = useRef(false); 17 | const nativeInputRef = useRef(null); 18 | 19 | useImperativeHandle(ref, () => nativeInputRef.current!); 20 | 21 | return ( 22 | (imeWaiting.current = true)} 25 | onCompositionEnd={(ev) => { 26 | imeWaiting.current = false; 27 | // special case: press Enter open IME panel will not trigger onChange 28 | props.onChange(ev.currentTarget.value); 29 | }} 30 | onFocus={props.onFocus} 31 | onBlur={props.onBlur} 32 | onMouseEnter={props.onMouseEnter} 33 | onKeyDown={(ev) => { 34 | if (['ArrowDown', 'ArrowUp'].includes(ev.key)) ev.preventDefault(); 35 | // esc to blur input 36 | if (ev.key === 'Escape' && !imeWaiting.current) ev.currentTarget.blur(); 37 | }} 38 | onChange={(ev) => { 39 | // wait for onCompositionEnd event be triggered 40 | const value = ev.target.value; 41 | setTimeout(() => { 42 | if (!imeWaiting.current) { 43 | props.onChange(value); 44 | } 45 | }, 1); 46 | }} 47 | placeholder={intl.formatMessage({ id: 'header.search.placeholder' })} 48 | ref={nativeInputRef} 49 | /> 50 | ); 51 | }); 52 | -------------------------------------------------------------------------------- /.dumi/theme/slots/Contribution/index.less: -------------------------------------------------------------------------------- 1 | @import (reference) '.dumi/theme/styles/variables.less'; 2 | 3 | .@{prefix}-Contribution { 4 | margin: -@s-header-height auto 0 auto; 5 | height: 100%; 6 | display: flex; 7 | flex-direction: column; 8 | justify-content: center; 9 | 10 | @media @mobile { 11 | margin-top: -@s-header-height-m - 20; 12 | padding-top: 160px; 13 | height: 660px; 14 | } 15 | 16 | +* { 17 | position: relative; 18 | } 19 | 20 | .banner { 21 | position: relative; 22 | padding-top: 30px; 23 | height: 380px; 24 | width: 100%; 25 | box-sizing: border-box; 26 | // background: linear-gradient(-225deg, #7085B6 0%, #87A7D9 50%, #DEF3F8 100%); 27 | background-image: url('https://mdn.alipayobjects.com/huamei_bvbxju/afts/img/A*EamVR52XxpUAAAAAAAAAAAAADlHYAQ/original'); 28 | background-repeat: no-repeat; 29 | background-size: cover; 30 | background-position: center; 31 | display: flex; 32 | align-items: center; 33 | justify-content: center; 34 | flex-direction: column; 35 | 36 | .bannerContent { 37 | width: 1200px; 38 | padding: 0px 24px; 39 | margin-top: 30px; 40 | width: 900px; 41 | color: #f8f9fa; 42 | font-size: 2.7rem; 43 | display: flex; 44 | justify-content: center; 45 | 46 | 47 | img { 48 | width: 35%; 49 | } 50 | } 51 | } 52 | 53 | .content { 54 | width: 1000px; 55 | margin: 0 auto; 56 | color: #e4e9ec; 57 | line-height: 30px; 58 | padding: 24px; 59 | 60 | .contentTitle { 61 | margin: 40px 0 32px; 62 | text-align: left; 63 | font-weight: 400; 64 | font-size: 32px; 65 | } 66 | 67 | .contentText { 68 | font-size: 16px; 69 | letter-spacing: 0.61px; 70 | padding-left: 20px; 71 | line-height: 30px; 72 | 73 | li { 74 | margin-top: 5px; 75 | } 76 | } 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /docs/docs/developer-docs/CodeFuse-ChatBot/master/quickstart.en-US.md: -------------------------------------------------------------------------------- 1 | --- 2 | nav: 3 | title: Docs 4 | order: -1 5 | second: 6 | title: Developer-Docs 7 | order: -1 8 | store: 9 | title: CodeFuse-ChatBot 10 | version: master 11 | group: 12 | title: 🌱 CodeFuse-ChatBot 13 | order: -1 14 | title: QuickStart 15 | order: -1 16 | toc: content 17 | github: https://github.com/codefuse-ai/codefuse-chatbot 18 | --- 19 | 20 | ## 🚀 Quick Start 21 | 22 | To deploy private models, please install the NVIDIA driver by yourself. 23 | This project has been tested on Python 3.9.18 and CUDA 11.7 environments, as well as on Windows and macOS systems with x86 architecture. 24 | For Docker installation, private LLM access, and related startup issues, see: [Start-detail...](/docs/developer-docs/CodeFuse-ChatBot/master/start-detail) 25 | 26 | 1. Preparation of Python environment 27 | 28 | - It is recommended to use conda to manage the python environment (optional) 29 | 30 | ```bash 31 | # Prepare conda environment 32 | conda create --name Codefusegpt python=3.9 33 | conda activate Codefusegpt 34 | ``` 35 | 36 | - Install related dependencies 37 | 38 | ```bash 39 | cd Codefuse-ChatBot 40 | # python=3.9，use notebook-latest，python=3.8 use notebook==6.5.5 41 | pip install -r requirements.txt 42 | ``` 43 | 44 | 2. Start the Service 45 | 46 | ```bash 47 | # After configuring server_config.py, you can start with just one click. 48 | cd examples 49 | bash start.sh 50 | # you can config your llm model and embedding model, then choose the "启动对话服务" 51 | ``` 52 | 53 |
54 | 图片 55 |
56 | 57 | Or `python start.py` by [old version to start](/docs/developer-docs/CodeFuse-ChatBot/master/start-detail) 58 | More details about accessing LLM Moldes[More Details...](/docs/developer-docs/CodeFuse-ChatBot/master/fastchat) 59 |
60 | -------------------------------------------------------------------------------- /docs/docs/developer-docs/CodeFuse-DevOps-Model/main/traindetail.zh-CN.md: -------------------------------------------------------------------------------- 1 | --- 2 | store: 3 | title: CodeFuse-DevOps-Model 4 | version: main 5 | group: 6 | title: 🌱 CodeFuse-DevOps-Model 7 | order: -1 8 | title: 训练解析 9 | order: 0 10 | toc: content 11 | github: https://github.com/codefuse-ai/CodeFuse-DevOps-Model 12 | --- 13 | 14 | ## 训练流程 15 | 16 | 根据查阅文献可知，大部分领域模型都是在对话模型的基础上，通过 SFT 微调来进行知识注入。而 SFT 微调所需要 QA 预料基本都来自于 ChatGPT 生成。然而，该方案可能存在 QA 语料无法完全覆盖领域知识的情况。 17 | 因此，DevOps-Model 采用的是预训练加训 + SFT 微调的方案，如图 2.1 所示。我们认为针对领域大模型，预训练的加训是必要的，因为其可以将领域内的一些知识在预训练阶段注入到大模型，如果这些知识在通用大模型预训练时没有出现过，那会让大模型学习到新的知识；如果出现过，就可以让大模型进一步加深印象。第二步则是大模型对齐，目的是让大模型可以根据问题来回答最合适的内容。 18 | 19 | ![](https://mdn.alipayobjects.com/huamei_bvbxju/afts/img/A*66DWSbAXqRAAAAAAAAAAAAAADlHYAQ/original) 20 | 21 | ## 训练数据 22 | 23 | ### 数据收集 24 | 25 | 模型的定位是中文 DevOps 领域大模型，因此收集与中文 DevOps 相关的预训练数据和 QA 数据。 26 | 27 | - 预训练数据主要来自互联网技术博客、技术文档、技术书籍等，最终收集到了 50G+ 的预训练语料数据； 28 | - 针对 QA 数据，我们的目的是想让模型不但对齐到通用的问答能力，而且针对 DevOps 领域也可以学会如何更好的回答问题，因此不但收集了通用领域的单轮和多轮对话数据，还针对 DevOps 领域，通过爬取和 ChatGPT 生成的方式产出了属于 DevOps 领域的问答数据。最终我们精心筛选了约 200K 的 QA 数据进行 SFT 微调训练，具体数据量如下表所示。 29 | 30 | | 数据类型 | 数据量级 | 31 | | -------------- | -------- | 32 | | 通用单轮 QA | 50K | 33 | | 通用多轮 QA | 20K | 34 | | DevOps 领域 QA | 130K | 35 | 36 | ### 数据筛选 37 | 38 | ![](https://mdn.alipayobjects.com/huamei_bvbxju/afts/img/A*jKlFTp3GWg8AAAAAAAAAAAAADlHYAQ/original) 39 | 40 | 由于预训练数据大部分是从互联网上收集的数据，质量会参差不齐，而大模型训练中数据是最重要的一环，我们建立了如上图所示的清洗 Pipeline，来针对收集到的数据进行质量的全面过滤。 41 | 42 | 1. 首先，由专家经验和人工筛选，总结出来了一批文档级别的 Heuristic 过滤规则，这一步主要用来过滤掉那些质量非常差的文档； 43 | 2. 然后，即便是一篇质量稍差的文章中，也有可能还是含有一些有价值的领域知识，我们也需要尽可能的进行收集。此处，我们对文章进行段落拆分，将文章拆分成一个个段落； 44 | 3. 然后，我们将拆分后的段落会再次通过步骤 1 进行过滤，便得到了一批经过规则过滤后的段落； 45 | 4. 然后，我们摘取了其中 1000 个段落，由经验丰富的专业开发人员来进行打标，获得高质量的打标数据； 46 | 5. 最后，我们根据打标后的结果来训练了一个打分模型来针对段落进行质量的打分，段落的向量模型选用了预训练好的中文版本的 Sentence-Bert，打分算法选用了逻辑回归，为了避免打分模型的误差，会再通过帕累托分布来根据段落的质量打分进行采样来决定要不要过滤这个段落。 47 | 经过这个 Pipeline 后，我们最终沉淀下 15G 左右的数据来进行大模型的预训练加训。 48 | -------------------------------------------------------------------------------- /.dumi/theme/locales/zh-CN.json: -------------------------------------------------------------------------------- 1 | { 2 | "header.search.placeholder": "输入关键字搜索...", 3 | "header.color.mode.light": "亮色模式", 4 | "header.color.mode.dark": "暗色模式", 5 | "header.color.mode.auto": "跟随系统", 6 | "header.social.github": "GitHub", 7 | "header.social.weibo": "微博", 8 | "header.social.twitter": "Twitter", 9 | "header.social.gitlab": "GitLab", 10 | "header.social.facebook": "Facebook", 11 | "header.social.zhihu": "知乎", 12 | "header.social.yuque": "语雀", 13 | "header.social.linkedin": "Linkedin", 14 | "previewer.actions.code.expand": "展开代码", 15 | "previewer.actions.code.shrink": "收起代码", 16 | "previewer.actions.codesandbox": "在 CodeSandbox 中打开", 17 | "previewer.actions.sketch": "拷贝到 Sketch", 18 | "previewer.actions.sketch.group": "拷贝为 Sketch Group", 19 | "previewer.actions.sketch.symbol": "拷贝为 Sketch Symbol", 20 | "previewer.actions.sketch.divider": "----------------------", 21 | "previewer.actions.sketch.guide": "如何粘贴到 SKetch？", 22 | "previewer.actions.stackblitz": "在 StackBlitz 中打开", 23 | "previewer.actions.separate": "在独立页面中打开", 24 | "404.title": "页面未找到", 25 | "404.back": "返回首页", 26 | "api.component.name": "属性名", 27 | "api.component.description": "描述", 28 | "api.component.type": "类型", 29 | "api.component.default": "默认值", 30 | "api.component.required": "(必选)", 31 | "api.component.unavailable": "必须启用 apiParser 才能使用自动 API 特性", 32 | "api.component.loading": "属性定义正在解析中，稍等片刻...", 33 | "api.component.not.found": "未找到 {id} 组件的属性定义", 34 | "content.tabs.default": "文档", 35 | "content.footer.last.updated": "最后更新时间：", 36 | "content.footer.actions.edit": "帮助改进此文档", 37 | "content.footer.actions.previous": "上一篇", 38 | "content.footer.actions.next": "下一篇", 39 | "search.not.found": "未找到相关内容", 40 | "layout.sidebar.btn": "侧边菜单", 41 | "content.menus.docs.developer": "开发者文档", 42 | "content.menus.docs.api": "API文档", 43 | "content.menus.docs.product": "产品文档", 44 | "content.menus.docs.about": "关于 CodeFuse" 45 | } 46 | -------------------------------------------------------------------------------- /docs/docs/api-docs/MuAgent/llm_models/embedding_config.en-US.md: -------------------------------------------------------------------------------- 1 | --- 2 | group: 3 | title: llm_models 4 | order: 1 5 | title: Embedding Config 6 | order: 0 7 | toc: content 8 | --- 9 | 10 | ## Prepare Relevant Parameters 11 | 12 | First, add the OpenAI configuration; this could also be a model similar to the OpenAI interface (launched via fastchat). 13 | 14 | ``` 15 | import os, sys 16 | 17 | api_key = "sk-xxx" 18 | api_base_url= "https://api.openai.com/v1" 19 | embed_model = "{{embed_model_name}}" 20 | embed_model_path = "{{embed_model_path}}" 21 | ``` 22 | 23 | ## Build LLM Config 24 | 25 | - Constructing with a local model file 26 | 27 | ``` 28 | from muagent.llm_models.llm_config import EmbedConfig, LLMConfig 29 | 30 | embed_config = EmbedConfig( 31 | embed_engine="model", embed_model=embed_model, embed_model_path=embed_model_path 32 | ) 33 | ``` 34 | 35 | - Constructing via OpenAI 36 | 37 | ``` 38 | from muagent.llm_models.llm_config import EmbedConfig, LLMConfig 39 | 40 | embed_config = EmbedConfig( 41 | embed_engine="openai", api_key=api_key, api_base_url=api_base_url, 42 | ) 43 | ``` 44 | 45 | - Customizing and inputting langchain embeddings 46 | 47 | ``` 48 | from muagent.llm_models.llm_config import EmbedConfig, LLMConfig 49 | 50 | class CustomizedEmbeddings(Embeddings): 51 | def embed_documents(self, texts: List[str]) -> List[List[float]]: 52 | embeddings = [] 53 | # add your embedding code 54 | return embeddings 55 | def embed_query(self, text: str) -> List[float]: 56 | """Compute query embeddings using a HuggingFace transformer model. 57 | Args: 58 | text: The text to embed. 59 | Returns: 60 | Embeddings for the text. 61 | """ 62 | # add your embedding code 63 | return embedding 64 | 65 | 66 | embeddings = CustomizedEmbeddings() 67 | embed_config = EmbedConfig( 68 | embed_model="default", 69 | langchain_embeddings=embeddings 70 | ) 71 | ``` 72 | -------------------------------------------------------------------------------- /.dumi/theme/constants/index.ts: -------------------------------------------------------------------------------- 1 | /** 一级导航入口 */ 2 | export const NavbarEnums = { 3 | 'CodeFuse-Query': '/zh-CN/docs/developer-docs/CodeFuse-Query/main/CodeFuseQuery', 4 | 'MFTCoder': '/zh-CN/docs/developer-docs/MFTCoder/main/MFTCoder', 5 | 'CodeFuse-MFT-VLM': '/zh-CN/docs/developer-docs/CodeFuse-MFT-VLM/main/mftvlm', 6 | 'Test-Agent': '/zh-CN/docs/developer-docs/Test-Agent/main/TestAgent', 7 | 'CodeFuse-ModelCache': '/zh-CN/docs/developer-docs/CodeFuse-ModelCache/main/CodeFuseModelCache', 8 | 'CodeFuse-ChatBot': '/zh-CN/docs/developer-docs/CodeFuse-ChatBot/master/quickstart', 9 | 'CodeFuse-DevOps-Eval': '/zh-CN/docs/developer-docs/CodeFuse-DevOps-Eval/master/data', 10 | 'CodeFuse-DevOps-Model': '/zh-CN/docs/developer-docs/CodeFuse-DevOps-Model/main/codefuseDevopsModel', 11 | 'CodeFuse-evalution': '/zh-CN/docs/developer-docs/CodeFuse-evalution/main/codefuse-evalution', 12 | 'MuAgent': '/zh-CN/docs/api-docs/MuAgent/overview/multi-agent', 13 | '整体介绍': '/zh-CN/docs/about/overview', 14 | '关于':'/zh-CN/aboutDocs/aboutdocs' 15 | } 16 | 17 | /** 一级导航 英文 */ 18 | export const NavbarEnumsEn = { 19 | 'CodeFuse-Query': '/docs/developer-docs/CodeFuse-Query/main/CodeFuseQuery', 20 | 'MFTCoder': '/docs/developer-docs/MFTCoder/main/MFTCoder', 21 | 'CodeFuse-MFT-VLM': '/docs/developer-docs/CodeFuse-MFT-VLM/main/mftvlm', 22 | 'Test-Agent': '/docs/developer-docs/Test-Agent/main/TestAgent', 23 | 'CodeFuse-ModelCache': '/docs/developer-docs/CodeFuse-ModelCache/main/CodeFuseModelCache', 24 | 'CodeFuse-ChatBot': '/docs/developer-docs/CodeFuse-ChatBot/master/quickstart', 25 | 'CodeFuse-DevOps-Eval': '/docs/developer-docs/CodeFuse-DevOps-Eval/master/data', 26 | 'CodeFuse-DevOps-Model': '/docs/developer-docs/CodeFuse-DevOps-Model/main/codefuseDevopsModel', 27 | 'CodeFuse-evalution': '/docs/developer-docs/CodeFuse-evalution/main/codefuse-evalution', 28 | 'MuAgent': '/docs/api-docs/MuAgent/overview/multi-agent', 29 | 'Overview': '/docs/about/overview', 30 | 'AboutDocs':'/aboutDocs/aboutdocs' 31 | } 32 | 33 | -------------------------------------------------------------------------------- /.dumi/theme/slots/IntelligentInference/index.less: -------------------------------------------------------------------------------- 1 | .codeAnalysis { 2 | margin-top: 180px; 3 | display: flex; 4 | justify-content: center; 5 | 6 | .IntelligentInference-center { 7 | display: flex; 8 | justify-content: space-between; 9 | flex-direction: row; 10 | width: 1200px; 11 | 12 | img { 13 | height: 628px; 14 | height: 507px; 15 | border-radius: 25px; 16 | } 17 | 18 | .generationContent { 19 | margin-right: 57px; 20 | padding-top: 21px; 21 | 22 | .desc { 23 | width: 542px; 24 | margin-top: 48px; 25 | font-size: 16px; 26 | color: #ffffff; 27 | letter-spacing: 1px; 28 | opacity: 0.8; 29 | 30 | @{dark-selector} & { 31 | font-size: 16px; 32 | color: #171616; 33 | } 34 | } 35 | 36 | .buttom { 37 | cursor: pointer; 38 | width: 150px; 39 | height: 37px; 40 | background-image: url("https://mdn.alipayobjects.com/huamei_bvbxju/afts/img/A*eMQHT670srQAAAAAAAAAAAAADlHYAQ/original"); 41 | background-repeat: no-repeat; 42 | background-size: contain; 43 | background-position: center; 44 | border-radius: 7px; 45 | margin-top: 32px; 46 | color: #a8abff; 47 | display: flex; 48 | align-items: center; 49 | justify-content: center; 50 | position: relative; 51 | 52 | .anticon { 53 | svg { 54 | width: 30px; 55 | height: 23px; 56 | position: absolute; 57 | bottom: 5px; 58 | } 59 | 60 | margin-left: 8px; 61 | width: 25px; 62 | height: 25px; 63 | } 64 | 65 | &:hover { 66 | background-image: url("https://mdn.alipayobjects.com/huamei_bvbxju/afts/img/A*Pm85SKQpQ88AAAAAAAAAAAAADlHYAQ/original"); 67 | background-repeat: no-repeat; 68 | background-size: contain; 69 | background-position: center; 70 | color: #fff; 71 | 72 | @{dark-selector} & { 73 | color: #7031FD; 74 | } 75 | } 76 | } 77 | } 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /.dumi/theme/slots/Publication/index.tsx: -------------------------------------------------------------------------------- 1 | import { useRouteMeta } from 'dumi'; 2 | import React, { type FC } from 'react'; 3 | import './index.less'; 4 | 5 | const Publication: FC = () => { 6 | const { frontmatter } = useRouteMeta(); 7 | return ( 8 |
9 |
10 |
11 | 12 |
13 |
14 |
15 |
{frontmatter.contentTitle}
16 |
    17 |
    18 |

    {frontmatter?.titleConDirectly}

    19 | { 20 | frontmatter?.contentDirectly?.map((item: any) => { 21 | return
  • 22 |
    23 | {item?.titleCon.replace(/\\/g, "")} 24 |
    25 |

    {item?.desc}

    26 |
    • 27 | }) 28 | } 29 |
    30 |
    31 |

    {frontmatter?.titleConPreprint}

    32 | { 33 | frontmatter?.contentPreprint?.map((item: any) => { 34 | return
  • 35 |
    36 | {item?.titleCon.replace(/\\/g, "")} 37 |
    38 |

    {item?.desc}

    39 |
    • 40 | }) 41 | } 42 |
    43 |
    44 |

    {frontmatter?.titleConRelated}

    45 | { 46 | frontmatter?.contentRelated?.map((item: any) => { 47 | return
  • 48 |
    49 | {item?.titleCon.replace(/\\/g, "")} 50 |
    51 |

    {item?.desc}

    52 |
    • 53 | }) 54 | } 55 |
    56 |
57 |
58 |
59 | ); 60 | }; 61 | export default Publication; 62 | -------------------------------------------------------------------------------- /.dumi/theme/slots/AutomatedTesting/index.less: -------------------------------------------------------------------------------- 1 | .automatedTesting { 2 | margin-top: 180px; 3 | display: flex; 4 | justify-content: center; 5 | 6 | .automatedTesting-center { 7 | display: flex; 8 | justify-content: space-between; 9 | flex-direction: row; 10 | width: 1200px; 11 | margin: 41px 0 25px 0; 12 | 13 | img { 14 | height: 628px; 15 | height: 507px; 16 | border-radius: 25px; 17 | } 18 | 19 | .automatedTestingContent { 20 | margin-left: 57px; 21 | padding-top: 41px; 22 | 23 | .desc { 24 | width: 542px; 25 | margin-top: 48px; 26 | font-size: 16px; 27 | color: #ffffff; 28 | letter-spacing: 1px; 29 | opacity: 0.8; 30 | 31 | @{dark-selector} & { 32 | font-size: 16px; 33 | color: #171616; 34 | } 35 | } 36 | 37 | .buttom { 38 | cursor: pointer; 39 | width: 150px; 40 | height: 37px; 41 | background-image: url("https://mdn.alipayobjects.com/huamei_bvbxju/afts/img/A*eMQHT670srQAAAAAAAAAAAAADlHYAQ/original"); 42 | background-repeat: no-repeat; 43 | background-size: contain; 44 | background-position: center; 45 | border-radius: 7px; 46 | margin-top: 32px; 47 | color: #a8abff; 48 | display: flex; 49 | align-items: center; 50 | justify-content: center; 51 | position: relative; 52 | 53 | .anticon { 54 | svg { 55 | width: 30px; 56 | height: 23px; 57 | position: absolute; 58 | bottom: 5px; 59 | } 60 | 61 | margin-left: 8px; 62 | width: 25px; 63 | height: 25px; 64 | } 65 | 66 | &:hover { 67 | background-image: url("https://mdn.alipayobjects.com/huamei_bvbxju/afts/img/A*Pm85SKQpQ88AAAAAAAAAAAAADlHYAQ/original"); 68 | background-repeat: no-repeat; 69 | background-size: contain; 70 | background-position: center; 71 | color: #fff; 72 | 73 | @{dark-selector} & { 74 | color: #7031FD; 75 | } 76 | } 77 | } 78 | } 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /.dumi/theme/slots/Navbar/menu.tsx: -------------------------------------------------------------------------------- 1 | import type { MenuProps } from 'antd'; 2 | import { ConfigProvider, Menu, theme } from 'antd'; 3 | import { useLocale, useNavigate, usePrefersColor } from 'dumi'; 4 | import React from 'react'; 5 | import { NavbarEnums, NavbarEnumsEn } from '../../constants'; 6 | 7 | type MenuItem = Required['items'][number]; 8 | 9 | const linkUrl = (title: string, link: string) => { 10 | const navigate = useNavigate(); 11 | return ( 12 | navigate(link)} title={title}> 13 | {title} 14 | 15 | ); 16 | }; 17 | 18 | export function getItem( 19 | label: keyof typeof NavbarEnums | keyof typeof NavbarEnumsEn, 20 | key?: any, 21 | icon?: React.ReactNode, 22 | children?: MenuItem[], 23 | type?: 'group', 24 | ): MenuItem { 25 | const locale = useLocale(); 26 | const isEn = locale.id === 'en-US'; 27 | const Enums: any = isEn ? NavbarEnumsEn : NavbarEnums; 28 | const uniqueKey: keyof typeof NavbarEnums = key ?? label; 29 | return { 30 | key: key ?? label, 31 | icon, 32 | children, 33 | label: linkUrl(label, Enums[uniqueKey]), 34 | type, 35 | } as MenuItem; 36 | } 37 | 38 | export const NavbarMenus = (items: MenuProps['items']) => { 39 | const [color] = usePrefersColor(); 40 | return ( 41 | 58 | 69 | 70 | ); 71 | }; 72 | -------------------------------------------------------------------------------- /.dumi/theme/slots/Toc/index.less: -------------------------------------------------------------------------------- 1 | @import (reference) '.dumi/theme/styles/variables.less'; 2 | 3 | .@{prefix}-content-tool { 4 | color: @c-text-note; 5 | font-size: 14px; 6 | 7 | @{dark-selector} & { 8 | color: @c-text-note-dark; 9 | } 10 | 11 | >dl { 12 | &:empty { 13 | display: none; 14 | } 15 | 16 | dd { 17 | margin-inline-start: 0px; 18 | padding-bottom: 10px; 19 | 20 | >a { 21 | color: @c-text-note; 22 | 23 | @{dark-selector} & { 24 | color: @c-primary-dark; 25 | } 26 | 27 | &:not(:hover) { 28 | text-decoration: none; 29 | } 30 | 31 | >svg { 32 | fill: @c-primary; 33 | 34 | @{dark-selector} & { 35 | fill: @c-primary-dark; 36 | } 37 | } 38 | } 39 | } 40 | } 41 | } 42 | 43 | .@{prefix}-toc { 44 | list-style: none; 45 | margin: 12px 0 0; 46 | padding: 0; 47 | border-inline-start: 2px solid @c-border; 48 | 49 | @{dark-selector} & { 50 | border-inline-start-color:#d0d5d8; 51 | } 52 | 53 | &:empty { 54 | display: none; 55 | } 56 | 57 | >li { 58 | >a { 59 | display: block; 60 | margin: 6px 0; 61 | padding: 3px 16px; 62 | color: @c-text-secondary; 63 | font-size: 14px; 64 | line-height: 1; 65 | text-decoration: none; 66 | white-space: nowrap; 67 | overflow: hidden; 68 | text-overflow: ellipsis; 69 | 70 | @{dark-selector} & { 71 | color: @c-text-secondary-dark; 72 | } 73 | 74 | &:hover { 75 | color: @c-text; 76 | 77 | @{dark-selector} & { 78 | color: @c-text-dark; 79 | } 80 | } 81 | 82 | &.active { 83 | margin-inline-start: -2px; 84 | // color: #fff ; 85 | color: #9999FF !important; 86 | border-inline-start: 2px solid @c-primary; 87 | 88 | @{dark-selector} & { 89 | color: #9999FF; 90 | // color: @c-text-dark; 91 | border-inline-start-color: @c-primary-dark; 92 | } 93 | } 94 | } 95 | 96 | &[data-depth='3']>a { 97 | padding-inline-start: 32px; 98 | } 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /docs/docs/developer-docs/CodeFuse-ChatBot/master/roadmap.zh-CN.md: -------------------------------------------------------------------------------- 1 | --- 2 | store: 3 | title: CodeFuse-ChatBot 4 | version: master 5 | group: 6 | title: 🌱 CodeFuse-ChatBot 7 | order: -1 8 | title: ChatBot 技术路线 9 | order: 2 10 | toc: content 11 | github: https://github.com/codefuse-ai/codefuse-chatbot 12 | --- 13 | 14 | ## RoadMap 15 | 16 |
17 | 图片 18 |
19 |
20 | 21 | 完整路线 22 | 23 | - [x] Sandbox 环境 24 | - [x] 环境隔离的 sandbox 环境与代码执行 25 | - [x] 上传、下载文件 26 | - [ ] 支持 java 执行环境 27 | - [ ] Vector Database & Retrieval 28 | - [x] task retrieval 29 | - [x] tool retrieval 30 | - [x] Prompt Management 31 | - [x] memory Management 32 | - [x] Multi Agent 33 | - [ ] PRD 需求文档、系分、接口设计 34 | - [ ] 根据需求文档、系分、接口设计生产代码 35 | - [ ] 自动测试、自动 debugger 36 | - [ ] 运维流程接入（ToolLearning 37 | - [ ] 全流程自动 38 | - [x] 基于 fastchat 接入 LLM 39 | - [x] 基于 sentencebert 接入 Text Embedding 40 | - [x] 向量加载速度提升 41 | - [x] Connector 42 | - [x] 基于 langchain 的 react 模式 43 | - [x] 基于 langchain 完成 tool 检索 44 | - [x] Web Crawl 通用能力 45 | - [x] 技术文档: 知乎、csdn、阿里云开发者论坛、腾讯云开发者论坛等 46 | - [ ] issue document 47 | - [ ] SDK Library Document 48 | 49 |

50 | 51 | - v0.0 52 | - [x] Sandbox 环境 53 | - [x] 环境隔离的 sandbox 环境与代码执行 54 | - [x] 基于 fastchat 接入 LLM 55 | - [x] 基于 sentencebert 接入 Text Embedding 56 | - [x] Web Crawl 通用能力：技术文档: 知乎、csdn、阿里云开发者论坛、腾讯云开发者论坛等 57 |
58 | - v0.1 59 | - [x] Sandbox 环境: 上传、下载文件 60 | - [x] Vector Database & Retrieval 61 | - [x] task retrieval 62 | - [x] tool retrieval 63 | - [x] Connector 64 | - [x] 基于 langchain 的 react 模式 65 | - [x] 基于 sentencebert 接入 Text Embedding: 向量加载速度提升 66 | 67 | Done 68 |
69 | 70 | - v0.2 71 | - [x] Prompt Management 72 | - [x] memory Management 73 | - [x] Vector Database & Retrieval 74 | 75 | DDL： 2024.01.31 76 |
77 | 78 | - v0.3 79 | - [x] Sandbox 环境 80 | - [ ] 支持 java 执行环境 81 | - [x] Multi Agent Framework 82 | - [ ] PRD 需求文档、系分、接口设计 83 | - [ ] 根据需求文档、系分、接口设计生产代码 84 | - [ ] 自动测试、自动 debugger 85 | - [ ] 运维流程接入（ToolLearning） 86 | - [ ] 全流程自动 87 | - [x] Web Crawl 通用能力 88 | - [ ] issue document 89 | - [ ] SDK Library Document 90 | 91 | DDL： 2024.12.31 92 |
93 | -------------------------------------------------------------------------------- /.dumi/theme/locales/en-US.json: -------------------------------------------------------------------------------- 1 | { 2 | "header.search.placeholder": "Search", 3 | "header.color.mode.light": "Light Mode", 4 | "header.color.mode.dark": "Dark Mode", 5 | "header.color.mode.auto": "Follow System", 6 | "header.social.github": "GitHub", 7 | "header.social.weibo": "Weibo", 8 | "header.social.twitter": "Twitter", 9 | "header.social.gitlab": "GitLab", 10 | "header.social.facebook": "Facebook", 11 | "header.social.zhihu": "Zhihu", 12 | "header.social.yuque": "Yuque", 13 | "header.social.linkedin": "Linkedin", 14 | "previewer.actions.code.expand": "Show Code", 15 | "previewer.actions.code.shrink": "Hide Code", 16 | "previewer.actions.sketch": "Copy to Sketch app", 17 | "previewer.actions.sketch.group": "Copy as Sketch Group", 18 | "previewer.actions.sketch.symbol": "Copy as Sketch Symbol", 19 | "previewer.actions.sketch.divider": "------------------------", 20 | "previewer.actions.sketch.guide": "How to paste to Sketch?", 21 | "previewer.actions.codesandbox": "Open in CodeSandbox", 22 | "previewer.actions.codepen": "Open in CodePen (Not implemented)", 23 | "previewer.actions.stackblitz": "Open in StackBlitz", 24 | "previewer.actions.separate": "Open in separate page", 25 | "404.title": "PAGE NOT FOUND", 26 | "404.back": "Back to homepage", 27 | "api.component.name": "Name", 28 | "api.component.description": "Description", 29 | "api.component.type": "Type", 30 | "api.component.default": "Default", 31 | "api.component.required": "(required)", 32 | "api.component.unavailable": "apiParser must be enabled to use auto-generated API", 33 | "api.component.loading": "Properties definition is resolving, wait a moment...", 34 | "api.component.not.found": "Properties definition not found for {id} component", 35 | "content.tabs.default": "Doc", 36 | "content.footer.last.updated": "Last updated: ", 37 | "content.footer.actions.edit": "Improve this documentation", 38 | "content.footer.actions.previous": "PREV", 39 | "content.footer.actions.next": "NEXT", 40 | "search.not.found": "No content was found", 41 | "layout.sidebar.btn": "Sidebar", 42 | "content.menus.docs.developer": "Developer-Docs", 43 | "content.menus.docs.api": "API-Docs", 44 | "content.menus.docs.product": "Product-Docs", 45 | "content.menus.docs.about": "About CodeFuse" 46 | } 47 | -------------------------------------------------------------------------------- /docs/docs/developer-docs/CodeFuse-Query/main/CodeFuseQuery.zh-CN.md: -------------------------------------------------------------------------------- 1 | --- 2 | nav: 3 | title: 文档 4 | order: -1 5 | second: 6 | title: 开发者文档 7 | order: -1 8 | store: 9 | title: CodeFuse-Query 10 | version: main 11 | group: 12 | title: 🌱 CodeFuse-Query 13 | index: true 14 | order: -1 15 | title: CodeFuse-Query 16 | order: -1 17 | toc: content 18 | github: https://github.com/codefuse-ai/CodeFuse-Query 19 | --- 20 | 21 | ## CodeFuse-Query 22 | 23 | 随着大规模软件开发的普及，对可扩展且易于适应的静态代码分析技术的需求正在加大。传统的静态分析工具，如 Clang Static Analyzer (CSA) 或 PMD，在检查编程规则或样式问题方面已经展现出了良好的效果。然而，这些工具通常是为了满足特定的目标而设计的，往往无法满足现代软件开发环境中多变和多元化的需求。这些需求可以涉及服务质量 (QoS)、各种编程语言、不同的算法需求，以及各种性能需求。例如，安全团队可能需要复杂的算法，如上下文敏感的污点分析，来审查较小的代码库，而项目经理可能需要一种相对较轻的算法，例如计算圈复杂度的算法，以在较大的代码库上测量开发人员的生产力。 24 | 25 | 这些多元化的需求，加上大型组织中常见的计算资源限制，构成了一项重大的挑战。由于传统工具采用的是问题特定的计算方式，往往无法在这种环境中实现扩展。因此，我们推出了 CodeQuery，这是一个专为大规模静态分析设计的集中式数据平台。 26 | 在 CodeQuery 的实现中，我们把源代码和分析结果看作数据，把执行过程看作大数据处理，这与传统的以工具为中心的方法有着显著的不同。我们利用大型组织中的常见系统，如数据仓库、MaxCompute 和 Hive 等数据计算设施、OSS 对象存储和 Kubernetes 等灵活计算资源，让 CodeQuery 能够无缝地融入这些系统中。这种方法使 CodeQuery 高度可维护和可扩展，能够支持多元化的需求，并有效应对不断变化的需求。此外，CodeQuery 的开放架构鼓励各种内部系统之间的互操作性，实现了无缝的交互和数据交换。这种集成和交互能力不仅提高了组织内部的自动化程度，也提高了效率，降低了手动错误的可能性。通过打破信息孤岛，推动更互联、更自动化的环境，CodeQuery 显著提高了软件开发过程的整体生产力和效率。 27 | 此外，CodeQuery 的以数据为中心的方法在处理静态源代码分析的领域特定挑战时具有独特的优势。例如，源代码通常是一个高度结构化和互联的数据集，与其他代码和配置文件有强烈的信息和连接。将代码视为数据，CodeQuery 可以巧妙地处理这些问题，这使得它特别适合在大型组织中使用，其中代码库持续但逐步地进行演变，大部分代码在每天进行微小的改动同时保持稳定。 CodeQuery 还支持如基于代码数据的商业智能 (BI) 这类用例，能生成报告和仪表板，协助监控和决策过程。此外，CodeQuery 在分析大型语言模型 (LLM) 的训练数据方面发挥了重要作用，提供了增强这些模型整体效果的深入见解。 28 | 29 | 在当前的静态分析领域，CodeQuery 带来了一种新的范式。它不仅满足了大规模、复杂的代码库分析需求，还能适应不断变化和多元化的静态分析场景。CodeQuery 的以数据为中心的方法，使得其在处理大数据环境中的代码分析问题时具有独特优势。CodeQuery 的设计，旨在解决大规模软件开发环境中的静态分析问题。它能够将源代码和分析结果视作数据，使得其可以灵活地融入大型组织的各种系统中。这种方法不仅可以有效地处理大规模的代码库，还可以应对各种复杂的分析需求，从而使得静态分析工作变得更加高效和准确。 30 | 31 | CodeQuery 的特点和优势可以概括为以下几点： 32 | 33 | - **高度可扩展**：CodeQuery 可以处理大规模的代码库，且能够适应不同的分析需求。这种高度的可扩展性使得 CodeQuery 可以在大型组织中发挥重要作用。 34 | - **以数据为中心**：CodeQuery 将源代码和分析结果视作数据，这种以数据为中心的方法使其在处理大数据环境中的代码分析问题时具有独特优势。 35 | - **高度集成**：CodeQuery 能够无缝地融入大型组织的各种系统中，包括数据仓库、数据计算设施、对象存储和灵活计算资源等。这种高度的集成性使得 CodeQuery 在大型组织中的使用变得更加方便和高效。 36 | - **支持多元化的需求**：CodeQuery 不仅可以处理大规模的代码库，还可以应对各种复杂的分析需求，包括服务质量分析需求、跨编程语言分析需求、算法需求和性能需求等。 37 | 38 | CodeQuery 是一种强大的静态代码分析平台，适合大规模、复杂的代码库分析场景。它的以数据为中心的方法和高度的可扩展性使得它在现代软件开发环境中具有独特的优势。未来，随着静态代码分析技术的不断发展，CodeQuery 有望在这个领域中扮演更加重要的角色。 39 | -------------------------------------------------------------------------------- /docs/blogDetails/20231211.zh-CN.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: '蚂蚁CodeFuse新版发布，前端能力优化，支持安卓开发' 3 | time: '2023-12-11' 4 | toc: content 5 | --- 6 | 7 | 蚂蚁百灵研发助手 CodeFuse 插件发布新版，本版本新增支持 Android Studio，并针对 JavaScript、TypeScript 等前端语言优化了模型效果，同时还将输出 Token 增加到最多 1024 个。 8 | 9 | 目前 CodeFuse 处于邀请测试阶段，欢迎各位开发者前往官网申请资格参与测试。在之前已安装插件的用户需要下载最新版本，才可享受 CodeFuse 插件最新能力。 10 | 11 | CodeFuse 产品官网：[https://codefuse.yuque.com/](https://link.zhihu.com/?target=https%3A//codefuse.yuque.com/) 12 | 13 | # 新版功能更新一览 14 | 15 | ## 新增 Android Studio 支持 16 | 17 | CodeFuse 插件新增兼容 Android Studio，目前支持的 IDE 达到 11 款，包括 Visual Studio Code、IDEA 等主流 IDE。在官网下载安装对应的插件即可使用。 18 | ![](https://mdn.alipayobjects.com/huamei_bvbxju/afts/img/A*CtcTRabVhgMAAAAAAAAAAAAADlHYAQ/original) 19 | 20 | ## 优化前端代码能力 21 | 22 | 新版插件针对 JavaScript、TypeScript 等前端语言优化了模型效果，提升代码补全的准确率。据蚂蚁内部使用情况统计，前端代码生成的采纳率相比旧版提升了 20%。 23 | ![image.png](https://mdn.alipayobjects.com/huamei_bvbxju/afts/img/A*g0T-T7MiwzMAAAAAAAAAAAAADlHYAQ/original) 24 | 25 | 如上例子所示，根据注释，CodeFuse 可直接生成较复杂的 JavaScript 完整函数，实现字符串匹配算法，并在测试中取得预期结果。 26 | 27 | 其它新功能包括：插件面板顶部新增“研发小蜜”产品答疑入口和“退出登录”功能；增加输出 Token，目前最大输出不超过 1024 Tokens；插件面板适配 VS Code 浅色主题等。 28 | 29 | # CodeFuse 让研发变得更简单 30 | 31 | 10 月 24 日，蚂蚁百灵研发助手 CodeFuse 对外发布 IDE 插件，面向全体开发者开启邀请测试。蚂蚁很早就开始在代码智能生成领域发力，从零到一，最终打造了 CodeFuse。CodeFuse 已经率先在蚂蚁内部广泛使用，汲取工程师们的反馈不断成长，这些积累多年的实战经验也融合在 CodeFuse 的每一行代码里。 32 | 33 | 目前 CodeFuse 插件能力包括： 34 | 35 | 1. 代码补全，基于海量数据提供实时地代码补全服务，提升编码效率。 36 | 2. 代码优化，基于代码理解能力和静态源码分析能力，对选定代码段进行分析理解，提出优化和改进建议。 37 | 3. 代码注释，通过智能分析，CodeFuse 可以准确解释代码含义、添加代码注释。 38 | 4. 解释代码，基于大量高质量的代码数据训练，准确解释代码含义。 39 | 5. 生成单测，智能生成具备业务语义的测试用例。 40 | 41 | 自 CodeFuse 对外发布以来，在开发者群体获得广泛反响。 42 | 43 | 有参与测试的用户表示，CodeFuse 生成代码的效果很好，基本能够满足需求。 44 | ![](https://mdn.alipayobjects.com/huamei_bvbxju/afts/img/A*cbl1Q6k9JAcAAAAAAAAAAAAADlHYAQ/original) 45 | 46 | 在蚂蚁内部，CodeFuse 更是成为很多工程师的日常必备工具，很多技术同学利用它提升工作效率，挖掘它的更多玩法。 47 | 48 | ![](https://mdn.alipayobjects.com/huamei_bvbxju/afts/img/A*IFgxT7sY9B8AAAAAAAAAAAAADlHYAQ/original) 49 | 50 | ![](https://mdn.alipayobjects.com/huamei_bvbxju/afts/img/A*IFgxT7sY9B8AAAAAAAAAAAAADlHYAQ/original) 51 | 52 | ![](https://mdn.alipayobjects.com/huamei_bvbxju/afts/img/A*mDsYQ4Q9RRQAAAAAAAAAAAAADlHYAQ/original) 53 | 54 | 更多使用实践，推荐阅读这篇对蚂蚁研发工程师悟鸣的用户访谈： 55 | 56 | 新时代的程序员，已经在用大模型写代码了 57 | 58 | # 邀测火热进行中 59 | 60 | 过去的一个多月时间，除了大家能够看到的新功能，在背后研发和产品同学也做出了很多不为人知的努力，以给大家提供更好的体验。CodeFuse 也将持续进化，让研发变得更简单。 61 | 62 | CodeFuse 的邀请测试正在火热进行中，欢迎大家前往官网申请测试资格： 63 | -------------------------------------------------------------------------------- /docs/docs/developer-docs/CodeFuse-DevOps-Eval/master/evaluate.zh-CN.md: -------------------------------------------------------------------------------- 1 | --- 2 | store: 3 | title: CodeFuse-DevOps-Eval 4 | version: master 5 | group: 6 | title: 🌱 CodeFuse-DevOps-Eval 7 | order: -1 8 | title: 评测 9 | order: 0 10 | toc: content 11 | github: https://github.com/codefuse-ai/codefuse-devops-eval 12 | --- 13 | 14 | ## 🚀 如何进行测试 15 | 16 | 如果需要在自己的 HuggingFace 格式的模型上进行测试的话，总的步骤分为如下几步: 17 | 18 | 1. 编写 Model 的 loader 函数 19 | 2. 编写 Model 的 context_builder 函数 20 | 3. 注册模型到配置文件中 21 | 4. 执行测试脚本 22 | 如果模型在加载进来后不需要特殊的处理，而且输入也不需要转换为特定的格式（e.g. chatml 格式或者其他的 human-bot 格式），请直接跳转到第四步直接发起测试。 23 | 24 | #### 1. 编写 loader 函数 25 | 26 | 模型加载时还需要做一些额外的处理（e.g. tokenizer 调整），需要继承 `ModelAndTokenizerLoader` 类来覆写对应的 `load_model` 和 `load_tokenizer` 函数， 如下所示： 27 | 28 | ```python 29 | class QwenModelAndTokenizerLoader(ModelAndTokenizerLoader): 30 | def __init__(self): 31 | super().__init__() 32 | pass 33 | 34 | @override 35 | def load_model(self, model_path: str): 36 | # Implementation of the method 37 | pass 38 | 39 | @override 40 | def load_tokenizer(self, model_path: str): 41 | # Implementation of the method 42 | pass 43 | ``` 44 | 45 | #### 2. 编写 Model 的 context_builder 函数 46 | 47 | 如果输入需要转换为特定的格式（e.g. chatml 格式或者其他的 human-bot 格式），则需要继承 ContextBuilder 类来覆写 make_context 函数，如下所示： 48 | 49 | ```python 50 | class QwenChatContextBuilder(ContextBuilder): 51 | def __init__(self): 52 | super().__init__() 53 | 54 | @override 55 | def make_context(self, model, tokenizer, query: str, system: str = "hello！"): 56 | # Implementation of the method 57 | pass 58 | ``` 59 | 60 | #### 3. 注册模型到配置文件中 61 | 62 | 去 conf 中的 `model_conf.json`，注册对应的模型名和这个模型将要使用的 loader 和 context_builder，示例如下： 63 | 64 | ```json 65 | { 66 | "Qwen-Chat": { 67 | "loader": "QwenModelAndTokenizerLoader", 68 | "context_builder": "QwenChatContextBuilder" 69 | } 70 | } 71 | ``` 72 | 73 | #### 4. 执行测试脚本 74 | 75 | 直接运行以下代码发起测试 76 | 77 | ```Bash 78 | python src/run_eval.py \ 79 | --model_path path_to_model \ 80 | --model_name model_name_in_conf \ 81 | --model_conf_path path_to_model_conf \ 82 | --eval_dataset_list all \ 83 | --eval_dataset_fp_conf_path path_to_dataset_conf \ 84 | --eval_dataset_type test \ 85 | --data_path path_to_downloaded_devops_eval_data \ 86 | --k_shot 0 87 | ``` 88 | 89 | 👀 👀 具体评测流程见 📖 [**数据集评测教程**](/zh-CN/docs/developer-docs/CodeFuse-DevOps-Eval/master/tutorial) 90 |
91 | -------------------------------------------------------------------------------- /docs/docs/developer-docs/CodeFuse-ModelCache/main/release_note.zh-CN.md: -------------------------------------------------------------------------------- 1 | --- 2 | store: 3 | title: CodeFuse-ModelCache 4 | version: main 5 | group: 6 | title: 🌱 CodeFuse-ModelCache 7 | order: -1 8 | title: 版本记录 9 | order: 3 10 | toc: content 11 | github: https://github.com/codefuse-ai/CodeFuse-ModelCache 12 | --- 13 | 14 | | 时间 | 功能 | 版本号 | 15 | | -------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------ | 16 | | 20230430 | 完成 GPTCache 调研，开源流程在 OpenAI 接口上跑通，单节点形式 | 无 | 17 | | 20230509 | 1、完成技术选型及上下游交互方案
2、重新开发数据库模块，替换 SQLalchemy 框架
3、重构 llm_handler 模块，兼容 codegpt，适配 codegpt 模型参数 | V0.1.0 | 18 | | 20230519 | 1、根据环境动态选择 codegpt 服务模式
2、模型本地加载能力，以及预加载能力
3、增加本地路径依据环境动态加载能力 | V0.1.1 | 19 | | 20230522 | 1、架构优化，调整为类 redis 结构，解藕大模型调用
2、关系数据库由 sqlite 切换至 OceanBase
3、向量数据库由 faiss 切换至 milvus
4、模型数据隔离能力
5、增加核心模块 adapter_query、adapter_insert | V0.2.0 | 20 | | 20230531 | 1、线上环境上线，动态感知能力
2、embedding 模型评测及选型
3、增加预发环境及数据隔离能力
4、增加原始 query 字段透出能力 | V0.2.1 | 21 | | 20230607 | 1、优化关系数据库访问性能
2、优化环境和模型隔离能力 | V0.2.2 | 22 | | 20230630 | 1、在 modelCache 中增加大模型 embedding 层适配模块
2、增加采纳率统计能力 | V0.2.3 | 23 | | 20230730 | 1、增加缓存统计功能
2、增加数据删除功能接口
3、缓存一键清空能力上线
4、多轮会话能力研发，支持 system 指令和多轮对话 | v0.3.0 | 24 | | 20230830 | 1、增加异步处理能力，性能提升超 20%
2、架构变更，解藕 embedding 推理和业务处理逻辑
3、黑名单过滤功能 | V0.3.1 | 25 | -------------------------------------------------------------------------------- /.dumi/theme/slots/Sidebar/index.less: -------------------------------------------------------------------------------- 1 | @import (reference) '.dumi/theme/styles/variables.less'; 2 | 3 | .@{prefix}-sidebar { 4 | position: sticky; 5 | top: @s-header-height; 6 | width: @s-sidebar-width; 7 | max-height: calc(100vh - @s-header-height); 8 | padding-top: 20px; 9 | padding-bottom: 24px; 10 | // padding-inline-start: 8px; 11 | padding-inline-end: 32px; 12 | box-sizing: border-box; 13 | overflow: auto; 14 | 15 | .@{prefix}-sidebar-version { 16 | display: flex; 17 | width: 100%; 18 | margin-top: 10px; 19 | justify-content: space-between; 20 | 21 | margin-bottom: 16px; 22 | padding-bottom: 16px; 23 | border-bottom: 1px solid @c-border; 24 | 25 | @{dark-selector} & { 26 | border-top-color: #d0d5d8; 27 | } 28 | } 29 | 30 | @{dark-selector} & { 31 | border-top-color: @c-border-dark; 32 | } 33 | 34 | @media @mobile { 35 | position: fixed; 36 | z-index: 20; 37 | top: 0; 38 | left: 0; 39 | bottom: 0; 40 | max-height: initial; 41 | padding-inline-start: 32px; 42 | background-color: @c-site-bg; 43 | border-top: 1px solid @c-border-light; 44 | box-shadow: 0 0 20px rgba(0, 0, 0, 10%); 45 | transition: 0.2s all; 46 | 47 | @{dark-selector} & { 48 | // background-color: @c-site-bg-dark; 49 | background-color: @c-site-bg-dark; 50 | border-top-color: @c-border-less-dark; 51 | } 52 | 53 | .@{prefix}-doc-layout:not([data-mobile-sidebar-active]) & { 54 | opacity: 0; 55 | visibility: hidden; 56 | transform: translateX(-100%); 57 | } 58 | } 59 | 60 | .ant-menu-light .ant-menu-item:not(.ant-menu-item-selected):not(.ant-menu-submenu-selected):hover { 61 | color: #99f; 62 | background-color: #181d29; 63 | 64 | @{dark-selector} & { 65 | 66 | background-color: #f0f4ff; 67 | } 68 | } 69 | 70 | } 71 | 72 | 73 | // .ant-select-dropdown { 74 | // background-color: #0d0d15; 75 | // border: 1px solid #3f414c; 76 | // border-radius: 10px; 77 | // } 78 | 79 | // .ant-select-dropdown .ant-select-item-option-selected:not(.ant-select-item-option-disabled) { 80 | // background: #181d29; 81 | // border-radius: 5px; 82 | // } 83 | // .ant-select-dropdown .ant-select-item { 84 | // color: #b5b5b5; 85 | // } 86 | // .ant-select-single.ant-select-open .ant-select-selection-item { 87 | // color: #fff; 88 | // } 89 | // .ant-btn-default { 90 | // background: transparent; 91 | // color: #fff; 92 | // } 93 | -------------------------------------------------------------------------------- /.dumi/theme/slots/PerformanceEvaluation/index.less: -------------------------------------------------------------------------------- 1 | .Performance { 2 | margin-top: 100px; 3 | display: flex; 4 | justify-content: center; 5 | 6 | @{dark-selector} & { 7 | margin-bottom: 150px; 8 | } 9 | 10 | .Performance-center { 11 | display: flex; 12 | justify-content: center; 13 | flex-direction: column; 14 | width: 1200px; 15 | 16 | .PerformanceTitle { 17 | margin: 0 auto; 18 | text-align: center; 19 | min-width: 343px; 20 | font-size: 40px; 21 | color: #ffffff; 22 | letter-spacing: 1.35px; 23 | font-weight: 600; 24 | background-image: linear-gradient(90deg, #d8d8d8 0%, #545eff 100%); 25 | display: inline-block; 26 | background-clip: text; 27 | -webkit-background-clip: text; 28 | -webkit-text-fill-color: transparent; 29 | 30 | 31 | .more { 32 | cursor: pointer; 33 | margin-top: 20px; 34 | font-size: 18px; 35 | color: #a8abfa; 36 | letter-spacing: 0.54px; 37 | text-align: center; 38 | 39 | .anticon { 40 | svg { 41 | width: 32px; 42 | height: 30px; 43 | } 44 | 45 | position: absolute; 46 | margin-left: 8px; 47 | width: 35px; 48 | height: 28px; 49 | } 50 | } 51 | } 52 | 53 | .PerformanceContent { 54 | display: flex; 55 | justify-content: center; 56 | flex-direction: column; 57 | margin-top: 42px; 58 | width: 1200px; 59 | border-radius: 13px; 60 | background-image: url('https://mdn.alipayobjects.com/huamei_bvbxju/afts/img/A*uh7FRKajlBIAAAAAAAAAAAAADlHYAQ/original'); 61 | background-repeat: no-repeat; 62 | background-size: cover; 63 | background-position: center; 64 | 65 | @{dark-selector} & { 66 | background-image: url('https://mdn.alipayobjects.com/huamei_bvbxju/afts/img/A*4scPTrdP_xIAAAAAAAAAAAAADlHYAQ/original'); 67 | } 68 | 69 | img { 70 | margin: 21px auto 41px auto; 71 | border-radius: 5px; 72 | width: 1050px; 73 | height: 341px; 74 | } 75 | 76 | .desc { 77 | margin: 41px auto 0; 78 | width: 1050px; 79 | text-align: left; 80 | font-size: 16px; 81 | color: #ffffff; 82 | letter-spacing: 1px; 83 | 84 | @{dark-selector} & { 85 | font-size: 16px; 86 | color: #171616; 87 | } 88 | } 89 | } 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /docs/docs/developer-docs/CodeFuse-ModelCache/main/CodeFuseModelCache.en-US.md: -------------------------------------------------------------------------------- 1 | --- 2 | nav: 3 | title: Docs 4 | order: -1 5 | second: 6 | title: Developer-Docs 7 | order: -1 8 | store: 9 | title: CodeFuse-ModelCache 10 | version: main 11 | group: 12 | title: 🌱 CodeFuse-ModelCache 13 | index: true 14 | order: -1 15 | title: CodeFuse-ModelCache 16 | order: -1 17 | toc: content 18 | github: https://github.com/codefuse-ai/CodeFuse-ModelCache 19 | --- 20 | 21 | ## Contents 22 | 23 | - [news](#news) 24 | - [Introduction](#Introduction) 25 | - [Modules](#Modules) 26 | - [Acknowledgements](#Acknowledgements) 27 | - [Contributing](#Contributing) 28 | 29 | ## news 30 | 31 | - 🔥🔥[2023.12.10] we integrate LLM embedding frameworks such as 'llmEmb', 'ONNX', 'PaddleNLP', 'FastText', alone with the image embedding framework 'timm', to bolster embedding functionality. 32 | - 🔥🔥[2023.11.20] codefuse-ModelCache has integrated local storage, such as sqlite and faiss, providing users with the convenience of quickly initiating tests. 33 | - [2023.08.26] codefuse-ModelCache... 34 | 35 | ## Introduction 36 | 37 | Codefuse-ModelCache is a semantic cache for large language models (LLMs). By caching pre-generated model results, it reduces response time for similar requests and improves user experience.
This project aims to optimize services by introducing a caching mechanism. It helps businesses and research institutions reduce the cost of inference deployment, improve model performance and efficiency, and provide scalable services for large models. Through open-source, we aim to share and exchange technologies related to large model semantic cache. 38 | 39 | ## modules 40 | 41 | ![modelcache modules](https://mdn.alipayobjects.com/huamei_bvbxju/afts/img/A*Z-6cSr6udKAAAAAAAAAAAAAADlHYAQ/original) 42 | 43 | ## Acknowledgements 44 | 45 | This project has referenced the following open-source projects. We would like to express our gratitude to the projects and their developers for their contributions and research.
[GPTCache](https://github.com/zilliztech/GPTCache) 46 | 47 | ## Contributing 48 | 49 | ModelCache is a captivating and invaluable project, whether you are an experienced developer or a novice just starting out, your contributions to this project are warmly welcomed. Your involvement in this project, be it through raising issues, providing suggestions, writing code, or documenting and creating examples, will enhance the project's quality and make a significant contribution to the open-source community. 50 | -------------------------------------------------------------------------------- /docs/publication/publication.zh-CN.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Publication 3 | nav: 4 | title: 论文 5 | order: 2 6 | bannerTitle: https://mdn.alipayobjects.com/huamei_bvbxju/afts/img/A*ApP-TL00Dw8AAAAAAAAAAAAADlHYAQ/original 7 | contentTitle: 精选论文 8 | 9 | titleConDirectly: CodeFuse 相关 10 | contentDirectly: 11 | - titleCon: \[ICSE-SEIP 2024] CodeFuse-13B:\ A Pretrained Multi-lingual Code Large Language Model 12 | desc: Peng Di, Jianguo Li, Hang Yu, Wei Jiang, Wenting Cai, ..., Xianying Zhu 13 | - titleCon: \[KDD 2024] MFTCoder:\ Boosting Code LLMs with Multitask Fine-Tuning 14 | desc: Bingchang Liu, Chaoyu Chen, Zi Gong, Cong Liao, Wanghuanh, Zhichao Lei, Ming Liang, Chen Dajun, Min Shen, Hailian Zhou, wei jiang, Hang Yu, Jianguo Li 15 | - titleCon: \[ACL 2024] D2LLM:\ Decomposed and Distilled Large Language Models for Semantic Search 16 | desc: Zihan Liao, Hang Yu, Jianguo Li, Jun Wang, Wei Zhang 17 | - titleCon: \[ACL 2024] CoCA:\ Fusing Position Embedding with Collinear Constrained Attention in Transformers for Long Context Window Extending 18 | desc: Shiyi Zhu, Jing Ye, Wei Jiang, Siqiao Xue, Qi Zhang, Yifan Wu, Jianguo Li 19 | - titleCon: \[ICWS 2024] Enabling Efficient Batch Serving for LMaaS via Generation Length Prediction 20 | desc: Ke Chen, Wen Hu, Zhi Wang, Peng Du, Jianguo Li, Sheng Zhang 21 | - titleCon: \[ICDE 2024] VDTuner:\ Automated Performance Tuning for Vector Data Management Systems. 22 | desc: Tiannuo Yang, Wen Hu, Wangqi Peng, Yusen Li, Jianguo Li, Gang Wang, Xiaoguang Liu 23 | 24 | titleConRelated: LLM 相关 25 | contentRelated: 26 | - titleCon: \[ICML 2024] DUPLEX:\ Dual GAT for Complex Embedding of Directed Graphs 27 | desc: Zhaoru Ke, Hang Yu, Jianguo Li, Haipeng Zhang 28 | - titleCon: \[ACL 2023 Findings] ZeroAE:\ Pre-trained Language Model based Autoencoder for Transductive Zero-shot Text Classification 29 | desc: Kaihao Guo, Hang Yu, Cong Liao, Jianguo Li, Haipeng Zhang 30 | 31 | titleConPreprint: 预印本 32 | contentPreprint: 33 | - titleCon: \Unifying the Perspectives of NLP and Software Engineering:\ A Survey on Language Models for Code 34 | desc: Ziyin Zhang, Chaoyu Chen, Bingchang Liu, Cong Liao, Zi Gong, Hang Yu, Jianguo Li, Rui Wang 35 | - titleCon: \RepoFuse:\ Repository-Level Code Completion with Fused Dual Context 36 | desc: Ming Liang, Xiaoheng Xie, Gehao Zhang, Xunjin Zheng, Peng Di, wei jiang, Hongwei Chen, Chengpeng Wang, Gang Fan 37 | - titleCon: \Prompting Frameworks for Large Language Models:\ A Survey 38 | desc: Xiaoxia Liu, Jingyi Wang, Jun Sun, Xiaohan Yuan, Guoliang Dong, Peng Di, Wenhai Wang, Dongxia Wang 39 | --- 40 | -------------------------------------------------------------------------------- /docs/blogs/blogs.en-US.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Blogs 3 | nav: 4 | title: Blogs 5 | order: 1 6 | bannerTitle: https://mdn.alipayobjects.com/huamei_bvbxju/afts/img/A*Id1KTZ75cPIAAAAAAAAAAAAADlHYAQ/original 7 | toc: content 8 | 9 | # 发布 10 | publish: 11 | - time: '2024-04-23' 12 | title: Multi-Agent Framework MuAgent Unlocks New Paradigms in Code Development 13 | desc: For complex SOPs requiring constant redefinition of agents and numerous post-processing stages, this procedure can become cumbersome and challenging. Striving to lift this weight and accelerate the execution of SOP workflows，this system streamlines the construction process with a suite of core components, enabling a more convenient and rapid build procedure. It spares users the need to delve into the intricacies of internal prompt construction logic. At last, The paper highlights muAgent's implementation for automating Code Q&A functionalities within Java code repositories, enabling query execution, API documentation, and test case generation. 14 | link: /blogDetails/20240423 15 | 16 | # 技术 17 | develop: 18 | - time: '2024-09-05' 19 | title: Not updated yet 20 | desc: Not updated yet 21 | link: /blogDetails/001 22 | 23 | # 产品 24 | products: 25 | - time: '2024-09-05' 26 | title: Not updated yet 27 | desc: Not updated yet 28 | link: /blogDetails/001 29 | 30 | # 使用 31 | use: 32 | - time: '2024-04-23' 33 | title: Multi-Agent Framework MuAgent Unlocks New Paradigms in Code Development 34 | desc: For complex SOPs requiring constant redefinition of agents and numerous post-processing stages, this procedure can become cumbersome and challenging. Striving to lift this weight and accelerate the execution of SOP workflows，this system streamlines the construction process with a suite of core components, enabling a more convenient and rapid build procedure. It spares users the need to delve into the intricacies of internal prompt construction logic. At last, The paper highlights muAgent's implementation for automating Code Q&A functionalities within Java code repositories, enabling query execution, API documentation, and test case generation. 35 | link: /blogDetails/20240423 36 | 37 | # 活动咨询 38 | EventConsultation: 39 | - time: '2024-09-14' 40 | title: Not updated yet 41 | desc: Not updated yet 42 | link: /blogDetails/001 43 | 44 | - time: '2024-11-11' 45 | title: CodeFuse 编码挑战季」 46 | desc: 本次「编码挑战季」活动，共涉及 muAgent、MFTCoder、ModelCache、CodeFuse-IDE四个代码仓库，社区任务会在 https://github.com/orgs/codefuse-ai/projects 中进行发放。每个代码仓库将会有难、中、易不同程度的任务分类，开发者可以自由选择自己要参与的项目任务，并跟帖，我们将在第一时间分配任务，完成任务即获得所对应任务的积分。 47 | link: /blogDetails/20241111 48 | --- 49 | -------------------------------------------------------------------------------- /docs/publication/publication.en-US.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Publication 3 | nav: 4 | title: Publication 5 | order: 3 6 | bannerTitle: https://mdn.alipayobjects.com/huamei_bvbxju/afts/img/A*r31CSbR3uFUAAAAAAAAAAAAADlHYAQ/original 7 | contentTitle: Selected Publications 8 | 9 | titleConDirectly: CodeFuse Directly Related 10 | contentDirectly: 11 | - titleCon: \[ICSE-SEIP 2024] CodeFuse-13B:\ A Pretrained Multi-lingual Code Large Language Model 12 | desc: Peng Di, Jianguo Li, Hang Yu, Wei Jiang, Wenting Cai, ..., Xianying Zhu 13 | - titleCon: \[KDD 2024] MFTCoder:\ Boosting Code LLMs with Multitask Fine-Tuning 14 | desc: Bingchang Liu, Chaoyu Chen, Zi Gong, Cong Liao, Wanghuanh, Zhichao Lei, Ming Liang, Chen Dajun, Min Shen, Hailian Zhou, wei jiang, Hang Yu, Jianguo Li 15 | - titleCon: \[ACL 2024] D2LLM:\ Decomposed and Distilled Large Language Models for Semantic Search 16 | desc: Zihan Liao, Hang Yu, Jianguo Li, Jun Wang, Wei Zhang 17 | - titleCon: \[ACL 2024] CoCA:\ Fusing Position Embedding with Collinear Constrained Attention in Transformers for Long Context Window Extending 18 | desc: Shiyi Zhu, Jing Ye, Wei Jiang, Siqiao Xue, Qi Zhang, Yifan Wu, Jianguo Li 19 | - titleCon: \[ICWS 2024] Enabling Efficient Batch Serving for LMaaS via Generation Length Prediction 20 | desc: Ke Chen, Wen Hu, Zhi Wang, Peng Du, Jianguo Li, Sheng Zhang 21 | - titleCon: \[ICDE 2024] VDTuner:\ Automated Performance Tuning for Vector Data Management Systems. 22 | desc: Tiannuo Yang, Wen Hu, Wangqi Peng, Yusen Li, Jianguo Li, Gang Wang, Xiaoguang Liu 23 | 24 | titleConRelated: LLM Related 25 | contentRelated: 26 | - titleCon: \[ICML 2024] DUPLEX:\ Dual GAT for Complex Embedding of Directed Graphs 27 | desc: Zhaoru Ke, Hang Yu, Jianguo Li, Haipeng Zhang 28 | - titleCon: \[ACL 2023 Findings] ZeroAE:\ Pre-trained Language Model based Autoencoder for Transductive Zero-shot Text Classification 29 | desc: Kaihao Guo, Hang Yu, Cong Liao, Jianguo Li, Haipeng Zhang 30 | 31 | titleConPreprint: Preprint 32 | contentPreprint: 33 | - titleCon: \Unifying the Perspectives of NLP and Software Engineering:\ A Survey on Language Models for Code 34 | desc: Ziyin Zhang, Chaoyu Chen, Bingchang Liu, Cong Liao, Zi Gong, Hang Yu, Jianguo Li, Rui Wang 35 | - titleCon: \RepoFuse:\ Repository-Level Code Completion with Fused Dual Context 36 | desc: Ming Liang, Xiaoheng Xie, Gehao Zhang, Xunjin Zheng, Peng Di, wei jiang, Hongwei Chen, Chengpeng Wang, Gang Fan 37 | - titleCon: \Prompting Frameworks for Large Language Models:\ A Survey 38 | desc: Xiaoxia Liu, Jingyi Wang, Jun Sun, Xiaohan Yuan, Guoliang Dong, Peng Di, Wenhai Wang, Dongxia Wang 39 | --- 40 | -------------------------------------------------------------------------------- /docs/docs/developer-docs/Test-Agent/main/quickstart.zh-CN.md: -------------------------------------------------------------------------------- 1 | --- 2 | store: 3 | title: Test-Agent 4 | version: main 5 | group: 6 | title: 🌱 Test-Agent 7 | order: -1 8 | title: 快速开始 9 | order: 0 10 | toc: content 11 | github: https://github.com/codefuse-ai/Test-Agent 12 | --- 13 | 14 | ## 快速使用（QuickStart） 15 | 16 | ### 前置准备 17 | 18 | #### 模型下载 19 | 20 | 您可在[modelscope](https://modelscope.cn/models/codefuse-ai/TestGPT-7B)或[huggingface](https://huggingface.co/codefuse-ai/TestGPT-7B)上获取到模型的详细信息并下载模型文件。 21 | 需要注意的是： 22 | 1）如果您通过 modelscope 下载模型，下载方式可参考：[下载说明](https://www.modelscope.cn/docs/%E6%A8%A1%E5%9E%8B%E7%9A%84%E4%B8%8B%E8%BD%BD#%E4%BD%BF%E7%94%A8Git%E4%B8%8B%E8%BD%BD%E6%A8%A1%E5%9E%8B)； 23 | 2）如果您通过 huggingface 下载模型，请确保您可以正常访问 huggingface。 24 | 25 | #### 环境安装 26 | 27 | - python>=3.8 28 | - transformers==4.33.2 29 | 30 | ```plain 31 | git clone https://github.com/codefuse-ai/Test-Agent 32 | cd Test-Agent 33 | pip install -r requirements.txt 34 | ``` 35 | 36 | 在开始运行 TestGPT-7B 模型之前，请确保你的执行环境拥有大约 14GB 的显存。 37 | 38 | ### 启动服务 39 | 40 | 项目提供了网页端快速搭建 UI 的能力能够更直观的展示模型交互和效果，我们可以使用简单的几个命令把前端页面唤醒并实时调用模型能力。在项目目录下，依次启动以下服务： 41 | 42 | 1.**启动 controller** 43 | ![controller](https://github.com/codefuse-ai/Test-Agent/assets/103973989/e68ce187-c9f1-4ce8-9d59-ff9d8348d0ac) 44 | python3 -m chat.server.controller 45 | 46 | 2.**启动模型 worker** 47 | ![work](https://github.com/codefuse-ai/Test-Agent/assets/103973989/073e4e79-4005-4c98-87f7-0eaa0b2b1e22) 48 | python3 -m chat.server.model_worker --model-path models/TestGPT-7B --device mps 49 | 50 | （models/TestGPT-7B 为实际模型文件路径） 51 | 52 | 对于启动方式，可以按需选择以下几种配置选项： 53 | 54 | - --device mps 用于在 Mac 电脑上开启 GPU 加速的选项（Apple Silicon 或 AMD GPUs）； 55 | - --device xpu 用于在 Intel XPU 上开启加速的选项（Intel Data Center and Arc A-Series GPUs）； 56 | - 需安装[Intel Extension for PyTorch](https://intel.github.io/intel-extension-for-pytorch/xpu/latest/tutorials/installation.html) 57 | - 设置 OneAPI 环境变量：source /opt/intel/oneapi/setvars.sh 58 | - --device npu 用于在华为 AI 处理器上开启加速的选项； 59 | - 需安装[Ascend PyTorch Adapter](https://github.com/Ascend/pytorch) 60 | - 设置 CANN 环境变量：source /usr/local/Ascend/ascend-toolkit/set_env.sh 61 | - --device cpu 单独使用 CPU 运行的选项，不需要 GPU； 62 | - --num-gpus 2 指定并发 gpu 运行的选项。 63 | 64 | 3. **启动 web 服务** 65 | python3 -m chat.server.gradio_testgpt 66 | ![web](https://github.com/codefuse-ai/Test-Agent/assets/103973989/340dae35-573b-4046-a3e8-e87a91453601) 67 | 待服务准备就绪后，我们可以打开本地启动的 web 服务地址 http://0.0.0.0:7860 ，就能看到完整的前端页面了。在页面下方包含了【单测生成】和【Assert 补全】的两个例子，点击按钮后会自动生成一段样例文本到输入框中，点击 Send 按钮就会触发模型运行，之后耐心等待一段时间后（运行时间视本机性能而定）即可看到完整的回答了。 68 | ![demo](https://github.com/codefuse-ai/Test-Agent/assets/103973989/fd24274c-729b-4ce7-8763-a083b39300fb) 69 | -------------------------------------------------------------------------------- /docs/docs/developer-docs/CodeFuse-DevOps-Model/main/quickstart.zh-CN.md: -------------------------------------------------------------------------------- 1 | --- 2 | store: 3 | title: CodeFuse-DevOps-Model 4 | version: main 5 | group: 6 | title: 🌱 CodeFuse-DevOps-Model 7 | order: -1 8 | title: 快速使用 9 | order: 1 10 | toc: content 11 | github: https://github.com/codefuse-ai/CodeFuse-DevOps-Model 12 | --- 13 | 14 | ## 依赖安装 15 | 16 | 需要先 PIP 安装一下 Github 地址下的 requirement.txt 中的包，可以参考一下代码 17 | pip install -r requirements.txt 18 | 19 | ## 模型下载 20 | 21 | 模型下载相关信息如下： 22 | 🤗 Huggingface 地址 23 | 24 | | - | 基座模型 | 对齐模型 | 25 | | --- | --------------------- | --------------------- | 26 | | 7B | DevOps-Model-7B-Base | DevOps-Model-7B-Chat | 27 | | 14B | DevOps-Model-14B-Base | DevOps-Model-14B-Chat | 28 | 29 | 🤖 ModelScope 地址 30 | | - | 基座模型 |对齐模型| 31 | | -- | ---------- | ------- | 32 | |7B | DevOps-Model-7B-Base |DevOps-Model-7B-Chat| 33 | |14B| DevOps-Model-14B-Base| DevOps-Model-14B-Chat| 34 | 35 | 找到自己想要下载的 Chat 模型版本，当前提供了 7B 和 14B 的模型 36 | 37 | ## 模型使用 38 | 39 | 根据以下代码来和 Chat 模型进行交互 40 | 41 | ``` 42 | from transformers import AutoModelForCausalLM, AutoTokenizer 43 | from transformers.generation import GenerationConfig 44 | 45 | tokenizer = AutoTokenizer.from_pretrained("path_to_DevOps-Model-Chat", trust_remote_code=True) 46 | 47 | model = AutoModelForCausalLM.from_pretrained("path_to_DevOps-Model-Chat", device_map="auto", trust_remote_code=True, bf16=True).eval() 48 | 49 | # 指定 generation_config 50 | model.generation_config = GenerationConfig.from_pretrained("path_to_DevOps-Model-Chat", trust_remote_code=True) 51 | 52 | # 第一轮对话 53 | resp, hist = model.chat(query='你是谁', tokenizer=tokenizer, history=None) 54 | print(resp) 55 | # 我是 DevOps-Model，一个由蚂蚁集团平台技术事业群风险智能团队和北京大学联合研发的人工智能机器人，可以与用户进行自然语言交互，并协助解答 DevOps 全生命周期中的各种问题。如果您有任何需要协助的问题或者想要进行闲聊，都可以和我交流哦。 56 | 57 | # 第二轮对话 58 | resp2, hist2 = model.chat(query='Java 中 HashMap 和 Hashtable 有什么区别', tokenizer=tokenizer, history=hist) 59 | print(resp2) 60 | # HashMap 和 Hashtable 都是 Java 中常用的哈希表实现，它们的主要区别在于： 61 | # 1. Hashtable 是线程安全的，而 HashMap 不是线程安全的，因此在多线程环境下，Hashtable 的性能更稳定。 62 | # 2. Hashtable 中的方法都是同步的，而 HashMap 的方法不是同步的，因此在多线程环境下，Hashtable 的性能更好。 63 | # 3. Hashtable 中的 key 和 value 都必须实现 Serializable 接口，而 HashMap 中的 key 和 value 可以是任何对象，包括基本数据类型。 64 | # 4. Hashtable 的初始容量是 11，而 HashMap 的初始容量是 16。 65 | # 总之，如果需要在多线程环境下使用哈希表，并且需要保证线程安全，那么应该使用 Hashtable；如果不需要考虑线程安全，或者需要快速地进行哈希表操作，那么应该使用 HashMap。 66 | 67 | # 第三轮对话 68 | resp3, hist3 = model.chat(query='线程安全代表什么', tokenizer=tokenizer, history=hist2) 69 | print(resp3) 70 | # 线程安全是指在多线程环境下，程序能够正确地处理并发访问，并且不会出现数据竞争、死锁、饥饿等异常情况。线程安全的程序可以保证在不同的线程之间共享同一个数据结构时，数据的正确性和一致性。线程安全的实现通常需要使用同步机制，如锁、原子操作等，来保证对共享数据的访问是线程安全的。在 Java 中，可以通过 synchronized 关键字、Lock 接口等机制来实现线程安全。 71 | ``` 72 | -------------------------------------------------------------------------------- /docs/docs/developer-docs/CodeFuse-DevOps-Model/main/codefuseDevopsModel.zh-CN.md: -------------------------------------------------------------------------------- 1 | --- 2 | nav: 3 | title: Docs 4 | order: -1 5 | second: 6 | title: Developer-Docs 7 | order: -1 8 | store: 9 | title: CodeFuse-DevOps-Model 10 | version: main 11 | group: 12 | title: 🌱 CodeFuse-DevOps-Model 13 | index: true 14 | order: -1 15 | title: CodeFuse-DevOps-Model 16 | order: -1 17 | toc: content 18 | github: https://github.com/codefuse-ai/CodeFuse-DevOps-Model 19 | --- 20 | 21 | ## codeFuse-devops-model 22 | 23 | DevOps-Model 是蚂蚁集团联合北京大学发布面向中文 DevOps 领域的大语言模型，通过收集 DevOps 领域相关的专业数据，再针对模型进行语言模型的加训和对齐训练，产出可以帮助工程师在整个开发运维生命周期提效的大模型。弥补当前大模型在 DevOps 领域的缺失，旨在做到有问题，问 DevOps-Model ! 24 | 25 | 当前我们已经开源了 7B 和 14B 两种规格的经过加训得 Base 模型和经过对齐后的 Chat 模型，同时还开源了对应的训练代码，欢迎大家一起合作建设！ 26 | 27 | ## 项目地址 28 | 29 | Github 地址：https://github.com/codefuse-ai/CodeFuse-DevOps-Model/tree/main 30 | 31 | ModelScope 地址： 32 | 33 | - DevOps-Model-7B-Base：https://modelscope.cn/models/codefuse-ai/CodeFuse-DevOps-Model-7B-Base/summary 34 | - DevOps-Model-7B-Chat：https://modelscope.cn/models/codefuse-ai/CodeFuse-DevOps-Model-7B-Chat/summary 35 | - DevOps-Model-14B-Base：https://modelscope.cn/models/codefuse-ai/CodeFuse-DevOps-Model-14B-Base/summary 36 | - DevOps-Model-14B-Chat：https://modelscope.cn/models/codefuse-ai/CodeFuse-DevOps-Model-14B-Chat/summary 37 | 38 | ## 评测考题 39 | 40 | 针对模型评测，最初并没有这样的一个 benchmark 用来 DevOps 领域进行测试，所以我们首先选用了一些通用开源测试中和 DevOps 领域相关的选择题进行测试，具体测试数据如下： 41 | |数据集 |考试科目 |题目总数| 42 | | ---- | --------- | ----- | 43 | |CMMLU |Computer science 204| 44 | |Computer |security |171| 45 | |Machine |learning |122| 46 | |CEval |college programming| 37| 47 | |CEval |computer_architecture| 21| 48 | |CEval |computer_network |19| 49 | |总计 |总计题目数 |574| 50 | 51 | ## 评测方式 52 | 53 | 由于都是单选题,我们采用的是选取模型产出的第一个 Token 中四个选项 Token 中得分最高的作为模型对于问题的回答。同时我们还测试了 Zero-shot 和 Five-shot 的结果。 54 | 55 | ## 评测结果 56 | 57 | ![](https://mdn.alipayobjects.com/huamei_bvbxju/afts/img/A*8RCfS6OraH4AAAAAAAAAAAAADlHYAQ/original) 58 | 59 | 具体的得分如下表所示： 60 | |参数量级| 模型 |模型大小 |Zero-shot 得分 |Five-shot 得分| 61 | | - | ---- | --- | ---- | ---- | 62 | |10+ B| DevOps-Model-14B-Base |14B |70.73 |73.00| 63 | |10+ B|Qwen-14B-Base |14B |69.16| 71.25| 64 | |10+ B|Baichuan2-13B-Base |13B |55.75| 61.15| 65 | |10+ B|DevOps-Model-14B-Chat| 14B |74.04 |75.96| 66 | |10+ B|Qwen-14B-Chat |14B |69.16| 70.03| 67 | |10+ B|Baichuan2-13B-Chat |13B |52.79 |55.23| 68 | |7B| DevOps-Model-7B-Base| 7B |62.72| 62.02| 69 | |7B|Qwen-7B-Base| 7B| 55.75| 56.0| 70 | |7B|Baichuan2-7B-Base| 7B |49.30| 55.4| 71 | |7B|Internlm-7B-Base |7B |47.56 |52.6| 72 | |7B|DevOps-Model-7B-Chat| 7B |62.20| 64.11| 73 | |7B|Qwen-7B-Chat| 7B |46.00 |52.44| 74 | |7B|Baichuan2-7B-Chat| 7B| 52.26| 54.46| 75 | |7B|Internlm-7B-Chat |7B |52.61 |55.75| 76 | -------------------------------------------------------------------------------- /docs/docs/developer-docs/CodeFuse-ChatBot/master/codefusechatbot.zh-CN.md: -------------------------------------------------------------------------------- 1 | --- 2 | store: 3 | title: CodeFuse-ChatBot 4 | version: master 5 | group: 6 | title: 🌱 CodeFuse-ChatBot 7 | index: true 8 | order: -1 9 | title: CodeFuse-ChatBot 10 | toc: content 11 | github: https://github.com/codefuse-ai/codefuse-chatbot 12 | --- 13 | 14 | DevOps-ChatBot 是由蚂蚁 CodeFuse 团队开发的开源 AI 智能助手，致力于简化和优化软件开发生命周期中的各个环节。该项目结合了 Multi-Agent 的协同调度机制，并集成了丰富的工具库、代码库、知识库和沙盒环境，使得 LLM 模型能够在 DevOps 领域内有效执行和处理复杂任务。 15 | 16 | ## 📜 目录 17 | 18 | - [🤝 介绍](#-介绍) 19 | - [🎥 演示视频](#-演示视频) 20 | - [🧭 技术路线](#-技术路线) 21 | 22 | ## 🤝 介绍 23 | 24 | 💡 本项目旨在通过检索增强生成（Retrieval Augmented Generation，RAG）、工具学习（Tool Learning）和沙盒环境来构建软件开发全生命周期的 AI 智能助手，涵盖设计、编码、测试、部署和运维等阶段。 逐渐从各处资料查询、独立分散平台操作的传统开发运维模式转变到大模型问答的智能化开发运维模式，改变人们的开发运维习惯。 25 | 26 | 本项目核心差异技术、功能点： 27 | 28 | - **🧠 智能调度核心：** 构建了体系链路完善的调度核心，支持多模式一键配置，简化操作流程。 [使用说明](/zh-CN/docs/api-docs/MuAgent/overview/multi-agent) 29 | - **💻 代码整库分析：** 实现了仓库级的代码深入理解，以及项目文件级的代码编写与生成，提升了开发效率。 30 | - **📄 文档分析增强：** 融合了文档知识库与知识图谱，通过检索和推理增强，为文档分析提供了更深层次的支持。 31 | - **🔧 垂类专属知识：** 为 DevOps 领域定制的专属知识库，支持垂类知识库的自助一键构建，便捷实用。 32 | - **🤖 垂类模型兼容：** 针对 DevOps 领域的小型模型，保证了与 DevOps 相关平台的兼容性，促进了技术生态的整合。 33 | 34 | 🌍 依托于开源的 LLM 与 Embedding 模型，本项目可实现基于开源模型的离线私有部署。此外，本项目也支持 OpenAI API 的调用。[接入 Demo](/zh-CN/docs/developer-docs/CodeFuse-ChatBot/master/fastchat) 35 | 36 | 👥 核心研发团队长期专注于 AIOps + NLP 领域的研究。我们发起了 Codefuse-ai 项目，希望大家广泛贡献高质量的开发和运维文档，共同完善这套解决方案，以实现“让天下没有难做的开发”的目标。 37 | 38 |
39 | 图片 40 |
41 | 42 | ## 🎥 演示视频 43 | 44 | 为了帮助您更直观地了解 Codefuse-ChatBot 的功能和使用方法，我们录制了一系列演示视频。您可以通过观看这些视频，快速了解本项目的主要特性和操作流程。 45 | 46 | - 知识库导入和问答：[演示视频](https://www.youtube.com/watch?v=UGJdTGaVnNY&t=2s&ab_channel=HaotianZhu) 47 | - 本地代码库导入和问答：[演示视频](https://www.youtube.com/watch?v=ex5sbwGs3Kg) 48 | 49 | ## 🧭 技术路线 50 | 51 |
52 | Image 53 |
54 | 55 | - 🧠 **Multi-Agent Schedule Core:** 多智能体调度核心，简易配置即可打造交互式智能体。 56 | - 🕷️ **Multi Source Web Crawl:** 多源网络爬虫，提供对指定 URL 的爬取功能，以搜集所需信息。 57 | - 🗂️ **Data Processor:** 数据处理器，轻松完成文档载入、数据清洗，及文本切分，整合不同来源的数据。 58 | - 🔤 **Text Embedding & Index:**：文本嵌入索引，用户可以轻松上传文件进行文档检索，优化文档分析过程。 59 | - 🗄️ **Vector Database & Graph Database:** 向量与图数据库，提供灵活强大的数据管理解决方案。 60 | - 📝 **Prompt Control & Management:**：Prompt 控制与管理，精确定义智能体的上下文环境。 61 | - 🚧 **SandBox:**：沙盒环境，安全地执行代码编译和动作。 62 | - 💬 **LLM:**：智能体大脑，支持多种开源模型和 LLM 接口。 63 | - 🛠️ **API Management:：** API 管理工具，实现对开源组件和运维平台的快速集成。 64 | 65 | 具体实现明细见：[技术路线明细](/zh-CN/docs/developer-docs/CodeFuse-ChatBot/master/roadmap) 66 | -------------------------------------------------------------------------------- /.dumi/theme/slots/Publication/index.less: -------------------------------------------------------------------------------- 1 | @import (reference) '.dumi/theme/styles/variables.less'; 2 | 3 | .@{prefix}-Contribution { 4 | margin: -@s-header-height auto 0 auto; 5 | height: 100%; 6 | display: flex; 7 | flex-direction: column; 8 | justify-content: center; 9 | 10 | @media @mobile { 11 | margin-top: -@s-header-height-m - 20; 12 | padding-top: 160px; 13 | height: 660px; 14 | } 15 | 16 | +* { 17 | position: relative; 18 | } 19 | 20 | .banner { 21 | position: relative; 22 | padding-top: 30px; 23 | height: 380px; 24 | width: 100%; 25 | box-sizing: border-box; 26 | // background: linear-gradient(-225deg, #7085B6 0%, #87A7D9 50%, #DEF3F8 100%); 27 | background-image: url('https://mdn.alipayobjects.com/huamei_bvbxju/afts/img/A*EamVR52XxpUAAAAAAAAAAAAADlHYAQ/original'); 28 | background-repeat: no-repeat; 29 | background-size: cover; 30 | background-position: center; 31 | display: flex; 32 | align-items: center; 33 | justify-content: center; 34 | flex-direction: column; 35 | 36 | .bannerContent { 37 | width: 1200px; 38 | padding: 0px 24px; 39 | margin-top: 30px; 40 | width: 900px; 41 | color: #f8f9fa; 42 | font-size: 2.7rem; 43 | display: flex; 44 | justify-content: center; 45 | 46 | 47 | img { 48 | width: 35%; 49 | } 50 | } 51 | } 52 | 53 | .content { 54 | width: 1000px; 55 | margin: 0 auto; 56 | color: #e4e9ec; 57 | line-height: 30px; 58 | padding: 24px; 59 | 60 | .contentTitle { 61 | margin: 40px 0 32px; 62 | text-align: left; 63 | font-weight: 400; 64 | font-size: 32px; 65 | 66 | @{dark-selector} & { 67 | font-size: 32px; 68 | color: #5c6cf7; 69 | font-weight: 600; 70 | } 71 | } 72 | 73 | .contentText { 74 | font-size: 16px; 75 | letter-spacing: 0.61px; 76 | padding-left: 20px; 77 | line-height: 30px; 78 | h2 { 79 | font-style: italic; 80 | font-weight: 700; 81 | vertical-align: baseline; 82 | 83 | } 84 | .titleCon { 85 | font-style: italic; 86 | font-weight: 700; 87 | vertical-align: baseline; 88 | } 89 | .descCon { 90 | margin: 0; 91 | font-style: italic; 92 | font-weight: 400; 93 | color: #b6b9bb; 94 | @{dark-selector} & { 95 | color: #242424; 96 | } 97 | } 98 | 99 | @{dark-selector} & { 100 | font-size: 16px; 101 | color: #1b1b1b; 102 | } 103 | 104 | li { 105 | margin: 10px 0; 106 | } 107 | } 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /docs/docs/api-docs/MuAgent/overview/agent-flow.en-US.md: -------------------------------------------------------------------------------- 1 | --- 2 | group: 3 | title: ❤️ Codefuse-muAgent 4 | order: -1 5 | title: Agent Flow 6 | order: 0 7 | toc: content 8 | --- 9 | 10 | ## Introduction to Core Connectors 11 | 12 | To facilitate everyone's understanding of the entire CoAgent link, we use a Flow format to detail how to build through configuration settings. 13 | 14 |
15 | 图片 16 |
17 | 18 |
Below, we first introduce the related core components
19 | 20 | ### Agent 21 | 22 | On the design level of the Agent, we provide four basic types of Agents, with Role settings for these Agents that can meet the interactions and uses of various common scenarios: 23 | 24 | 1. BaseAgent: Provides basic question answering, tool usage, and code execution functions, and realizes input => output according to the Prompt format. 25 | 2. ReactAgent: Provides standard React functionality, accomplishing current tasks based on questions. 26 | 3. ExecutorAgent: Performs sequential execution of task lists, completing related tasks according to plans arranged by the User or the previous Agent. 27 | 4. SelectorAgent: Provides the function of selecting an Agent, choosing the appropriate Agent to respond according to the question from the User or the previous Agent. After output, the message is pushed into the memory pool, which will later be managed by the Memory Manager. 28 | 29 | It selects the appropriate Agent to respond based on the question from the User or the previous Agent. After output, the message is pushed into the memory pool, which is subsequently managed by the Memory Manager. 30 | 31 | ### Chain 32 | 33 | Basic Chain: BaseChain, connects the interactions of agents, manages the related messages and memory. 34 | 35 | ### Phase 36 | 37 | Basic Phase: BasePhase, connects the interactions of chains, and manages the related messages and memory. 38 | 39 | ### Prompt Manager 40 | 41 | The prompt creation for each agent in the Mutli-Agent link: 42 | 43 | - By setting simple prompt_input_keys and prompt_output_keys, the preset Prompt Context creation logic can be followed to quickly configure the agent prompt. 44 | - It is also possible to design a new key-context in the prompt manager module, achieving personalized Agent Prompt. 45 | 46 | ### Memory Manager 47 | 48 | Mainly used for the management of chat history: 49 | 50 | - Manages the reading and writing of chat history in a database, including user input, llm output, doc retrieval, code retrieval, search retrieval. 51 | - Summarizes the key information in chat history to create a summary context, which serves as a prompt context. 52 | - Provides a retrieval function to search for information related to the question in the chat history or summary context, assisting with question answering. 53 | -------------------------------------------------------------------------------- /docs/docs/developer-docs/CodeFuse-MFT-VLM/main/quickstart.zh-CN.md: -------------------------------------------------------------------------------- 1 | --- 2 | store: 3 | title: CodeFuse-MFT-VLM 4 | version: main 5 | group: 6 | title: 🌱 CodeFuse-MFT-VLM 7 | order: -1 8 | title: 快速使用 9 | order: 0 10 | toc: content 11 | github: https://github.com/codefuse-ai/CodeFuse-MFT-VLM 12 | --- 13 | 14 | ## Contents 15 | 16 | - [Install](#Install) 17 | - [Datasets](#Datasets) 18 | - [Multimodal Alignment](#Multimodal-Alignment) 19 | - [Visual Instruction Tuning](#Visual-Instruction-Tuning) 20 | - [Evaluation](#Evaluation) 21 | 22 | ## Install 23 | 24 | 请执行 sh init_env.sh 25 | 26 | ## Datasets 27 | 28 | 使用了以下数据集训练模型: 29 | 30 | | 数据集 | 任务种类 | 样本量 | 31 | | ----------------------------------------------- | --------------------------- | ------- | 32 | | synthdog-en | OCR | 800,000 | 33 | | synthdog-zh | OCR | 800,000 | 34 | | cc3m(downsampled) | Image Caption | 600,000 | 35 | | cc3m(downsampled) | Image Caption | 600,000 | 36 | | SBU | Image Caption | 850,000 | 37 | | Visual Genome VQA (Downsampled) | Visual Question Answer(VQA) | 500,000 | 38 | | Visual Genome Region descriptions (Downsampled) | Reference Grouding | 500,000 | 39 | | Visual Genome objects (Downsampled) | Grounded Caption | 500,000 | 40 | | OCR VQA (Downsampled) | OCR and VQA | 500,000 | 41 | 42 | 请到各个数据集的官网上下载这些数据。 43 | 44 | ## Multimodal Alignment 45 | 46 | 请执行 sh scripts/pretrain.sh 或者 sh scripts/pretrain_multinode.sh 47 | 48 | ## Visual Instruction Tuning 49 | 50 | 请执行 sh scripts/finetune.sh 或者 sh scripts/finetune_multinode.sh 51 | 52 | ## Evaluation 53 | 54 | 请执行 llava/eval/ 当中的 python 脚本. 可以通过下面的代码来加载我们预训练的 CodeFuse-VLM-14B: 55 | 56 | ``` 57 | import os 58 | from llava.model.builder import load_mixed_pretrained_model 59 | 60 | model_path = '/pretrained/model/path' 61 | tokenizer, model, image_processor, context_len = load_mixed_pretrained_model(model_path, None, 'qwen-vl-14b', os.path.join(model_path, 'Qwen-VL-visual'), 'cross_attn', os.path.join(model_path, 'mm_projector/mm_projector.bin')) 62 | ``` 63 | 64 | 您也可以先运行下面的脚本来合并各个模型组件：scripts/merge_qwen_vl_weights.sh，然后通过下面的代码加载合并后的模型： 65 | 66 | ``` 67 | from llava.model import LlavaQWenForCausalLM 68 | 69 | model = LlavaQWenForCausalLM.from_pretrained('/path/to/our/pretrained/model') 70 | ``` 71 | 72 | ## CodeFuse-VLM 产品视频 73 | 74 | 这是我们模型支持的产品的视频 75 | 76 | 79 | -------------------------------------------------------------------------------- /docs/docs/developer-docs/CodeFuse-ModelCache/main/quickstart.zh-CN.md: -------------------------------------------------------------------------------- 1 | --- 2 | store: 3 | title: CodeFuse-ModelCache 4 | version: main 5 | group: 6 | title: 🌱 CodeFuse-ModelCache 7 | order: -1 8 | title: 快速开始 9 | order: 0 10 | toc: content 11 | github: https://github.com/codefuse-ai/CodeFuse-ModelCache 12 | --- 13 | 14 | ModelCache 易于使用，只需 1 步骤即可构建缓存测试 Demo 15 | 16 | ## 快速开始 17 | 18 | ### Demo 服务启动 19 | 20 | - 离线模型 bin 文件下载， 参考地址：[https://huggingface.co/shibing624/text2vec-base-chinese/tree/main](https://huggingface.co/shibing624/text2vec-base-chinese/tree/main)，并将下载的 bin 文件，放到 model/text2vec-base-chinese 文件夹中。 21 | - 执行 flask4modelcache_demo.py 启动服务。 22 | 23 | ```shell 24 | cd CodeFuse-ModelCache 25 | ``` 26 | 27 | ```shell 28 | python flask4modelcache_demo.py 29 | ``` 30 | 31 | ### 正常服务启动 32 | 33 | 在启动服务前，应该进行如下环境配置： 34 | 35 | 1. 安装关系数据库 mysql， 导入 sql 创建数据表，sql 文件:`reference_doc/create_table.sql` 36 | 2. 安装向量数据库 milvus 37 | 3. 在配置文件中添加数据库访问信息，配置文件为： 38 | 1. `modelcache/config/milvus_config.ini` 39 | 2. `modelcache/config/mysql_config.ini` 40 | 4. 离线模型 bin 文件下载， 参考地址：[https://huggingface.co/shibing624/text2vec-base-chinese/tree/main](https://huggingface.co/shibing624/text2vec-base-chinese/tree/main)，并将下载的 bin 文件，放到 model/text2vec-base-chinese 文件夹中 41 | 5. 通过 flask4modelcache.py 脚本启动后端服务。 42 | 43 | ## 服务访问 44 | 45 | 当前服务以 restful API 方式提供 3 个核心功能：数据写入，cache 查询和 cache 数据清空。请求 demo 如下： 46 | 47 | ### cache 写入 48 | 49 | ```python 50 | import json 51 | import requests 52 | url = 'http://127.0.0.1:5000/modelcache' 53 | type = 'insert' 54 | scope = {"model": "CODEGPT-1008"} 55 | chat_info = [{"query": [{"role": "system", "content": "你是一个AI代码助手, 你必须提供中立的、无害的答案帮助用户解决代码相关的问题"}, {"role": "user", "content": "你是谁?"}], 56 | "answer": "你好，我是智能助手，请问有什么能帮您!"}] 57 | data = {'type': type, 'scope': scope, 'chat_info': chat_info} 58 | headers = {"Content-Type": "application/json"} 59 | res = requests.post(url, headers=headers, json=json.dumps(data)) 60 | ``` 61 | 62 | ### cache 查询 63 | 64 | ```python 65 | import json 66 | import requests 67 | url = 'http://127.0.0.1:5000/modelcache' 68 | type = 'query' 69 | scope = {"model": "CODEGPT-1008"} 70 | query = [{"role": "system", "content": "你是一个AI代码助手, 你必须提供中立的、无害的答案帮助用户解决代码相关的问题"}, {"role": "user", "content": "你是谁?"}] 71 | data = {'type': type, 'scope': scope, 'query': query} 72 | 73 | headers = {"Content-Type": "application/json"} 74 | res = requests.post(url, headers=headers, json=json.dumps(data)) 75 | ``` 76 | 77 | ### cache 清空 78 | 79 | ```python 80 | import json 81 | import requests 82 | url = 'http://127.0.0.1:5000/modelcache' 83 | type = 'remove' 84 | scope = {"model": "CODEGPT-1008"} 85 | remove_type = 'truncate_by_model' 86 | data = {'type': type, 'scope': scope, 'remove_type': remove_type} 87 | 88 | headers = {"Content-Type": "application/json"} 89 | res = requests.post(url, headers=headers, json=json.dumps(data)) 90 | ``` 91 | -------------------------------------------------------------------------------- /docs/docs/developer-docs/CodeFuse-Query/main/user_case.zh-CN.md: -------------------------------------------------------------------------------- 1 | --- 2 | store: 3 | title: CodeFuse-Query 4 | version: main 5 | group: 6 | title: 🌱 CodeFuse-Query 7 | order: -1 8 | title: 用户案例 9 | order: 4 10 | toc: content 11 | github: https://github.com/codefuse-ai/CodeFuse-Query 12 | --- 13 | 14 | # 使用场景 15 | 16 | ## 查询代码特征 17 | 18 | 小开发同学想知道 Repo A 里面使用了哪些 String 型的变量，所以他写了一个 Gödel 如下，交给 CodeFuse-Query 系统给他返回了结果。 19 | 20 | ```rust 21 | // script 22 | use coref::java::* 23 | 24 | fn out(var: string) -> bool { 25 | for(v in Variable(JavaDB::load("coref_java_src.db"))) { 26 | if (v.getType().getName() = "String" && var = v.getName()) { 27 | return true 28 | } 29 | } 30 | } 31 | 32 | fn main() { 33 | output(out()) 34 | } 35 | ``` 36 | 37 | 类似需求：查询：类，函数，变量，返回值，调用图，类继承等等。 38 | 39 | ## 代码规则检查器 40 | 41 | 小 TL 同学发现团队总是写出很多类似的 Bug A，**他想针对 Bug A 制定一个代码规则和其检查器**，并在 CodeReview 阶段做个卡点。小 TL 通过在 CodeFuse-Query 平台上面编写了一段分析 Query，在平台上面测试符合要求，把这段分析 Query 固化下来作为一个代码规则，并上线到了 CodeReview/CI 阶段。从此这个 Bug 再也没发生过了。 42 | 类似需求：编写静态缺陷扫描规则进行代码风险拦截。 43 | 44 | ## 获取统计数据 45 | 46 | 小研究发现传统的代码复杂度指标很难准确地衡量代码的复杂情况，通过学习国际先进经验加上自我灵光一闪，设计了一套复杂度指标和算法。通过 Gödel 实现出来以后，**发现不怎么优化就已经性能非常高了**，很快就应用到了 10 几种语言，11+万个仓库当中去了。马上就对代码仓库整体的复杂度有了深入的了解。相比较以前需要自己解析代码，分析语法树，对接系统，**不知道方便了多少。** 47 | 类似需求：代码统计，代码度量，算法设计，学术研究。 48 | 49 | # 应用领域 50 | 51 | 目前，CodeFuse-Query 在蚂蚁集团已经支持 **CodeFuse 大语言模型数据清洗**、**代码度量评估**、**研发风险控制**、**隐私安全分析**、**代码智能**、**终端包大小治理 **等多个场景的落地应用，服务月均调用量超过百万。 52 | 53 | ## 高质量代码数据清洗 - CodeFuse 代码大模型 54 | 55 | CodeFuse 代码大模型是蚂蚁集团对外开源的处理代码相关问题的模型，对于 CodeFuse 大语言模型而言，训练的数据质量直接影响模型的推理结果。低质量的代码数据会直接污染语言模型的输出，例如：模型可能会学习到错误的代码模式，从而生成错误的代码；数据中只包含某种编程语言的代码，模型可能无法很好地适应其他编程语言的代码。 56 | 为了把控进入模型的代码数据质量，进而提升模型的推理能力。我们基于蚂蚁程序分析团队多年的实践积累结合业界共识，梳理了高质量代码的定义方式，并利用已有程序分析技术实现了自动化、大规模的代码数据清洗。 57 | CodeFuse-Query 为 CodeFuse 代码大模型提供了以下数据清洗能力： 58 | 59 | - 高质量代码数据清洗：对代码数据进行清洗，包括对 Python，Java，JavaScript，TypeScript，Go，C，C++ 7 种语言进行漏洞扫描，对语言种类 / star 数进行筛选，过滤有效代码行数为 0 的数据等。目前已沉淀清洗后的 GitHub 和蚂蚁内部代码数据总共约 **2TB**。 60 | - 代码画像：实现对大规模代码进行高性能多维度的自动标注，支持 Java, Scala, Kotlin, JavaScript, JSX, TypeScript, TSX, Vue, Python, Go 等 **10** 种语言，**77** 种通用标签，**40** 种蚂蚁特有标签，共 **117** 种标签。目前自动标注性能能够达到 **40MB/s**。 61 | - 其他原子能力 62 | - 高级代码特征提取，包括提取 AST（抽象语法树），DFG（数据流图）数据等。目前 AST 信息已用于 SFT 训练，准确率 97% 左右。 63 | - 代码片段识别，用于针对文本数据中的代码进行提取，方便进行代码格式化或加上 Markdown 格式： 64 | - 文本提取代码：从文本中提取代码块信息，支持主流语言的解析，函数及类定义，仅验证二分类问题，就是说仅验证文本是否含有代码块准确率 83% 左右。 65 | - 识别代码片段的编程语言种类：识别任意代码片段的编程语言种类，支持 30+ 种语言，准确率 80%左右。 66 | - 代码注释对提取：支持提取方法级别的注释-代码对信息，覆盖 **15 种** GitHub 最流行的语言，用于 Text To Code/Code To Text 的 SFT 训练。 67 | 68 | ## 变更分析-优酷服务端研发效能 69 | 70 | 优酷质量保障团队从 2023 年开始针对服务端精准测试的探索，经过半年的技术沉淀和体系搭建，形成了具备**变更内容识别、变更影响分析、测试能力推荐、测试覆盖评估**的精准测试体系。 71 | 在此过程中，CodeFuse-Query 能提供的能力主要有： 72 | 73 | - 根据代码变更内容（文件+行号），分析出影响的对象：方法、入口（http 入口、hsf 入口）、调用链路（从入口到变更方法的所有调用链路）、数据库操作（表、操作类型） 74 | - 结合线上动态调用链路（方法链路）、CodeFuse-Query 静态分析调用链路的影响面精准分析能力，提升变更分析影响面的有效性、准备率 75 | 76 | 到目前为止，优酷已通过 CodeFuse-Query 接入所有核心应用，并基于静态分析采集数据，构建了服务端完整的代码知识库和流量知识库。 77 | -------------------------------------------------------------------------------- /.dumi/theme/slots/AboutDocs/index.less: -------------------------------------------------------------------------------- 1 | @import (reference) '.dumi/theme/styles/variables.less'; 2 | 3 | .@{prefix}-about { 4 | margin: -@s-header-height auto 0 auto; 5 | // margin: 0 auto ; 6 | height: 100%; 7 | display: flex; 8 | flex-direction: column; 9 | justify-content: center; 10 | 11 | @media @mobile { 12 | margin-top: -@s-header-height-m - 20; 13 | padding-top: 160px; 14 | height: 660px; 15 | } 16 | 17 | +* { 18 | position: relative; 19 | } 20 | 21 | .banner { 22 | position: relative; 23 | padding-top: 30px; 24 | height: 380px; 25 | width: 100%; 26 | box-sizing: border-box; 27 | background-image: url('https://mdn.alipayobjects.com/huamei_bvbxju/afts/img/A*wCCKR7plqDMAAAAAAAAAAAAADlHYAQ/original'); 28 | background-repeat: no-repeat; 29 | background-size: cover; 30 | background-position: center; 31 | display: flex; 32 | align-items: center; 33 | justify-content: center; 34 | flex-direction: column; 35 | 36 | .bannerContent { 37 | padding: 0px 24px; 38 | margin-top: 30px; 39 | width: 900px; 40 | color: #f8f9fa; 41 | font-size: 2.7rem; 42 | display: flex; 43 | justify-content: center; 44 | 45 | img { 46 | width: 35%; 47 | } 48 | } 49 | } 50 | 51 | p { 52 | font-weight: 500; 53 | text-align: center; 54 | line-height: 72px; 55 | margin-top: 0px; 56 | font-size: 36px; 57 | color: #ffffff; 58 | letter-spacing: 1.21px; 59 | 60 | @{dark-selector} & { 61 | color: @c-text-secondary-dark; 62 | } 63 | 64 | @media @mobile { 65 | font-size: 16px; 66 | } 67 | } 68 | 69 | &-actions { 70 | margin-top: 48px; 71 | display: flex; 72 | justify-content: center; 73 | 74 | >a { 75 | display: inline-block; 76 | height: 52px; 77 | font-size: 18px; 78 | line-height: 52px; 79 | text-decoration: none; 80 | min-width: 168px; 81 | 82 | border-radius: 16px; 83 | box-sizing: border-box; 84 | transition: opacity 0.2s; 85 | 86 | @media @mobile { 87 | font-size: 16px; 88 | height: 42px; 89 | line-height: 40px; 90 | min-width: 128px; 91 | } 92 | 93 | &:hover { 94 | opacity: 0.8; 95 | } 96 | 97 | &:not(:first-child) { 98 | margin-inline-start: 48px; 99 | color: @c-primary; 100 | border: 2px solid @c-primary; 101 | 102 | @{dark-selector} & { 103 | color: @c-primary-dark; 104 | border-color: @c-primary-dark; 105 | } 106 | } 107 | 108 | &:first-child { 109 | color: #fff; 110 | background-color: @c-primary; 111 | 112 | @{dark-selector} & { 113 | background-color: @c-primary-dark; 114 | } 115 | } 116 | } 117 | } 118 | } 119 | -------------------------------------------------------------------------------- /.dumi/theme/slots/DevOps/index.less: -------------------------------------------------------------------------------- 1 | .devOps { 2 | margin-top: 71px; 3 | display: flex; 4 | justify-content: center; 5 | 6 | .devOps-center { 7 | display: flex; 8 | justify-content: center; 9 | flex-direction: column; 10 | width: 1200px; 11 | 12 | .DevOpsContent { 13 | width: 100%; 14 | 15 | .DevOpsUl { 16 | display: flex; 17 | justify-content: space-between; 18 | flex-direction: row; 19 | flex: 1; 20 | margin-top: 83px; 21 | padding: 0; 22 | list-style: none; 23 | 24 | 25 | .DevOpsLi { 26 | position: relative; 27 | width: 379px; 28 | height: 383px; 29 | border-radius: 33px; 30 | cursor: pointer; 31 | 32 | img { 33 | position: absolute; 34 | width: 379px; 35 | height: 383px; 36 | border-radius: 33px; 37 | } 38 | 39 | .title { 40 | position: absolute; 41 | font-size: 36px; 42 | color: #ffffff; 43 | letter-spacing: 1.21px; 44 | font-weight: 600; 45 | top: 38px; 46 | left: 43px; 47 | 48 | @{dark-selector} & { 49 | color: #171616; 50 | } 51 | } 52 | 53 | .desc { 54 | display: none; 55 | position: absolute; 56 | width: 323px; 57 | font-size: 16px; 58 | color: #ffffff; 59 | letter-spacing: 1px; 60 | top: 38px; 61 | left: 31px; 62 | } 63 | 64 | .more { 65 | display: none; 66 | font-size: 16px; 67 | position: absolute; 68 | color: #e1e2ff; 69 | bottom: 46px; 70 | left: 31px; 71 | 72 | .anticon { 73 | svg { 74 | width: 30px; 75 | height: 23px; 76 | } 77 | 78 | position: absolute; 79 | margin-left: 8px; 80 | width: 25px; 81 | height: 25px; 82 | } 83 | } 84 | 85 | &:hover { 86 | @{dark-selector} & { 87 | background-image: linear-gradient(138deg, #5aefff -10%, #6646f9 42%, #ffcf8a 99%); 88 | border-radius: 33px; 89 | 90 | } 91 | 92 | background-image: linear-gradient(138deg, #5aefff -10%, #502fe6 42%, #92764d 99%); 93 | border-radius: 33px; 94 | 95 | img { 96 | display: none; 97 | } 98 | 99 | .title { 100 | display: none; 101 | } 102 | 103 | .desc { 104 | display: block; 105 | } 106 | 107 | .more { 108 | display: block; 109 | } 110 | } 111 | } 112 | } 113 | } 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /.dumi/theme/slots/CodeAnalysis/index.less: -------------------------------------------------------------------------------- 1 | .codeAnalysis { 2 | margin-top: 71px; 3 | display: flex; 4 | justify-content: center; 5 | 6 | .codeAnalysis-center { 7 | display: flex; 8 | justify-content: space-between; 9 | flex-direction: row; 10 | width: 1200px; 11 | 12 | img { 13 | width: 614px; 14 | height: 383px; 15 | border-radius: 25px; 16 | } 17 | 18 | .codeAnalysisContent { 19 | margin-left: 57px; 20 | 21 | .codeAnalysisTitle { 22 | margin: 0 auto; 23 | text-align: left; 24 | min-width: 343px; 25 | font-size: 40px; 26 | color: #ffffff; 27 | letter-spacing: 1.35px; 28 | font-weight: 600; 29 | background-image: linear-gradient(90deg, #d8d8d8 0%, #545eff 100%); 30 | display: inline-block; 31 | background-clip: text; 32 | -webkit-background-clip: text; 33 | -webkit-text-fill-color: transparent; 34 | 35 | .line { 36 | margin-top: 19px; 37 | background-image: linear-gradient(90deg, #e5b2ca 0%, #7546f3 100%); 38 | height: 4px; 39 | width: 323px; 40 | } 41 | 42 | @{dark-selector} & { 43 | background-image: linear-gradient(90deg, #e5b2ca 0%, #545eff 100%); 44 | } 45 | } 46 | 47 | .desc { 48 | width: 542px; 49 | margin-top: 48px; 50 | font-size: 16px; 51 | color: #ffffff; 52 | letter-spacing: 1px; 53 | opacity: 0.8; 54 | 55 | @{dark-selector} & { 56 | color: #171616; 57 | } 58 | } 59 | 60 | .buttom { 61 | cursor: pointer; 62 | width: 150px; 63 | height: 37px; 64 | background-image: url("https://mdn.alipayobjects.com/huamei_bvbxju/afts/img/A*eMQHT670srQAAAAAAAAAAAAADlHYAQ/original"); 65 | background-repeat: no-repeat; 66 | background-size: contain; 67 | background-position: center; 68 | border-radius: 7px; 69 | margin-top: 32px; 70 | color: #a8abff; 71 | display: flex; 72 | align-items: center; 73 | justify-content: center; 74 | position: relative; 75 | 76 | .anticon { 77 | svg { 78 | width: 30px; 79 | height: 23px; 80 | position: absolute; 81 | bottom: 5px; 82 | } 83 | 84 | margin-left: 8px; 85 | width: 25px; 86 | height: 25px; 87 | } 88 | 89 | &:hover { 90 | background-image: url("https://mdn.alipayobjects.com/huamei_bvbxju/afts/img/A*Pm85SKQpQ88AAAAAAAAAAAAADlHYAQ/original"); 91 | background-repeat: no-repeat; 92 | background-size: contain; 93 | background-position: center; 94 | color: #fff; 95 | 96 | @{dark-selector} & { 97 | color: #7031FD; 98 | } 99 | } 100 | } 101 | } 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /docs/contribution/pr.en-US.md: -------------------------------------------------------------------------------- 1 | --- 2 | group: 3 | title: ❤️ Contribution Guide 4 | order: -1 5 | title: Pull Request 6 | order: 1 7 | toc: content 8 | --- 9 | 10 | ## Contribution 11 | 12 | ### Pre-Checklist 13 | 14 | - Confirm if you have checked the publicly available document information, such as documents, issues, and discussions (GitHub features). 15 | - Find the GitHub issue you wish to address. If it does not exist, create an issue or a draft PR and request a review from the maintainers. 16 | - Check for related, similar, or duplicate pull requests. 17 | - Create a draft pull request. 18 | - Complete the description in the PR template. 19 | - Link any GitHub issues your PR resolves. 20 | 21 | ### Description 22 | 23 | A concise language should be used to describe what the PR achieves, with specific standards available in the [Commit Format Standards](#commit-format-standards). 24 | 25 | ### Related Issue 26 | 27 | `#xx` if applicable 28 | 29 | ### Test Code with Result 30 | 31 | Please provide relevant test code if necessary. 32 | 33 | ## Commit Format Standards 34 | 35 | Commit is divided into a "title" and a "body." In principle, the title is all in lowercase. The first letter of the body should be capitalized. 36 | 37 | ### Title 38 | 39 | The title of the commit message: `[]() (#pr)` 40 | 41 | ### type options (all lowercase) 42 | 43 | Type of this commit, constrained to the following: 44 | 45 | - fix: bug fixes 46 | - feature: new features 47 | - feature-wip: Features in development, such as parts of a function. 48 | - improvement: Optimization and enhancement of existing features 49 | - style: Code style adjustments 50 | - typo: Typos in code or documentation 51 | - refactor: Code refactoring (not involving feature changes) 52 | - performance/optize: Performance optimization 53 | - test: Adding or repairing unit tests 54 | - deps: Modifications to third-party dependencies 55 | - community: Modifications related to the community, such as editing GitHub Issue templates, etc. 56 | Notes: 57 | If multiple types appear in one commit, add multiple types. 58 | If code refactoring leads to performance improvements, you can add both [refactor][optimize] 59 | It is not allowed to have types other than those listed above. If necessary, new types should be added to this document. 60 | 61 | ### scope options 62 | 63 | The module scope involved in this submission. As there are many functional modules, only a part is listed here, which will be continuously improved according to needs. 64 |
For example, the framework of a chatbot: 65 | 66 | - connector 67 | - codechat 68 | - sandbox 69 | - ... 70 | Notes: 71 | Try to use options already listed in this document. If you need to add a new option, please update this document promptly. 72 | 73 | ### subject content 74 | 75 | The title should clearly explain the main content of the submission. 76 | 77 | ## Example 78 | 79 | Coming soon 80 | 81 | ## Reference 82 | 83 | [doris-commit-format](https://doris.apache.org/zh-CN/community/how-to-contribute/commit-format-specification) 84 | -------------------------------------------------------------------------------- /docs/contribution/issue.en-US.md: -------------------------------------------------------------------------------- 1 | --- 2 | group: 3 | title: ❤️ Contribution Guide 4 | order: 0 5 | title: Issue Report 6 | order: 0 7 | toc: content 8 | --- 9 | 10 | ## Issue Type 11 | 12 | Issues can be categorized into three types: 13 | 14 | - Bug: Issues where code or execution examples contain bugs or lack dependencies, resulting in incorrect execution. 15 | - Documentation: Discrepancies in documentation, inconsistencies between documentation content and code, etc. 16 | - Feature: New functionalities that evolve from the current codebase. 17 | 18 | ## Issue Template 19 | 20 | ### Issue: Bug Template 21 | 22 | **Checklist before submitting an issue** 23 |
Please confirm that you have checked the document, issues, discussions (GitHub feature), and other publicly available documentation. 24 | 25 | - I have searched through all documentation related to Codefuse. 26 | - I used GitHub search to find a similar issue, but did not find one. 27 | - I have added a very descriptive title for this issue. 28 | 29 | **System Information** 30 |
Please confirm your operating system, such as mac-xx, windows-xx, linux-xx. 31 | 32 | **Code Version** 33 |
Please confirm the code version or branch, such as master, release, etc. 34 | 35 | **Problem Description** 36 |
Describe the problem you encountered, what you want to achieve, or the bug encountered during code execution. 37 | 38 | **Code Example** 39 |
Attach your execution code and relevant configuration to facilitate rapid intervention and reproduction. 40 | 41 | **Error Information, Logs** 42 |
The error logs and related information after executing the above code example. 43 | 44 | **Related Dependencies** 45 |
Taking the chatbot project as an example: 46 | 47 | - connector 48 | - codechat 49 | - sandbox 50 | - ... 51 | 52 | ### Issue: Documentation Template 53 | 54 | **Issue with current documentation:** 55 |
Please point out any problems, typos, or confusing points in the current documentation. 56 | 57 | **Idea or request for content** 58 |
What do you think would be a reasonable way to express the documentation? 59 | 60 | ### Issue: Feature Template 61 | 62 | **Checklist before submitting an issue** 63 |
Please confirm that you have checked the document, issues, discussions (GitHub feature), and other publicly available documentation. 64 | 65 | - I have searched through all documentation related to Codefuse. 66 | - I used GitHub Issue search to find a similar issue, but did not find one. 67 | - I have added a very descriptive title for this issue. 68 | 69 | **Feature Description** 70 |
Describe the purpose of this feature. 71 | 72 | **Related Examples** 73 |
Provide references to documents, repositories, etc., Please provide links to any relevant GitHub repos, papers, or other resources if relevant. 74 | 75 | **Motivation** 76 |
Describe the motivation for this feature. Why is it needed? Provide enough context information to help understand the demand for this feature. 77 | 78 | **Contribution** 79 |
How you can contribute to the building of this feature (if you are participating). 80 | -------------------------------------------------------------------------------- /.dumi/theme/slots/ColorSwitch/index.tsx: -------------------------------------------------------------------------------- 1 | import { useIntl, usePrefersColor, useSiteData, } from 'dumi'; 2 | import React, { useEffect, useState, type FC } from 'react'; 3 | import './index.less'; 4 | 5 | const IconDark = ({ 6 | onClick, 7 | }: { 8 | onClick: React.MouseEventHandler; 9 | }) => ( 10 | 11 | 12 | 13 | ); 14 | 15 | const IconLight = ({ 16 | onClick, 17 | }: { 18 | onClick: React.MouseEventHandler; 19 | }) => ( 20 | 21 | 22 | 23 | ); 24 | 25 | const IconAuto = () => ( 26 | 27 | 28 | 29 | ); 30 | 31 | const ICON_MAPPING = { 32 | light: IconDark, 33 | dark: IconLight, 34 | auto: IconAuto, 35 | }; 36 | 37 | const ColorSwitch: FC = () => { 38 | const { 39 | themeConfig: { 40 | prefersColor: { default: defaultColor }, 41 | }, 42 | } = useSiteData(); 43 | const intl = useIntl(); 44 | const [, prefersColor = defaultColor, setPrefersColor] = usePrefersColor(); 45 | const Icon = ICON_MAPPING[prefersColor]; 46 | 47 | // useEffect(() => { 48 | // setPrefersColor(prefersColor); 49 | // sessionStorage.setItem('prefersColortest', prefersColor); 50 | // }, [prefersColor]) 51 | // 切换颜色模式的函数 52 | 53 | const switchColorMode = () => { 54 | // 根据当前模式切换到另一模式 55 | const nextMode = prefersColor === 'light' ? 'dark' : 'light'; 56 | setPrefersColor(nextMode); 57 | sessionStorage.setItem('prefersColortest', prefersColor); 58 | }; 59 | 60 | return ( 67 | {Icon && } 68 | 69 | ); 70 | }; 71 | 72 | export default ColorSwitch; 73 | -------------------------------------------------------------------------------- /.dumi/theme/slots/LangSwitch/index.tsx: -------------------------------------------------------------------------------- 1 | import { ReactComponent as IconDown } from '@ant-design/icons-svg/inline-svg/outlined/down.svg'; 2 | import { 3 | history, 4 | Link, 5 | useIntl, 6 | useLocale, 7 | useLocation, 8 | useSiteData, 9 | usePrefersColor, 10 | } from 'dumi'; 11 | import React, { useEffect, useState, type FC } from 'react'; 12 | import './index.less'; 13 | import { set } from 'lodash'; 14 | 15 | type ILocaleItem = ReturnType['locales'][0]; 16 | 17 | function getTargetLocalePath({ 18 | pathname, 19 | current, 20 | target, 21 | }: { 22 | pathname: string; 23 | current: ILocaleItem; 24 | target: ILocaleItem; 25 | }) { 26 | const clearPath = 27 | 'base' in current 28 | ? // handle '/en-US/a' => '/a' or '/en-US' => '' => '/' 29 | pathname.replace(current.base.replace(/\/$/, ''), '') || '/' 30 | : pathname.replace(new RegExp(`${current.suffix}$`), ''); 31 | 32 | return 'base' in target 33 | ? `${ 34 | // for `/` base, strip duplicated leading slash 35 | target.base.replace(/\/$/, '') 36 | }${clearPath}` 37 | // for `/` clearPath, strip duplicated ending slash 38 | .replace(/([^/])\/$/, '$1') 39 | : `${clearPath}${target.suffix}`; 40 | } 41 | 42 | const SingleSwitch: FC<{ locale: ILocaleItem; current: ILocaleItem }> = ({ 43 | locale, 44 | current, 45 | }) => { 46 | const { pathname } = useLocation(); 47 | const [path, setPath] = useState(() => 48 | getTargetLocalePath({ pathname, current, target: locale }), 49 | ); 50 | 51 | useEffect(() => { 52 | setPath(getTargetLocalePath({ pathname, current, target: locale })); 53 | }, [pathname, current.id, locale.id]); 54 | 55 | return ( 56 | 57 | 58 | {locale.name} 59 | 60 | 61 | ); 62 | }; 63 | 64 | const LangSwitch: FC = () => { 65 | const { locales } = useSiteData(); 66 | const { locale } = useIntl(); 67 | const current = useLocale(); 68 | 69 | // do not render in single language 70 | if (locales.length <= 1) return null; 71 | 72 | return locales.length > 2 ? ( 73 |
74 | 92 | 93 |
94 | ) : ( 95 | // single language switch 96 | id !== locale)!} 98 | current={current} 99 | /> 100 | ); 101 | }; 102 | 103 | export default LangSwitch; 104 | -------------------------------------------------------------------------------- /docs/docs/developer-docs/CodeFuse-DevOps-Model/main/quickstart.en-US.md: -------------------------------------------------------------------------------- 1 | --- 2 | store: 3 | title: CodeFuse-DevOps-Model 4 | version: main 5 | group: 6 | title: 🌱 CodeFuse-DevOps-Model 7 | order: -1 8 | title: QuickStart 9 | order: 1 10 | toc: content 11 | github: https://github.com/codefuse-ai/CodeFuse-DevOps-Model 12 | --- 13 | 14 | ## Dependency Installation 15 | 16 | Please install the packages listed in the requirements.txt file from the GitHub address first. You can refer to the following code: 17 | 18 | ``` 19 | pip install -r requirements.txt 20 | ``` 21 | 22 | ## Model Download 23 | 24 | Model download information is as follows: 25 | 26 | 🤗 Huggingface Address 27 | 28 | | - | Base Model | Aligned Model | 29 | | --- | --------------------- | --------------------- | 30 | | 7B | DevOps-Model-7B-Base | DevOps-Model-7B-Chat | 31 | | 14B | DevOps-Model-14B-Base | DevOps-Model-14B-Chat | 32 | 33 | 🤖 ModelScope Address 34 | 35 | | - | Base Model | Aligned Model | 36 | | --- | --------------------- | --------------------- | 37 | | 7B | DevOps-Model-7B-Base | DevOps-Model-7B-Chat | 38 | | 14B | DevOps-Model-14B-Base | DevOps-Model-14B-Chat | 39 | 40 | Find the version of the Chat model you want to download; currently, 7B and 14B models are provided. 41 | 42 | ## Model Usage 43 | 44 | Interact with the Chat model using the following code: 45 | 46 | ``` 47 | from transformers import AutoModelForCausalLM, AutoTokenizer 48 | from transformers.generation import GenerationConfig 49 | 50 | tokenizer = AutoTokenizer.from_pretrained("path_to_DevOps-Model-Chat", trust_remote_code=True) 51 | 52 | model = AutoModelForCausalLM.from_pretrained("path_to_DevOps-Model-Chat", device_map="auto", trust_remote_code=True, bf16=True).eval() 53 | 54 | # 指定 generation_config 55 | model.generation_config = GenerationConfig.from_pretrained("path_to_DevOps-Model-Chat", trust_remote_code=True) 56 | 57 | # First round of conversation 58 | resp, hist = model.chat(query='你是谁', tokenizer=tokenizer, history=None) 59 | print(resp) 60 | # 我是 DevOps-Model，一个由蚂蚁集团平台技术事业群风险智能团队和北京大学联合研发的人工智能机器人，可以与用户进行自然语言交互，并协助解答 DevOps 全生命周期中的各种问题。如果您有任何需要协助的问题或者想要进行闲聊，都可以和我交流哦。 61 | 62 | # Second round of conversation 63 | resp2, hist2 = model.chat(query='Java 中 HashMap 和 Hashtable 有什么区别', tokenizer=tokenizer, history=hist) 64 | print(resp2) 65 | # HashMap 和 Hashtable 都是 Java 中常用的哈希表实现，它们的主要区别在于： 66 | # 1. Hashtable 是线程安全的，而 HashMap 不是线程安全的，因此在多线程环境下，Hashtable 的性能更稳定。 67 | # 2. Hashtable 中的方法都是同步的，而 HashMap 的方法不是同步的，因此在多线程环境下，Hashtable 的性能更好。 68 | # 3. Hashtable 中的 key 和 value 都必须实现 Serializable 接口，而 HashMap 中的 key 和 value 可以是任何对象，包括基本数据类型。 69 | # 4. Hashtable 的初始容量是 11，而 HashMap 的初始容量是 16。 70 | # 总之，如果需要在多线程环境下使用哈希表，并且需要保证线程安全，那么应该使用 Hashtable；如果不需要考虑线程安全，或者需要快速地进行哈希表操作，那么应该使用 HashMap。 71 | 72 | # Third round of conversation 73 | resp3, hist3 = model.chat(query='线程安全代表什么', tokenizer=tokenizer, history=hist2) 74 | print(resp3) 75 | # 线程安全是指在多线程环境下，程序能够正确地处理并发访问，并且不会出现数据竞争、死锁、饥饿等异常情况。线程安全的程序可以保证在不同的线程之间共享同一个数据结构时，数据的正确性和一致性。线程安全的实现通常需要使用同步机制，如锁、原子操作等，来保证对共享数据的访问是线程安全的。在 Java 中，可以通过 synchronized 关键字、Lock 接口等机制来实现线程安全。 76 | ``` 77 | -------------------------------------------------------------------------------- /docs/docs/developer-docs/CodeFuse-MFT-VLM/main/mftvlm.en-US.md: -------------------------------------------------------------------------------- 1 | --- 2 | nav: 3 | title: Docs 4 | order: -1 5 | second: 6 | title: Developer-Docs 7 | order: -1 8 | store: 9 | title: CodeFuse-MFT-VLM 10 | version: main 11 | group: 12 | title: 🌱 CodeFuse-MFT-VLM 13 | index: true 14 | order: -1 15 | title: CodeFuse-MFT-VLM 16 | order: -1 17 | toc: content 18 | github: https://github.com/codefuse-ai/CodeFuse-MFT-VLM 19 | --- 20 | 21 | ## CodeFuse-VLM 22 | 23 | CodeFuse-VLM is a Multimodal LLM(MLLM) framework that provides users with multiple vision encoders, multimodal alignment adapters, and LLMs. Through CodeFuse-VLM framework, users are able to customize their own MLLM model to adapt their own tasks. 24 | As more and more models are published on Huggingface community, there will be more open-source vision encoders and LLMs. Each of these models has their own specialties, e.g. Code-LLama is good at code-related tasks but has poor performance for Chinese tasks. Therefore, we built CodeFuse-VLM framework to support multiple vision encoders, multimodal alignment adapters, and LLMs to adapt different types of tasks. 25 | ![img.jpg](https://mdn.alipayobjects.com/huamei_bvbxju/afts/img/A*t7OIS58EJmIAAAAAAAAAAAAADlHYAQ/original) 26 | 27 | Under CodeFuse-VLM framework, we use cross attention multimodal adapter, Qwen-14B LLM, and Qwen-VL's vision encoder to train CodeFuse-VLM-14B model. On multiple benchmarks, our CodeFuse-VLM-14B shows superior performances over Qwen-VL and LLAVA-1.5. 28 | ![img.jpg](https://mdn.alipayobjects.com/huamei_bvbxju/afts/img/A*BuIrRZd62ssAAAAAAAAAAAAADlHYAQ/original) 29 | 30 | Here is the table for different MLLM model's performance on benchmarks 31 | Model | MMBench | MMBench-CN | VqaV2 | GQA | TextVQA | Vizwiz 32 | | ------------- | ------------- | ------------- | ------------- | ------------- | ------------- | ------------- | 33 | LLAVA-1.5 | 67.7 | 63.6 | 80.0 | 63.3 | 61.3 | 53.6 34 | Qwen-VL | 60.6 | 56.7 | 78.2 | 57.5 | 63.8 | 38.9 35 | CodeFuse-VLM-14B | 75.7 | 69.8 | 79.3 | 59.4 | 63.9 | 45.3 36 | 37 | Our model achieved high ranking on MMBenchmark: https://mmbench.opencompass.org.cn/leaderboard 38 | 39 | Here's our model's demo video 40 | 41 | 44 | 45 | -------------------------------------------------------------------------------- /docs/docs/developer-docs/CodeFuse-ChatBot/master/roadmap.en-US.md: -------------------------------------------------------------------------------- 1 | --- 2 | store: 3 | title: CodeFuse-ChatBot 4 | version: master 5 | group: 6 | title: 🌱 CodeFuse-ChatBot 7 | order: -1 8 | title: ChatBot-RoadMap 9 | order: 2 10 | toc: content 11 | github: https://github.com/codefuse-ai/codefuse-chatbot 12 | --- 13 | 14 | ## RoadMap 15 | 16 |
17 | 图片 18 |
19 |
20 | 21 | Roadmap Overview 22 | 23 | - [x] Sandbox Environment 24 | - [x] Isolated sandbox environment for code execution 25 | - [x] File upload and download 26 | - [ ] Support for Java execution environment 27 | - [x] Vector Database & Retrieval 28 | - [x] Task retrieval 29 | - [x] Tool retrieval 30 | - [x] Prompt Management 31 | - [x] Memory Management 32 | - [x] Multi Agent Framework 33 | - [ ] PRD (Product Requirement Document), system analysis, interface design 34 | - [ ] Generate code based on requirement documents, system analysis, and interface design 35 | - [ ] Automated testing, automated debugger 36 | - [ ] Operations process integration (ToolLearning) 37 | - [ ] Fully automated end-to-end process 38 | - [x] Integration with LLM based on fastchat 39 | - [x] Integration with Text Embedding based on sentencebert 40 | - [x] Improved vector loading speed 41 | - [x] Connector 42 | - [x] React Mode based on langchain 43 | - [x] Tool retrieval completed with langchain 44 | - [ ] General Capability for Web Crawl 45 | - [x] Technical documentation: Zhihu, CSDN, Alibaba Cloud Developer Forum, Tencent Cloud Developer Forum, etc. 46 | - [ ] Issue document 47 | - [ ] SDK Library Document 48 | 49 | v0.0 50 | 51 | - [x] Sandbox Environment 52 | - [x] Isolated sandbox environment for code execution 53 | - [x] Integration with LLM based on fastchat 54 | - [x] Integration with Text Embedding based on sentencebert 55 | - [x] General Capability for Web Crawl: Technical documentation: Zhihu, CSDN, Alibaba Cloud Developer Forum, Tencent Cloud Developer Forum, etc. 56 | 57 | Done 58 |
59 | 60 | v0.1 61 | 62 | - [x] Sandbox Environment: File upload and download 63 | - [x] Vector Database & Retrieval 64 | - [x] Task retrieval 65 | - [x] Tool retrieval 66 | - [x] Connector 67 | - [x] React Mode based on langchain 68 | - [x] Integration with Text Embedding based on sentencebert: Improved vector loading speed 69 | 70 | Done 71 |
72 | 73 | v0.2 74 | 75 | - [x] Prompt Management 76 | - [x] Memory Management 77 | - [x] Vector Database & Retrieval 78 | 79 | Done 80 |
81 | 82 | v0.3 83 | 84 | - [x] Sandbox Environment 85 | - [ ] Support for Java execution environment 86 | - [x] Multi Agent 87 | - [ ] PRD (Product Requirement Document), system analysis, interface design 88 | - [ ] Generate code based on requirement documents, system analysis, and interface design 89 | - [ ] Automated testing, automated debugger 90 | - [ ] Operations process integration (ToolLearning) 91 | - [ ] Fully automated end-to-end process 92 | - [x] General Capability for Web Crawl 93 | - [ ] Issue document 94 | - [ ] SDK Library Document 95 | 96 | DDL： 2024.12.31 97 |
98 | -------------------------------------------------------------------------------- /docs/docs/developer-docs/CodeFuse-MFT-VLM/main/quickstart.en-US.md: -------------------------------------------------------------------------------- 1 | --- 2 | store: 3 | title: CodeFuse-MFT-VLM 4 | version: main 5 | group: 6 | title: 🌱 CodeFuse-MFT-VLM 7 | order: -1 8 | title: QuickStart 9 | order: 0 10 | toc: content 11 | github: https://github.com/codefuse-ai/CodeFuse-MFT-VLM 12 | --- 13 | 14 | ## Contents 15 | 16 | - [Install](#Install) 17 | - [Datasets](#Datasets) 18 | - [Multimodal Alignment](#Multimodal-Alignment) 19 | - [Visual Instruction Tuning](#Visual-Instruction-Tuning) 20 | - [Evaluation](#Evaluation) 21 | 22 | ## Install 23 | 24 | Please run sh init_env.sh 25 | 26 | ## Datasets 27 | 28 | Here's the table of datasets we used to train CodeFuse-VLM-14B: 29 | 30 | | Dataset | Task Type | Number of Samples | 31 | | ----------------------------------------------- | --------------------------- | ----------------- | 32 | | synthdog-en | OCR | 800,000 | 33 | | synthdog-zh | OCR | 800,000 | 34 | | cc3m(downsampled) | Image Caption | 600,000 | 35 | | cc3m(downsampled) | Image Caption | 600,000 | 36 | | SBU | Image Caption | 850,000 | 37 | | Visual Genome VQA (Downsampled) | Visual Question Answer(VQA) | 500,000 | 38 | | Visual Genome Region descriptions (Downsampled) | Reference Grouding | 500,000 | 39 | | Visual Genome objects (Downsampled) | Grounded Caption | 500,000 | 40 | | OCR VQA (Downsampled) | OCR and VQA | 500,000 | 41 | 42 | Please download these datasets on their own official websites. 43 | 44 | ## Multimodal Alignment 45 | 46 | Please run sh scripts/pretrain.sh or sh scripts/pretrain_multinode.sh 47 | 48 | ## Visual Instruction Tuning 49 | 50 | Please run sh scripts/finetune.sh or sh scripts/finetune_multinode.sh 51 | 52 | ## Evaluation 53 | 54 | Please run python scripts in directory llava/eval/. Our pre-trained CodeFuse-VLM-14B can be loaded with the following code: 55 | 56 | ``` 57 | import os 58 | from llava.model.builder import load_mixed_pretrained_model 59 | 60 | model_path = '/pretrained/model/path' 61 | tokenizer, model, image_processor, context_len = load_mixed_pretrained_model(model_path, None, 'qwen-vl-14b', os.path.join(model_path, 'Qwen-VL-visual'), 'cross_attn', os.path.join(model_path, 'mm_projector/mm_projector.bin')) 62 | ``` 63 | 64 | You can also run scripts/merge_qwen_vl_weights.sh first and load the merged model by the following code: 65 | 66 | ``` 67 | from llava.model import LlavaQWenForCausalLM 68 | 69 | model = LlavaQWenForCausalLM.from_pretrained('/path/to/our/pretrained/model') 70 | ``` 71 | 72 | ## CodeFuse-VLM Product Video 73 | 74 | Here's the demo video of front-end code copilot backed by our VLM model 75 | 76 | 79 | -------------------------------------------------------------------------------- /docs/docs/developer-docs/MFTCoder/main/introduction.en-US.md: -------------------------------------------------------------------------------- 1 | --- 2 | store: 3 | title: MFTCoder 4 | version: main 5 | group: 6 | title: 🌱 MFTCoder 7 | order: -1 8 | title: Introduction 9 | order: 0 10 | toc: content 11 | github: https://github.com/codefuse-ai/MFTCoder 12 | --- 13 | 14 | ## Introduction 15 | 16 | **High Accuracy and efficiency Multi-task Fine-tuning framework for Code LLMs.** 17 | 18 | **MFTCoder** is an open-source project of CodeFuse for accurate and efficient Multi-task Fine-tuning(MFT) on Large Language Models(LLMs), especially on Code-LLMs(large language model for code tasks). 19 | Moreover, we open source Code LLM models and code-related datasets along with the MFTCoder framework. 20 | 21 | In MFTCoder, we released two codebases for finetuning Large Language Models: 22 | 23 | - **`MFTCoder-accelerate`** is a framework with accelerate and DeepSpeed/FSDP. All tech-stacks are open-source and vibrant. We highly recommend you try this framework and make your fintuning accurate and efficient. 24 | - `MFTCoder-atorch` is based on the [ATorch frameworks](https://github.com/intelligent-machine-learning/dlrover), which is a fast distributed training framework of LLM. 25 | 26 | The aim of this project is to foster collaboration and share advancements in large language models, particularly within the domain of code development. 27 | 28 | ### Frameworks 29 | 30 | ![img.jpg](https://mdn.alipayobjects.com/huamei_bvbxju/afts/img/A*p8ahSYMtrwsAAAAAAAAAAAAADlHYAQ/original) 31 | 32 | ### Highlights 33 | 34 | :white_check_mark: **Multi-task**: Train models on multiple tasks while maintaining a balance between them. The models can even generalize to new, previously unseen tasks. 35 | 36 | :white_check_mark: **Multi-model**: It integrates state-of-the-art open-source models such as gpt-neox, llama, llama-2, baichuan, Qwen, chatglm2, and more. (These finetuned models will be released in the near future.) 37 | 38 | :white_check_mark: **Multi-framework**: It provides support for both Accelerate (with Deepspeed and FSDP) and ATorch 39 | 40 | :white_check_mark: **Efficient fine-tuning**: It supports LoRA, QLoRA as well as Full-parameters training, enabling fine-tuning of large models with minimal resources. The training speed meets the demands of almost all fine-tuning scenarios. 41 | 42 | The main components of this project include: 43 | 44 | - Support for both SFT (Supervised FineTuning) and MFT (Multi-task FineTuning). The current MFTCoder achieves data balance among multiple tasks, and future releases will achieve a balance between task difficulty and convergence speed during training. 45 | - Support for QLoRA instruction fine-tuning, LoRA fine-tuning as well as Full-parameters fine-tuning. 46 | - Support for most mainstream open-source large models, particularly those relevant to Code-LLMs, such as DeepSeek-coder, Mistral, Mixtral, Chatglm3, Code-LLaMA, Starcoder, Codegeex2, Qwen, GPT-Neox, and more. 47 | - Support for weight merging between the LoRA adaptor and base models, simplifying the inference process. 48 | - Release of 2 high-quality code-related instruction fine-tuning datasets: [Evol-instruction-66k](https://huggingface.co/datasets/codefuse-ai/Evol-instruction-66k) and [CodeExercise-Python-27k](https://huggingface.co/datasets/codefuse-ai/CodeExercise-Python-27k). 49 | - Release of many Code LLMs, please refer to organizations: [codefuse-ai on huggingface](https://huggingface.co/codefuse-ai) or [codefuse-ai on modelscope](https://modelscope.cn/organization/codefuse-ai). 50 | -------------------------------------------------------------------------------- /README_ZH.md: -------------------------------------------------------------------------------- 1 |

2 | 3 |

4 | 5 |
6 | 7 | [**简体中文**](./README_ZH.md)|[**HuggingFace**](https://huggingface.co/codefuse-ai)|[**ModelScope**](https://modelscope.cn/organization/codefuse-ai) 8 | 9 | Hello! 这里是 CodeFuse！ 10 | CodeFuse 的使命是开发专门设计用于支持整个软件开发生命周期的大型代码语言模型（Code LLMs），涵盖设计、需求、编码、测试、部署、运维等关键阶段。我们致力于打造创新的解决方案，让软件开发者们在研发的过程中如丝般顺滑。 11 | 12 |
13 | 14 | ## CodeFuse 开源官网 15 | 16 | ### 快速访问 17 | 18 | 通过访问 https://codefuse-ai.github.io/ 即可快速访问 CodeFuse 开源官网 19 | 20 | 在这里，你可以了解到以下内容: 21 | ● 我们会分享 CodeFuse 项目的背景故事，是如何启动这个项目，以及背后的理念 22 | ● 在主页面展示了一系列围绕整个软件开发生命周期所涉及的 AI 开发项目 23 | ● 文档页编写了详细的文档来帮助各位更深入地了解项目 24 | ● 在项目发展过程中，也会实时同步 CodeFuse 项目的最新版本更新和进展，并展示项目的关键功能和特性 25 | ● 我们也欢迎提问和贡献，在这里提供了贡献指南和指导原则，说明如何来提交问题、改进代码或文档 26 | ● 同样地，我们也会在 Publication 里展示 CodeFuse 最新的学术成果和技术文章 27 | 28 | 希望以上这些内容能够好帮助大家了解 CodeFuse！ 29 | 30 | ![](./static/codefuse_web.webp) 31 | 32 | ### 快速修改 33 | 34 | 1、拉取代码 35 | 36 | ``` 37 | git clone https://github.com/codefuse-ai/codefuse-ai.github.io.git 38 | 39 | cd codefuse-ai.github.io.git 40 | ``` 41 | 42 | 2、确认 node 版本 43 | 44 | ```bash 45 | node -v 46 | 47 | # if your version is wrong 48 | nvm install 49 | nvm use 50 | ``` 51 | 52 | 3、安装相关依赖 53 | 54 | ```bash 55 | npm cache clean --force # 清除缓存 56 | npm install 57 | ``` 58 | 59 | 4、本地启动 60 | 61 | ```bash 62 | #you can check it by npm-start if you have any changes before you push 63 | npm start 64 | ``` 65 | 66 | ### 基本介绍 67 | 68 | | CodeFuse 域 | 功能 | 理念 | 69 | | ----------- | ------------------------------------------------------------------------------------- | ---------------------------------------------------- | 70 | | 代码域 | 旨在提升大预言模型的多任务能力 | 融会贯通，代码天成 | 71 | | 运维域 | 旨在融合 RAG、工具学习、EKG 等技术对软件开发生命周期进行提效 | 面向过程编程，过程即是结果 | 72 | | 分析域 | 满足大规模、复杂的代码库分析需求，适应多元化的静态分析场景 | 如你所愿，洞察代码 | 73 | | 测试域 | 构建测试领域的“智能体”，融合大模型和质量领域工程化技术，促进质量技术代系升级。 | TestAgent：您的 24 小时在线测试助手 | 74 | | 推理域 | 缓存已生成的模型结果，降低类似请求的响应时间，提升用户体验 | 致力高吞吐，缔造即刻智能 | 75 | | 评测域 | 构建开发编程领域的评测基准， 用于评估模型在代码补全，代码生成，测例生成等多类任务性能 | 高效测准、精确反馈 -代码大模型的企业级多任务评估基准 | 76 | 77 | ## 后续更新 78 | 79 | 1. 我们会不断完善文档内容来提供更好的技术支持。 80 | 2. 后续会增加 Blog 页面，来持续同步 CodeFuse 的最新技术/技术应用/学术前沿文章。 81 | 3. CodeFuse 的线上社区活动，包括新手任务计划参考案例、社区共建计划参考案例等，也会在这里进行同步发布 82 | 83 | ## 关于 CodeFuse 团队 84 | 85 | CodeFuse 团队由一群充满热情的成员组成，我们的目标是构建大型编码语言模型（Code LLMs），以支持和提升在整个软件开发生命周期中的 AI 原生开发工作。我们的工作覆盖了从设计需求、编写代码、测试、构建、部署、运维到洞察分析等关键环节。我们积极推广开源精神，目前已经推出了 15 个代码相关的模型，并开源了一系列技术工具，例如 MFTCoder、CodeFuse-VLM、CodeFuse-DevOps、CodeFuse-Query、TestGPT 和 CodeFuse-muAgent。此外，为了让用户在日常编码工作中能够直接受益，我们还开发了 CodeFuse-IDE 插件。同时 CodeFuse 团队至今有 6 篇技术论文被顶会接收，技术栈涉及语义检索、上下文增长、大模型训练微调、大模型推理加速等大模型技术领域。另外还参与了信通院、ITU 国际电联相关标准制定，获得多项国内行业奖励，总计获批软件专著 2 项。随时欢迎和我们交流，如果您对这项激动人心的工作感兴趣，欢迎加入我们！ 86 | 87 | ## 联系我们 88 | 89 | CodeFuse 的相关模型和数据集也在陆续开源中，如果您喜欢我们的工作，欢迎试用、指正错误和贡献代码，可以的话请给我们的项目增加 Star 来支持我们。 90 | ● GitHub 项目主页：https://github.com/codefuse-ai 91 | ● HuggingFace 主页：https://huggingface.co/codefuse-ai 92 | ● 魔搭社区主页：https://modelscope.cn/organization/codefuse-ai 93 | 94 | 如果您想更快地获取到最新信息，欢迎加入我们的企业微信群. 95 | 96 | ![](./static/wechat.webp) 97 | -------------------------------------------------------------------------------- /.dumi/theme/slots/Header/index.tsx: -------------------------------------------------------------------------------- 1 | import type { SocialTypes } from '@/client/theme-api/types'; 2 | import { ReactComponent as IconClose } from '@ant-design/icons-svg/inline-svg/outlined/close.svg'; 3 | import { ReactComponent as IconMenu } from '@ant-design/icons-svg/inline-svg/outlined/menu.svg'; 4 | import { useRouteMeta, useSiteData, useLocation } from 'dumi'; 5 | import ColorSwitch from '../ColorSwitch'; 6 | import HeaderExtra from 'dumi/theme/slots/HeaderExtra'; 7 | import LangSwitch from 'dumi/theme/slots/LangSwitch'; 8 | import Logo from 'dumi/theme/slots/Logo'; 9 | import Navbar from 'dumi/theme/slots/Navbar'; 10 | import RtlSwitch from 'dumi/theme/slots/RtlSwitch'; 11 | import SearchBar from 'dumi/theme/slots/SearchBar'; 12 | import SocialIcon from 'dumi/theme/slots/SocialIcon'; 13 | import React, { useMemo, useState, type FC } from 'react'; 14 | import './index.less'; 15 | const Header: FC = () => { 16 | const { frontmatter } = useRouteMeta(); 17 | const [showMenu, setShowMenu] = useState(false); 18 | const { themeConfig } = useSiteData(); 19 | const { hash, pathname } = useLocation(); 20 | const blogDet = pathname.includes("/blogDetails"); 21 | const socialIcons = useMemo( 22 | () => 23 | themeConfig.socialLinks 24 | ? Object.keys(themeConfig.socialLinks) 25 | .slice(0, 5) 26 | .map((key) => ({ 27 | icon: key as SocialTypes, 28 | link: themeConfig.socialLinks[key as SocialTypes], 29 | })) 30 | : [], 31 | [themeConfig.socialLinks], 32 | ); 33 | return ( 34 |
setShowMenu(false)} 39 | > 40 |
41 |
42 | 43 | {/* 文档信息下拉弹框 */} 44 | 45 |
46 | 47 |
48 | {/* 导航➕国际化 */} 49 |
50 | { 51 | !blogDet && <> 52 |
53 | } 54 | 55 | 56 | {/* 亮度显示 */} 57 | { 58 | themeConfig.prefersColor.switch && 59 | } 60 |
61 | {socialIcons.map((item) => ( 62 | 63 | ))} 64 |
{ window.open('https://huggingface.co/codefuse-ai') }}> 65 | 66 |
67 |
{ window.open('https://modelscope.cn/organization/codefuse-ai') }}> 68 |
69 | 70 |
71 | 72 |
73 |
74 | {/* 移动端导航栏 */} 75 | 85 |
86 |
87 | ); 88 | }; 89 | 90 | export default Header; 91 | -------------------------------------------------------------------------------- /.dumi/theme/slots/CodeGeneration/index.tsx: -------------------------------------------------------------------------------- 1 | import { useLocale, usePrefersColor, useRouteMeta, } from 'dumi'; 2 | import './index.less'; 3 | import React, { type FC } from 'react'; 4 | import "slick-carousel/slick/slick.css"; 5 | import "slick-carousel/slick/slick-theme.css"; 6 | import Slider from "react-slick"; 7 | import HomeTitle from '../HomeTitle'; 8 | import { SwapRightOutlined } from '@ant-design/icons'; 9 | 10 | const CodeGeneration: FC = () => { 11 | const locale = useLocale(); 12 | const { frontmatter } = useRouteMeta(); 13 | const [color] = usePrefersColor(); 14 | const settings = { 15 | dots: true, 16 | infinite: true, 17 | speed: 900, 18 | slidesToShow: 1, 19 | slidesToScroll: 1, 20 | nextArrow: , 21 | prevArrow: , 22 | appendDots: (dots: string | number | boolean | React.ReactElement> | Iterable | React.ReactPortal | null | undefined) => ( 23 |
{dots}
36 | ), 37 | customPaging: (i: number) => ( 38 |
39 | { 40 | frontmatter?.CodeGeneration.map((item: any, index: number) => { 41 | return
42 | {index === i && item.buttom} 43 |
44 | }) 45 | } 46 |
47 | ) 48 | }; 49 | return
50 |
51 | 52 |
53 | 54 | { 55 | frontmatter?.CodeGeneration.map((item: any, index: number) => { 56 | return
57 | 58 |
59 |
60 | {item.title} 61 |
62 |
63 | {item.description} 64 |
65 |
{ window.open(item.link) }}> 66 | {locale.id === 'zh-CN' ? '了解更多' : 'Learn more'} 67 | 68 |
69 |
70 |
71 | }) 72 | } 73 | 74 |
75 |
76 |
77 | }; 78 | export default CodeGeneration; 79 | -------------------------------------------------------------------------------- /docs/docs/developer-docs/CodeFuse-DevOps-Eval/master/evaluate.en-US.md: -------------------------------------------------------------------------------- 1 | --- 2 | store: 3 | title: CodeFuse-DevOps-Eval 4 | version: master 5 | group: 6 | title: 🌱 CodeFuse-DevOps-Eval 7 | order: -1 8 | title: Evaluate 9 | order: 0 10 | toc: content 11 | github: https://github.com/codefuse-ai/codefuse-devops-eval 12 | --- 13 | 14 | ## 🚀 How to Evaluate 15 | 16 | If you need to test your own huggingface-formatted model, the overall steps are as follows: 17 | 18 | 1. Write the loader function for the model. 19 | 2. Write the context_builder function for the model. 20 | 3. Register the model in the configuration file. 21 | 4. Run the testing script. 22 | If the model does not require any special processing after loading, and the input does not need to be converted to a specific format (e.g. chatml format or other human-bot formats), you can directly proceed to step 4 to initiate the testing. 23 | 24 | #### 1. Write the loader function 25 | 26 | If the model requires additional processing after loading (e.g. adjusting the tokenizer), you need to inherit the `ModelAndTokenizerLoader` class in `src.context_builder.context_builder_family.py` and override the corresponding `load_model` and `load_tokenizer` functions. You can refer to the following example: 27 | 28 | ```python 29 | class QwenModelAndTokenizerLoader(ModelAndTokenizerLoader): 30 | def __init__(self): 31 | super().__init__() 32 | pass 33 | 34 | @override 35 | def load_model(self, model_path: str): 36 | # Implementation of the method 37 | pass 38 | 39 | @override 40 | def load_tokenizer(self, model_path: str): 41 | # Implementation of the method 42 | pass 43 | ``` 44 | 45 | #### 2. Write the context_builder function for the Model 46 | 47 | If the input needs to be converted to a specific format (e.g. chatml format or other human-bot formats), you need to inherit the ContextBuilder class in `src.context_builder.context_builder_family` and override the make_context function. This function is used to convert the input to the corresponding required format. An example is shown below: 48 | 49 | ```python 50 | class QwenChatContextBuilder(ContextBuilder): 51 | def __init__(self): 52 | super().__init__() 53 | 54 | @override 55 | def make_context(self, model, tokenizer, query: str, system: str = "hello！"): 56 | # Implementation of the method 57 | pass 58 | ``` 59 | 60 | #### 3. Register the model in the configuration file 61 | 62 | Go to the `model_conf.json` file in the conf directory and register the corresponding model name and the loader and context_builder that will be used for this model. Simply write the class names defined in the first and second steps for the loader and context_builder. Here is an example: 63 | 64 | ```json 65 | { 66 | "Qwen-Chat": { 67 | "loader": "QwenModelAndTokenizerLoader", 68 | "context_builder": "QwenChatContextBuilder" 69 | } 70 | } 71 | ``` 72 | 73 | #### 4. Execute the testing script 74 | 75 | Run the following code to initiate the test: 76 | 77 | ```Bash 78 | python src/run_eval.py \ 79 | --model_path path_to_model \ 80 | --model_name model_name_in_conf \ 81 | --model_conf_path path_to_model_conf \ 82 | --eval_dataset_list all \ 83 | --eval_dataset_fp_conf_path path_to_dataset_conf \ 84 | --eval_dataset_type test \ 85 | --data_path path_to_downloaded_devops_eval_data \ 86 | --k_shot 0 87 | ``` 88 | 89 | 👀 👀 The specific evaluation process is as follows 📖 [**Evaluate Tutorial**](/docs/developer-docs/CodeFuse-DevOps-Eval/master/tutorial) 90 | 91 |
92 | -------------------------------------------------------------------------------- /docs/blogDetails/001.zh-CN.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: 'ACL 2024｜D2LLM：将Causal LLM改造成向量搜索模型的黑科技' 3 | time: '2024-06-05' 4 | toc: content 5 | --- 6 | 7 | Thank you for your interest in the Codefuse project. We warmly welcome any suggestions, opinions (including criticisms), comments, and contributions to the Codefuse project. 8 | 9 | Your suggestions, opinions, and comments on Codefuse can be directly submitted through GitHub Issues. 10 | 11 | There are many ways to participate in the Codefuse project and contribute to it: code implementation, test writing, process tool improvement, documentation enhancement, and more. We welcome any contributions and will add you to our list of contributors. 12 | 13 | Furthermore, with enough contributions, you may have the opportunity to become a Committer for Codefuse. 14 | 15 | For any questions, you can contact us for timely answers through various means including WeChat, Gitter (an instant messaging tool provided by GitHub), email, and more. 16 | 17 | ## Getting Started 18 | 19 | If you are new to the Codefuse community, you can: 20 | 21 | - Follow the Codefuse GitHub repository. 22 | - Join related WeChat groups for Codefuse to ask questions at any time; 23 | 24 | Through the above methods, you can stay up-to-date with the development dynamics of the Codefuse project and express your opinions on topics of interest. 25 | 26 | ## Contributation Ways 27 | 28 | This contribution guide is not just about writing code. We value and appreciate help in all areas. Here are some ways you can contribute: 29 | 30 | - Documentation 31 | - Issues 32 | - Pull Requests (PR) 33 | 34 | ### Improve Documentation 35 | 36 | Documentation is the main way for you to understand Codefuse and is also where we need the most help! 37 | 38 | By browsing the documentation, you can deepen your understanding of Codefuse and also help you grasp the features and technical details of Codefuse. If you find any issues with the documentation, please contact us in time; 39 | 40 | If you are interested in improving the quality of the documentation, whether it is revising an address of a page, correcting a link, or writing a better introductory document, we are very welcoming! 41 | 42 | Most of our documentation is written in markdown format. You can directly modify and submit documentation changes in the docs/ directory on GitHub. For submitting code changes, please refer to Pull Requests. 43 | 44 | ### If You Discover a Bug or Issue 45 | 46 | If you discover a bug or issue, you can directly submit a new Issue through GitHub Issues, and someone will handle it regularly. For more details, see Issue Template.[Issue Template](/contribution/issue) 47 | 48 | You can also choose to read and analyze the code to fix it yourself (it is best to communicate with us before doing so, as someone might already be working on the same issue), and then submit a Pull Request. 49 | 50 | ### Modify Code and Submit a PR (Pull Request) 51 | 52 | You can download the code, compile, install, and deploy to try it out (you can refer to the compilation documentation to see if it works as you expected). If there are any issues, you can directly contact us, submit an Issue, or fix it yourself by reading and analyzing the source code. For more details, see[How to Submit a PR.](/contribution/pr) 53 | 54 | Whether it's fixing a bug or adding a feature, we warmly welcome it. If you wish to submit code to Doris, you need to fork the code repository to your project space on GitHub, create a new branch for your submitted code, add the original project as an upstream, and submit a PR. The method for submitting a PR can be referenced in the Pull Request documentation. 55 | -------------------------------------------------------------------------------- /docs/blogDetails/20241111.en-US.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: 'CodeFuse 「编码挑战季」' 3 | time: '2024-11-11' 4 | toc: content 5 | --- 6 | 7 | ![](https://mdn.alipayobjects.com/huamei_bvbxju/afts/img/A*Rep6SLNlOy0AAAAAAAAAAAAADlHYAQ/fmt.webp) 8 | 9 | 众人拾柴火焰高，CodeFuse 自发布之初，就定义了推动研发生命周期自动化的目标：让研发变得更简单。一年来，我们开源了多个自研和微调的代码大模型，模型总下载数近 200 万。2024 年 1 月还在 **Big Code Model LeaderBoard 的评测中实现登顶**，受到众多开发者的欢迎。未来，我们还将以更开源开放的形式持续输出，欢迎开发者们与我们一路同行，一起参与共建 CodeFuse 社区。 10 | 11 | ## 社区任务 12 | 13 | 本次「编码挑战季」活动，共涉及 [muAgent](https://github.com/codefuse-ai/CodeFuse-muAgent)、[MFTCoder](https://github.com/codefuse-ai/MFTCoder)、[ModelCache](https://github.com/codefuse-ai/ModelCache)、[CodeFuse-IDE](https://github.com/codefuse-ai/codefuse-ide) 四个代码仓库，社区任务会在 https://github.com/orgs/codefuse-ai/projects 中进行发放。每个代码仓库将会有难、中、易不同程度的任务分类，开发者可以自由选择自己要参与的项目任务，并跟帖，我们将在第一时间分配任务，完成任务即获得所对应任务的积分。 14 | ● 简单：1 分 15 | ● 中度: 3 分 16 | ● 困难: 5 分 17 | 18 | **积分: 可兑换奖品** 19 | 20 | > 1 CodeFuse 定制鼠标垫 21 | > 2 CodeFuse 定制 U 型枕 22 | > 3 CodeFuse 定制单肩包 23 | > 4 CodeFuse 文化衫 24 | > 5 CodeFuse 定制双肩包 25 | 26 | ## 参与方式看这里 27 | 28 | 1. 👉 报名： 29 | ○ 点击[[这里](https://www.wjx.top/vm/Q77be70.aspx#)]完成报名 30 | ○ If you are an overseas developer, you can complete the event registration by sending an email to **codefuse_ai_native@service.alipay.com** 31 | ■ Title：[CodeFuse] sign up + GitHub ID 32 | ■ Text：Name + GitHub ID + Current school/Company + Major/Technical fields 33 | 2. 👉 加群： 添加小助手微信：codefuse01 ，备注【CodeFuse】，即可加入群聊 34 | 3. 👉 任务认领：你可以在 [GitHub Project](https://github.com/orgs/codefuse-ai/projects) 中查看本次活动的所有任务，包括详细的任务要求以及任务认领进度。若你对该任务感兴趣，请在任务具体 ISSUE 下按格式回复，该任务 assign 给你后即为预成功认领～ 35 | ○ 认领回复格式：【GitHub ID + Give it to me】 36 | ![1_1026950018_171_85_3_928127326_5b87bfb3a6680de497f70a0ea014b074](https://github.com/user-attachments/assets/70a34bcf-b3e9-447e-b6ac-dff24f4728c3) 37 | 4. 👉 任务开发： 成功认领任务后，即可进入任务的具体实现阶段啦，请在 活动有效期内完成任务哟。 38 | 5. 👉PR 提交： 当按照任务要求全部完成后，以 PR 形式提交要求内容到指定位置。 39 | 说明：收到 PR 提交后，研发老师会在 2-3 个工作日进行验收并给到 Review 意见。 40 | 41 | ## ‼️ 请注意： 42 | 43 | 1. 对于“简单”类型的任务一次只能认领/提交一个任务。若想要认领多个任务，可以在第一个任务完成并 merger 后，再进行第二个任务的认领与提交。对于“中度”、“困难”类型的任务，开发者可以组团参与。 44 | 2. 任务认领建议：同类任务可能出现开发过程类型的情况，若想要探索更多任务类型，可以选择多种类型任务进行尝试哈～ 45 | 3. 任务开发注意事项：为避免小伙伴们走弯路以及保证任务的完成度，以下有几个注意点： 46 | a. 请在任务成功认领后，一周内完成以下操作，否则该任务将被重新释放： 47 | ⅰ. 需提交设计思路的任务：在任务 ISSUE 下提交设计思路 48 | ⅱ. 无需提交设计思路的任务：在任务 ISSUE 下提交目前开发进展 / 遇到困难 or 提交 PR 49 | b. 在任务认领后，请在规定时间内完成任务，若有特殊情况可以及时通过任务 ISSUE 评论或与小助手联系： 50 | ⅰ. 简单 🌟：1-5 天 51 | ⅱ. 中度 🌟🌟：1-2 周 52 | ⅲ. 困难 🌟🌟🌟：3-4 周 53 | c. 在完成任务过程中，有任何问题都可以在 GitHub 上任务 ISSUE 下、本 Discussion / 微信群中提问（鼓励先在 ISSUE 中提问～） 54 | ![CodeFuse-AI群](https://github.com/user-attachments/assets/d33272a7-a7db-434e-8b0f-b1ccf2e7513a) 55 | d. 说明： 56 | ■ 认领任务后，需按照任务要求时间执行，弹性时间一周，若在任务要求时间+一周内未提交 PR，则任务将会被回收。可在后期有时间的状态下再进行任务认领。 57 | ■ 提交 PR 后，若针对修改意见在 2 周内未完善的，将合入符合要求部分的代码，并按照一定比例给予 🌟 奖励 58 | 4. PR 提交注意事项：提交 PR 时，请满足以下两点，以便快速进入 PR review 阶段： 59 | a. PR 标题参考 “【CodeFuse】 + PR 内容 ” ，如 “【CodeFuse】 xxxxx” 60 | b. PR 正文需包含“ Fixed + 认领的任务 ISSUE 链接”，能实现 PR 与对应任务 ISSUE 进行关联 61 | 5. 其他参考：📖 Git 操作是参与本次任务共建的必备技能哈～可以通过 [using-git](https://docs.github.com/en/get-started/using-git) 了解 git 的基本操作哈～ 62 | 6. PS：若有任何疑问，欢迎在微信群/对应任务 ISSUE /本条 Discussion 下留言，与我们交流。 63 | 7. 本次活动，用户完成任务后获得积分及不同奖品最终将通过社群、官网、Github 网站的信息公示发出，请积极参与哦~ 64 | 65 | ## 本期任务 66 | 67 | 任务认领进度以及任务详情均可在 [[GitHub Project](https://github.com/orgs/codefuse-ai/projects/3)] 中查看。 68 | 本期共有 20 左右 个任务，分为三个难度： 69 | ● [simple] 简单 🌟：1-5 天 欢迎新朋友认领，这是快速认识 CodeFuse 最好的方式哟～不要错过 70 | ● [moderate] 中度 🌟🌟：1-2 周 欢迎对 CodeFuse 有一定了解的朋友认领，这是深入与 CodeFuse 更熟悉的好机会～ 71 | ● [difficule] 困难 🌟🌟🌟：3-4 周 欢迎对 CodeFuse 有足够了解、想挑战技术深度、想要深度参与 CodeFuse 贡献的朋友们认领，逐步参与到 CodeFuse 核心能力的贡献啦～ 72 | 具体时间以任务详情中说明为准。 73 | -------------------------------------------------------------------------------- /docs/blogDetails/20241111.zh-CN.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: 'CodeFuse 「编码挑战季」' 3 | time: '2024-11-11' 4 | toc: content 5 | --- 6 | 7 | ![](https://mdn.alipayobjects.com/huamei_bvbxju/afts/img/A*Rep6SLNlOy0AAAAAAAAAAAAADlHYAQ/fmt.webp) 8 | 9 | 众人拾柴火焰高，CodeFuse 自发布之初，就定义了推动研发生命周期自动化的目标：让研发变得更简单。一年来，我们开源了多个自研和微调的代码大模型，模型总下载数近 200 万。2024 年 1 月还在 **Big Code Model LeaderBoard 的评测中实现登顶**，受到众多开发者的欢迎。未来，我们还将以更开源开放的形式持续输出，欢迎开发者们与我们一路同行，一起参与共建 CodeFuse 社区。 10 | 11 | ## 社区任务 12 | 13 | 本次「编码挑战季」活动，共涉及 [muAgent](https://github.com/codefuse-ai/CodeFuse-muAgent)、[MFTCoder](https://github.com/codefuse-ai/MFTCoder)、[ModelCache](https://github.com/codefuse-ai/ModelCache)、[CodeFuse-IDE](https://github.com/codefuse-ai/codefuse-ide) 四个代码仓库，社区任务会在 https://github.com/orgs/codefuse-ai/projects 中进行发放。每个代码仓库将会有难、中、易不同程度的任务分类，开发者可以自由选择自己要参与的项目任务，并跟帖，我们将在第一时间分配任务，完成任务即获得所对应任务的积分。 14 | ● 简单：1 分 15 | ● 中度: 3 分 16 | ● 困难: 5 分 17 | 18 | **积分: 可兑换奖品** 19 | 20 | > 1 CodeFuse 定制鼠标垫 21 | > 2 CodeFuse 定制 U 型枕 22 | > 3 CodeFuse 定制单肩包 23 | > 4 CodeFuse 文化衫 24 | > 5 CodeFuse 定制双肩包 25 | 26 | ## 参与方式看这里 27 | 28 | 1. 👉 报名： 29 | ○ 点击[[这里](https://www.wjx.top/vm/Q77be70.aspx#)]完成报名 30 | ○ If you are an overseas developer, you can complete the event registration by sending an email to **codefuse_ai_native@service.alipay.com** 31 | ■ Title：[CodeFuse] sign up + GitHub ID 32 | ■ Text：Name + GitHub ID + Current school/Company + Major/Technical fields 33 | 2. 👉 加群： 添加小助手微信：codefuse01 ，备注【CodeFuse】，即可加入群聊 34 | 3. 👉 任务认领：你可以在 [GitHub Project](https://github.com/orgs/codefuse-ai/projects) 中查看本次活动的所有任务，包括详细的任务要求以及任务认领进度。若你对该任务感兴趣，请在任务具体 ISSUE 下按格式回复，该任务 assign 给你后即为预成功认领～ 35 | ○ 认领回复格式：【GitHub ID + Give it to me】 36 | ![1_1026950018_171_85_3_928127326_5b87bfb3a6680de497f70a0ea014b074](https://github.com/user-attachments/assets/70a34bcf-b3e9-447e-b6ac-dff24f4728c3) 37 | 4. 👉 任务开发： 成功认领任务后，即可进入任务的具体实现阶段啦，请在 活动有效期内完成任务哟。 38 | 5. 👉PR 提交： 当按照任务要求全部完成后，以 PR 形式提交要求内容到指定位置。 39 | 说明：收到 PR 提交后，研发老师会在 2-3 个工作日进行验收并给到 Review 意见。 40 | 41 | ## ‼️ 请注意： 42 | 43 | 1. 对于“简单”类型的任务一次只能认领/提交一个任务。若想要认领多个任务，可以在第一个任务完成并 merger 后，再进行第二个任务的认领与提交。对于“中度”、“困难”类型的任务，开发者可以组团参与。 44 | 2. 任务认领建议：同类任务可能出现开发过程类型的情况，若想要探索更多任务类型，可以选择多种类型任务进行尝试哈～ 45 | 3. 任务开发注意事项：为避免小伙伴们走弯路以及保证任务的完成度，以下有几个注意点： 46 | a. 请在任务成功认领后，一周内完成以下操作，否则该任务将被重新释放： 47 | ⅰ. 需提交设计思路的任务：在任务 ISSUE 下提交设计思路 48 | ⅱ. 无需提交设计思路的任务：在任务 ISSUE 下提交目前开发进展 / 遇到困难 or 提交 PR 49 | b. 在任务认领后，请在规定时间内完成任务，若有特殊情况可以及时通过任务 ISSUE 评论或与小助手联系： 50 | ⅰ. 简单 🌟：1-5 天 51 | ⅱ. 中度 🌟🌟：1-2 周 52 | ⅲ. 困难 🌟🌟🌟：3-4 周 53 | c. 在完成任务过程中，有任何问题都可以在 GitHub 上任务 ISSUE 下、本 Discussion / 微信群中提问（鼓励先在 ISSUE 中提问～） 54 | ![CodeFuse-AI群](https://github.com/user-attachments/assets/d33272a7-a7db-434e-8b0f-b1ccf2e7513a) 55 | d. 说明： 56 | ■ 认领任务后，需按照任务要求时间执行，弹性时间一周，若在任务要求时间+一周内未提交 PR，则任务将会被回收。可在后期有时间的状态下再进行任务认领。 57 | ■ 提交 PR 后，若针对修改意见在 2 周内未完善的，将合入符合要求部分的代码，并按照一定比例给予 🌟 奖励 58 | 4. PR 提交注意事项：提交 PR 时，请满足以下两点，以便快速进入 PR review 阶段： 59 | a. PR 标题参考 “【CodeFuse】 + PR 内容 ” ，如 “【CodeFuse】 xxxxx” 60 | b. PR 正文需包含“ Fixed + 认领的任务 ISSUE 链接”，能实现 PR 与对应任务 ISSUE 进行关联 61 | 5. 其他参考：📖 Git 操作是参与本次任务共建的必备技能哈～可以通过 [using-git](https://docs.github.com/en/get-started/using-git) 了解 git 的基本操作哈～ 62 | 6. PS：若有任何疑问，欢迎在微信群/对应任务 ISSUE /本条 Discussion 下留言，与我们交流。 63 | 7. 本次活动，用户完成任务后获得积分及不同奖品最终将通过社群、官网、Github 网站的信息公示发出，请积极参与哦~ 64 | 65 | ## 本期任务 66 | 67 | 任务认领进度以及任务详情均可在 [[GitHub Project](https://github.com/orgs/codefuse-ai/projects/3)] 中查看。 68 | 本期共有 20 左右 个任务，分为三个难度： 69 | ● [simple] 简单 🌟：1-5 天 欢迎新朋友认领，这是快速认识 CodeFuse 最好的方式哟～不要错过 70 | ● [moderate] 中度 🌟🌟：1-2 周 欢迎对 CodeFuse 有一定了解的朋友认领，这是深入与 CodeFuse 更熟悉的好机会～ 71 | ● [difficule] 困难 🌟🌟🌟：3-4 周 欢迎对 CodeFuse 有足够了解、想挑战技术深度、想要深度参与 CodeFuse 贡献的朋友们认领，逐步参与到 CodeFuse 核心能力的贡献啦～ 72 | 具体时间以任务详情中说明为准。 73 | -------------------------------------------------------------------------------- /docs/docs/developer-docs/CodeFuse-ModelCache/main/quickstart.en-US.md: -------------------------------------------------------------------------------- 1 | --- 2 | store: 3 | title: CodeFuse-ModelCache 4 | version: main 5 | group: 6 | title: 🌱 CodeFuse-ModelCache 7 | order: -1 8 | title: QuickStart 9 | order: 0 10 | toc: content 11 | github: https://github.com/codefuse-ai/CodeFuse-ModelCache 12 | --- 13 | 14 | ModelCache is easy to use, and you can build a cache testing demo in just one step. 15 | 16 | ## Quick Start 17 | 18 | ### Demo Service Startup 19 | 20 | 1. Download the embedding model bin file from the following address: [https://huggingface.co/shibing624/text2vec-base-chinese/tree/main](https://huggingface.co/shibing624/text2vec-base-chinese/tree/main). Place the downloaded bin file in the model/text2vec-base-chinese folder. 21 | 2. Start the backend service using the flask4modelcache_dome.py script. 22 | 23 | ```shell 24 | cd CodeFuse-ModelCache 25 | ``` 26 | 27 | ```shell 28 | python flask4modelcache_demo.py 29 | ``` 30 | 31 | ### Normal Service Startup 32 | 33 | Before starting the service, the following environment configurations should be performed: 34 | 35 | 1. Install the relational database MySQL and import the SQL file to create the data tables. The SQL file can be found at: `reference_doc/create_table.sql` 36 | 2. Install the vector database Milvus. 37 | 3. Add the database access information to the configuration files: 38 | 1. `modelcache/config/milvus_config.ini ` 39 | 2. `modelcache/config/mysql_config.ini` 40 | 4. Download the embedding model bin file from the following address: [https://huggingface.co/shibing624/text2vec-base-chinese/tree/main](https://huggingface.co/shibing624/text2vec-base-chinese/tree/main). Place the downloaded bin file in the model/text2vec-base-chinese folder. 41 | 5. Start the backend service using the flask4modelcache.py script. 42 | 43 | ## Service-Access 44 | 45 | The current service provides three core functionalities through RESTful API.: Cache-Writing, Cache-Querying, and Cache-Clearing. Demos: 46 | 47 | ### Cache-Writing 48 | 49 | ```python 50 | import json 51 | import requests 52 | url = 'http://127.0.0.1:5000/modelcache' 53 | type = 'insert' 54 | scope = {"model": "CODEGPT-1008"} 55 | chat_info = [{"query": [{"role": "system", "content": "You are an AI code assistant and you must provide neutral and harmless answers to help users solve code-related problems."}, {"role": "user", "content": "你是谁?"}], 56 | "answer": "Hello, I am an intelligent assistant. How can I assist you?"}] 57 | data = {'type': type, 'scope': scope, 'chat_info': chat_info} 58 | headers = {"Content-Type": "application/json"} 59 | res = requests.post(url, headers=headers, json=json.dumps(data)) 60 | ``` 61 | 62 | ### Cache-Querying 63 | 64 | ```python 65 | import json 66 | import requests 67 | url = 'http://127.0.0.1:5000/modelcache' 68 | type = 'query' 69 | scope = {"model": "CODEGPT-1008"} 70 | query = [{"role": "system", "content": "You are an AI code assistant and you must provide neutral and harmless answers to help users solve code-related problems."}, {"role": "user", "content": "Who are you?"}] 71 | data = {'type': type, 'scope': scope, 'query': query} 72 | 73 | headers = {"Content-Type": "application/json"} 74 | res = requests.post(url, headers=headers, json=json.dumps(data)) 75 | ``` 76 | 77 | ### Cache-Clearing 78 | 79 | ```python 80 | import json 81 | import requests 82 | url = 'http://127.0.0.1:5000/modelcache' 83 | type = 'remove' 84 | scope = {"model": "CODEGPT-1008"} 85 | remove_type = 'truncate_by_model' 86 | data = {'type': type, 'scope': scope, 'remove_type': remove_type} 87 | 88 | headers = {"Content-Type": "application/json"} 89 | res = requests.post(url, headers=headers, json=json.dumps(data)) 90 | ``` 91 | --------------------------------------------------------------------------------