├── .gitattributes ├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ ├── feature_request.md │ └── new-framework-support.md └── workflows │ └── ci.yml ├── .gitignore ├── .idea └── .name ├── LICENSE ├── README.md ├── collector ├── README.md ├── fintech │ ├── pdf-to-text.py │ └── requirements.txt └── swagger │ ├── .gitignore │ ├── README.md │ ├── index.js │ ├── package-lock.json │ └── package.json ├── docs └── .gitkeep ├── index ├── .gitignore ├── build_index.py ├── examples │ └── .gitkeep └── requirements.txt ├── processor ├── .editorconfig ├── .gitignore ├── .gitkeep ├── README.md ├── build.gradle.kts ├── common │ ├── cli-core │ │ ├── build.gradle.kts │ │ └── src │ │ │ └── main │ │ │ └── kotlin │ │ │ └── cc │ │ │ └── unitmesh │ │ │ └── core │ │ │ ├── cli │ │ │ ├── ProcessorConfig.kt │ │ │ └── ProcessorUtils.kt │ │ │ └── git │ │ │ ├── Command.kt │ │ │ ├── ExecListeners.kt │ │ │ └── GitCommandManager.kt │ ├── core │ │ ├── build.gradle.kts │ │ └── src │ │ │ ├── main │ │ │ └── kotlin │ │ │ │ └── cc │ │ │ │ └── unitmesh │ │ │ │ └── core │ │ │ │ ├── Instruction.kt │ │ │ │ └── model │ │ │ │ ├── ApiCollection.kt │ │ │ │ └── Usecase.kt │ │ │ └── test │ │ │ └── kotlin │ │ │ └── cc │ │ │ └── unitmesh │ │ │ └── core │ │ │ └── model │ │ │ └── UsecaseKtTest.kt │ ├── prompter │ │ ├── build.gradle.kts │ │ └── src │ │ │ └── main │ │ │ └── kotlin │ │ │ └── cc │ │ │ └── unitmesh │ │ │ └── core │ │ │ └── prompter │ │ │ └── OpenAiPrompter.kt │ └── verifier │ │ ├── build.gradle.kts │ │ └── src │ │ ├── main │ │ └── kotlin │ │ │ └── cc │ │ │ └── unitmesh │ │ │ └── verifier │ │ │ ├── drawio │ │ │ ├── DrawioSerial.kt │ │ │ ├── DrawioVerifier.kt │ │ │ └── MxGraph.kt │ │ │ ├── markdown │ │ │ ├── MarkdownExt.kt │ │ │ ├── MarkdownVerifier.kt │ │ │ └── UsecaseParser.kt │ │ │ ├── plantuml │ │ │ └── PlantUmlParser.kt │ │ │ └── sql │ │ │ └── SqlParser.kt │ │ └── test │ │ └── kotlin │ │ └── cc │ │ └── unitmesh │ │ └── verifier │ │ └── markdown │ │ ├── MarkdownVerifierTest.kt │ │ ├── TableHeaderVisitorTest.kt │ │ └── UsecaseParserTest.kt ├── docs │ ├── debug.md │ └── kotlin.md ├── examples │ └── unit-demo │ │ ├── .gitignore │ │ ├── build.gradle.kts │ │ ├── docs │ │ ├── dsl.draft.md │ │ └── prompt-template.md │ │ ├── gradle.properties │ │ └── src │ │ ├── main │ │ ├── kotlin │ │ │ └── unitmesh │ │ │ │ └── cc │ │ │ │ ├── Application.kt │ │ │ │ └── plugins │ │ │ │ ├── Databases.kt │ │ │ │ ├── Routing.kt │ │ │ │ ├── Security.kt │ │ │ │ ├── Serialization.kt │ │ │ │ └── UsersSchema.kt │ │ └── resources │ │ │ └── logback.xml │ │ └── test │ │ └── kotlin │ │ └── unitmesh │ │ └── cc │ │ └── ApplicationTest.kt ├── gradle.properties ├── gradle │ ├── libs.versions.toml │ └── wrapper │ │ ├── gradle-wrapper.jar │ │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── java-modules │ ├── .gitignore │ ├── codegen-processor │ │ ├── build.gradle.kts │ │ └── src │ │ │ └── main │ │ │ └── kotlin │ │ │ └── cc │ │ │ └── unitmesh │ │ │ └── processor │ │ │ └── codegen │ │ │ └── Main.kt │ ├── core-analysis │ │ ├── build.gradle.kts │ │ └── src │ │ │ ├── main │ │ │ └── kotlin │ │ │ │ └── cc │ │ │ │ └── unitmesh │ │ │ │ └── core │ │ │ │ ├── JvmProcessor.kt │ │ │ │ └── java │ │ │ │ ├── JavaProcessor.kt │ │ │ │ ├── ShortClass.kt │ │ │ │ ├── SpringProcessor.kt │ │ │ │ └── TestProcessor.kt │ │ │ └── test │ │ │ └── kotlin │ │ │ └── cc │ │ │ └── unitmesh │ │ │ └── core │ │ │ └── java │ │ │ ├── JavaProcessorTest.kt │ │ │ ├── ShortClassTest.kt │ │ │ ├── SpringProcessorTest.kt │ │ │ └── TestProcessorTest.kt │ ├── spring-processor │ │ ├── build.gradle.kts │ │ └── src │ │ │ └── main │ │ │ └── kotlin │ │ │ └── cc │ │ │ └── unitmesh │ │ │ └── spring │ │ │ └── Main.kt │ └── test-processor │ │ ├── build.gradle.kts │ │ └── src │ │ └── main │ │ └── kotlin │ │ └── cc │ │ └── unitmesh │ │ └── processor │ │ └── Main.kt ├── kotlin-modules │ ├── analysis │ │ ├── build.gradle.kts │ │ └── src │ │ │ └── main │ │ │ └── kotlin │ │ │ └── ktlint │ │ │ └── analysis │ │ │ ├── Code.kt │ │ │ ├── KotlinPsiFileFactory.kt │ │ │ ├── KtLintParseException.kt │ │ │ └── PositionInTextLocator.kt │ └── importer │ │ ├── .gitignore │ │ ├── README.md │ │ ├── build.gradle.kts │ │ └── src │ │ ├── main │ │ ├── kotlin │ │ │ └── cc │ │ │ │ └── unitmesh │ │ │ │ └── importer │ │ │ │ ├── Main.kt │ │ │ │ ├── SnippetPrompt.kt │ │ │ │ ├── Snippets.kt │ │ │ │ ├── model │ │ │ │ ├── CodeSnippet.kt │ │ │ │ ├── PackageUtil.kt │ │ │ │ └── RawDump.kt │ │ │ │ └── processor │ │ │ │ ├── KotlinCodeProcessor.kt │ │ │ │ └── KotlinParserWrapper.kt │ │ └── resources │ │ │ ├── .gitkeep │ │ │ └── prompt.txt │ │ └── test │ │ ├── kotlin │ │ └── cc │ │ │ └── unitmesh │ │ │ └── importer │ │ │ ├── SnippetsKtTest.kt │ │ │ ├── filter │ │ │ └── KotlinCodeProcessorTest.kt │ │ │ └── model │ │ │ ├── PackageUtilTest.kt │ │ │ └── RawDumpTest.kt │ │ └── resources │ │ ├── snippets.json │ │ └── types.json ├── modules │ ├── api-processor │ │ ├── .gitignore │ │ ├── README.md │ │ ├── build.gradle.kts │ │ ├── docs │ │ │ ├── domain │ │ │ │ ├── banking.csv │ │ │ │ ├── domains-2.csv │ │ │ │ └── domains.md │ │ │ ├── domains.csv │ │ │ └── prompter │ │ │ │ ├── domain-model.prompt.txt │ │ │ │ ├── domain-reprompter.txt │ │ │ │ ├── introduction-service.prompt.txt │ │ │ │ ├── introduction.prompt.txt │ │ │ │ ├── uml-to-usecases.txt │ │ │ │ └── uml-to-userstory.txt │ │ └── src │ │ │ ├── main │ │ │ ├── kotlin │ │ │ │ └── cc │ │ │ │ │ └── unitmesh │ │ │ │ │ └── processor │ │ │ │ │ └── api │ │ │ │ │ ├── ApiProcessorDetector.kt │ │ │ │ │ ├── Main.kt │ │ │ │ │ ├── base │ │ │ │ │ ├── ApiDetailRender.kt │ │ │ │ │ └── ApiProcessor.kt │ │ │ │ │ ├── command │ │ │ │ │ ├── CommandExt.kt │ │ │ │ │ ├── Generating.kt │ │ │ │ │ ├── Modeling.kt │ │ │ │ │ ├── Prompting.kt │ │ │ │ │ └── Usecase.kt │ │ │ │ │ ├── parser │ │ │ │ │ ├── PostmanParser.kt │ │ │ │ │ └── PostmanProcessor.kt │ │ │ │ │ ├── postman │ │ │ │ │ ├── LICENSE │ │ │ │ │ ├── PostmanAuth.kt │ │ │ │ │ ├── PostmanBody.kt │ │ │ │ │ ├── PostmanCollection.kt │ │ │ │ │ ├── PostmanEnvValue.kt │ │ │ │ │ ├── PostmanEnvironment.kt │ │ │ │ │ ├── PostmanEvent.kt │ │ │ │ │ ├── PostmanFile.kt │ │ │ │ │ ├── PostmanFolder.kt │ │ │ │ │ ├── PostmanHeader.kt │ │ │ │ │ ├── PostmanInfo.kt │ │ │ │ │ ├── PostmanItem.kt │ │ │ │ │ ├── PostmanQuery.kt │ │ │ │ │ ├── PostmanReader.kt │ │ │ │ │ ├── PostmanRequest.kt │ │ │ │ │ ├── PostmanResponse.kt │ │ │ │ │ ├── PostmanScript.kt │ │ │ │ │ ├── PostmanUrl.kt │ │ │ │ │ ├── PostmanUrlEncoded.kt │ │ │ │ │ ├── PostmanVariable.kt │ │ │ │ │ ├── PostmanVariables.kt │ │ │ │ │ └── PoyntHttpHeaders.kt │ │ │ │ │ ├── render │ │ │ │ │ ├── MarkdownTableRender.kt │ │ │ │ │ └── SimpleApiRender.kt │ │ │ │ │ └── swagger │ │ │ │ │ └── Swagger3Processor.kt │ │ │ └── resources │ │ │ │ ├── logback-test.xml │ │ │ │ └── logback.xml │ │ │ └── test │ │ │ ├── kotlin │ │ │ └── cc │ │ │ │ └── unitmesh │ │ │ │ └── processor │ │ │ │ └── api │ │ │ │ ├── ApiProcessorDetectorTest.kt │ │ │ │ ├── ModelSerializeTest.kt │ │ │ │ ├── parser │ │ │ │ └── PostmanParserTest.kt │ │ │ │ └── render │ │ │ │ ├── MarkdownTableRenderTest.kt │ │ │ │ └── SimpleApiRenderTest.kt │ │ │ └── resources │ │ │ ├── openapi │ │ │ ├── Box.json │ │ │ └── CircleCI.postman_collection.json │ │ │ ├── postman │ │ │ └── Wechat-Simple.postman_collection.json │ │ │ └── testsets │ │ │ ├── postman.json │ │ │ ├── swagger-3.json │ │ │ └── swagger-3.yaml │ └── plantuml-verifier │ │ ├── build.gradle.kts │ │ └── src │ │ └── main │ │ ├── kotlin │ │ └── cc │ │ │ └── unitmesh │ │ │ └── verifier │ │ │ └── uml │ │ │ └── Main.kt │ │ └── resources │ │ ├── logback-test.xml │ │ └── logback.xml └── settings.gradle.kts ├── prompter └── prepare │ ├── .gitignore │ ├── README.md │ ├── prompts │ ├── create-user-story-name.md │ ├── create-user-story.md │ ├── create_api_prompt.txt │ ├── create_userstory_from_swagger.md │ └── test_to_code.md │ ├── repository-to-prompt.py │ ├── requirements.txt │ ├── swagger-user-story.py │ ├── test-to-code.py │ ├── user-story.py │ └── utils.py └── train ├── README.md ├── alpaca-lora.ipynb └── chatglm-tuning.ipynb /.gitattributes: -------------------------------------------------------------------------------- 1 | *.ipynb linguist-vendored 2 | *.ipynb linguist-detectable=false 3 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **Env (please complete the following information):** 14 | - OS: [e.g. iOS] 15 | - Env: Docker compose? Local dev? 16 | - Browser [e.g. chrome, safari] 17 | - Version [e.g. 22] 18 | 19 | 20 | **To Reproduce** 21 | Steps to reproduce the behavior: 22 | 1. Go to '...' 23 | 2. Click on '....' 24 | 3. Scroll down to '....' 25 | 4. See error 26 | 27 | **Expected behavior** 28 | A clear and concise description of what you expected to happen. 29 | 30 | **Screenshots** 31 | If applicable, add screenshots to help explain your problem. 32 | 33 | 34 | **Additional context** 35 | Add any other context about the problem here. 36 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/new-framework-support.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: New framework support 3 | about: new framework support 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Which language would you to support** 11 | xxx 12 | 13 | **Example of this language HTTP API syntax** 14 | xxx 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: [ push ] 4 | 5 | jobs: 6 | build: 7 | strategy: 8 | matrix: 9 | os: [ macos-latest, ubuntu-latest ] 10 | runs-on: ${{ matrix.os }} 11 | 12 | steps: 13 | - uses: actions/checkout@v3 14 | - uses: actions/setup-java@v3 15 | with: 16 | distribution: 'zulu' 17 | java-version: '17' 18 | 19 | - name: Setup Gradle 20 | uses: gradle/gradle-build-action@v2 21 | with: 22 | arguments: build 23 | build-root-directory: processor 24 | 25 | - name: Execute Gradle build 26 | run: ./gradlew build 27 | working-directory: processor 28 | 29 | - name: Execute Gradle Coverage 30 | run: ./gradlew check 31 | working-directory: processor 32 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .gradle 2 | build/ 3 | !gradle/wrapper/gradle-wrapper.jar 4 | !**/src/main/**/build/ 5 | !**/src/test/**/build/ 6 | 7 | ### IntelliJ IDEA ### 8 | .idea/modules.xml 9 | .idea/jarRepositories.xml 10 | .idea/compiler.xml 11 | .idea/libraries/ 12 | *.iws 13 | *.iml 14 | *.ipr 15 | out/ 16 | !**/src/main/**/out/ 17 | !**/src/test/**/out/ 18 | 19 | ### Eclipse ### 20 | .apt_generated 21 | .classpath 22 | .factorypath 23 | .project 24 | .settings 25 | .springBeans 26 | .sts4-cache 27 | bin/ 28 | !**/src/main/**/bin/ 29 | !**/src/test/**/bin/ 30 | 31 | ### NetBeans ### 32 | /nbproject/private/ 33 | /nbbuild/ 34 | /dist/ 35 | /nbdist/ 36 | /.nb-gradle/ 37 | 38 | ### VS Code ### 39 | .vscode/ 40 | 41 | ### Mac OS ### 42 | .DS_Store 43 | origindatasets 44 | datasets 45 | processor.yml 46 | *.jsonl 47 | .idea/workspace.xml 48 | .idea/uiDesigner.xml 49 | .idea 50 | -------------------------------------------------------------------------------- /.idea/.name: -------------------------------------------------------------------------------- 1 | UnitProcessor -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # DevTi - 开发钛 2 | 3 | [![CI](https://github.com/unit-mesh/devti/actions/workflows/ci.yml/badge.svg)](https://github.com/unit-mesh/devti/actions/workflows/ci.yml) 4 | 5 | > DevTi（Development + Titanium）一款基于大语言模型的研发效能提升的开源项目。旨在基于 LLM 的微调来提供全面智能化解决方案，助力开发人员高效完成开发任务，以实现自动化用户任务拆解、用户故事生成、自动化代码生成、自动化测试生成等等。 6 | 7 | ## About DevTi 8 | 9 | ![DevTi](https://unitmesh.cc/images/devti-processes.png) 10 | 11 | 特性: 12 | 13 | - 全流程研发效能提升 14 | - 端到端的 AI 辅助生成 15 | - 需求拆解 16 | - 拆分、分解用户故事 17 | - 测试代码生成 18 | - 代码辅助生成 19 | - …… 20 | - 研发实践辅助导入 21 | - 研发规范自检内嵌 22 | 23 | ## Usage 24 | 25 | Instruction: 26 | 27 | - 拆分任务。instruction：split user story tasks，input：折分用户故事任务 28 | - 需求细化。instruction：create Agile user story for following topic，input：功能的基本信息 29 | - 代码生成。instruction：Implement the method xxx，input：类的基本信息 30 | - 测试生成。instruction：Write test for follow code，input：类的基本信息 31 | - Repository 生成。instruction：text to repository with class，input：功能的基本信息 32 | 33 | ## Development 34 | 35 | ![DevTi](https://unitmesh.cc/images/devti.png) 36 | 37 | 文档详细见各模块的 README.md 38 | 39 | 模块： 40 | 41 | - [Collector](./collector)（Python, JavaScript），数据收集 42 | - [Processor](./processor)（Kotlin），数据处理 43 | - [Prompter](./prompter)（Python），Prompt 设计、调整、优化等 44 | - [Train](./train)（Python），训练相关的 Notebook 45 | - Chain（Python），开发框架 46 | 47 | Todos： 48 | 49 | - [x] 端到端的 AI 辅助生成 50 | - [x] 需求拆解 51 | - [x] 分析用户故事 52 | - [x] 测试代码生成 53 | - [x] 代码辅助生成 54 | - SQL 生成 55 | - ... 56 | - [ ] 研发实践辅助导入 57 | - [ ] 研发规范自检内嵌 58 | 59 | 更新中... 60 | 61 | ## LICENSE 62 | 63 | This code is distributed under the MPL 2.0 license. See `LICENSE` in this directory. 64 | -------------------------------------------------------------------------------- /collector/README.md: -------------------------------------------------------------------------------- 1 | # Collector 2 | 3 | - Swagger Crawler: [swagger](./swagger) 4 | -------------------------------------------------------------------------------- /collector/fintech/pdf-to-text.py: -------------------------------------------------------------------------------- 1 | import glob 2 | import json 3 | import re 4 | 5 | import PyPDF2 6 | 7 | title_pattern = r'^[一二三四五六七八九十零〇百千万亿]+、(.*?)$' 8 | subtitle_pattern = r'^（[一二三四五六七八九十零〇百千万亿]+）(.*?)$' 9 | 10 | 11 | def is_valid_subtitle(title: str) -> bool: 12 | # Check if the title matches the pattern 13 | if re.match(subtitle_pattern, title): 14 | return True 15 | else: 16 | return False 17 | 18 | 19 | def get_valid_subtitle(title: str) -> str: 20 | if re.match(subtitle_pattern, title): 21 | return title.strip().split('）')[1].strip() 22 | else: 23 | return "" 24 | 25 | 26 | # a valid title should start with a chinese_numbers then a 、, like `十一、基金的财产` 27 | def is_valid_title(title: str) -> bool: 28 | # Check if the title matches the pattern 29 | if re.match(title_pattern, title): 30 | return True 31 | else: 32 | return False 33 | 34 | 35 | def get_valid_title(title: str) -> str: 36 | if re.match(title_pattern, title): 37 | return title.strip().split('、')[1].strip() 38 | else: 39 | return "" 40 | 41 | 42 | def read_pdf_file(file: str, fund_name: str) -> list[str]: 43 | text_list = [] 44 | with open(file, "rb") as fp: 45 | # Create a PDF object 46 | pdf = PyPDF2.PdfReader(fp) 47 | 48 | # Get the number of pages in the PDF document 49 | num_pages = len(pdf.pages) 50 | 51 | # Iterate over every page 52 | for page in range(num_pages): 53 | # Extract the text from the page 54 | page_text = pdf.pages[page].extract_text() 55 | 56 | # use regex remove footnotes, which are in the form of `招商基金管理有限公司 57 | # 招募说明书 \n29` 58 | page_text = re.sub(r'\s+招募说明书\s+\d+', '', page_text) 59 | 60 | # remove continues empty lines which are in the form of `\n\n\n\n` 61 | page_text = re.sub(r'\n\n', '', page_text) 62 | 63 | text_list.append(page_text) 64 | 65 | return text_list 66 | 67 | 68 | question_answers = [] 69 | 70 | 71 | # use regex to match 72 | # input: 2017-10-10-005230.OF---长盛货币B-长盛货币市场基金托管协议.pdf, output: 长盛货币B 73 | # input: 2022-09-09-007725.OF---招商瑞文A-招商瑞文混合型证券投资基金招募说明书更新.pdf, output: 招商瑞文A 74 | def get_fund_name(file: str) -> str: 75 | # 定义匹配基金名称的正则表达式 76 | pattern = re.compile(r'[\u4e00-\u9fa5\w]+\-\w+') 77 | 78 | # 在文件名中搜索匹配的字符串 79 | match = pattern.search(file) 80 | 81 | if match: 82 | # 返回匹配的基金名称 83 | return match.group(0) 84 | else: 85 | # 如果找不到匹配的基金名称，则返回空字符串 86 | return "" 87 | 88 | 89 | def process_pdf_to_question(file: str): 90 | fund_name = get_fund_name(file) 91 | 92 | text_list = read_pdf_file(file, fund_name) 93 | # merge all the text into one string 94 | page_text = ''.join(text_list) 95 | # write page_text to a file 96 | with open("domain-pdf.txt", "w") as fp: 97 | fp.write(page_text) 98 | 99 | lines = page_text.split('\n') 100 | start_question = False 101 | answer = "" 102 | last_question = "" 103 | 104 | for line in lines: 105 | if is_valid_title(line.strip()): 106 | start_question = False 107 | continue 108 | 109 | if is_valid_subtitle(line.strip()): 110 | if len(answer) > 0: 111 | create_questions(answer, fund_name, last_question) 112 | answer = "" 113 | 114 | last_question = get_valid_subtitle(line) 115 | start_question = True 116 | 117 | if start_question: 118 | answer += line 119 | 120 | create_questions(answer, fund_name, last_question) 121 | 122 | 123 | def create_questions(answer, fund_name, last_question): 124 | question_answers.append({ 125 | "instruction": '介绍一下' + fund_name + '的' + last_question + '?', 126 | "input": "", 127 | "output": answer 128 | }) 129 | question_answers.append({ 130 | "instruction": '什么是' + fund_name + last_question + '?', 131 | "input": "", 132 | "output": answer 133 | }) 134 | 135 | 136 | if __name__ == '__main__': 137 | # fetch all files in the directory testsets/*.pdf 138 | for file in glob.glob("testsets/*.pdf"): 139 | process_pdf_to_question(file) 140 | 141 | print("question size: ", len(question_answers)) 142 | # write question_answers to jsonl file 143 | with open("domain-pdf.jsonl", "w") as fp: 144 | for qa in question_answers: 145 | fp.write(json.dumps(qa)) 146 | fp.write('\n') 147 | -------------------------------------------------------------------------------- /collector/fintech/requirements.txt: -------------------------------------------------------------------------------- 1 | PyPDF2 -------------------------------------------------------------------------------- /collector/swagger/.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | lerna-debug.log* 8 | 9 | # Diagnostic reports (https://nodejs.org/api/report.html) 10 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json 11 | 12 | # Runtime data 13 | pids 14 | *.pid 15 | *.seed 16 | *.pid.lock 17 | 18 | # Directory for instrumented libs generated by jscoverage/JSCover 19 | lib-cov 20 | 21 | # Coverage directory used by tools like istanbul 22 | coverage 23 | *.lcov 24 | 25 | # nyc test coverage 26 | .nyc_output 27 | 28 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) 29 | .grunt 30 | 31 | # Bower dependency directory (https://bower.io/) 32 | bower_components 33 | 34 | # node-waf configuration 35 | .lock-wscript 36 | 37 | # Compiled binary addons (https://nodejs.org/api/addons.html) 38 | build/Release 39 | 40 | # Dependency directories 41 | node_modules/ 42 | jspm_packages/ 43 | 44 | # TypeScript v1 declaration files 45 | typings/ 46 | 47 | # TypeScript cache 48 | *.tsbuildinfo 49 | 50 | # Optional npm cache directory 51 | .npm 52 | 53 | # Optional eslint cache 54 | .eslintcache 55 | 56 | # Microbundle cache 57 | .rpt2_cache/ 58 | .rts2_cache_cjs/ 59 | .rts2_cache_es/ 60 | .rts2_cache_umd/ 61 | 62 | # Optional REPL history 63 | .node_repl_history 64 | 65 | # Output of 'npm pack' 66 | *.tgz 67 | 68 | # Yarn Integrity file 69 | .yarn-integrity 70 | 71 | # dotenv environment variables file 72 | .env 73 | .env.test 74 | 75 | # parcel-bundler cache (https://parceljs.org/) 76 | .cache 77 | 78 | # Next.js build output 79 | .next 80 | 81 | # Nuxt.js build / generate output 82 | .nuxt 83 | dist 84 | 85 | # Gatsby files 86 | .cache/ 87 | # Comment in the public line in if your project uses Gatsby and *not* Next.js 88 | # https://nextjs.org/blog/next-9-1#public-directory-support 89 | # public 90 | 91 | # vuepress build output 92 | .vuepress/dist 93 | 94 | # Serverless directories 95 | .serverless/ 96 | 97 | # FuseBox cache 98 | .fusebox/ 99 | 100 | # DynamoDB Local files 101 | .dynamodb/ 102 | 103 | # TernJS port file 104 | .tern-port 105 | 106 | results 107 | results.* -------------------------------------------------------------------------------- /collector/swagger/README.md: -------------------------------------------------------------------------------- 1 | # github-swagger-crawler 2 | 3 | ``` 4 | npm install 5 | echo "GITHUB_TOKEN=YOUR_GITHUB_ACCESS_TOKEN" > .env 6 | node index.js [START_PAGE] 7 | ``` 8 | -------------------------------------------------------------------------------- /collector/swagger/index.js: -------------------------------------------------------------------------------- 1 | import { Octokit, App } from "octokit"; 2 | import * as dotenv from 'dotenv'; // see https://github.com/motdotla/dotenv#how-do-i-use-dotenv-with-import 3 | dotenv.config(); 4 | import { writeFileSync, existsSync, mkdirSync } from 'fs'; 5 | import YAML from 'yaml'; 6 | 7 | // Octokit.js 8 | // https://github.com/octokit/core.js#readme 9 | const octokit = new Octokit({ 10 | auth: process.env.GITHUB_TOKEN 11 | }) 12 | 13 | if (!existsSync("./results")) 14 | mkdirSync("./results"); 15 | let currentPage = process.argv[2] || 1; 16 | let continueFlag = true; 17 | while (continueFlag) { 18 | let result = undefined; 19 | try { 20 | console.log("Current page: " + currentPage); 21 | result = await octokit.rest.search.code({ 22 | q: 'swagger openapi paths+in:file+language:yml', 23 | per_page: 100, 24 | page: currentPage 25 | }); 26 | } 27 | catch(e) { 28 | console.error(e); 29 | continueFlag = false; 30 | break; 31 | } 32 | if (result && "data" in result && "items" in result.data) { 33 | const items = result.data.items; 34 | for (const i in items) { 35 | try { 36 | const item = items[i]; 37 | const url = item.html_url.replace("https://github.com/", "https://raw.githubusercontent.com/").replace("/blob", ""); 38 | const response = await fetch(url); 39 | 40 | if (!item.html_url.endsWith(".json")) { 41 | const text = await response.text(); 42 | const documents = YAML.parseAllDocuments(text); 43 | if (documents && documents.length) { 44 | for (const i in documents) { 45 | const document = documents[i]; 46 | if ((document.has("swagger") || document.has("openapi"))&& document.has("paths")) { 47 | const filename = "./results/" + item.repository.full_name.replaceAll("/", "_") + "_" + i + "_" + item.name; 48 | console.log(filename); 49 | writeFileSync(filename, document.toString()); 50 | } 51 | } 52 | } 53 | } else { 54 | const node = await response.json(); 55 | if ((node["swagger"] || node["openapi"])&& node["paths"]) { 56 | const filename = "./results/" + item.repository.full_name.replaceAll("/", "_") + "_" + i + "_" + item.name; 57 | console.log(filename); 58 | writeFileSync(filename, JSON.stringify(node, null, 2)); 59 | } 60 | } 61 | } 62 | catch(e) { 63 | console.error(e); 64 | } 65 | } 66 | currentPage++; 67 | } 68 | await new Promise(resolve => setTimeout(resolve, 30000)); 69 | } -------------------------------------------------------------------------------- /collector/swagger/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "github-swagger-crawler", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "start": "node index.js", 8 | "test": "echo \"Error: no test specified\" && exit 1" 9 | }, 10 | "type": "module", 11 | "author": "", 12 | "license": "MIT", 13 | "dependencies": { 14 | "dotenv": "^16.0.3", 15 | "octokit": "^2.0.14", 16 | "yaml": "^2.2.2" 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /docs/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unit-mesh/devti/9ad45cf571a40867e2d5fa1289a4aff16a03d4f5/docs/.gitkeep -------------------------------------------------------------------------------- /index/.gitignore: -------------------------------------------------------------------------------- 1 | venv 2 | -------------------------------------------------------------------------------- /index/build_index.py: -------------------------------------------------------------------------------- 1 | from llama_index import ( 2 | GPTKeywordTableIndex, 3 | SimpleDirectoryReader, 4 | LLMPredictor, 5 | ServiceContext 6 | ) 7 | from langchain import OpenAI 8 | 9 | documents = SimpleDirectoryReader('data').load_data() 10 | 11 | # define LLM 12 | llm_predictor = LLMPredictor(llm=OpenAI(temperature=0, model_name="text-davinci-003")) 13 | service_context = ServiceContext.from_defaults(llm_predictor=llm_predictor) 14 | -------------------------------------------------------------------------------- /index/examples/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unit-mesh/devti/9ad45cf571a40867e2d5fa1289a4aff16a03d4f5/index/examples/.gitkeep -------------------------------------------------------------------------------- /index/requirements.txt: -------------------------------------------------------------------------------- 1 | openai==0.27.4 2 | llama-index 3 | tiktoken -------------------------------------------------------------------------------- /processor/.editorconfig: -------------------------------------------------------------------------------- 1 | [*.{kt,kts}] 2 | ktlint_code_style = ktlint_official 3 | ktlint_standard_no-wildcard-imports = disabled 4 | ktlint_standard_max-line-length = disabled -------------------------------------------------------------------------------- /processor/.gitignore: -------------------------------------------------------------------------------- 1 | .gradle 2 | build/ 3 | !gradle/wrapper/gradle-wrapper.jar 4 | !**/src/main/**/build/ 5 | !**/src/test/**/build/ 6 | 7 | ### IntelliJ IDEA ### 8 | .idea/modules.xml 9 | .idea/jarRepositories.xml 10 | .idea/compiler.xml 11 | .idea/libraries/ 12 | *.iws 13 | *.iml 14 | *.ipr 15 | out/ 16 | !**/src/main/**/out/ 17 | !**/src/test/**/out/ 18 | 19 | ### Eclipse ### 20 | .apt_generated 21 | .classpath 22 | .factorypath 23 | .project 24 | .settings 25 | .springBeans 26 | .sts4-cache 27 | bin/ 28 | !**/src/main/**/bin/ 29 | !**/src/test/**/bin/ 30 | 31 | ### NetBeans ### 32 | /nbproject/private/ 33 | /nbbuild/ 34 | /dist/ 35 | /nbdist/ 36 | /.nb-gradle/ 37 | 38 | ### VS Code ### 39 | .vscode/ 40 | 41 | ### Mac OS ### 42 | .DS_Store 43 | origindatasets 44 | datasets 45 | processor.yml 46 | *.jsonl 47 | .idea/workspace.xml 48 | .idea/uiDesigner.xml 49 | prompt.txt 50 | source.csv 51 | output 52 | .env 53 | prompt2.txt 54 | domains.csv 55 | input 56 | domain-prompt.txt 57 | usecase-prompt.txt -------------------------------------------------------------------------------- /processor/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unit-mesh/devti/9ad45cf571a40867e2d5fa1289a4aff16a03d4f5/processor/.gitkeep -------------------------------------------------------------------------------- /processor/README.md: -------------------------------------------------------------------------------- 1 | # Processor 2 | 3 | > Processor 是一个代码分析和处理引擎，用于在 AI 编程的前半部分的代码分析和处理。 4 | 5 | 对源码进行语法分析，分解出不同的 Unit 6 | 7 | - Common 8 | - Cli-Core. core cli 9 | - Prompter. LLM prompter 10 | - Core. core models 11 | - Verifier. UML, SQL, Markdown, etc. 12 | - Java 13 | - Codegen Processor. 14 | - Core Analysis. Java Syntax Analysis 15 | - Spring Processor. Controller, Service, Model, Repository, etc. 16 | - Test Processor. 17 | - Kotlin 18 | - Syntax Analysis 19 | - Importer（Text to Repository） 20 | - Modules 21 | - API Processor 22 | - PlantUML Verifier 23 | 24 | ## LICENSE 25 | 26 | API Processor module postman based on [https://github.com/poynt/postman-runner](https://github.com/poynt/postman-runner) 27 | with Apache 2.0 license. 28 | 29 | This code is distributed under the MPL 2.0 license. See `LICENSE` in this directory. 30 | -------------------------------------------------------------------------------- /processor/build.gradle.kts: -------------------------------------------------------------------------------- 1 | @Suppress("DSL_SCOPE_VIOLATION") 2 | plugins { 3 | alias(libs.plugins.kotlin.jvm) 4 | } 5 | 6 | val ktlint by configurations.creating 7 | 8 | dependencies { 9 | ktlint("com.pinterest:ktlint:0.49.0") { 10 | attributes { 11 | attribute(Bundling.BUNDLING_ATTRIBUTE, objects.named(Bundling.EXTERNAL)) 12 | } 13 | } 14 | // ktlint(project(":custom-ktlint-ruleset")) // in case of custom ruleset 15 | } 16 | 17 | val ktlintCheck by tasks.registering(JavaExec::class) { 18 | group = LifecycleBasePlugin.VERIFICATION_GROUP 19 | description = "Check Kotlin code style" 20 | classpath = ktlint 21 | mainClass.set("com.pinterest.ktlint.Main") 22 | // see https://pinterest.github.io/ktlint/install/cli/#command-line-usage for more information 23 | args( 24 | "**/src/**/*.kt", 25 | "**.kts", 26 | "!**/build/**", 27 | ) 28 | } 29 | 30 | // tasks.check { 31 | // dependsOn(ktlintCheck) 32 | // } 33 | 34 | tasks.register("ktlintFormat") { 35 | group = LifecycleBasePlugin.VERIFICATION_GROUP 36 | description = "Check Kotlin code style and format" 37 | classpath = ktlint 38 | mainClass.set("com.pinterest.ktlint.Main") 39 | jvmArgs("--add-opens=java.base/java.lang=ALL-UNNAMED") 40 | // see https://pinterest.github.io/ktlint/install/cli/#command-line-usage for more information 41 | args( 42 | "-F", 43 | "**/src/**/*.kt", 44 | "**.kts", 45 | "!**/build/**", 46 | ) 47 | } 48 | 49 | group = "cc.unitmesh" 50 | version = "1.0-SNAPSHOT" 51 | 52 | repositories { 53 | mavenCentral() 54 | } 55 | -------------------------------------------------------------------------------- /processor/common/cli-core/build.gradle.kts: -------------------------------------------------------------------------------- 1 | @Suppress("DSL_SCOPE_VIOLATION") 2 | plugins { 3 | alias(libs.plugins.kotlin.jvm) 4 | alias(libs.plugins.shadow) 5 | alias(libs.plugins.serialization) 6 | application 7 | } 8 | 9 | repositories { 10 | mavenCentral() 11 | } 12 | 13 | dependencies { 14 | implementation(libs.serialization.json) 15 | 16 | // Logging 17 | implementation(libs.logging.slf4j.api) 18 | implementation(libs.logging.logback.classic) 19 | 20 | implementation(libs.kaml) 21 | 22 | testImplementation(kotlin("test")) 23 | testImplementation(libs.bundles.test) 24 | } 25 | 26 | tasks.test { 27 | useJUnitPlatform() 28 | } 29 | 30 | kotlin { 31 | jvmToolchain(11) 32 | } 33 | 34 | application { 35 | mainClass.set("cc.unitmesh.processor.MainKt") 36 | } 37 | 38 | tasks { 39 | shadowJar { 40 | manifest { 41 | attributes(Pair("Main-Class", "cc.unitmesh.processor.MainKt")) 42 | } 43 | // minimize() 44 | dependencies { 45 | exclude(dependency("org.junit.jupiter:.*:.*")) 46 | exclude(dependency("org.junit:.*:.*")) 47 | exclude(dependency("junit:.*:.*")) 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /processor/common/cli-core/src/main/kotlin/cc/unitmesh/core/cli/ProcessorConfig.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.core.cli 2 | 3 | import kotlinx.serialization.Serializable 4 | 5 | @Serializable 6 | data class PreProcessorConfig( 7 | val scm: List, 8 | ) 9 | 10 | @Serializable 11 | data class Scm( 12 | val repository: String, 13 | val branch: String, 14 | val language: String, 15 | ) 16 | -------------------------------------------------------------------------------- /processor/common/cli-core/src/main/kotlin/cc/unitmesh/core/cli/ProcessorUtils.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.core.cli 2 | 3 | import cc.unitmesh.core.git.GitCommandManager 4 | import com.charleskorn.kaml.Yaml 5 | import org.slf4j.Logger 6 | import java.io.File 7 | import kotlin.system.exitProcess 8 | 9 | object ProcessorUtils { 10 | private val logger: Logger = org.slf4j.LoggerFactory.getLogger(ProcessorUtils::class.java) 11 | private fun clonedPath(path: String) = "origindatasets" + File.separator + path 12 | 13 | fun cloneAllRepositories(config: PreProcessorConfig) { 14 | logger.info("Start to Clone code from GitHub") 15 | config.scm.forEach { 16 | val path = it.repository.split("/").last() 17 | val targetPath = clonedPath(path) 18 | // if directory exits and contains .git, then skip 19 | if (File(targetPath).exists() && File(targetPath + File.separator + ".git").exists()) { 20 | logger.info("Skip $targetPath") 21 | return@forEach 22 | } 23 | 24 | File(targetPath).mkdirs() 25 | 26 | val gitCommandManager = GitCommandManager(targetPath) 27 | gitCommandManager.shallowClone(it.repository, it.branch) 28 | } 29 | } 30 | 31 | fun loadConfig(): PreProcessorConfig { 32 | val file = File("processor.yml").let { 33 | if (!it.exists()) { 34 | logger.error("Config file not found: ${it.absolutePath}") 35 | exitProcess(1) 36 | } 37 | 38 | it 39 | } 40 | 41 | val content = file.readText() 42 | return Yaml.default.decodeFromString(deserializer = PreProcessorConfig.serializer(), content) 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /processor/common/cli-core/src/main/kotlin/cc/unitmesh/core/git/Command.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.core.git 2 | 3 | import org.slf4j.Logger 4 | import org.slf4j.LoggerFactory 5 | import java.io.File 6 | 7 | /** 8 | * Executes a command and returns the output. 9 | */ 10 | class Command { 11 | fun execJar(args: List, workdir: String, options: ExecOptions = ExecOptions(cwd = workdir)): Int { 12 | val processBuilder = ProcessBuilder("java", "-jar", *args.toTypedArray()) 13 | return doExecute(processBuilder, options) 14 | } 15 | 16 | fun exec(commandLine: String, args: List = listOf(), options: ExecOptions): Int { 17 | if (!options.silent) { 18 | logger.info("Executing: $commandLine ${args.joinToString(" ")}") 19 | } 20 | 21 | val processBuilder = ProcessBuilder(commandLine, *args.toTypedArray()) 22 | return doExecute(processBuilder, options) 23 | } 24 | 25 | private fun doExecute(processBuilder: ProcessBuilder, options: ExecOptions): Int { 26 | val process = processBuilder 27 | .directory(File(options.cwd)) 28 | .start() 29 | 30 | val exitCode = process.waitFor() 31 | 32 | process.inputStream.bufferedReader().use { it -> 33 | it.forEachLine { line -> 34 | options.listeners.stdout(line) 35 | } 36 | } 37 | 38 | if (exitCode != 0 && !options.ignoreReturnCode) { 39 | process.errorStream.bufferedReader().use { 40 | it.forEachLine { line -> 41 | options.listeners.stderr(line) 42 | } 43 | } 44 | } 45 | 46 | return exitCode 47 | } 48 | 49 | companion object { 50 | private val logger: Logger = LoggerFactory.getLogger(Command::class.java) 51 | 52 | fun exec(commandLine: String, args: List = listOf(), options: ExecOptions = ExecOptions()): Int { 53 | return Command().exec(commandLine, args, options) 54 | } 55 | } 56 | } 57 | 58 | class ExecOptions( 59 | val cwd: String = "", 60 | val env: MutableMap = mutableMapOf(), 61 | val silent: Boolean = false, 62 | val ignoreReturnCode: Boolean = false, 63 | val listeners: ExecListeners = object : ExecListeners {}, 64 | ) 65 | -------------------------------------------------------------------------------- /processor/common/cli-core/src/main/kotlin/cc/unitmesh/core/git/ExecListeners.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.core.git 2 | 3 | import org.slf4j.Logger 4 | 5 | interface ExecListeners { 6 | fun stdout(data: String) {} 7 | fun stderr(data: String) {} 8 | } 9 | 10 | class LoggerExecListeners(private val logger: Logger) : ExecListeners { 11 | override fun stdout(data: String) { 12 | logger.info(data) 13 | } 14 | 15 | override fun stderr(data: String) { 16 | logger.error(data) 17 | } 18 | } 19 | 20 | class StringListExecListeners(private var stdout: MutableList, private var stderr: MutableList) : 21 | ExecListeners { 22 | override fun stdout(data: String) { 23 | stdout += data 24 | } 25 | 26 | override fun stderr(data: String) { 27 | stderr += data 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /processor/common/cli-core/src/main/kotlin/cc/unitmesh/core/git/GitCommandManager.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.core.git 2 | 3 | class GitCommandManager(var workingDirectory: String = ".") { 4 | private val gitEnv: MutableMap = mutableMapOf( 5 | "GIT_TERMINAL_PROMPT" to "0", // Disable git prompt 6 | "GCM_INTERACTIVE" to "Never", // Disable prompting for git credential manager 7 | ) 8 | 9 | private var gitPath = "git" 10 | 11 | private val exec = Command() 12 | 13 | fun shallowClone(url: String, branch: String = "master"): GitOutput { 14 | val args = mutableListOf("clone", url, "--branch", branch, "--single-branch", "--depth", "1", ".") 15 | return execGit(args) 16 | } 17 | 18 | private fun execGit(args: List, allowAllExitCodes: Boolean = false, silent: Boolean = false): GitOutput { 19 | val result = GitOutput() 20 | 21 | val env = mutableMapOf() 22 | for ((key, value) in System.getenv()) { 23 | env[key] = value 24 | } 25 | for ((key, value) in gitEnv) { 26 | env[key] = value 27 | } 28 | 29 | val stdout = mutableListOf() 30 | val stderr = mutableListOf() 31 | val options = ExecOptions( 32 | cwd = workingDirectory, 33 | env = env, 34 | silent = silent, 35 | ignoreReturnCode = allowAllExitCodes, 36 | listeners = StringListExecListeners(stdout, stderr), 37 | ) 38 | 39 | result.exitCode = exec.exec(gitPath, args, options) 40 | result.stdout = stdout.joinToString("\n") 41 | 42 | return result 43 | } 44 | } 45 | 46 | data class GitOutput( 47 | var stdout: String = "", 48 | var exitCode: Int = 0, 49 | ) 50 | -------------------------------------------------------------------------------- /processor/common/core/build.gradle.kts: -------------------------------------------------------------------------------- 1 | @Suppress("DSL_SCOPE_VIOLATION") 2 | plugins { 3 | alias(libs.plugins.kotlin.jvm) 4 | alias(libs.plugins.serialization) 5 | } 6 | 7 | repositories { 8 | mavenCentral() 9 | } 10 | 11 | dependencies { 12 | implementation(libs.clikt) 13 | 14 | implementation(libs.serialization.json) 15 | 16 | testImplementation(kotlin("test")) 17 | testImplementation(libs.bundles.test) 18 | } 19 | 20 | tasks.test { 21 | useJUnitPlatform() 22 | } 23 | 24 | kotlin { 25 | jvmToolchain(11) 26 | } 27 | -------------------------------------------------------------------------------- /processor/common/core/src/main/kotlin/cc/unitmesh/core/Instruction.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.core 2 | 3 | import kotlinx.serialization.Serializable 4 | 5 | @Serializable 6 | data class Instruction( 7 | val instruction: String, 8 | val input: String, 9 | val output: String, 10 | ) 11 | -------------------------------------------------------------------------------- /processor/common/core/src/main/kotlin/cc/unitmesh/core/model/ApiCollection.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.core.model 2 | 3 | import kotlinx.serialization.Serializable 4 | 5 | @Serializable 6 | data class ApiCollection( 7 | val name: String, 8 | val description: String, 9 | val items: List, 10 | ) { 11 | override fun toString(): String { 12 | return "$name: ${items.joinToString(", ") { it.toString() }}" 13 | } 14 | } 15 | 16 | @Serializable 17 | data class ApiItem( 18 | val path: String, 19 | val method: String, 20 | var description: String, 21 | val operationId: String, 22 | val tags: List, 23 | val request: Request? = null, 24 | val response: List = listOf(), 25 | ) { 26 | override fun toString(): String { 27 | val request = request.toString() 28 | val response = response.joinToString(", ") { it.toString() } 29 | return "$method $path $description $request $response" 30 | } 31 | } 32 | 33 | @Serializable 34 | data class Parameter( 35 | val name: String, 36 | val type: String, 37 | ) { 38 | override fun toString() = "$name: $type" 39 | } 40 | 41 | enum class BodyMode { 42 | RAW_TEXT, 43 | TYPED, 44 | } 45 | 46 | @Serializable 47 | data class Request( 48 | val parameters: List = listOf(), 49 | val body: List = listOf(), 50 | val bodyMode: BodyMode = BodyMode.TYPED, 51 | var bodyString: String = "", 52 | ) { 53 | override fun toString(): String { 54 | val params = parameters.joinToString(", ") { it.toString() } 55 | val body = body.joinToString(", ") { it.toString() } 56 | if (params.isEmpty() && body.isEmpty()) return "" 57 | if (params.isEmpty()) return body 58 | if (body.isEmpty()) return params 59 | 60 | return "$params, ($body)" 61 | } 62 | } 63 | 64 | @Serializable 65 | data class Response( 66 | val code: Int, 67 | val parameters: List = listOf(), 68 | var bodyMode: BodyMode = BodyMode.TYPED, 69 | var bodyString: String = "", 70 | ) { 71 | override fun toString(): String = when (bodyMode) { 72 | BodyMode.RAW_TEXT -> { 73 | // TODO: 256 is a magic number 74 | if (bodyString.length > 256) { 75 | "$code: {}" 76 | } else { 77 | "$code: ${bodyString.replace("\r\n", "").replace("\n", "")}" 78 | } 79 | } 80 | 81 | BodyMode.TYPED -> { 82 | if (parameters.isEmpty()) { 83 | "$code: {}" 84 | } else { 85 | "$code: {${parameters.joinToString(", ") { it.toString() }}}" 86 | } 87 | } 88 | } 89 | } 90 | 91 | @Serializable 92 | data class ApiTagOutput(val string: String) { 93 | override fun toString() = string 94 | } 95 | -------------------------------------------------------------------------------- /processor/common/core/src/main/kotlin/cc/unitmesh/core/model/Usecase.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.core.model 2 | 3 | import kotlinx.serialization.Serializable 4 | import kotlinx.serialization.decodeFromString 5 | import kotlinx.serialization.json.Json 6 | 7 | // | 用例名称 | 前置条件 | 后置条件 | 主成功场景 | 扩展场景 | 8 | @Serializable 9 | data class Usecase( 10 | val name: String, 11 | val preCondition: String, 12 | val postCondition: String, 13 | val mainSuccessScenario: String, 14 | val extensionScenario: String, 15 | ) 16 | 17 | object Usecases { 18 | fun fromString(json: String): List { 19 | return Json.decodeFromString(json) 20 | } 21 | 22 | fun toMarkdown(usecases: List): String { 23 | return buildString { 24 | append("| 用例名称 | 前置条件 | 后置条件 | 主成功场景 | 扩展场景 |\n") 25 | append("| --- | --- | --- | --- | --- |\n") 26 | usecases.forEach { 27 | append("| ${it.name} | ${it.preCondition} | ${it.postCondition} | ${it.mainSuccessScenario} | ${it.extensionScenario} |\n") 28 | } 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /processor/common/core/src/test/kotlin/cc/unitmesh/core/model/UsecaseKtTest.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.core.model 2 | 3 | import org.junit.jupiter.api.Assertions.* 4 | import org.junit.jupiter.api.Test 5 | 6 | class UsecaseKtTest { 7 | @Test 8 | fun `should parse usecase from string`() { 9 | val json = """ 10 | [{ 11 | "name": "usecase name", 12 | "preCondition": "pre condition", 13 | "postCondition": "post condition", 14 | "mainSuccessScenario": "main success scenario", 15 | "extensionScenario": "extension scenario" 16 | }] 17 | """.trimIndent() 18 | 19 | val usecase = Usecase( 20 | name = "usecase name", 21 | preCondition = "pre condition", 22 | postCondition = "post condition", 23 | mainSuccessScenario = "main success scenario", 24 | extensionScenario = "extension scenario", 25 | ) 26 | 27 | assertEquals(listOf(usecase), Usecases.fromString(json)) 28 | } 29 | 30 | @Test 31 | fun `should convert usecases to markdown`() { 32 | val usecases = listOf( 33 | Usecase( 34 | name = "usecase name", 35 | preCondition = "pre condition", 36 | postCondition = "post condition", 37 | mainSuccessScenario = "main success scenario", 38 | extensionScenario = "extension scenario", 39 | ), 40 | ) 41 | 42 | val markdown = """ 43 | | 用例名称 | 前置条件 | 后置条件 | 主成功场景 | 扩展场景 | 44 | | --- | --- | --- | --- | --- | 45 | | usecase name | pre condition | post condition | main success scenario | extension scenario | 46 | 47 | """.trimIndent() 48 | 49 | Usecases.toMarkdown(usecases).also { 50 | assertEquals(markdown, it) 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /processor/common/prompter/build.gradle.kts: -------------------------------------------------------------------------------- 1 | @Suppress("DSL_SCOPE_VIOLATION") 2 | plugins { 3 | alias(libs.plugins.kotlin.jvm) 4 | alias(libs.plugins.serialization) 5 | } 6 | 7 | repositories { 8 | mavenCentral() 9 | } 10 | 11 | dependencies { 12 | implementation(libs.serialization.json) 13 | 14 | // Logging 15 | implementation(libs.logging.slf4j.api) 16 | implementation(libs.logging.logback.classic) 17 | 18 | implementation(libs.bundles.openai) 19 | 20 | testImplementation(kotlin("test")) 21 | testImplementation(libs.bundles.test) 22 | } 23 | 24 | tasks.test { 25 | useJUnitPlatform() 26 | } 27 | 28 | kotlin { 29 | jvmToolchain(11) 30 | } 31 | -------------------------------------------------------------------------------- /processor/common/prompter/src/main/kotlin/cc/unitmesh/core/prompter/OpenAiPrompter.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.core.prompter 2 | 3 | import com.theokanning.openai.OpenAiApi 4 | import com.theokanning.openai.completion.chat.ChatCompletionRequest 5 | import com.theokanning.openai.completion.chat.ChatMessage 6 | import com.theokanning.openai.completion.chat.ChatMessageRole 7 | import com.theokanning.openai.service.OpenAiService 8 | import retrofit2.Retrofit 9 | import retrofit2.adapter.rxjava2.RxJava2CallAdapterFactory 10 | import retrofit2.converter.jackson.JacksonConverterFactory 11 | import java.time.Duration 12 | 13 | class OpenAiPrompter(val openAiKey: String, val openAiProxy: String) { 14 | private val openAiVersion: String = "gpt-3.5-turbo" 15 | private var service: OpenAiService 16 | private val timeout = Duration.ofSeconds(600) 17 | 18 | init { 19 | if (openAiProxy.trim().isEmpty()) { 20 | service = OpenAiService(openAiKey, timeout) 21 | } else { 22 | val mapper = OpenAiService.defaultObjectMapper() 23 | val client = OpenAiService.defaultClient(openAiKey, timeout) 24 | 25 | val retrofit = Retrofit.Builder() 26 | .baseUrl(openAiProxy) 27 | .client(client) 28 | .addConverterFactory(JacksonConverterFactory.create(mapper)) 29 | .addCallAdapterFactory(RxJava2CallAdapterFactory.create()) 30 | .build() 31 | 32 | val api = retrofit.create(OpenAiApi::class.java) 33 | service = OpenAiService(api) 34 | } 35 | } 36 | 37 | fun prompt(instruction: String): String { 38 | val messages: MutableList = ArrayList() 39 | val systemMessage = ChatMessage(ChatMessageRole.USER.value(), instruction) 40 | messages.add(systemMessage) 41 | 42 | val completionRequest = ChatCompletionRequest.builder() 43 | .model(openAiVersion) 44 | .messages(messages) 45 | .build() 46 | 47 | val completion = service.createChatCompletion(completionRequest) 48 | 49 | return completion 50 | .choices[0].message.content 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /processor/common/verifier/build.gradle.kts: -------------------------------------------------------------------------------- 1 | @Suppress("DSL_SCOPE_VIOLATION") 2 | plugins { 3 | alias(libs.plugins.kotlin.jvm) 4 | alias(libs.plugins.serialization) 5 | } 6 | 7 | repositories { 8 | mavenCentral() 9 | } 10 | 11 | dependencies { 12 | implementation(libs.serialization.json) 13 | 14 | implementation(libs.plantuml) 15 | implementation(libs.jsqlparser) 16 | 17 | implementation(libs.bundles.markdown) 18 | 19 | // Logging 20 | implementation(libs.logging.slf4j.api) 21 | implementation(libs.logging.logback.classic) 22 | 23 | testImplementation(kotlin("test")) 24 | testImplementation(libs.bundles.test) 25 | } 26 | 27 | tasks.test { 28 | useJUnitPlatform() 29 | } 30 | 31 | kotlin { 32 | jvmToolchain(11) 33 | } 34 | -------------------------------------------------------------------------------- /processor/common/verifier/src/main/kotlin/cc/unitmesh/verifier/drawio/DrawioSerial.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.verifier.drawio 2 | 3 | class DrawioSerial { 4 | fun decode(diagrams: String): String { 5 | val data = diagrams 6 | .map { it.toInt() } 7 | .map { it.toChar() } 8 | .joinToString("") 9 | return data 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /processor/common/verifier/src/main/kotlin/cc/unitmesh/verifier/drawio/DrawioVerifier.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.verifier.drawio 2 | 3 | class DrawioVerifier 4 | -------------------------------------------------------------------------------- /processor/common/verifier/src/main/kotlin/cc/unitmesh/verifier/drawio/MxGraph.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.verifier.drawio 2 | 3 | import kotlinx.serialization.Serializable 4 | 5 | @Serializable 6 | data class MxFileRoot( 7 | val mxfile: Mxfile, 8 | ) 9 | 10 | @Serializable 11 | data class Mxfile( 12 | val diagram: Diagram, 13 | val attributes: Attributes?, 14 | ) 15 | 16 | @Serializable 17 | data class Diagram( 18 | val _text: String, 19 | val attributes: Attributes?, 20 | ) 21 | 22 | @Serializable 23 | data class MxGraph( 24 | val mxGraphModel: MxGraphModel, 25 | ) 26 | 27 | @Serializable 28 | data class MxGraphModel( 29 | val root: RootNode, 30 | val attributes: Attributes?, 31 | ) 32 | 33 | @Serializable 34 | data class RootNode( 35 | val mxCell: List, 36 | ) 37 | 38 | @Serializable 39 | data class MXCell( 40 | val mxGeometry: MXGeometry?, 41 | val attributes: Attributes?, 42 | ) 43 | 44 | @Serializable 45 | data class MXGeometry( 46 | val mxPoint: List?, 47 | val Array: MXGeometry?, 48 | val attributes: Attributes?, 49 | ) 50 | 51 | @Serializable 52 | data class MxPoint( 53 | val attributes: Attributes?, 54 | ) 55 | 56 | @Serializable 57 | data class Attributes( 58 | val id: String, 59 | val name: String?, 60 | val host: String?, 61 | val modified: String?, 62 | val agent: String?, 63 | val etag: String?, 64 | val version: String?, 65 | val type: String?, 66 | val label: String?, 67 | val style: String?, 68 | val source: String?, 69 | val target: String?, 70 | val edge: String?, 71 | val value: String?, 72 | val vertex: String?, 73 | val parent: String?, 74 | val children: List?, 75 | val visible: Boolean?, 76 | val connectable: String?, 77 | val x: Double?, 78 | val y: Double?, 79 | val relative: String?, 80 | val `as`: String?, 81 | val width: Double?, 82 | val height: Double?, 83 | val offset: List?, 84 | ) 85 | -------------------------------------------------------------------------------- /processor/common/verifier/src/main/kotlin/cc/unitmesh/verifier/markdown/MarkdownExt.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.verifier.markdown 2 | 3 | import org.commonmark.ext.gfm.tables.TablesExtension 4 | import org.commonmark.parser.Parser 5 | 6 | fun createParser(): Parser { 7 | return Parser.builder() 8 | .extensions(listOf(TablesExtension.create())) 9 | .build() 10 | } 11 | -------------------------------------------------------------------------------- /processor/common/verifier/src/main/kotlin/cc/unitmesh/verifier/markdown/MarkdownVerifier.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.verifier.markdown 2 | 3 | import org.commonmark.ext.gfm.tables.TableCell 4 | import org.commonmark.ext.gfm.tables.TableHead 5 | import org.commonmark.node.* 6 | import org.commonmark.parser.Parser 7 | 8 | class MarkdownVerifier { 9 | private val parser: Parser = createParser() 10 | 11 | fun tableVerifier(markdown: String, headers: List): Boolean { 12 | var content = markdown 13 | // 1. if there is inside code block, remove block, like: ```markdown xxx ``` 14 | if (markdown.contains("```")) { 15 | val code = parseMarkdownCodeBlock(markdown) 16 | if (code.isEmpty()) { 17 | return false 18 | } 19 | 20 | content = code.first() 21 | } 22 | 23 | // 2. if no table, return false 24 | if (!markdown.contains("|")) { 25 | return false 26 | } 27 | 28 | // 3. parse table header and verify table header 29 | val tableHeader = parseTableHeader(content) 30 | if (tableHeader.size != headers.size) { 31 | return false 32 | } 33 | 34 | // 4. verify table header 35 | tableHeader.forEachIndexed { index, s -> 36 | if (s != headers[index]) { 37 | return false 38 | } 39 | } 40 | 41 | return true 42 | } 43 | 44 | private fun parseTableHeader(content: String): List { 45 | val node = parser.parse(content) 46 | val visitor = TableHeaderVisitor() 47 | node.accept(visitor) 48 | return visitor.headers 49 | } 50 | 51 | private fun parseMarkdownCodeBlock(markdown: String): List { 52 | val node = parser.parse(markdown) 53 | val visitor = CodeFilter(lang = "markdown") 54 | node.accept(visitor) 55 | return visitor.code 56 | } 57 | } 58 | 59 | internal class TableHeaderVisitor : AbstractVisitor() { 60 | val headers = mutableListOf() 61 | private var isBeforeHeadLine = true 62 | override fun visit(customNode: CustomNode?) { 63 | super.visit(customNode) 64 | 65 | when (customNode) { 66 | is TableHead -> { 67 | isBeforeHeadLine = false 68 | } 69 | 70 | is TableCell -> { 71 | if (isBeforeHeadLine) { 72 | headers += (customNode.firstChild as Text).literal 73 | } 74 | } 75 | } 76 | } 77 | 78 | override fun visit(customBlock: CustomBlock?) { 79 | super.visit(customBlock) 80 | } 81 | } 82 | 83 | internal class CodeFilter(val lang: String) : AbstractVisitor() { 84 | var code = listOf() 85 | 86 | override fun visit(fencedCodeBlock: FencedCodeBlock?) { 87 | if (fencedCodeBlock?.literal != null) { 88 | if (fencedCodeBlock.info == lang) { 89 | this.code += fencedCodeBlock.literal 90 | } 91 | } 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /processor/common/verifier/src/main/kotlin/cc/unitmesh/verifier/markdown/UsecaseParser.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.verifier.markdown 2 | 3 | import org.commonmark.ext.gfm.tables.TableCell 4 | import org.commonmark.ext.gfm.tables.TableHead 5 | import org.commonmark.ext.gfm.tables.TableRow 6 | import org.commonmark.node.AbstractVisitor 7 | import org.commonmark.node.CustomNode 8 | import org.commonmark.node.Node 9 | import org.commonmark.node.Text 10 | import org.commonmark.parser.Parser 11 | 12 | class UsecaseParser { 13 | private val parser: Parser = createParser() 14 | 15 | fun filterTableColumn(markdown: String, headerName: String): List { 16 | val node = parser.parse(markdown) 17 | val visitor = TableHeaderVisitor() 18 | node.accept(visitor) 19 | val headers = visitor.headers 20 | val columnIndex = headers.indexOf(headerName) 21 | 22 | if (columnIndex == -1) { 23 | return emptyList() 24 | } 25 | 26 | val filter = TableColumnFilter(columnIndex) 27 | node.accept(filter) 28 | return filter.values 29 | } 30 | } 31 | 32 | internal class TableColumnFilter(private val columnIndex: Int) : AbstractVisitor() { 33 | val values = mutableListOf() 34 | 35 | // | a | b | c | 36 | // |---|---|---| <--- head line 37 | private var isBeforeHeadLine = true 38 | 39 | override fun visit(customNode: CustomNode?) { 40 | super.visit(customNode) 41 | 42 | // | a | b | c | 43 | // |---|---|---| 44 | // | col | .. | .. | 45 | when (customNode) { 46 | is TableHead -> { 47 | isBeforeHeadLine = false 48 | } 49 | 50 | is TableRow -> { 51 | // skip header 52 | if (isBeforeHeadLine) { 53 | return 54 | } 55 | 56 | var node: Node? = customNode.getFirstChild() 57 | var index = 0 58 | while (node != null) { 59 | if (node is TableCell) { 60 | if (index == columnIndex) { 61 | values += node.literal() 62 | } 63 | } 64 | 65 | val next: Node = node.next ?: break 66 | node = next 67 | index += 1 68 | } 69 | } 70 | } 71 | } 72 | } 73 | 74 | private fun TableCell.literal(): String { 75 | if (this.firstChild is Text) { 76 | return (this.firstChild as Text).literal 77 | } 78 | 79 | return "" 80 | } 81 | -------------------------------------------------------------------------------- /processor/common/verifier/src/main/kotlin/cc/unitmesh/verifier/plantuml/PlantUmlParser.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.verifier.plantuml 2 | 3 | import net.sourceforge.plantuml.FileFormat 4 | import net.sourceforge.plantuml.FileFormatOption 5 | import net.sourceforge.plantuml.SourceStringReader 6 | import org.slf4j.Logger 7 | import org.slf4j.LoggerFactory 8 | import java.io.ByteArrayOutputStream 9 | import java.io.File 10 | import java.nio.charset.Charset 11 | 12 | class PlantUmlParser(private val file: File) { 13 | fun isCorrect(): Boolean { 14 | val source = file.readText() 15 | 16 | val reader = SourceStringReader(source) 17 | 18 | try { 19 | val os = ByteArrayOutputStream() 20 | reader.generateImage(os, FileFormatOption(FileFormat.SVG)) 21 | val svg = String(os.toByteArray(), Charset.forName("UTF-8")) 22 | 23 | if (svg.contains("Syntax Error?")) { 24 | logger.error("failed to parse ${file.absolutePath}") 25 | // remove file 26 | file.delete() 27 | return false 28 | } 29 | 30 | // parentFile + "svg" + name 31 | val svgFile = File(file.parentFile, "svg/${file.nameWithoutExtension}.svg") 32 | svgFile.writeText(svg) 33 | } catch (e: Exception) { 34 | logger.error("failed to parse ${file.absolutePath}", e) 35 | return false 36 | } 37 | 38 | return true 39 | } 40 | 41 | companion object { 42 | val logger: Logger = LoggerFactory.getLogger(PlantUmlParser::class.java) 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /processor/common/verifier/src/main/kotlin/cc/unitmesh/verifier/sql/SqlParser.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.verifier.sql 2 | 3 | import net.sf.jsqlparser.parser.CCJSqlParserUtil 4 | 5 | class SqlParser { 6 | companion object { 7 | fun isCorrect(sqlText: String): Boolean { 8 | return try { 9 | CCJSqlParserUtil.parse(sqlText) 10 | true 11 | } catch (e: Exception) { 12 | false 13 | } 14 | } 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /processor/common/verifier/src/test/kotlin/cc/unitmesh/verifier/markdown/MarkdownVerifierTest.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.verifier.markdown 2 | 3 | import org.junit.jupiter.api.Assertions.* 4 | import org.junit.jupiter.api.Test 5 | 6 | class MarkdownVerifierTest { 7 | @Test 8 | fun should_return_true_when_given_headers() { 9 | val markdown = """ 10 | | a | b | c | 11 | |---|---|---| 12 | | 1 | 2 | 3 | 13 | """.trimIndent() 14 | 15 | val verifier = MarkdownVerifier() 16 | val result = verifier.tableVerifier(markdown, listOf("a", "b", "c")) 17 | assertTrue(result) 18 | } 19 | 20 | @Test 21 | fun should_return_true_when_given_table_inside_markdown_code_block() { 22 | val markdown = """ 23 | ```markdown 24 | | a | b | c | 25 | |---|---|---| 26 | | 1 | 2 | 3 | 27 | ``` 28 | """.trimIndent() 29 | 30 | val verifier = MarkdownVerifier() 31 | val result = verifier.tableVerifier(markdown, listOf("a", "b", "c")) 32 | assertTrue(result) 33 | } 34 | 35 | @Test 36 | fun should_return_false_when_dont_have_table() { 37 | val markdown = """ 38 | # a 39 | ## b 40 | ### c 41 | """.trimIndent() 42 | 43 | val verifier = MarkdownVerifier() 44 | val result = verifier.tableVerifier(markdown, listOf("a", "b", "c")) 45 | assertFalse(result) 46 | } 47 | 48 | @Test 49 | fun should_return_false_when_given_error_table() { 50 | val markdown = """ 51 | | a | b | c | 52 | |---|---|---| 53 | | 1 | 2 | 3 | 54 | """.trimIndent() 55 | 56 | val verifier = MarkdownVerifier() 57 | val result = verifier.tableVerifier(markdown, listOf("a", "b", "d")) 58 | assertFalse(result) 59 | } 60 | 61 | @Test 62 | fun should_return_false_when_code_block_is_empty() { 63 | val markdown = """ 64 | ```markdown 65 | ``` 66 | """.trimIndent() 67 | 68 | val verifier = MarkdownVerifier() 69 | val result = verifier.tableVerifier(markdown, listOf("a", "b", "d")) 70 | assertFalse(result) 71 | } 72 | 73 | @Test 74 | fun should_return_false_when_table_size_not_equal() { 75 | val markdown = """ 76 | | a | b | c | 77 | |---|---|---| 78 | | 1 | 2 | 3 | 79 | """.trimIndent() 80 | 81 | val verifier = MarkdownVerifier() 82 | val result = verifier.tableVerifier(markdown, listOf("a", "b")) 83 | assertFalse(result) 84 | } 85 | 86 | @Test 87 | fun should_return_false_when_have_error_code_block() { 88 | val markdown = """ 89 | ``` 90 | demo 91 | """.trimIndent() 92 | 93 | val verifier = MarkdownVerifier() 94 | val result = verifier.tableVerifier(markdown, listOf("a", "b")) 95 | assertFalse(result) 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /processor/common/verifier/src/test/kotlin/cc/unitmesh/verifier/markdown/TableHeaderVisitorTest.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.verifier.markdown 2 | 3 | import org.junit.jupiter.api.Assertions.assertEquals 4 | import org.junit.jupiter.api.Test 5 | 6 | class TableHeaderVisitorTest { 7 | @Test 8 | fun should_return_correct_headers_when_given_a_markdown() { 9 | val markdown = """ 10 | | a | b | c | 11 | |---|---|---| 12 | | 1 | 2 | 3 | 13 | """.trimIndent() 14 | 15 | val node = createParser().parse(markdown) 16 | val visitor = TableHeaderVisitor() 17 | node.accept(visitor) 18 | 19 | assertEquals(3, visitor.headers.size) 20 | assertEquals("a", visitor.headers[0]) 21 | assertEquals("b", visitor.headers[1]) 22 | assertEquals("c", visitor.headers[2]) 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /processor/common/verifier/src/test/kotlin/cc/unitmesh/verifier/markdown/UsecaseParserTest.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.verifier.markdown 2 | 3 | import org.junit.jupiter.api.Assertions.* 4 | import org.junit.jupiter.api.Test 5 | 6 | class UsecaseParserTest { 7 | @Test 8 | fun should_return_columns_value_when_given_a_usecases() { 9 | val markdown = """ 10 | | a | b | c | 11 | |---|---|---| 12 | | 1 | 2 | 3 | 13 | | 4 | 5 | 6 | 14 | """.trimIndent() 15 | 16 | val parser = UsecaseParser() 17 | val values = parser.filterTableColumn(markdown, "a") 18 | assertEquals(listOf("1", "4"), values) 19 | } 20 | 21 | // fitler by b 22 | @Test 23 | fun should_return_columns_value_when_given_a_usecases_and_column_is_b() { 24 | val markdown = """ 25 | | a | b | c | 26 | |---|---|---| 27 | | 1 | 2 | 3 | 28 | | 4 | 5 | 6 | 29 | """.trimIndent() 30 | 31 | val parser = UsecaseParser() 32 | val values = parser.filterTableColumn(markdown, "b") 33 | assertEquals(listOf("2", "5"), values) 34 | } 35 | 36 | @Test 37 | fun should_return_empty_list_when_given_a_usecases_and_column_not_exist() { 38 | val markdown = """ 39 | | a | b | c | 40 | |---|---|---| 41 | | 1 | 2 | 3 | 42 | | 4 | 5 | 6 | 43 | """.trimIndent() 44 | 45 | val parser = UsecaseParser() 46 | val values = parser.filterTableColumn(markdown, "d") 47 | assertEquals(emptyList(), values) 48 | } 49 | 50 | @Test 51 | fun should_return_empty_list_when_given_a_usecases_and_column_is_empty() { 52 | val markdown = """ 53 | | a | b | c | 54 | |---|---|---| 55 | | 1 | 2 | 3 | 56 | | 4 | 5 | 6 | 57 | """.trimIndent() 58 | 59 | val parser = UsecaseParser() 60 | val values = parser.filterTableColumn(markdown, "") 61 | assertEquals(emptyList(), values) 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /processor/docs/kotlin.md: -------------------------------------------------------------------------------- 1 | # Datasource 2 | 3 | 4 | - Leetcode to Kotlin 5 | - https://github.com/sshpark/LeetCode-Kotlin 6 | - https://github.com/yuanliu666/Leetcode-Kotlin 单个项目 7 | - https://github.com/wkxjc/Kotlin-leetcode 使用 Markdown 生成 8 | - Kotlin translate code 9 | - Kotlin Math 10 | - https://github.com/amazon-science/mxeval/blob/main/data/mbxp/mbkp_release_v1.jsonl 11 | - Kotless 12 | 13 | 14 | Projects: 15 | 16 | ## MxEval 17 | 18 | ![Execution-based evaluation of code in 10+ languages](https://github.com/amazon-science/mxeval) 19 | 20 | > Our paper describes the language conversion framework, the synthetic solution generation, and many other types of evaluation beyond the traditional function completion evaluation such as translation, code insertion, summarization, and robustness evaluation. 21 | 22 | ![](https://github.com/amazon-science/mxeval/blob/main/graphics/paper_summary.png) 23 | 24 | ![Language conversion of execution-based function completion datasets](https://github.com/amazon-science/mxeval/blob/main/graphics/mbxp_java_conversion.png) 25 | -------------------------------------------------------------------------------- /processor/examples/unit-demo/.gitignore: -------------------------------------------------------------------------------- 1 | .gradle 2 | build/ 3 | !gradle/wrapper/gradle-wrapper.jar 4 | !**/src/main/**/build/ 5 | !**/src/test/**/build/ 6 | 7 | ### STS ### 8 | .apt_generated 9 | .classpath 10 | .factorypath 11 | .project 12 | .settings 13 | .springBeans 14 | .sts4-cache 15 | bin/ 16 | !**/src/main/**/bin/ 17 | !**/src/test/**/bin/ 18 | 19 | ### IntelliJ IDEA ### 20 | .idea 21 | *.iws 22 | *.iml 23 | *.ipr 24 | out/ 25 | !**/src/main/**/out/ 26 | !**/src/test/**/out/ 27 | 28 | ### NetBeans ### 29 | /nbproject/private/ 30 | /nbbuild/ 31 | /dist/ 32 | /nbdist/ 33 | /.nb-gradle/ 34 | 35 | ### VS Code ### 36 | .vscode/ -------------------------------------------------------------------------------- /processor/examples/unit-demo/build.gradle.kts: -------------------------------------------------------------------------------- 1 | val ktor_version: String by project 2 | val kotlin_version: String by project 3 | val logback_version: String by project 4 | val exposed_version: String by project 5 | val h2_version: String by project 6 | 7 | plugins { 8 | kotlin("jvm") version "1.8.20" 9 | id("io.ktor.plugin") version "2.2.4" 10 | id("org.jetbrains.kotlin.plugin.serialization") version "1.8.20" 11 | } 12 | 13 | group = "unitmesh.cc" 14 | version = "0.0.1" 15 | application { 16 | mainClass.set("unitmesh.cc.ApplicationKt") 17 | 18 | val isDevelopment: Boolean = project.ext.has("development") 19 | applicationDefaultJvmArgs = listOf("-Dio.ktor.development=$isDevelopment") 20 | } 21 | 22 | repositories { 23 | mavenCentral() 24 | } 25 | 26 | dependencies { 27 | implementation("io.ktor:ktor-server-core-jvm:$ktor_version") 28 | implementation("io.ktor:ktor-serialization-kotlinx-json-jvm:$ktor_version") 29 | implementation("io.ktor:ktor-server-content-negotiation-jvm:$ktor_version") 30 | implementation("org.jetbrains.exposed:exposed-core:$exposed_version") 31 | implementation("org.jetbrains.exposed:exposed-jdbc:$exposed_version") 32 | implementation("com.h2database:h2:$h2_version") 33 | implementation("io.ktor:ktor-server-auth-jvm:$ktor_version") 34 | implementation("io.ktor:ktor-server-auth-jwt-jvm:$ktor_version") 35 | implementation("io.ktor:ktor-server-netty-jvm:$ktor_version") 36 | implementation("ch.qos.logback:logback-classic:$logback_version") 37 | testImplementation("io.ktor:ktor-server-tests-jvm:$ktor_version") 38 | testImplementation("org.jetbrains.kotlin:kotlin-test-junit:$kotlin_version") 39 | } 40 | -------------------------------------------------------------------------------- /processor/examples/unit-demo/docs/dsl.draft.md: -------------------------------------------------------------------------------- 1 | # Unit Mesh DSL 2 | 3 | ## API 要素 ？ 4 | 5 | ### URI 6 | 7 | 一个 API 请求 8 | 9 | ``` 10 | GET /users/{id} 11 | request: 12 | { 13 | "id": 1 14 | } 15 | response: 16 | { 17 | "id": 1, 18 | "name": "John", 19 | "age": 20 20 | } 21 | ``` 22 | 23 | ### Endpoint-Action 24 | 25 | Endpoint 26 | 27 | ```java 28 | 29 | @RequestMapping("/users") 30 | class UserController { 31 | constructor(UserService userService) { 32 | this.userService = userService; 33 | } 34 | 35 | @RequestMapping("/{id}") 36 | public User getUser(@PathVariable("id") int id) { 37 | return userService.getUser(id); 38 | } 39 | } 40 | ``` 41 | 42 | Action 43 | 44 | ```java 45 | class UserService { 46 | public User getUser(int id) { 47 | return new User(id, "John", 20); 48 | } 49 | } 50 | ``` 51 | 52 | ### 那么 53 | 54 | ```java 55 | // Controller/Endpoint 56 | // Service/Logic 57 | // Repository/DataSchema 58 | ``` 59 | 60 | - Controller 模板 61 | - AI 将逻辑转换为代码 62 | - Repository 模板 + SQL 查询 63 | 64 | ## Output 65 | 66 | [https://github.com/square/kotlinpoet](https://github.com/square/kotlinpoet) 67 | 68 | ## Step 1. Text to Repository 69 | 70 | 1. SQL to Text 71 | 2. SQL Prompter 72 | 3. Text to Prompt Template (with DB), Prompt to SQL 73 | 74 | Format: 75 | 76 | ```java 77 | @Service 78 | public class UserService(Database database) { 79 | public User getUser(int id) { 80 | return database.queryForObject("SELECT * FROM users WHERE id = ?", User.class, id); 81 | } 82 | } 83 | ``` 84 | 85 | 如果都是模板代码，就可以直接生成代码了 + Usecases 86 | 87 | ```java 88 | @Service 89 | public class CreateUserUsecase(Database database) { 90 | public User createUser(User user) { 91 | return database.queryForObject("INSERT INTO users (name, age) VALUES (?, ?)", User.class, user.name, user.age); 92 | } 93 | } 94 | ``` 95 | 96 | 或者 CreateBlogComment 97 | 98 | ```java 99 | @Service 100 | public class CreateBlogCommentUsecase(Database database) { 101 | public BlogComment createBlogComment(BlogComment blogComment) { 102 | return database.queryForObject("INSERT INTO blog_comments (blog_id, user_id, content) VALUES (?, ?, ?)", BlogComment.class, blogComment.blogId, blogComment.userId, blogComment.content); 103 | } 104 | } 105 | ``` 106 | 107 | ## Step 2. Text to API 108 | 109 | 110 | ## Prompt 111 | 112 | ``` 113 | 请编写用户故事，能覆盖下面的代码功能，要求：1. 突出重点 2. 你返回的内容只有： 作为 xx，我想 xxx，以便于 xxx。 114 | 115 | ### 116 | interface ItemRepository: JpaRepository { 117 | @Query("select i from Item i where i.user = :user and :before <= i.receiptDate and i.receiptDate <= :after") 118 | fun searchItems(@Param("user") user: User, @Param("before") before: LocalDate, @Param("after") after: LocalDate): List 119 | } 120 | ### 121 | ``` 122 | 123 | Demo 2： 124 | 125 | 126 | ``` 127 | 请编写用户故事，能覆盖下面的代码功能，要求：1. 突出重点 2. 你返回的内容只有：我想 xxx，以便于 xxx。 128 | 129 | ### 130 | @Repository 131 | interface StatisticsRepository : CrudRepository { 132 | @Query("select s from Statistic s where chatId = ?1 and postedDate >= ?2 and postedDate <= ?3 " + 133 | "order by postedDate, messageCount desc") 134 | fun findStatisticsBetweenDateRange(chatId: Long, since: LocalDate, until: LocalDate): List 135 | } 136 | ### 137 | ``` 138 | 139 | ## 还需要考虑： 140 | 141 | ``` 142 | fun findStatisticByPostedDate(postedDate: LocalDate): List 143 | fun findStatisticByChatId(chatId: Long): List 144 | ``` 145 | -------------------------------------------------------------------------------- /processor/examples/unit-demo/docs/prompt-template.md: -------------------------------------------------------------------------------- 1 | # Prompt Template 2 | 3 | ## 领域词汇表 4 | 5 | | 中文 | 英文 | 说明 | 6 | |---------|--------------------------|----| 7 | | 业务 | Business | | 8 | | 所有xxx | All xxx | | 9 | | 时间内 | In time | | 10 | | 特定时间范围内 | In a specific time range | | 11 | | 查询 | Query | | 12 | | 某个用户 | A specific user | | 13 | 14 | ## 表信息 15 | 16 | | 表名 | 字段 | 说明 | 17 | |------|---------------|----| 18 | | user | id, name, age | | 19 | 20 | 21 | 作为一个用户，我想查找某个用户拥有的在一个特定时间范围内收到的所有项目，以便了解他们在这段时间内的支出情况。 22 | 23 | - 为此，我需要将该用户的用户名作为参数传递给查询，以及指定查询的时间范围。然后， 24 | - 系统将从项目表中选择所有符合条件的项目，并以列表的形式返回给我 25 | - 如果没有符合条件的项目，系统将返回一个空列表。 26 | 27 | 28 | ## Prompt 29 | 30 | 原始 SQL: 31 | 32 | ```sqlite 33 | select i from Item i where i.user = :user and :before <= i.receiptDate and i.receiptDate <= :after 34 | ``` 35 | 36 | ```markdown 37 | > 围绕下面的用户故事，编写一段 SQL 代码： 38 | 39 | ### 40 | 41 | 作为一个用户，我想查找某个用户拥有的在一个特定时间范围内收到的所有项目，以便了解他们在这段时间内的支出情况。 42 | 43 | 为此，我需要将该用户的用户名作为参数传递给查询，以及指定查询的时间范围。然后， 44 | 系统将从项目表中选择所有符合条件的项目，并以列表的形式返回给我 45 | 如果没有符合条件的项目，系统将返回一个空列表。 46 | ### 47 | ``` 48 | 49 | 返回 50 | 51 | ``` 52 | 你好，我可以为你提供一段 SQL 代码来查找某个用户拥有的在一个特定时间范围内收到的所有项目，以便了解他们在这段时间内的支出情况： 53 | 54 | SELECT * 55 | FROM items 56 | WHERE user_name = [username] 57 | AND date_received BETWEEN [start_date] AND [end_date]; 58 | 59 | 如果没有符合条件的项目，这段代码将返回一个空列表。 60 | ``` 61 | 62 | 更新模板 63 | 64 | ```markdown 65 | > 围绕下面的用户故事，编写一段 SQL 代码： 66 | 67 | ### 68 | 69 | 作为一个用户，我想查找某个用户（User）拥有的在一个特定时间范围内（receiptDate）收到的所有项目（Item），以便了解他们在这段时间内的支出情况。 70 | 71 | 为此，我需要将该用户的用户名（username）作为参数传递给查询，以及指定查询的时间范围。然后， 72 | 系统将从项目表中选择所有符合条件的项目，并以列表的形式返回给我 73 | 如果没有符合条件的项目，系统将返回一个空列表。 74 | ### 75 | ``` 76 | 77 | 结果： 78 | 79 | ``` 80 | 你好，我可以为你提供一段 SQL 代码来查找某个用户拥有的在一个特定时间范围内收到的所有项目，以便了解他们在这段时间内的支出情况： 81 | 82 | SELECT * 83 | FROM items 84 | WHERE username = [username] 85 | AND receiptDate BETWEEN [start_date] AND [end_date]; 86 | 87 | 如果没有符合条件的项目，这段代码将返回一个空列表。 88 | ``` 89 | 90 | 91 | ``` 92 | 请编写用户故事，能覆盖下面的代码功能，要求：1. 分析其业务含义，突出重点 2. 你返回的内容只有： 我想 xxx。 93 | 94 | ### 95 | interface ZhuanlanDao { 96 | @Insert(onConflict = OnConflictStrategy.IGNORE) 97 | fun insert(zhuanlanBean: ZhuanlanBean): Long 98 | } 99 | ### 100 | ``` 101 | -------------------------------------------------------------------------------- /processor/examples/unit-demo/gradle.properties: -------------------------------------------------------------------------------- 1 | ktor_version=2.2.4 2 | kotlin_version=1.8.20 3 | logback_version=1.2.11 4 | kotlin.code.style=official 5 | exposed_version=0.41.1 6 | h2_version=2.1.214 7 | -------------------------------------------------------------------------------- /processor/examples/unit-demo/src/main/kotlin/unitmesh/cc/Application.kt: -------------------------------------------------------------------------------- 1 | package unitmesh.cc 2 | 3 | import io.ktor.serialization.kotlinx.json.* 4 | import io.ktor.server.application.* 5 | import io.ktor.server.engine.* 6 | import io.ktor.server.netty.* 7 | import io.ktor.server.plugins.contentnegotiation.* 8 | import org.jetbrains.exposed.sql.Database 9 | import unitmesh.cc.plugins.* 10 | 11 | fun main() { 12 | embeddedServer(Netty, port = 8080, host = "0.0.0.0", module = Application::module) 13 | .start(wait = true) 14 | } 15 | 16 | fun Application.module() { 17 | install(ContentNegotiation) { 18 | json() 19 | } 20 | val database = Database.connect( 21 | url = "jdbc:h2:mem:test;DB_CLOSE_DELAY=-1", 22 | user = "root", 23 | driver = "org.h2.Driver", 24 | password = "", 25 | ) 26 | 27 | configureSerialization() 28 | configureDatabases(database) 29 | configureSecurity() 30 | configureRouting() 31 | } 32 | -------------------------------------------------------------------------------- /processor/examples/unit-demo/src/main/kotlin/unitmesh/cc/plugins/Databases.kt: -------------------------------------------------------------------------------- 1 | package unitmesh.cc.plugins 2 | 3 | import io.ktor.http.* 4 | import io.ktor.server.application.* 5 | import io.ktor.server.request.* 6 | import io.ktor.server.response.* 7 | import io.ktor.server.routing.* 8 | import org.jetbrains.exposed.sql.* 9 | 10 | fun Application.configureDatabases(database: Database) { 11 | val userService = UserService(database) 12 | 13 | routing { 14 | // Create user 15 | post("/users") { 16 | val user = call.receive() 17 | val id = userService.create(user) 18 | call.respond(HttpStatusCode.Created, id) 19 | } 20 | 21 | // Read user 22 | get("/users/{id}") { 23 | val id = call.parameters["id"]?.toInt() ?: throw IllegalArgumentException("Invalid ID") 24 | val user = userService.read(id) 25 | if (user != null) { 26 | call.respond(HttpStatusCode.OK, user) 27 | } else { 28 | call.respond(HttpStatusCode.NotFound) 29 | } 30 | } 31 | 32 | // Update user 33 | put("/users/{id}") { 34 | val id = call.parameters["id"]?.toInt() ?: throw IllegalArgumentException("Invalid ID") 35 | val user = call.receive() 36 | userService.update(id, user) 37 | call.respond(HttpStatusCode.OK) 38 | } 39 | 40 | // Delete user 41 | delete("/users/{id}") { 42 | val id = call.parameters["id"]?.toInt() ?: throw IllegalArgumentException("Invalid ID") 43 | userService.delete(id) 44 | call.respond(HttpStatusCode.OK) 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /processor/examples/unit-demo/src/main/kotlin/unitmesh/cc/plugins/Routing.kt: -------------------------------------------------------------------------------- 1 | package unitmesh.cc.plugins 2 | 3 | import io.ktor.server.application.* 4 | import io.ktor.server.response.* 5 | import io.ktor.server.routing.* 6 | 7 | fun Application.configureRouting() { 8 | routing { 9 | get("/") { 10 | call.respondText("Hello World!") 11 | } 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /processor/examples/unit-demo/src/main/kotlin/unitmesh/cc/plugins/Security.kt: -------------------------------------------------------------------------------- 1 | package unitmesh.cc.plugins 2 | 3 | import io.ktor.server.application.* 4 | import io.ktor.server.auth.* 5 | import io.ktor.server.auth.jwt.* 6 | 7 | fun Application.configureSecurity() { 8 | authentication { 9 | // jwt { 10 | // val jwtAudience = this@configureSecurity.environment.config.property("jwt.audience").getString() 11 | // realm = this@configureSecurity.environment.config.property("jwt.realm").getString() 12 | // verifier( 13 | // JWT 14 | // .require(Algorithm.HMAC256("secret")) 15 | // .withAudience(jwtAudience) 16 | // .withIssuer(this@configureSecurity.environment.config.property("jwt.domain").getString()) 17 | // .build() 18 | // ) 19 | // validate { credential -> 20 | // if (credential.payload.audience.contains(jwtAudience)) JWTPrincipal(credential.payload) else null 21 | // } 22 | // } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /processor/examples/unit-demo/src/main/kotlin/unitmesh/cc/plugins/Serialization.kt: -------------------------------------------------------------------------------- 1 | package unitmesh.cc.plugins 2 | 3 | import io.ktor.serialization.kotlinx.json.* 4 | import io.ktor.server.application.* 5 | import io.ktor.server.plugins.contentnegotiation.* 6 | import io.ktor.server.response.* 7 | import io.ktor.server.routing.* 8 | 9 | fun Application.configureSerialization() { 10 | routing { 11 | get("/json/kotlinx-serialization") { 12 | call.respond(mapOf("hello" to "world")) 13 | } 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /processor/examples/unit-demo/src/main/kotlin/unitmesh/cc/plugins/UsersSchema.kt: -------------------------------------------------------------------------------- 1 | package unitmesh.cc.plugins 2 | 3 | import kotlinx.coroutines.Dispatchers 4 | import kotlinx.serialization.Serializable 5 | import org.jetbrains.exposed.sql.* 6 | import org.jetbrains.exposed.sql.SqlExpressionBuilder.eq 7 | import org.jetbrains.exposed.sql.transactions.experimental.newSuspendedTransaction 8 | import org.jetbrains.exposed.sql.transactions.transaction 9 | 10 | @Serializable 11 | data class User(val name: String, val age: Int) 12 | class UserService(private val database: Database) { 13 | object Users : Table() { 14 | val id = integer("id").autoIncrement() 15 | val name = varchar("name", length = 50) 16 | val age = integer("age") 17 | 18 | override val primaryKey = PrimaryKey(id) 19 | } 20 | 21 | init { 22 | transaction(database) { 23 | SchemaUtils.create(Users) 24 | } 25 | } 26 | 27 | suspend fun dbQuery(block: suspend () -> T): T = 28 | newSuspendedTransaction(Dispatchers.IO) { block() } 29 | 30 | suspend fun create(user: User): Int = dbQuery { 31 | Users.insert { 32 | it[name] = user.name 33 | it[age] = user.age 34 | }[Users.id] 35 | } 36 | 37 | suspend fun read(id: Int): User? { 38 | return dbQuery { 39 | Users.select { Users.id eq id } 40 | .map { User(it[Users.name], it[Users.age]) } 41 | .singleOrNull() 42 | } 43 | } 44 | 45 | suspend fun update(id: Int, user: User) { 46 | dbQuery { 47 | Users.update({ Users.id eq id }) { 48 | it[name] = user.name 49 | it[age] = user.age 50 | } 51 | } 52 | } 53 | 54 | suspend fun delete(id: Int) { 55 | dbQuery { 56 | Users.deleteWhere { Users.id.eq(id) } 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /processor/examples/unit-demo/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | %d{YYYY-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /processor/examples/unit-demo/src/test/kotlin/unitmesh/cc/ApplicationTest.kt: -------------------------------------------------------------------------------- 1 | package unitmesh.cc 2 | 3 | import io.ktor.client.request.* 4 | import io.ktor.client.statement.* 5 | import io.ktor.http.* 6 | import io.ktor.server.testing.* 7 | import unitmesh.cc.plugins.* 8 | import kotlin.test.* 9 | 10 | class ApplicationTest { 11 | @Test 12 | fun testRoot() = testApplication { 13 | application { 14 | configureRouting() 15 | } 16 | client.get("/").apply { 17 | assertEquals(HttpStatusCode.OK, status) 18 | assertEquals("Hello World!", bodyAsText()) 19 | } 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /processor/gradle.properties: -------------------------------------------------------------------------------- 1 | kotlin.code.style=official 2 | -------------------------------------------------------------------------------- /processor/gradle/libs.versions.toml: -------------------------------------------------------------------------------- 1 | [versions] 2 | kotlin = "1.8.20" 3 | 4 | kotlinxSerialization = "1.5.0" 5 | 6 | junit = "5.9.1" 7 | kotlinTest = "5.5.4" 8 | mockk = "1.13.3" 9 | assertj = "3.22.0" 10 | javaparser = "3.25.1" 11 | 12 | logback = "1.4.5" 13 | slf4j = "2.0.5" 14 | 15 | clikt = "3.5.1" 16 | 17 | kaml = "0.51.0" 18 | 19 | [plugins] 20 | kotlin-jvm = { id = "org.jetbrains.kotlin.jvm", version.ref = "kotlin" } 21 | checksum = "org.gradle.crypto.checksum:1.4.0" 22 | shadow = "com.github.johnrengelman.shadow:8.1.1" 23 | serialization = { id = "org.jetbrains.kotlin.plugin.serialization", version.ref = "kotlin" } 24 | # id("org.jetbrains.kotlinx.dataframe") version "0.9.1" 25 | dataframe = { id = "org.jetbrains.kotlinx.dataframe", version = "0.9.1" } 26 | ksp = { id = "com.google.devtools.ksp", version = "1.8.20-1.0.10" } 27 | 28 | [libraries] 29 | kotlin-compiler = { module = "org.jetbrains.kotlin:kotlin-compiler-embeddable", version.ref = "kotlin" } 30 | kotlin-plugin = { module = "org.jetbrains.kotlin:kotlin-gradle-plugin", version.ref = "kotlin" } 31 | kotlin-plugin-dev = { module = "org.jetbrains.kotlin:kotlin-gradle-plugin", version.ref = "kotlin" } 32 | dokka = "org.jetbrains.dokka:dokka-gradle-plugin:1.7.20" 33 | # a mighty tiny command line interface 34 | picocli = "info.picocli:picocli:4.7.1" 35 | logging = "io.github.microutils:kotlin-logging-jvm:3.0.5" 36 | # Use logback-classic as the logger for kotlin-logging / slf4j as it allow changing the log level at runtime. 37 | logback = "ch.qos.logback:logback-classic:1.3.5" 38 | # Required for logback.xml conditional configuration 39 | janino = "org.codehaus.janino:janino:3.1.9" 40 | # Testing libraries 41 | junit5 = "org.junit.jupiter:junit-jupiter:5.9.2" 42 | assertj = "org.assertj:assertj-core:3.24.2" 43 | sarif4k = "io.github.detekt.sarif4k:sarif4k:0.2.0" 44 | jimfs = "com.google.jimfs:jimfs:1.2" 45 | 46 | serialization-json = { group = "org.jetbrains.kotlinx", name = "kotlinx-serialization-json", version.ref = "kotlinxSerialization" } 47 | 48 | javaparser = { module = "com.github.javaparser:javaparser-core", version.ref = "javaparser" } 49 | javaparser-serialization = { module = "com.github.javaparser:javaparser-core-serialization", version.ref = "javaparser" } 50 | javaparser-symbol-solver-core = { module = "com.github.javaparser:javaparser-symbol-solver-core", version.ref = "javaparser" } 51 | 52 | logging-slf4j-api = { group = "org.slf4j", name = "slf4j-api", version.ref = "slf4j" } 53 | logging-logback-classic = { group = "ch.qos.logback", name = "logback-classic", version.ref = "logback" } 54 | 55 | clikt = { group = "com.github.ajalt.clikt", name = "clikt", version.ref = "clikt" } 56 | 57 | kaml = { group = "com.charleskorn.kaml", name = "kaml", version.ref = "kaml" } 58 | 59 | commonmark-core = "org.commonmark:commonmark:0.21.0" 60 | commonmark-gfm-tables = "org.commonmark:commonmark-ext-gfm-tables:0.21.0" 61 | 62 | # com.github.jsqlparser:jsqlparser:4.6 63 | jsqlparser = "com.github.jsqlparser:jsqlparser:4.6" 64 | 65 | # "net.sourceforge.plantuml:plantuml:1.2023.6" 66 | plantuml = "net.sourceforge.plantuml:plantuml:1.2023.8" 67 | 68 | # "com.theokanning.openai-gpt3-java:service:0.12.0" 69 | openai-gpt3 = "com.theokanning.openai-gpt3-java:service:0.12.0" 70 | 71 | # "com.squareup.retrofit2:converter-jackson:2.9.0" 72 | retrofit2-converter-jackson = "com.squareup.retrofit2:converter-jackson:2.9.0" 73 | 74 | # "io.github.cdimascio:dotenv-kotlin:6.4.1" 75 | dotenv = "io.github.cdimascio:dotenv-kotlin:6.4.1" 76 | 77 | # "org.jetbrains.kotlinx:dataframe:0.10.0" 78 | dataframe = "org.jetbrains.kotlinx:dataframe:0.10.0" 79 | 80 | # "com.google.devtools.ksp:symbol-processing-api:1.8.20-1.0.10" 81 | ksp = "com.google.devtools.ksp:symbol-processing-api:1.8.20-1.0.10" 82 | 83 | # Testing 84 | test-junit-api = { group = "org.junit.jupiter", name = "junit-jupiter-api", version.ref = "junit" } 85 | test-junit-params = { group = "org.junit.jupiter", name = "junit-jupiter-params", version.ref = "junit" } 86 | test-junit-engine = { group = "org.junit.jupiter", name = "junit-jupiter-engine", version.ref = "junit" } 87 | test-kotlintest-assertions = { module = "io.kotest:kotest-assertions-core", version.ref = "kotlinTest" } 88 | test-mockk = { group = "io.mockk", name = "mockk", version.ref = "mockk" } 89 | test-assertj = { group = "org.assertj", name = "assertj-core", version.ref = "assertj" } 90 | 91 | # // https://mvnrepository.com/artifact/io.swagger.parser.v3/swagger-parser-v3 92 | swagger-parser-v3 = "io.swagger.parser.v3:swagger-parser-v3:2.1.12" 93 | swagger-parser = "io.swagger:swagger-parser:2.0.0-rc1" 94 | 95 | [bundles] 96 | test = ["test-junit-params", "test-kotlintest-assertions", "test-mockk", "test-assertj"] 97 | openai = ["openai-gpt3", "retrofit2-converter-jackson"] 98 | markdown = ["commonmark-core", "commonmark-gfm-tables"] 99 | -------------------------------------------------------------------------------- /processor/gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unit-mesh/devti/9ad45cf571a40867e2d5fa1289a4aff16a03d4f5/processor/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /processor/gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionSha256Sum=1b6b558be93f29438d3df94b7dfee02e794b94d9aca4611a92cdb79b6b88e909 4 | distributionUrl=https\://services.gradle.org/distributions/gradle-8.0.1-bin.zip 5 | networkTimeout=10000 6 | zipStoreBase=GRADLE_USER_HOME 7 | zipStorePath=wrapper/dists 8 | -------------------------------------------------------------------------------- /processor/gradlew.bat: -------------------------------------------------------------------------------- 1 | @rem 2 | @rem Copyright 2015 the original author or authors. 3 | @rem 4 | @rem Licensed under the Apache License, Version 2.0 (the "License"); 5 | @rem you may not use this file except in compliance with the License. 6 | @rem You may obtain a copy of the License at 7 | @rem 8 | @rem https://www.apache.org/licenses/LICENSE-2.0 9 | @rem 10 | @rem Unless required by applicable law or agreed to in writing, software 11 | @rem distributed under the License is distributed on an "AS IS" BASIS, 12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | @rem See the License for the specific language governing permissions and 14 | @rem limitations under the License. 15 | @rem 16 | 17 | @if "%DEBUG%" == "" @echo off 18 | @rem ########################################################################## 19 | @rem 20 | @rem Gradle startup script for Windows 21 | @rem 22 | @rem ########################################################################## 23 | 24 | @rem Set local scope for the variables with windows NT shell 25 | if "%OS%"=="Windows_NT" setlocal 26 | 27 | set DIRNAME=%~dp0 28 | if "%DIRNAME%" == "" set DIRNAME=. 29 | set APP_BASE_NAME=%~n0 30 | set APP_HOME=%DIRNAME% 31 | 32 | @rem Resolve any "." and ".." in APP_HOME to make it shorter. 33 | for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi 34 | 35 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 36 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" 37 | 38 | @rem Find java.exe 39 | if defined JAVA_HOME goto findJavaFromJavaHome 40 | 41 | set JAVA_EXE=java.exe 42 | %JAVA_EXE% -version >NUL 2>&1 43 | if "%ERRORLEVEL%" == "0" goto execute 44 | 45 | echo. 46 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 47 | echo. 48 | echo Please set the JAVA_HOME variable in your environment to match the 49 | echo location of your Java installation. 50 | 51 | goto fail 52 | 53 | :findJavaFromJavaHome 54 | set JAVA_HOME=%JAVA_HOME:"=% 55 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 56 | 57 | if exist "%JAVA_EXE%" goto execute 58 | 59 | echo. 60 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 61 | echo. 62 | echo Please set the JAVA_HOME variable in your environment to match the 63 | echo location of your Java installation. 64 | 65 | goto fail 66 | 67 | :execute 68 | @rem Setup the command line 69 | 70 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 71 | 72 | 73 | @rem Execute Gradle 74 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* 75 | 76 | :end 77 | @rem End local scope for the variables with windows NT shell 78 | if "%ERRORLEVEL%"=="0" goto mainEnd 79 | 80 | :fail 81 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 82 | rem the _cmd.exe /c_ return code! 83 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 84 | exit /b 1 85 | 86 | :mainEnd 87 | if "%OS%"=="Windows_NT" endlocal 88 | 89 | :omega 90 | -------------------------------------------------------------------------------- /processor/java-modules/.gitignore: -------------------------------------------------------------------------------- 1 | processor.yml -------------------------------------------------------------------------------- /processor/java-modules/codegen-processor/build.gradle.kts: -------------------------------------------------------------------------------- 1 | @Suppress("DSL_SCOPE_VIOLATION") 2 | plugins { 3 | alias(libs.plugins.kotlin.jvm) 4 | alias(libs.plugins.shadow) 5 | alias(libs.plugins.serialization) 6 | application 7 | } 8 | 9 | repositories { 10 | mavenCentral() 11 | } 12 | 13 | dependencies { 14 | implementation(projects.javaModules.coreAnalysis) 15 | implementation(projects.common.cliCore) 16 | implementation(libs.clikt) 17 | implementation(libs.kotlin.compiler) 18 | implementation(libs.serialization.json) 19 | 20 | // Logging 21 | implementation(libs.logging.slf4j.api) 22 | implementation(libs.logging.logback.classic) 23 | 24 | implementation(libs.kaml) 25 | 26 | // java parser 27 | implementation(libs.javaparser) 28 | implementation(libs.javaparser.serialization) 29 | implementation(libs.javaparser.symbol.solver.core) 30 | 31 | testImplementation(kotlin("test")) 32 | 33 | testImplementation(libs.bundles.test) 34 | } 35 | 36 | tasks.test { 37 | useJUnitPlatform() 38 | } 39 | 40 | kotlin { 41 | jvmToolchain(11) 42 | } 43 | 44 | application { 45 | mainClass.set("cc.unitmesh.processor.codegen.MainKt") 46 | } 47 | 48 | tasks { 49 | shadowJar { 50 | manifest { 51 | attributes(Pair("Main-Class", "cc.unitmesh.processor.codegen.MainKt")) 52 | } 53 | // minimize() 54 | dependencies { 55 | exclude(dependency("org.junit.jupiter:.*:.*")) 56 | exclude(dependency("org.junit:.*:.*")) 57 | exclude(dependency("junit:.*:.*")) 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /processor/java-modules/core-analysis/build.gradle.kts: -------------------------------------------------------------------------------- 1 | @Suppress("DSL_SCOPE_VIOLATION") 2 | plugins { 3 | alias(libs.plugins.kotlin.jvm) 4 | alias(libs.plugins.serialization) 5 | } 6 | 7 | repositories { 8 | mavenCentral() 9 | } 10 | 11 | dependencies { 12 | implementation(libs.clikt) 13 | implementation(libs.serialization.json) 14 | 15 | // Logging 16 | implementation(libs.logging.slf4j.api) 17 | implementation(libs.logging.logback.classic) 18 | 19 | implementation(libs.kaml) 20 | 21 | // java parser 22 | implementation(libs.javaparser) 23 | implementation(libs.javaparser.serialization) 24 | implementation(libs.javaparser.symbol.solver.core) 25 | 26 | testImplementation(kotlin("test")) 27 | 28 | testImplementation(libs.bundles.test) 29 | } 30 | 31 | tasks.test { 32 | useJUnitPlatform() 33 | } 34 | 35 | kotlin { 36 | jvmToolchain(11) 37 | } 38 | -------------------------------------------------------------------------------- /processor/java-modules/core-analysis/src/main/kotlin/cc/unitmesh/core/JvmProcessor.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.core 2 | 3 | import cc.unitmesh.core.java.ShortClass 4 | 5 | interface JvmProcessor { 6 | fun toShortClass(): ShortClass? 7 | } 8 | -------------------------------------------------------------------------------- /processor/java-modules/core-analysis/src/main/kotlin/cc/unitmesh/core/java/JavaProcessor.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.core.java 2 | 3 | import cc.unitmesh.core.JvmProcessor 4 | import com.github.javaparser.StaticJavaParser 5 | import com.github.javaparser.ast.CompilationUnit 6 | import com.github.javaparser.ast.ImportDeclaration 7 | import com.github.javaparser.ast.body.ClassOrInterfaceDeclaration 8 | import com.github.javaparser.ast.body.ConstructorDeclaration 9 | import com.github.javaparser.ast.body.FieldDeclaration 10 | 11 | val LICENSES = listOf( 12 | "Licensed under the Apache License,", 13 | "Licensed to the Apache Software Foundation (ASF) under one", 14 | "under the terms of the MIT License.", 15 | "Mozilla Public License", 16 | ) 17 | 18 | open class JavaProcessor(open val code: String) : JvmProcessor { 19 | var unit: CompilationUnit 20 | 21 | init { 22 | try { 23 | unit = StaticJavaParser.parse(code) 24 | } catch (e: Exception) { 25 | throw e 26 | } 27 | } 28 | 29 | override fun toShortClass(): ShortClass? { 30 | val cls = unit.findFirst(ClassOrInterfaceDeclaration::class.java).orElse(null) ?: return null 31 | 32 | val packageName = unit.packageDeclaration.map { it.nameAsString }.orElse(null) 33 | val fields = cls.fields?.map { 34 | ShortField(it.nameAsString, it.typeAsString) 35 | } ?: emptyList() 36 | val methods = cls.methods.map { method -> 37 | ShortMethod( 38 | method.nameAsString, 39 | method.typeAsString, 40 | method.parameters.map { ShortParameter(it.nameAsString, it.typeAsString) }, 41 | ) 42 | } 43 | val constructors = cls.constructors.map { ShortParameter(it.nameAsString, it.typeAsString) } 44 | return ShortClass(cls.nameAsString, packageName, fields, methods, constructors) 45 | } 46 | 47 | fun packageName(): String? { 48 | return unit.packageDeclaration.map { it.nameAsString }.orElse(null) 49 | } 50 | 51 | fun removeLicenseInfoBeforeImport(): JavaProcessor { 52 | unit.allComments.forEach { comment -> 53 | LICENSES.forEach { license -> 54 | if (comment.content.contains(license)) { 55 | comment.remove() 56 | } 57 | } 58 | } 59 | 60 | return this 61 | } 62 | 63 | fun removePackage(): JavaProcessor { 64 | unit.packageDeclaration.ifPresent { it.remove() } 65 | return this 66 | } 67 | 68 | fun removeAllImport(): JavaProcessor { 69 | unit.findAll(ImportDeclaration::class.java).forEach { it.remove() } 70 | return this 71 | } 72 | 73 | fun splitMethods(): MutableMap { 74 | // map to method name and method body 75 | val results = mutableMapOf() 76 | unit.findAll(ClassOrInterfaceDeclaration::class.java).forEach { cls -> 77 | cls.methods.map { method -> 78 | val unit = unit.clone() 79 | unit.findAll(ClassOrInterfaceDeclaration::class.java).forEach { cls -> 80 | cls.methods.filter { it != method }.forEach { it.remove() } 81 | } 82 | results[method.nameAsString] = unit.toString() 83 | } 84 | } 85 | 86 | return results 87 | } 88 | } 89 | 90 | private val FieldDeclaration.nameAsString: String 91 | get() = this.variables.first().nameAsString 92 | private val FieldDeclaration.typeAsString: String 93 | get() = this.elementType.toString() 94 | 95 | private val ConstructorDeclaration.typeAsString: String 96 | get() = this.parameters.joinToString(", ") { it.typeAsString } 97 | -------------------------------------------------------------------------------- /processor/java-modules/core-analysis/src/main/kotlin/cc/unitmesh/core/java/ShortClass.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.core.java 2 | 3 | import kotlinx.serialization.Serializable 4 | 5 | @Serializable 6 | data class ShortClass( 7 | val className: String, 8 | val packageName: String?, 9 | val fields: List = listOf(), 10 | val methods: List = listOf(), 11 | val constructors: List = listOf(), 12 | ) { 13 | val name = packageName?.let { "$it.$className" } ?: className 14 | 15 | override fun toString(): String { 16 | val classInfo = 17 | """${packageName?.let { "$it." } ?: ""}$className(${constructors.joinToString(", ") { it.dataType }})""" 18 | val fieldInfo = 19 | if (fields.isNotEmpty()) { 20 | "\n- fields: ${fields.joinToString(", ") { it.fieldName + ":" + it.dataType }}" 21 | } else { 22 | "" 23 | } 24 | val methodInfo = 25 | if (methods.isNotEmpty()) { 26 | "\n- methods: ${methods.joinToString(", ") { it.toString() }}" 27 | } else { 28 | "" 29 | } 30 | 31 | return classInfo + fieldInfo + methodInfo 32 | } 33 | } 34 | 35 | @Serializable 36 | data class ShortField( 37 | val fieldName: String, 38 | val dataType: String, 39 | ) 40 | 41 | @Serializable 42 | data class ShortMethod( 43 | val methodName: String, 44 | val returnType: String, 45 | val parameters: List = listOf(), 46 | ) { 47 | override fun toString(): String { 48 | return "$methodName(${parameters.joinToString(", ") { it.dataType }}): $returnType" 49 | } 50 | } 51 | 52 | @Serializable 53 | data class ShortParameter( 54 | val parameterName: String, 55 | val dataType: String, 56 | ) 57 | -------------------------------------------------------------------------------- /processor/java-modules/core-analysis/src/main/kotlin/cc/unitmesh/core/java/SpringProcessor.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.core.java 2 | 3 | import com.github.javaparser.ast.body.ClassOrInterfaceDeclaration 4 | import com.github.javaparser.ast.body.MethodDeclaration 5 | import com.github.javaparser.ast.expr.MarkerAnnotationExpr 6 | import com.github.javaparser.ast.expr.SingleMemberAnnotationExpr 7 | 8 | class SpringProcessor(code: String) : JavaProcessor(code) { 9 | fun isSpringController(): Boolean { 10 | for (annotationExpr in unit.findFirst(ClassOrInterfaceDeclaration::class.java).orElseThrow().annotations) { 11 | if (annotationExpr is SingleMemberAnnotationExpr || annotationExpr is MarkerAnnotationExpr) { 12 | val name = annotationExpr.name 13 | if (name.identifier == "RestController" || name.identifier == "Controller") { 14 | // Check if the class contains at least one method with a @RequestMapping annotation 15 | return unit.findFirst(MethodDeclaration::class.java) 16 | .filter { m -> m.getAnnotationByName("RequestMapping").isPresent() } 17 | .isPresent 18 | } 19 | } 20 | } 21 | 22 | return false 23 | } 24 | 25 | fun isAnnotationWith(name: String): Boolean { 26 | return unit.findFirst(ClassOrInterfaceDeclaration::class.java).orElseThrow().annotations.any { it.name.identifier == name } 27 | } 28 | 29 | fun splitControllerMethods(): List { 30 | val methods = mutableListOf() 31 | unit.findAll(ClassOrInterfaceDeclaration::class.java).forEach { cls -> 32 | cls.methods.filter { 33 | it.annotations.any { annotation -> 34 | annotation.nameAsString == "RequestMapping" 35 | } 36 | }.map { method -> 37 | val test = unit.clone() 38 | test.findAll(ClassOrInterfaceDeclaration::class.java).forEach { cls -> 39 | cls.methods.filter { it != method }.forEach { it.remove() } 40 | } 41 | methods.add(test.toString()) 42 | } 43 | } 44 | 45 | return methods 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /processor/java-modules/core-analysis/src/main/kotlin/cc/unitmesh/core/java/TestProcessor.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.core.java 2 | 3 | import com.github.javaparser.ast.body.ClassOrInterfaceDeclaration 4 | 5 | class TestProcessor(code: String) : JavaProcessor(code) { 6 | fun splitTests(): List { 7 | val tests = mutableListOf() 8 | unit.findAll(ClassOrInterfaceDeclaration::class.java).forEach { cls -> 9 | cls.methods.filter { 10 | it.annotations.any { annotation -> 11 | annotation.nameAsString == "Test" 12 | } 13 | }.map { method -> 14 | val test = unit.clone() 15 | test.findAll(ClassOrInterfaceDeclaration::class.java).forEach { cls -> 16 | cls.methods.filter { it != method }.forEach { it.remove() } 17 | } 18 | tests.add(test.toString()) 19 | } 20 | } 21 | 22 | return tests 23 | } 24 | 25 | fun output(): String { 26 | return unit.toString() 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /processor/java-modules/core-analysis/src/test/kotlin/cc/unitmesh/core/java/JavaProcessorTest.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.core.java 2 | 3 | import io.kotest.matchers.shouldBe 4 | import org.junit.jupiter.api.Test 5 | 6 | class JavaProcessorTest { 7 | private val SampleClass = """ 8 | package org.unitmesh.processor.java; 9 | 10 | import org.springframework.web.bind.annotation.RequestMapping; 11 | import org.springframework.web.bind.annotation.RestController; 12 | 13 | @RestController 14 | public class JavaProcessor { 15 | @RequestMapping("/test") 16 | public String test() { 17 | return "test"; 18 | } 19 | } 20 | """ 21 | 22 | @Test 23 | fun `to short class`() { 24 | val code = SampleClass.trimIndent() 25 | val processor = JavaProcessor(code) 26 | val shortClass = processor.toShortClass()!! 27 | val expected = """ 28 | org.unitmesh.processor.java.JavaProcessor() 29 | - methods: test(): String 30 | """.trimIndent() 31 | shortClass.toString() shouldBe expected 32 | } 33 | 34 | @Test 35 | fun `split methods`() { 36 | val code = """ 37 | class TestProcessorTest { 38 | @Test 39 | void test1() { 40 | } 41 | 42 | @Test 43 | void test2() { 44 | } 45 | } 46 | """.trimIndent() 47 | 48 | val processor = JavaProcessor(code) 49 | val methods = processor.splitMethods() 50 | methods.size shouldBe 2 51 | methods["test1"] shouldBe """ 52 | class TestProcessorTest { 53 | 54 | @Test 55 | void test1() { 56 | } 57 | } 58 | 59 | """.trimIndent() 60 | 61 | methods["test2"] shouldBe """ 62 | class TestProcessorTest { 63 | 64 | @Test 65 | void test2() { 66 | } 67 | } 68 | 69 | """.trimIndent() 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /processor/java-modules/core-analysis/src/test/kotlin/cc/unitmesh/core/java/SpringProcessorTest.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.core.java 2 | 3 | import io.kotest.matchers.shouldBe 4 | import org.junit.jupiter.api.Test 5 | 6 | class SpringProcessorTest { 7 | private val SampleClass = """ 8 | package org.unitmesh.processor.java; 9 | 10 | import org.springframework.web.bind.annotation.RequestMapping; 11 | import org.springframework.web.bind.annotation.RestController; 12 | 13 | @RestController 14 | public class JavaProcessor { 15 | @RequestMapping("/test") 16 | public String test() { 17 | return "test"; 18 | } 19 | } 20 | """ 21 | 22 | @Test 23 | fun `search spring class`() { 24 | val code = SampleClass.trimIndent() 25 | val processor = SpringProcessor(code) 26 | processor.isSpringController() shouldBe true 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /processor/java-modules/spring-processor/build.gradle.kts: -------------------------------------------------------------------------------- 1 | @Suppress("DSL_SCOPE_VIOLATION") 2 | plugins { 3 | alias(libs.plugins.kotlin.jvm) 4 | alias(libs.plugins.shadow) 5 | alias(libs.plugins.serialization) 6 | application 7 | } 8 | 9 | repositories { 10 | mavenCentral() 11 | } 12 | 13 | dependencies { 14 | implementation(projects.javaModules.coreAnalysis) 15 | implementation(projects.common.cliCore) 16 | implementation(libs.clikt) 17 | implementation(libs.kotlin.compiler) 18 | 19 | implementation(libs.serialization.json) 20 | 21 | // Logging 22 | implementation(libs.logging.slf4j.api) 23 | implementation(libs.logging.logback.classic) 24 | 25 | // java parser 26 | implementation(libs.swagger.parser) 27 | implementation(libs.swagger.parser.v3) 28 | implementation(libs.javaparser) 29 | implementation(libs.javaparser.serialization) 30 | implementation(libs.javaparser.symbol.solver.core) 31 | 32 | testImplementation(kotlin("test")) 33 | 34 | testImplementation(libs.bundles.test) 35 | } 36 | 37 | tasks.test { 38 | useJUnitPlatform() 39 | } 40 | 41 | kotlin { 42 | jvmToolchain(11) 43 | } 44 | 45 | application { 46 | mainClass.set("cc.unitmesh.spring.MainKt") 47 | } 48 | 49 | tasks { 50 | shadowJar { 51 | manifest { 52 | attributes(Pair("Main-Class", "cc.unitmesh.spring.MainKt")) 53 | } 54 | // minimize() 55 | dependencies { 56 | exclude(dependency("org.junit.jupiter:.*:.*")) 57 | exclude(dependency("org.junit:.*:.*")) 58 | exclude(dependency("junit:.*:.*")) 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /processor/java-modules/spring-processor/src/main/kotlin/cc/unitmesh/spring/Main.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.spring 2 | 3 | import cc.unitmesh.core.cli.ProcessorUtils 4 | import com.github.ajalt.clikt.core.CliktCommand 5 | import org.slf4j.Logger 6 | import java.io.File 7 | 8 | fun main(args: Array) = Runner().main(args) 9 | 10 | class Runner : CliktCommand(help = "Action Runner") { 11 | override fun run() { 12 | logger.info("Runner started") 13 | 14 | // 1. load config `processor.yml` and start to scm 15 | val config = ProcessorUtils.loadConfig() 16 | 17 | // 2. clone all repositories 18 | ProcessorUtils.cloneAllRepositories(config) 19 | 20 | // clean old datasets under datasets/origin 21 | val outputDir = File("datasets" + File.separator + "spring") 22 | if (outputDir.exists()) { 23 | outputDir.deleteRecursively() 24 | } 25 | outputDir.mkdirs() 26 | outputDir.walkTopDown().forEach { 27 | if (it.isFile) { 28 | it.delete() 29 | } 30 | } 31 | 32 | config.scm.forEach { _ -> 33 | File("origindatasets").walkTopDown().forEach { file -> 34 | val isJavaPath = file.absolutePath.contains("src" + File.separator + "main" + File.separator + "java") 35 | // check is a Spring Controller 36 | val isController = file.name.endsWith("Controller.java") 37 | if (file.isFile && isJavaPath && isController) { 38 | val outputFile = File(outputDir.absolutePath + File.separator + file.name) 39 | file.copyTo(outputFile, true) 40 | } 41 | } 42 | } 43 | } 44 | 45 | companion object { 46 | val logger: Logger = org.slf4j.LoggerFactory.getLogger(Runner::class.java) 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /processor/java-modules/test-processor/build.gradle.kts: -------------------------------------------------------------------------------- 1 | @Suppress("DSL_SCOPE_VIOLATION") 2 | plugins { 3 | alias(libs.plugins.kotlin.jvm) 4 | alias(libs.plugins.shadow) 5 | alias(libs.plugins.serialization) 6 | application 7 | } 8 | 9 | repositories { 10 | mavenCentral() 11 | } 12 | 13 | dependencies { 14 | implementation(projects.javaModules.coreAnalysis) 15 | implementation(projects.common.cliCore) 16 | implementation(libs.clikt) 17 | implementation(libs.kotlin.compiler) 18 | implementation(libs.serialization.json) 19 | 20 | // Logging 21 | implementation(libs.logging.slf4j.api) 22 | implementation(libs.logging.logback.classic) 23 | 24 | implementation(libs.kaml) 25 | 26 | // java parser 27 | implementation(libs.javaparser) 28 | implementation(libs.javaparser.serialization) 29 | implementation(libs.javaparser.symbol.solver.core) 30 | 31 | testImplementation(kotlin("test")) 32 | 33 | testImplementation(libs.bundles.test) 34 | } 35 | 36 | tasks.test { 37 | useJUnitPlatform() 38 | } 39 | 40 | kotlin { 41 | jvmToolchain(11) 42 | } 43 | 44 | application { 45 | mainClass.set("cc.unitmesh.processor.MainKt") 46 | } 47 | 48 | tasks { 49 | shadowJar { 50 | manifest { 51 | attributes(Pair("Main-Class", "cc.unitmesh.processor.MainKt")) 52 | } 53 | // minimize() 54 | dependencies { 55 | exclude(dependency("org.junit.jupiter:.*:.*")) 56 | exclude(dependency("org.junit:.*:.*")) 57 | exclude(dependency("junit:.*:.*")) 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /processor/kotlin-modules/analysis/build.gradle.kts: -------------------------------------------------------------------------------- 1 | @Suppress("DSL_SCOPE_VIOLATION") 2 | plugins { 3 | alias(libs.plugins.kotlin.jvm) 4 | alias(libs.plugins.serialization) 5 | } 6 | 7 | repositories { 8 | mavenCentral() 9 | } 10 | 11 | dependencies { 12 | implementation(libs.clikt) 13 | implementation(libs.kotlin.compiler) 14 | implementation(libs.serialization.json) 15 | 16 | // Logging 17 | implementation(libs.logging.slf4j.api) 18 | implementation(libs.logging.logback.classic) 19 | 20 | implementation(libs.kaml) 21 | 22 | testImplementation(kotlin("test")) 23 | 24 | testImplementation(libs.bundles.test) 25 | } 26 | 27 | tasks.test { 28 | useJUnitPlatform() 29 | } 30 | 31 | kotlin { 32 | jvmToolchain(11) 33 | } 34 | -------------------------------------------------------------------------------- /processor/kotlin-modules/analysis/src/main/kotlin/ktlint/analysis/Code.kt: -------------------------------------------------------------------------------- 1 | package ktlint.analysis 2 | 3 | import org.jetbrains.kotlin.konan.file.file 4 | import java.io.File 5 | import java.nio.file.Path 6 | import kotlin.io.path.pathString 7 | 8 | /** 9 | * A representation of a block of code. Use one of the factory methods [fromFile], [fromPath], [fromSnippet] or [fromStdin] to instantiate. 10 | */ 11 | public class Code constructor( 12 | public val content: String, 13 | public val fileName: String?, 14 | public val filePath: Path?, 15 | public val script: Boolean, 16 | public val isStdIn: Boolean, 17 | ) { 18 | public companion object { 19 | /** 20 | * Create [Code] from a [file] containing valid Kotlin code or script. The '.editorconfig' files on the path to [file] are taken 21 | * into account. 22 | */ 23 | public fun fromFile(file: File): Code = 24 | Code( 25 | content = file.readText(), 26 | fileName = file.name, 27 | filePath = file.toPath(), 28 | script = file.name.endsWith(".kts", ignoreCase = true), 29 | isStdIn = false, 30 | ) 31 | 32 | /** 33 | * Create [Code] from a [path] to a file containing valid Kotlin code or script. The '.editorconfig' files on the path to [file] are 34 | * taken into account. This method is intended to be used in unit tests. In order to work with the Ktlint test file system it needs 35 | * to make additional call to get the file system which makes it slower compared to [fromFile]. Prefer to use [fromFile]. 36 | */ 37 | public fun fromPath(path: Path): Code { 38 | // Resolve the file based on the file system of the original path given. 39 | val file = 40 | path 41 | .fileSystem 42 | .file(path.pathString) 43 | return Code( 44 | content = file.readStrings().joinToString(separator = "\n"), 45 | fileName = file.name, 46 | filePath = path, 47 | script = file.name.endsWith(".kts", ignoreCase = true), 48 | isStdIn = false, 49 | ) 50 | } 51 | 52 | /** 53 | * The [content] represent a valid piece of Kotlin code or Kotlin script. The '.editorconfig' files on the filesystem are ignored as 54 | * the snippet is not associated with a file path. Use [Code.fromFile] for scanning a file while at the same time respecting the 55 | * '.editorconfig' files on the path to the file. 56 | */ 57 | public fun fromSnippet( 58 | content: String, 59 | script: Boolean = false, 60 | ): Code = 61 | Code( 62 | content = content, 63 | filePath = null, 64 | fileName = null, 65 | script = script, 66 | isStdIn = true, 67 | ) 68 | 69 | /** 70 | * Create [Code] by reading the snippet from 'stdin'. No '.editorconfig' are taken into account. The '.editorconfig' files on the 71 | * filesystem are ignored as the snippet is not associated with a file path. Use [Code.fromFile] for scanning a file while at the 72 | * same time respecting the '.editorconfig' files on the path to the file. 73 | */ 74 | public fun fromStdin(): Code = fromSnippet(String(System.`in`.readBytes())) 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /processor/kotlin-modules/analysis/src/main/kotlin/ktlint/analysis/KtLintParseException.kt: -------------------------------------------------------------------------------- 1 | package ktlint.analysis 2 | 3 | /** 4 | * [KtLintParseException] is thrown whenever the kotlin code which is to be scanned contains a parsing error. Ensure 5 | * that the code which is to be scanned, does not contain compilation errors. 6 | * 7 | * @param line line number (one-based) 8 | * @param col column number (one-based) 9 | * @param message message 10 | */ 11 | public class KtLintParseException( 12 | public val line: Int, 13 | public val col: Int, 14 | message: String, 15 | ) : RuntimeException("$line:$col $message") 16 | -------------------------------------------------------------------------------- /processor/kotlin-modules/analysis/src/main/kotlin/ktlint/analysis/PositionInTextLocator.kt: -------------------------------------------------------------------------------- 1 | package ktlint.analysis 2 | 3 | typealias LineAndColumn = Pair 4 | 5 | const val UTF8_BOM = "\uFEFF" 6 | 7 | /** 8 | * Builds function to locate position in text - line and column based on offset from the text start. 9 | */ 10 | fun buildPositionInTextLocator(text: String): (offset: Int) -> LineAndColumn { 11 | val textLength = text.length 12 | val arr = ArrayList() 13 | 14 | var endOfLineIndex = -1 15 | do { 16 | arr.add(endOfLineIndex + 1) 17 | endOfLineIndex = text.indexOf('\n', endOfLineIndex + 1) 18 | } while (endOfLineIndex != -1) 19 | 20 | arr.add(textLength + if (arr.last() == textLength) 1 else 0) 21 | 22 | val segmentTree = SegmentTree(arr.toTypedArray()) 23 | 24 | return { offset -> 25 | val line = segmentTree.indexOf(offset) 26 | if (line != -1) { 27 | val col = offset - segmentTree.get(line).left 28 | line + 1 to col + 1 29 | } else { 30 | 1 to 1 31 | } 32 | } 33 | } 34 | 35 | private class SegmentTree( 36 | sortedArray: Array, 37 | ) { 38 | init { 39 | require(sortedArray.size > 1) { "At least two data points are required" } 40 | sortedArray.reduce { current, next -> 41 | require(current <= next) { "Data points are not sorted (ASC)" } 42 | next 43 | } 44 | } 45 | 46 | private val segments: List = 47 | sortedArray 48 | .dropLast(1) 49 | .mapIndexed { index: Int, element: Int -> 50 | Segment(element, sortedArray[index + 1] - 1) 51 | } 52 | 53 | fun get(i: Int): Segment = segments[i] 54 | fun indexOf(v: Int): Int = binarySearch(v, 0, segments.size - 1) 55 | 56 | private fun binarySearch( 57 | v: Int, 58 | l: Int, 59 | r: Int, 60 | ): Int = 61 | when { 62 | l > r -> -1 63 | else -> { 64 | val i = l + (r - l) / 2 65 | val s = segments[i] 66 | if (v < s.left) { 67 | binarySearch(v, l, i - 1) 68 | } else { 69 | if (s.right < v) binarySearch(v, i + 1, r) else i 70 | } 71 | } 72 | } 73 | } 74 | 75 | private data class Segment( 76 | val left: Int, 77 | val right: Int, 78 | ) 79 | -------------------------------------------------------------------------------- /processor/kotlin-modules/importer/.gitignore: -------------------------------------------------------------------------------- 1 | datasets/*.json -------------------------------------------------------------------------------- /processor/kotlin-modules/importer/README.md: -------------------------------------------------------------------------------- 1 | # Importer 2 | 3 | - Apache Arrow as Format for speed 4 | - Hashmap as Index for speed 5 | - JSON as Format for compatibility 6 | 7 | ## Prompt 8 | 9 | ### 默认情况 10 | 11 | ``` 12 | 请编写用户故事，能覆盖下面的代码功能，要求：1. 突出重点 2. 你返回的内容只有： 我想 xxx。 13 | 14 | @Insert(onConflict = OnConflictStrategy.IGNORE) 15 | fun insert(zhuanlanBean: ZhuanlanBean): Long 16 | ``` 17 | 18 | 输出结果： 19 | 20 | ``` 21 | 一个用户想要在数据库中插入一个ZhuanlanBean，但他们希望如果这个ZhuanlanBean已经存在，则忽略这次插入操作。 22 | ``` 23 | 24 | ### 理想情况 25 | 26 | ``` 27 | 请编写用户故事，能覆盖下面的代码功能，要求：1. 突出重点 2. 你返回的内容只有： 我想 xxx，以便于。 28 | 领域词汇：专栏（ZhuanlanBean） 29 | 30 | @Insert(onConflict = OnConflictStrategy.IGNORE) 31 | fun insert(zhuanlanBean: ZhuanlanBean): Long 32 | ``` 33 | 34 | 结果： 35 | 36 | ``` 37 | 一个用户想要将一篇专栏文章存入数据库，但他们希望如果这篇文章已经存在，则忽略这次存储操作，而不是覆盖原有的文章。 38 | ``` 39 | -------------------------------------------------------------------------------- /processor/kotlin-modules/importer/build.gradle.kts: -------------------------------------------------------------------------------- 1 | buildscript { 2 | dependencies { 3 | classpath(kotlin("gradle-plugin", version = "1.8.0")) 4 | } 5 | } 6 | 7 | @Suppress("DSL_SCOPE_VIOLATION") 8 | plugins { 9 | alias(libs.plugins.kotlin.jvm) 10 | alias(libs.plugins.shadow) 11 | alias(libs.plugins.serialization) 12 | alias(libs.plugins.dataframe) 13 | alias(libs.plugins.ksp) 14 | application 15 | } 16 | 17 | kotlin.sourceSets.getByName("main").kotlin.srcDir("build/generated/ksp/main/kotlin/") 18 | 19 | repositories { 20 | mavenCentral() 21 | } 22 | 23 | dependencies { 24 | implementation(projects.kotlinModules.analysis) 25 | 26 | implementation(libs.ksp) 27 | 28 | implementation(libs.clikt) 29 | implementation(libs.kotlin.compiler) 30 | implementation(libs.serialization.json) 31 | 32 | // Logging 33 | implementation(libs.logging.slf4j.api) 34 | implementation(libs.logging.logback.classic) 35 | 36 | implementation(libs.kaml) 37 | 38 | testImplementation(kotlin("test")) 39 | 40 | testImplementation(libs.bundles.test) 41 | } 42 | 43 | tasks.test { 44 | useJUnitPlatform() 45 | } 46 | 47 | kotlin { 48 | jvmToolchain(11) 49 | } 50 | 51 | application { 52 | mainClass.set("cc.unitmesh.importer.MainKt") 53 | } 54 | 55 | tasks { 56 | shadowJar { 57 | manifest { 58 | attributes(Pair("Main-Class", "cc.unitmesh.importer.MainKt")) 59 | } 60 | // minimize() 61 | dependencies { 62 | exclude(dependency("org.junit.jupiter:.*:.*")) 63 | exclude(dependency("org.junit:.*:.*")) 64 | exclude(dependency("junit:.*:.*")) 65 | } 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /processor/kotlin-modules/importer/src/main/kotlin/cc/unitmesh/importer/Main.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.importer 2 | 3 | import cc.unitmesh.importer.model.CodeSnippet 4 | import cc.unitmesh.importer.model.PackageUtil 5 | import cc.unitmesh.importer.model.RawDump 6 | import cc.unitmesh.importer.processor.KotlinCodeProcessor 7 | import cc.unitmesh.importer.processor.KotlinParserWrapper 8 | import com.github.ajalt.clikt.core.CliktCommand 9 | import com.github.ajalt.clikt.core.subcommands 10 | import kotlinx.serialization.decodeFromString 11 | import kotlinx.serialization.encodeToString 12 | import kotlinx.serialization.json.Json 13 | import ktlint.analysis.KtLintParseException 14 | import org.slf4j.Logger 15 | import org.slf4j.LoggerFactory 16 | import java.io.File 17 | 18 | fun main(args: Array) = Importer() 19 | .subcommands(Analysis(), Type(), Prompt()) 20 | .main(args) 21 | 22 | val logger: Logger = LoggerFactory.getLogger(Importer::class.java) 23 | 24 | class Importer : CliktCommand() { 25 | override fun run() = Unit 26 | } 27 | 28 | fun readDumpLists(): List { 29 | val jsonFiles = File("datasets" + File.separator + "rawdump").walkTopDown().filter { file -> 30 | file.name.endsWith(".json") 31 | }.toList() 32 | 33 | return jsonFiles.flatMap(File::readLines).map(Json.Default::decodeFromString) 34 | } 35 | 36 | val filteredFile = "datasets" + File.separator + "filtered.json" 37 | val splitFile = File("datasets" + File.separator + "split.json") 38 | 39 | class Analysis : CliktCommand(help = "Action Runner") { 40 | override fun run() { 41 | logger.info("Analysis Started") 42 | 43 | logger.info("Analysis Prepare filter data") 44 | 45 | val outputFile = File(filteredFile) 46 | if (!outputFile.exists()) { 47 | val codes: List = readDumpLists() 48 | 49 | val outputs = codes.filter { code -> 50 | val snippet: KotlinParserWrapper 51 | try { 52 | snippet = KotlinParserWrapper.createUnitContext(code.toCode()) 53 | } catch (e: KtLintParseException) { 54 | return@filter false 55 | } 56 | 57 | val processor = KotlinCodeProcessor(snippet.rootNode, code.content) 58 | processor.getMethodByAnnotationName("Query").isNotEmpty() 59 | } 60 | 61 | outputFile.writeText(Json.Default.encodeToString(outputs)) 62 | } else { 63 | logger.info("Skip analysis, because the output file already exists") 64 | } 65 | 66 | val results: MutableList = Snippets.fromFile(outputFile) 67 | 68 | splitFile.writeText(Json.Default.encodeToString(results)) 69 | 70 | logger.info("Analysis finished") 71 | } 72 | } 73 | 74 | private val typeFile = "datasets" + File.separator + "types.json" 75 | 76 | class Type : CliktCommand(help = "Generate Type Items") { 77 | override fun run() { 78 | val snippets: List = Json.decodeFromString(splitFile.readText()) 79 | 80 | val types = snippets.flatMap { snippet -> 81 | snippet.requiredType 82 | }.distinct() 83 | 84 | // write types for debug 85 | File("datasets" + File.separator + "required-types.json").writeText(Json.Default.encodeToString(types)) 86 | 87 | val rawdumpMap: Map = 88 | readDumpLists().associateBy { PackageUtil.pathToIdentifier(it.path) } 89 | 90 | val typeItems: List = types.map { type -> 91 | val rawDump = rawdumpMap[type] ?: return@map null 92 | rawDump.copy(path = type) 93 | }.filterNotNull() 94 | 95 | File(typeFile).writeText(Json.Default.encodeToString(typeItems)) 96 | } 97 | } 98 | 99 | class Prompt : CliktCommand(help = "Generate Prompt") { 100 | override fun run() { 101 | val snippets: List = Json.decodeFromString(splitFile.readText()) 102 | val typesDump: List = Json.decodeFromString(File(typeFile).readText()) 103 | 104 | val openAiPrompts = Snippets.toOpenAIPrompts(snippets) 105 | logger.info("Prompt sizes: ${openAiPrompts.size}") 106 | File("datasets" + File.separator + "prompts.json").writeText(Json.Default.encodeToString(openAiPrompts)) 107 | 108 | val prompts = Snippets.toLLMPrompts(typesDump, snippets) 109 | 110 | logger.info("Prompt sizes: ${prompts.size}") 111 | File("datasets" + File.separator + "llm-prompts.json").writeText(Json.Default.encodeToString(prompts)) 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /processor/kotlin-modules/importer/src/main/kotlin/cc/unitmesh/importer/SnippetPrompt.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.importer 2 | 3 | import kotlinx.serialization.Serializable 4 | 5 | @Serializable 6 | class SnippetPrompt( 7 | val id: Int, 8 | val identifierName: String, 9 | val requiredType: List, 10 | val content: String, 11 | val prompt: String, 12 | ) 13 | -------------------------------------------------------------------------------- /processor/kotlin-modules/importer/src/main/kotlin/cc/unitmesh/importer/model/CodeSnippet.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.importer.model 2 | 3 | import kotlinx.serialization.Serializable 4 | 5 | @Serializable 6 | data class CodeSnippet( 7 | val identifierName: String, 8 | val size: Int, 9 | val path: String, 10 | val content: String, 11 | val imports: List, 12 | // convert returnType and to String 13 | val requiredType: List = listOf(), 14 | ) 15 | -------------------------------------------------------------------------------- /processor/kotlin-modules/importer/src/main/kotlin/cc/unitmesh/importer/model/PackageUtil.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.importer.model 2 | 3 | object PackageUtil { 4 | /** input examples: 5 | * 1. storage/src/main/kotlin/com/waz/zclient/storage/db/userclients/UserClientDao.kt 6 | * 2. storage/src/main/java/com/waz/zclient/storage/db/userclients/UserClientDao.kt 7 | * output: 8 | * 1. com.waz.zclient.storage.db.userclients.UserClientDao 9 | * 2. com.waz.zclient.storage.db.userclients.UserClientDao 10 | **/ 11 | fun pathToIdentifier(path: String): String { 12 | val regex = Regex("(kotlin|java)/(.+)\\.kt") 13 | val match = regex.find(path) 14 | return match?.groupValues?.get(2)?.replace('/', '.') ?: "" 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /processor/kotlin-modules/importer/src/main/kotlin/cc/unitmesh/importer/model/RawDump.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.importer.model 2 | 3 | import kotlinx.serialization.Serializable 4 | import kotlinx.serialization.decodeFromString 5 | import kotlinx.serialization.json.Json 6 | import ktlint.analysis.Code 7 | import kotlin.io.path.Path 8 | 9 | @Serializable 10 | data class RawDump( 11 | val repo_name: String, 12 | var path: String, 13 | val copies: String, 14 | var size: Int, 15 | var content: String, 16 | val license: String, 17 | ) { 18 | fun identifierName(): String { 19 | // match the package name by regex `package com.cognifide.gradle.aem.instance.tasks` 20 | val packageName = Regex("package\\s+(\\S+)").find(content)?.groupValues?.get(1) 21 | // match class name by FilePath 22 | val className = path.substringAfterLast("/").substringBeforeLast(".") 23 | return "$packageName.$className" 24 | } 25 | 26 | fun toCode(): Code { 27 | return Code( 28 | content = content.trimIndent(), 29 | filePath = Path(path), 30 | fileName = path.substringAfterLast("/"), 31 | script = path.endsWith(".kts", ignoreCase = true), 32 | isStdIn = false, 33 | ) 34 | } 35 | 36 | companion object { 37 | fun fromString(text: String): RawDump { 38 | return Json.decodeFromString(text) 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /processor/kotlin-modules/importer/src/main/kotlin/cc/unitmesh/importer/processor/KotlinParserWrapper.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.importer.processor 2 | 3 | import ktlint.analysis.Code 4 | import ktlint.analysis.KOTLIN_PSI_FILE_FACTORY_PROVIDER 5 | import ktlint.analysis.KtLintParseException 6 | import ktlint.analysis.UTF8_BOM 7 | import ktlint.analysis.buildPositionInTextLocator 8 | import org.jetbrains.kotlin.com.intellij.lang.FileASTNode 9 | import org.jetbrains.kotlin.com.intellij.psi.PsiElement 10 | import org.jetbrains.kotlin.com.intellij.psi.PsiErrorElement 11 | import org.jetbrains.kotlin.idea.KotlinLanguage 12 | import org.jetbrains.kotlin.psi.KtFile 13 | 14 | class KotlinParserWrapper private constructor( 15 | val code: Code, 16 | val rootNode: FileASTNode, 17 | ) { 18 | companion object { 19 | fun createUnitContext(code: Code): KotlinParserWrapper { 20 | val psiFileFactory = KOTLIN_PSI_FILE_FACTORY_PROVIDER.getKotlinPsiFileFactory(true) 21 | 22 | val normalizedText = normalizeText(code.content) 23 | val positionInTextLocator = buildPositionInTextLocator(normalizedText) 24 | val psiFileName = 25 | code.fileName 26 | ?: if (code.script) { 27 | "File.kts" 28 | } else { 29 | "File.kt" 30 | } 31 | val psiFile = 32 | psiFileFactory.createFileFromText( 33 | psiFileName, 34 | KotlinLanguage.INSTANCE, 35 | normalizedText, 36 | ) as KtFile 37 | psiFile 38 | .findErrorElement() 39 | ?.let { errorElement -> 40 | val (line, col) = positionInTextLocator(errorElement.textOffset) 41 | throw KtLintParseException(line, col, errorElement.errorDescription) 42 | } 43 | 44 | val rootNode = psiFile.node 45 | 46 | return KotlinParserWrapper( 47 | code, 48 | rootNode, 49 | ) 50 | } 51 | 52 | private fun normalizeText(text: String): String { 53 | return text 54 | .replace("\r\n", "\n") 55 | .replace("\r", "\n") 56 | .replaceFirst(UTF8_BOM, "") 57 | } 58 | 59 | private fun PsiElement.findErrorElement(): PsiErrorElement? { 60 | if (this is PsiErrorElement) { 61 | return this 62 | } 63 | this.children.forEach { child -> 64 | val errorElement = child.findErrorElement() 65 | if (errorElement != null) { 66 | return errorElement 67 | } 68 | } 69 | return null 70 | } 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /processor/kotlin-modules/importer/src/main/resources/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/unit-mesh/devti/9ad45cf571a40867e2d5fa1289a4aff16a03d4f5/processor/kotlin-modules/importer/src/main/resources/.gitkeep -------------------------------------------------------------------------------- /processor/kotlin-modules/importer/src/main/resources/prompt.txt: -------------------------------------------------------------------------------- 1 | 用户故事来描述如下的代码。要求：1. 只返回一句话 2. 突出重点 2 | 3 | -------------------------------------------------------------------------------- /processor/kotlin-modules/importer/src/test/kotlin/cc/unitmesh/importer/SnippetsKtTest.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.importer 2 | 3 | import cc.unitmesh.importer.model.RawDump 4 | import io.kotest.matchers.shouldBe 5 | import kotlinx.serialization.decodeFromString 6 | import kotlinx.serialization.json.Json 7 | import org.junit.jupiter.api.Assertions.* 8 | import org.junit.jupiter.api.Test 9 | import java.io.File 10 | 11 | class SnippetsKtTest { 12 | @Test 13 | fun test() { 14 | /** 15 | * package com.meiji.daily.data.local.dao 16 | * 17 | * import android.arch.persistence.room.Dao 18 | * import android.arch.persistence.room.Insert 19 | * import android.arch.persistence.room.OnConflictStrategy 20 | * import android.arch.persistence.room.Query 21 | * 22 | * import com.meiji.daily.bean.ZhuanlanBean 23 | * 24 | * import io.reactivex.Maybe 25 | * 26 | * @Dao 27 | * interface ZhuanlanDao { 28 | * 29 | * @Insert(onConflict = OnConflictStrategy.IGNORE) 30 | * fun insert(zhuanlanBean: ZhuanlanBean): Long 31 | * 32 | * @Insert(onConflict = OnConflictStrategy.IGNORE) 33 | * fun insert(list: MutableList) 34 | * 35 | * @Query("SELECT * FROM zhuanlans WHERE type = :type") 36 | * fun query(type: Int): Maybe> 37 | * 38 | * @Query("DELETE FROM zhuanlans WHERE slug = :slug") 39 | * fun delete(slug: String) 40 | * } 41 | * 42 | */ 43 | val snippets = Snippets.fromFile(File("src/test/resources/snippets.json")) 44 | assertEquals(4, snippets.size) 45 | 46 | snippets[0].identifierName shouldBe "com.meiji.daily.data.local.dao.ZhuanlanDao" 47 | snippets[0].requiredType shouldBe listOf("com.meiji.daily.bean.ZhuanlanBean", "Long") 48 | } 49 | 50 | @Test 51 | fun should_generate_prompts() { 52 | val snippets = Snippets.fromFile(File("src/test/resources/snippets.json")) 53 | val typeStrings = File("src/test/resources/types.json").readText() 54 | val types: List = Json.decodeFromString(typeStrings) 55 | 56 | val prompts = Snippets.toLLMPrompts( 57 | types, 58 | snippets, 59 | ) 60 | 61 | assertEquals(4, prompts.size) 62 | prompts[0].prompt shouldBe """""" 63 | 64 | prompts[0].requiredType[0] shouldBe "data class ZhuanlanBean( var followersCount: Int, var creator: Creator, var topics: List, var activateState: String, var href: String, var acceptSubmission: Boolean, var firstTime: Boolean, var pendingName: String, var avatar: Avatar, var canManage: Boolean, var description: String, var nameCanEditUntil: Int, var reason: String, var banUntil: Int, @PrimaryKey var slug: String, var name: String, var url: String, var intro: String, var topicsCanEditUntil: Int, var activateAuthorRequested: String, var commentPermission: String, var following: Boolean, var postsCount: Int, var canPost: Boolean, var type: Int = 0 )" 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /processor/kotlin-modules/importer/src/test/kotlin/cc/unitmesh/importer/model/PackageUtilTest.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.importer.model 2 | 3 | import org.junit.jupiter.api.Assertions.* 4 | import org.junit.jupiter.api.Test 5 | 6 | class PackageUtilTest { 7 | @Test 8 | fun should_parse_package_from_path() { 9 | val path = "storage/src/main/kotlin/com/waz/zclient/storage/db/userclients/UserClientDao.kt" 10 | val identifier = PackageUtil.pathToIdentifier(path) 11 | assertEquals("com.waz.zclient.storage.db.userclients.UserClientDao", identifier) 12 | } 13 | 14 | @Test 15 | fun should_parser_java_path() { 16 | val path = "storage/src/main/java/com/waz/zclient/storage/db/userclients/UserClientDao.kt" 17 | val identifier = PackageUtil.pathToIdentifier(path) 18 | assertEquals("com.waz.zclient.storage.db.userclients.UserClientDao", identifier) 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /processor/kotlin-modules/importer/src/test/kotlin/cc/unitmesh/importer/model/RawDumpTest.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.importer.model 2 | 3 | import io.kotest.matchers.shouldBe 4 | import org.junit.jupiter.api.Test 5 | 6 | class RawDumpTest { 7 | @Test 8 | fun test() { 9 | val sourceCode = 10 | """package com.cognifide.gradle.aem.instance.tasks\n\nimport com.cognifide.gradle.aem.common.instance.action.AwaitUpAction\nimport com.cognifide.gradle.aem.common.instance.action.ReloadAction\nimport com.cognifide.gradle.aem.common.instance.names\nimport com.cognifide.gradle.aem.common.tasks.Instance\nimport org.gradle.api.tasks.TaskAction\n\nopen class InstanceReload : Instance() {\n\n private var reloadOptions: ReloadAction.() -\u003e Unit = {}\n\n fun reload(options: ReloadAction.() -\u003e Unit) {\n this.reloadOptions = options\n }\n\n private var awaitUpOptions: AwaitUpAction.() -\u003e Unit = {}\n\n fun awaitUp(options: AwaitUpAction.() -\u003e Unit) {\n this.awaitUpOptions = options\n }\n\n @TaskAction\n fun reload() {\n instanceManager.awaitReloaded(anyInstances, reloadOptions, awaitUpOptions)\n common.notifier.lifecycle(\"Instance(s) reloaded\", \"Which: ${"$"}{anyInstances.names}\")\n }\n\n init {\n description = \"Reloads all AEM instance(s).\"\n }\n\n companion object {\n const val NAME = \"instanceReload\"\n }\n}\n"""" 11 | val originText = 12 | """{"repo_name":"Cognifide/gradle-aem-plugin","path":"src/main/kotlin/com/cognifide/gradle/aem/instance/tasks/InstanceReload.kt","copies":"1","size":"1052","content":"$sourceCode,"license":"apache-2.0"}""" 13 | 14 | val dump = RawDump.fromString(originText) 15 | dump.repo_name shouldBe "Cognifide/gradle-aem-plugin" 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /processor/kotlin-modules/importer/src/test/resources/snippets.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "repo_name": "iMeiji/Daily", 4 | "path": "app/src/main/java/com/meiji/daily/data/local/dao/ZhuanlanDao.kt", 5 | "copies": "1", 6 | "size": "774", 7 | "content": "package com.meiji.daily.data.local.dao\n\n/**\n * Created by Meiji on 2017/11/28.\n */\n\nimport android.arch.persistence.room.Dao\nimport android.arch.persistence.room.Insert\nimport android.arch.persistence.room.OnConflictStrategy\nimport android.arch.persistence.room.Query\n\nimport com.meiji.daily.bean.ZhuanlanBean\n\nimport io.reactivex.Maybe\n\n@Dao\ninterface ZhuanlanDao {\n\n @Insert(onConflict = OnConflictStrategy.IGNORE)\n fun insert(zhuanlanBean: ZhuanlanBean): Long\n\n @Insert(onConflict = OnConflictStrategy.IGNORE)\n fun insert(list: MutableList)\n\n @Query(\"SELECT * FROM zhuanlans WHERE type = :type\")\n fun query(type: Int): Maybe>\n\n @Query(\"DELETE FROM zhuanlans WHERE slug = :slug\")\n fun delete(slug: String)\n}\n", 8 | "license": "apache-2.0" 9 | } 10 | ] 11 | -------------------------------------------------------------------------------- /processor/kotlin-modules/importer/src/test/resources/types.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "repo_name": "iMeiji/Daily", 4 | "path": "com.meiji.daily.bean.ZhuanlanBean", 5 | "copies": "1", 6 | "size": 3066, 7 | "content": "package com.meiji.daily.bean\n\nimport android.arch.persistence.room.Entity\nimport android.arch.persistence.room.PrimaryKey\n\n\n/**\n * Created by Meiji on 2018/2/9.\n */\n@Entity(tableName = \"zhuanlans\")\ndata class ZhuanlanBean(\n var followersCount: Int, //28569\n var creator: Creator,\n var topics: List,\n var activateState: String, //activated\n var href: String, ///api/columns/design\n var acceptSubmission: Boolean, //true\n var firstTime: Boolean, //false\n// var postTopics: List,\n var pendingName: String,\n var avatar: Avatar,\n var canManage: Boolean, //false\n var description: String, //关于用户体验、产品 、技术 和创业的干货集中地 | 最美应用：zuimeia.com\n// var pendingTopics: List,\n var nameCanEditUntil: Int, //0\n var reason: String,\n var banUntil: Int, //0\n @PrimaryKey var slug: String, //design\n var name: String, //可能性 | 产品与大设计\n var url: String, ///design\n var intro: String, //马力的互联网产品设计与用户体验专栏\n var topicsCanEditUntil: Int, //0\n var activateAuthorRequested: String, //none\n var commentPermission: String, //anyone\n var following: Boolean, //false\n var postsCount: Int, //200\n var canPost: Boolean, //false\n var type: Int = 0\n) {\n data class Topic(\n var url: String, //https://www.zhihu.com/topic/19550517\n var id: String, //19550517\n var name: String //互联网\n )\n\n data class PostTopic(\n var postsCount: Int, //1\n var id: Int, //2\n var name: String //知乎\n )\n\n data class Creator(\n var bio: String, //马力在招聘：zhuanlan.zhihu.com/p/31904197\n var isFollowing: Boolean, //false\n var hash: String, //c6e85ba5d5999df4c5ce2f2903b1ce0e\n var uid: Long, //26680571723776\n var isOrg: Boolean, //false\n var slug: String, //mali\n var isFollowed: Boolean, //false\n var description: String, //最美应用创始人 产品经理 设计师，欢迎关注微博：@Ma_Li | 他在好奇的注视着这个饶有趣味的世界 ｜ 感谢在评选知乎年度荣誉会员时为我投票的各位，一起认真！| 马力的互联网学习圈：http://mali.brixd.com | 文章索引： https://zhuanlan.zhihu.com/p/25493627\n var name: String, //马力\n var profileUrl: String, //https://www.zhihu.com/people/mali\n var avatar: Avatar,\n var isOrgWhiteList: Boolean, //false\n var isBanned: Boolean //false\n ) {\n data class Avatar(\n var id: String, //ba332a401\n var template: String //https://pic2.zhimg.com/{id}_{size}.jpg\n )\n }\n\n data class Avatar(\n var id: String, //v2-5410cdcdc7fb1556a27d0ddc4734e64b\n var template: String //https://pic2.zhimg.com/{id}_{size}.jpg\n )\n}", 8 | "license": "apache-2.0" 9 | } 10 | ] -------------------------------------------------------------------------------- /processor/modules/api-processor/.gitignore: -------------------------------------------------------------------------------- 1 | .env 2 | -------------------------------------------------------------------------------- /processor/modules/api-processor/README.md: -------------------------------------------------------------------------------- 1 | # API Processor 2 | 3 | Todos: 4 | 5 | - [x] API Parser 6 | - [x] Swagger 7 | - [ ] Split swagger to grouping 8 | - [x] Postman 9 | - [x] CLI for prompting 10 | - [x] API to Domains 11 | - [ ] Domain Knowledge Tree 12 | - [ ] Remove Version from API 13 | -------------------------------------------------------------------------------- /processor/modules/api-processor/build.gradle.kts: -------------------------------------------------------------------------------- 1 | @Suppress("DSL_SCOPE_VIOLATION") 2 | plugins { 3 | alias(libs.plugins.kotlin.jvm) 4 | alias(libs.plugins.shadow) 5 | alias(libs.plugins.serialization) 6 | application 7 | } 8 | 9 | repositories { 10 | mavenCentral() 11 | } 12 | 13 | dependencies { 14 | implementation(projects.common.core) 15 | implementation(projects.common.verifier) 16 | implementation(projects.common.prompter) 17 | 18 | implementation(libs.clikt) 19 | implementation(libs.serialization.json) 20 | 21 | // Logging 22 | implementation(libs.logging.slf4j.api) 23 | implementation(libs.logging.logback.classic) 24 | 25 | implementation(libs.bundles.openai) 26 | 27 | implementation(libs.bundles.markdown) 28 | implementation(libs.dotenv) 29 | implementation(libs.dataframe) 30 | 31 | // java parser 32 | implementation(libs.swagger.parser.v3) 33 | 34 | testImplementation(kotlin("test")) 35 | 36 | testImplementation(libs.bundles.test) 37 | } 38 | 39 | tasks.test { 40 | useJUnitPlatform() 41 | } 42 | 43 | kotlin { 44 | jvmToolchain(11) 45 | } 46 | 47 | application { 48 | mainClass.set("cc.unitmesh.processor.api.MainKt") 49 | } 50 | 51 | tasks { 52 | shadowJar { 53 | manifest { 54 | attributes(Pair("Main-Class", "cc.unitmesh.processor.api.MainKt")) 55 | } 56 | // minimize() 57 | dependencies { 58 | exclude(dependency("org.junit.jupiter:.*:.*")) 59 | exclude(dependency("org.junit:.*:.*")) 60 | exclude(dependency("junit:.*:.*")) 61 | } 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /processor/modules/api-processor/docs/domain/banking.csv: -------------------------------------------------------------------------------- 1 | name 2 | BBVA 3 | Barclays 4 | Bank of America 5 | JPMorgan Chase 6 | Capital One 7 | HSBC 8 | Lloyds Bank 9 | Royal Bank of Scotland 10 | Deutsche Bank 11 | Citibank 12 | Wells Fargo 13 | BNP Paribas 14 | Banco Santander 15 | ING Group 16 | Societe Generale 17 | Standard Chartered Bank 18 | UniCredit 19 | Nordea 20 | ABN AMRO 21 | Allied Irish Banks (AIB) 22 | Commerzbank 23 | Intesa Sanpaolo 24 | National Australia Bank (NAB) 25 | CaixaBank 26 | Danske Bank 27 | Swedbank 28 | KBC Bank 29 | Handelsbanken 30 | DNB Bank 31 | SEB Bank 32 | Bank of Ireland 33 | Banco Sabadell 34 | Banco Popular 35 | Westpac 36 | ANZ Bank 37 | Commonwealth Bank of Australia 38 | Bank of New Zealand 39 | Bank of Montreal 40 | Royal Bank of Canada 41 | Scotiabank 42 | TD Bank 43 | CIBC 44 | National Bank of Canada 45 | Banque Populaire 46 | Crédit Agricole 47 | Crédit Mutuel 48 | La Banque Postale 49 | Rabobank 50 | Credit Suisse 51 | UBS 52 | Raiffeisen Bank 53 | Zürcher Kantonalbank 54 | Erste Group Bank 55 | Bank Millennium 56 | OTP Bank 57 | PKO Bank Polski 58 | Bank Pekao 59 | Garanti BBVA 60 | Akbank 61 | ING Bank Turkey 62 | Isbank 63 | Yapı Kredi 64 | Emirates NBD 65 | Mashreq Bank 66 | Abu Dhabi Commercial Bank 67 | First Abu Dhabi Bank 68 | Bank Hapoalim 69 | Bank Leumi 70 | Israel Discount Bank 71 | Mizrahi Tefahot Bank 72 | Axis Bank 73 | HDFC Bank 74 | ICICI Bank 75 | Kotak Mahindra Bank 76 | State Bank of India 77 | Yes Bank 78 | Maybank 79 | CIMB Bank 80 | Bank Negara Indonesia (BNI) 81 | Bank Central Asia (BCA) 82 | Bank Rakyat Indonesia (BRI) 83 | Bank Mandiri 84 | Bangkok Bank 85 | Kasikornbank 86 | Siam Commercial Bank 87 | Krung Thai Bank 88 | Bank of the Philippine Islands (BPI) 89 | Metrobank 90 | Security Bank 91 | UnionBank of the Philippines 92 | Banco do Brasil 93 | Itaú Unibanco 94 | Bradesco 95 | Caixa Econômica Federal 96 | Santander Brasil 97 | Banorte 98 | BBVA Bancomer 99 | Scotiabank Mexico 100 | Banco de Bogotá 101 | Bancolombia -------------------------------------------------------------------------------- /processor/modules/api-processor/docs/domains.csv: -------------------------------------------------------------------------------- 1 | English,Chinese 2 | Account Information,账户信息 3 | Payment Initiation,支付发起 4 | Transaction Categorization and Analysis,交易分类与分析 5 | Confirmation of Funds,资金确认 6 | Direct Debit,直接扣款 7 | Standing Orders,长期委托付款 8 | Accounts,账户 9 | Payments,支付 10 | Loans,贷款 11 | Investments,投资 12 | Merchant,商家 13 | Transaction Risk Analysis,交易风险分析 14 | Bulk Payments,批量支付 15 | Authentication,验证 16 | Credit Scoring,信用评分 17 | Risk Management,风险管理 18 | Account Management,账户管理 19 | Payment,支付 20 | Loan Management,贷款管理 21 | Investment,投资 22 | Account Balance,账户余额 23 | Transaction,交易 24 | Credit Score,信用评分 25 | Foreign Exchange,外汇 26 | Investment Portfolio,投资组合 27 | Account,账户 28 | Credit,信用 29 | Exchange,兑换 30 | AccountBalance,账户余额 31 | AccountTransaction,账户交易 32 | Credit Risk Assessment,信用风险评估 33 | Investment Recommendation,投资建议 34 | Cards,卡 35 | Loan,贷款 36 | Insurance,保险 37 | Customer service,客户服务 38 | Payments service,支付服务 39 | Credit service,信用服务 40 | Investment service,投资服务 41 | Account Info,账户信息 42 | Transaction History,交易历史 43 | Branch Locator,分行查询 44 | Payment Validation,支付验证 45 | Transaction Categorization,交易分类 46 | Card,卡 47 | Customer,客户 48 | Customers,客户群 49 | Credit Card,信用卡 50 | Card Issuing,发卡 51 | Mobile and Online Banking,手机和在线银行 52 | Deposits,存款 53 | Card Management,卡管理 54 | Account information,账户信息 55 | Branch locator,分行查询 56 | FX,外汇 57 | Alert,提醒 58 | Fx,外汇 59 | Banking Products,银行产品 60 | Investment Products,投资产品 61 | Card Issuance,发卡 62 | Investment Management,投资管理 63 | Credit Card Management,信用卡管理 64 | Security Management,安全管理 65 | Payment Transfer,支付转账 66 | Forex,外汇 67 | Notifications,通知 68 | Analytics,分析 69 | Customer Support,客户支持 70 | Transaction Management,交易管理 71 | Funds Transfer,资金转账 72 | Wealth Management,财富管理 73 | Investment Banking,投资银行 74 | Asset Management,资产管理 75 | HSBC Corporate Banking,汇丰企业银行 76 | HSBC Retail Banking,汇丰零售银行 77 | HSBC Investment Banking,汇丰投资银行 78 | Account Balance Retrieval,账户余额查询 79 | Transaction Retrieval,交易查询 80 | Transfer Funds,转账 81 | Bill Payment,账单支付 82 | Account Balance Inquiry,账户余额查询 83 | Account Transactions Inquiry,账户交易查询 84 | Fund Transfer,资金转账 85 | Investment Portfolio Management,投资组合管理 86 | Account Information,账户信息 87 | P2P Money Transfer,网络互转款 88 | Mobile Banking,手机银行 89 | Online Banking,网上银行 90 | ATM Branch Locator,ATM和分支机构定位 91 | Currency Exchange,货币兑换 92 | Notification,通知 93 | Mortgages,抵押贷款 94 | Credit and Loan,信贷 95 | Payment and Transfers,付款和转账 96 | Loans and Mortgages,贷款和抵押贷款 97 | Credit Cards,信用卡 98 | Foreign Currency Exchange,外汇兑换 99 | Financial Planning,财务规划 100 | Customer Information,客户信息 101 | Transaction Status,交易状态 102 | Deposit Management,存款管理 103 | Payments and Transfers,付款和转账 104 | Payment Processing,付款处理 105 | Deposit,存款 106 | E-Banking,电子银行 107 | Payment Gateway,付款网关 108 | Loan Application,贷款申请 109 | Payment Status,付款状态 110 | FX Rates,外汇汇率 111 | Customer Authentication,客户认证 112 | Payment and Transfer,付款和转账 113 | ATM and Branch Locator,ATM和分支机构定位 114 | International Money Transfers,国际汇款 115 | ATM Locator,ATM定位 116 | Investment and Wealth Management,投资和财富管理 117 | Loan Origination,贷款起源 118 | Investment Advisory,投资咨询 -------------------------------------------------------------------------------- /processor/modules/api-processor/docs/prompter/domain-model.prompt.txt: -------------------------------------------------------------------------------- 1 | 请你使用 DDD 的思想，设计 Barclays 银行的 Standing Orders API 服务的 Open Banking API，并返回 API 列表。要求如下： 2 | 3 | 1. Standing Orders API 的用途是 Manages standing orders for recurring payments. 4 | 2. 尽可能围绕 Barclays 银行的业务进行设计，但不要求完全符合现实情况。 5 | 3. 不需要返回 Standing Orders API 之外的 API。 6 | 4. 请详细考虑 Request 和 Response 包含必要的字段。 7 | 5. 需要包含 Success 和 Error 的状态码和对应结果。 8 | 6. 使用 markdown 的表格返回，不需要解释。 9 | 7. 你的返回格式，示例如下： 10 | 11 | ### 12 | | API | Method | Description | Request | Response | Error Response | 13 | | --- | --- | --- | --- | --- | --- | 14 | | /standing-orders | GET | Get all standing orders | | 200 {"id": String } | 400: { "error": String} | 15 | ### 16 | 17 | -------------------------------------------------------------------------------- /processor/modules/api-processor/docs/prompter/domain-reprompter.txt: -------------------------------------------------------------------------------- 1 | 请根据下面的 API 信息，设计出更合理、丰富的领域模型。 2 | 3 | 1. 使用领域驱动设计的思想 4 | 2. 使用 PlantUML 表示核心的模型和行为，对关键的模型进行解释 5 | 3. 如果原有模型不完整，请丰富领域模型 6 | 4. 对于重复的参数，请使用值对象进行封装 7 | 5. 如果原始的 API 有问题，请按照你的理解进行修正 8 | 6. 只返回 PlantUML，不做解释 9 | 7. 返回示例格式如下： 10 | ### 11 | ```uml 12 | @startuml 13 | ' Consent（同意书）是指用户在使用某个金融服务时，需要先同意某些服务条款和隐私政策等内容，以便于金融机构可以合法地处理用户的数据和账户信息。 14 | class Consent {...} 15 | class ConsentService {...} 16 | ... 17 | Consent -> ConsentService 18 | @enduml 19 | ``` 20 | ### 21 | 22 | API 信息： 23 | 24 | ### 25 | {code} 26 | ### 27 | -------------------------------------------------------------------------------- /processor/modules/api-processor/docs/prompter/introduction-service.prompt.txt: -------------------------------------------------------------------------------- 1 | 你现在作为 {bankName} 银行的架构师，帮我设计的一个 {serviceName}。要求如下： 2 | 3 | 1. {serviceName} 的功能是 {serviceDescription}。 4 | 2. 根据该银行的技术栈特点，来设计 Open API。 5 | 3. 只使用表格返回 API 列表，不做解释。 6 | 4. 需要包含：API 名称、API 描述、API 请求 URL、API 请求方法、API 请求参数、API 返回结果、失败状态码列表。 7 | -------------------------------------------------------------------------------- /processor/modules/api-processor/docs/prompter/introduction.prompt.txt: -------------------------------------------------------------------------------- 1 | 你现在是 {name} 银行的 IT 负责人，你需要输出 {name} 相关的系统的介绍材料。要求如下： 2 | 3 | 1. 精炼银行的介绍，只介绍其核心业务。 4 | 2. 返回格式为 JSON，不返回的字段请填写 null。 5 | 3. 请以领域驱动设计中服务的粒度，尽可能详细地列出该银行的 API 服务。 6 | 4. 返回的格式如下： 7 | 8 | ### 9 | { 10 | name: "", 11 | fullName: "", 12 | description: "", 13 | openApiService: [ 14 | { name: "", description: "" }, 15 | ... 16 | ], 17 | otherService: ..., 18 | bankType: "" 19 | } 20 | ### 21 | -------------------------------------------------------------------------------- /processor/modules/api-processor/docs/prompter/uml-to-usecases.txt: -------------------------------------------------------------------------------- 1 | 仔细分析下面模型，从用户视角来编写需求用例。要求： 2 | 3 | 1. 返回的内容中不出现：token、调用、Service、ID、代码等技术词汇。 4 | 2. 从业务人员的视角分析，从用户的角度编写 5 | 3. 只返回 markdown 表格，不返回无关的内容 6 | 4. 扩展场景里需要考虑失败等情况 7 | 5. 你返回的格式如下所示： 8 | 9 | ### 10 | ```markdown 11 | | 用例名称 | 前置条件 | 后置条件 | 主成功场景 | 扩展场景 | 12 | | --- | --- | --- | --- | --- | 13 | ``` 14 | ### 15 | 16 | ### 17 | @startuml 18 | ' PaymentRequest (付款请求) 19 | class PaymentRequest { 20 | -token: string 21 | -amount: integer 22 | -description: string 23 | -createdDateTime: string 24 | -status: string 25 | +getPayments(pageNumber: integer, pageSize: integer, fromDateTime: string, toDateTime: string, includeRefunds: boolean): PaymentList 26 | +getPayment(paymentToken: string): Payment 27 | } 28 | ' Payment (付款) 29 | class Payment { 30 | -token: string 31 | -tikkieId: string 32 | -counterPartyName: string 33 | -counterPartyAccountNumber: string 34 | -amountInCents: integer 35 | -description: string 36 | -createdDateTime: string 37 | -refunds: array 38 | } 39 | ' PaymentList (付款列表) 40 | class PaymentList { 41 | -payments: array 42 | -totalElementCount: integer 43 | } 44 | ' PaymentService (付款服务) 45 | class PaymentService { 46 | -appToken: string 47 | +getPayments(paymentRequestToken: string, pageNumber: integer, pageSize: integer, fromDateTime: string, toDateTime: string, includeRefunds: boolean): PaymentList 48 | +getPayment(paymentRequestToken: string, paymentToken: string): Payment 49 | } 50 | PaymentRequest -> Payment : 包含多个 Payment 51 | PaymentRequest -> PaymentList : 组成 PaymentList 52 | PaymentList <|-- Payment : 由多个 Payment 组成 53 | PaymentService -> PaymentRequest : 根据 PaymentRequestToken 获取 PaymentList 和 Payment 54 | @enduml 55 | ### 56 | 57 | -------------------------------------------------------------------------------- /processor/modules/api-processor/docs/prompter/uml-to-userstory.txt: -------------------------------------------------------------------------------- 1 | 从纯业务视角，仔细分析下面遗留代码的业务需求，并使用用户故事描述。要求： 2 | 3 | 1. 不使用技术词汇，如 token 4 | 2. 用户故事之间应该是独立的，需求不应该重复，不应该有依赖关系 5 | 3. 你返回的格式如下所示： 6 | 7 | ### 8 | ```markdown 9 | 1. 作为一个用户，我可以xxx，以便于xxx 10 | ``` 11 | ### 12 | 13 | ### 14 | 15 | @startuml 16 | ' PaymentRequest (付款请求) 17 | class PaymentRequest { 18 | -token: string 19 | -amount: integer 20 | -description: string 21 | -createdDateTime: string 22 | -status: string 23 | +getPayments(pageNumber: integer, pageSize: integer, fromDateTime: string, toDateTime: string, includeRefunds: boolean): PaymentList 24 | +getPayment(paymentToken: string): Payment 25 | } 26 | ' Payment (付款) 27 | class Payment { 28 | -token: string 29 | -tikkieId: string 30 | -counterPartyName: string 31 | -counterPartyAccountNumber: string 32 | -amountInCents: integer 33 | -description: string 34 | -createdDateTime: string 35 | -refunds: array 36 | } 37 | ' PaymentList (付款列表) 38 | class PaymentList { 39 | -payments: array 40 | -totalElementCount: integer 41 | } 42 | ' PaymentService (付款服务) 43 | class PaymentService { 44 | -appToken: string 45 | +getPayments(paymentRequestToken: string, pageNumber: integer, pageSize: integer, fromDateTime: string, toDateTime: string, includeRefunds: boolean): PaymentList 46 | +getPayment(paymentRequestToken: string, paymentToken: string): Payment 47 | } 48 | PaymentRequest -> Payment : 包含多个 Payment 49 | PaymentRequest -> PaymentList : 组成 PaymentList 50 | PaymentList <|-- Payment : 由多个 Payment 组成 51 | PaymentService -> PaymentRequest : 根据 PaymentRequestToken 获取 PaymentList 和 Payment 52 | @enduml 53 | ### 54 | 55 | -------------------------------------------------------------------------------- /processor/modules/api-processor/src/main/kotlin/cc/unitmesh/processor/api/ApiProcessorDetector.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.processor.api 2 | 3 | import cc.unitmesh.processor.api.base.ApiProcessor 4 | import cc.unitmesh.processor.api.command.Command 5 | import cc.unitmesh.processor.api.parser.PostmanProcessor 6 | import cc.unitmesh.processor.api.swagger.Swagger3Processor 7 | import org.slf4j.Logger 8 | import org.slf4j.LoggerFactory 9 | import java.io.File 10 | 11 | object ApiProcessorDetector { 12 | private val logger: Logger = LoggerFactory.getLogger(Command::class.java) 13 | 14 | fun detectApiProcessor(file: File, withPostman: Boolean = false, postmanOnly: Boolean = false): ApiProcessor? { 15 | val content = file.readText() 16 | 17 | if (withPostman || postmanOnly) { 18 | if (file.extension == "json") { 19 | val isPostmanContent = content.contains("_postman_id") && content.contains("schema") 20 | if (isPostmanContent) { 21 | return PostmanProcessor(file) 22 | } 23 | } 24 | } 25 | 26 | if (postmanOnly) { 27 | return null 28 | } 29 | 30 | // if not json, yaml, yml file, skip 31 | if (!file.extension.matches(Regex("json|yaml|yml"))) { 32 | return null 33 | } 34 | 35 | return getSwaggerProcessor(file)?.let { 36 | return it 37 | } 38 | } 39 | 40 | private fun getSwaggerProcessor(it: File): ApiProcessor? { 41 | return try { 42 | val openAPI = Swagger3Processor.fromFile(it) ?: return null 43 | Swagger3Processor(openAPI) 44 | } catch (e: Exception) { 45 | logger.info("Failed to parse ${it.absolutePath}", e) 46 | null 47 | } 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /processor/modules/api-processor/src/main/kotlin/cc/unitmesh/processor/api/base/ApiDetailRender.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.processor.api.base 2 | 3 | import cc.unitmesh.core.model.ApiCollection 4 | 5 | interface ApiDetailRender { 6 | fun render(apiCollections: List): String { 7 | val apiDetailsByTag = apiCollections.map { renderCollection(it) }.filter { 8 | it.isNotBlank() 9 | } 10 | return apiDetailsByTag.joinToString("\n\n") { it } 11 | } 12 | 13 | fun renderCollection(collection: ApiCollection): String 14 | } 15 | -------------------------------------------------------------------------------- /processor/modules/api-processor/src/main/kotlin/cc/unitmesh/processor/api/base/ApiProcessor.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.processor.api.base 2 | 3 | import cc.unitmesh.core.model.ApiCollection 4 | 5 | interface ApiProcessor { 6 | fun convertApi(): List 7 | } 8 | -------------------------------------------------------------------------------- /processor/modules/api-processor/src/main/kotlin/cc/unitmesh/processor/api/command/CommandExt.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.processor.api.command 2 | 3 | import cc.unitmesh.core.prompter.OpenAiPrompter 4 | import io.github.cdimascio.dotenv.Dotenv 5 | import kotlinx.serialization.Serializable 6 | import org.jetbrains.kotlinx.dataframe.DataFrame 7 | import org.jetbrains.kotlinx.dataframe.api.rows 8 | import org.jetbrains.kotlinx.dataframe.io.readCSV 9 | import org.slf4j.Logger 10 | import org.slf4j.LoggerFactory 11 | import java.io.File 12 | import kotlin.random.Random 13 | 14 | class Command 15 | 16 | val logger: Logger = LoggerFactory.getLogger(Command::class.java) 17 | 18 | val GROUP_API_INSTRUCTION = "帮我设计一组 API：" 19 | val ONE_API_INSTRUCTION = "帮我设计一个银行的 API:" 20 | fun randomInstruction(serviceName: String): String { 21 | val instructions = listOf( 22 | "展示${serviceName}的 API 应该如何设计？", 23 | "如何实现${serviceName}功能的 API？", 24 | "设计一个可以查询${serviceName}的 API。", 25 | "如何设计一个支持${serviceName}的 API？", 26 | "你会如何设计一个可以处理${serviceName}的 API？", 27 | "设计一个可以查询${serviceName}的 API。", 28 | "如何设计一个可以处理${serviceName}的 API？", 29 | "如何设计一个可以处理${serviceName}的 API？", 30 | "如何实现${serviceName}的 API？", 31 | "设计一个可以${serviceName}的 API。", 32 | ) 33 | 34 | val random = Random.nextInt(0, instructions.size) 35 | return instructions[random] 36 | } 37 | 38 | val MIN_OUTPUT_LENGTH = 128 39 | fun String.simplifyApi(): String = this 40 | .replace(" Services", "") 41 | .replace(" Service", "") 42 | .replace(" API", "") 43 | .replace("API", "") 44 | .replace("服务", "") 45 | 46 | fun getDomainTranslate(domainFile: File): MutableMap { 47 | val domainTranslation = mutableMapOf() 48 | if (domainFile.exists()) { 49 | val englishToChinese = DataFrame.readCSV(domainFile.absolutePath) 50 | englishToChinese.rows().forEach { row -> 51 | val values = row.values() as List 52 | val english = values[0] 53 | domainTranslation[english] = values[1] + "($english)" + "服务" 54 | } 55 | } 56 | 57 | return domainTranslation 58 | } 59 | 60 | fun outputMarkdown( 61 | markdownApiOutputDir: File, 62 | index: Int, 63 | bank: Bank, 64 | service: OpenApiService, 65 | ) = File(markdownApiOutputDir, "$index-${bank.name}-${service.name}.md") 66 | 67 | @Serializable 68 | class Bank( 69 | val name: String, 70 | val fullName: String, 71 | val description: String, 72 | val openApiService: List, 73 | val otherService: List? = null, 74 | val bankType: String, 75 | ) 76 | 77 | @Serializable 78 | class OpenApiService( 79 | val name: String, 80 | val description: String, 81 | ) 82 | 83 | @Serializable 84 | class OtherService( 85 | val name: String, 86 | val description: String, 87 | ) 88 | 89 | fun createOpenAiPrompter(): OpenAiPrompter { 90 | logger.debug("loading dotenv") 91 | val dotenv = Dotenv.load() 92 | val proxy = dotenv.get("OPEN_AI_PROXY") 93 | val key = dotenv.get("OPEN_AI_KEY") 94 | return OpenAiPrompter(key, proxy) 95 | } 96 | 97 | object Workspace { 98 | fun usecases(dir: String) = File(dir, "usecases") 99 | fun puml(dir: String) = File(dir, "domain") 100 | } 101 | -------------------------------------------------------------------------------- /processor/modules/api-processor/src/main/kotlin/cc/unitmesh/processor/api/command/Usecase.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.processor.api.command 2 | 3 | import cc.unitmesh.core.Instruction 4 | import cc.unitmesh.verifier.markdown.MarkdownVerifier 5 | import com.github.ajalt.clikt.core.CliktCommand 6 | import com.github.ajalt.clikt.parameters.arguments.argument 7 | import com.github.ajalt.clikt.parameters.arguments.default 8 | import com.github.ajalt.clikt.parameters.types.file 9 | import kotlinx.serialization.encodeToString 10 | import kotlinx.serialization.json.Json 11 | import java.io.File 12 | 13 | class Usecase : CliktCommand() { 14 | private val prompt by argument().file().default(File("usecase-prompt.txt")) 15 | private val inputDir by argument().file().default(File("output")) 16 | private val outputDir by argument().file().default(File("output")) 17 | 18 | override fun run() { 19 | logger.info("Usecases Started") 20 | val prompter = createOpenAiPrompter() 21 | 22 | val pumlDir = Workspace.puml(inputDir.absolutePath) 23 | pumlDir.mkdirs() 24 | 25 | val usecaseDir = Workspace.usecases(inputDir.absolutePath) 26 | usecaseDir.mkdirs() 27 | 28 | val promptText = prompt.readText() 29 | 30 | val instructions: MutableList = mutableListOf() 31 | 32 | // load all uml files under output/domain/*.puml 33 | pumlDir.walk().forEachIndexed { index, file -> 34 | if (file.isFile && file.name.endsWith(".puml")) { 35 | val content = file.readText() 36 | 37 | val newPrompt = promptText.replace("{code}", content) 38 | 39 | val outputFile = File(usecaseDir, file.name.replace("puml", "md")) 40 | if (outputFile.exists()) { 41 | instructions += Instruction( 42 | instruction = "分析下面遗留代码的业务需求，并使用用户视角来编写需求用例。", 43 | input = content, 44 | output = outputFile.readText(), 45 | ) 46 | logger.info("Skipping ${file.absolutePath}") 47 | return@forEachIndexed 48 | } 49 | 50 | var output: String = try { 51 | prompter.prompt(newPrompt) 52 | } catch (e: Exception) { 53 | logger.info("Failed to prompt ${file.absolutePath}", e) 54 | return@forEachIndexed 55 | } 56 | output = clearOutput(output) 57 | 58 | instructions += Instruction( 59 | instruction = "分析下面遗留代码的业务需求，并使用用户视角来编写需求用例。", 60 | input = content, 61 | output = output, 62 | ) 63 | 64 | logger.debug("output to text: ${outputFile.absolutePath}") 65 | outputFile.writeText(output) 66 | } 67 | } 68 | 69 | val markdownVerifier = MarkdownVerifier() 70 | val headers = listOf("用例名称", "前置条件", "后置条件", "主成功场景", "扩展场景") 71 | 72 | // walk dir under usecaseDir filer all *.md 73 | usecaseDir.walk().forEachIndexed { index, file -> 74 | if (file.isFile && file.name.endsWith(".md")) { 75 | val content = file.readText() 76 | 77 | val isCorrect = markdownVerifier.tableVerifier(content, headers = headers) 78 | if (!isCorrect) { 79 | logger.warn("Failed to verify ${file.absolutePath}") 80 | logger.info("content: $content") 81 | file.delete() 82 | } 83 | } 84 | } 85 | 86 | val jsonl = File(this.outputDir, "usecase.jsonl") 87 | jsonl.writeText(instructions.joinToString("\n") { Json.encodeToString(it) }) 88 | } 89 | 90 | private fun clearOutput(output: String): String { 91 | // if content starts with ### or ends with ###, remove it 92 | var newOutput = output 93 | 94 | if (newOutput.startsWith("###")) { 95 | newOutput = newOutput.substring(3) 96 | } 97 | 98 | if (newOutput.endsWith("###")) { 99 | newOutput = newOutput.substring(0, newOutput.length - 3) 100 | } 101 | 102 | // remove continues empty line 103 | newOutput = newOutput.replace("\n\n\n", "\n\n") 104 | 105 | return newOutput 106 | } 107 | } 108 | -------------------------------------------------------------------------------- /processor/modules/api-processor/src/main/kotlin/cc/unitmesh/processor/api/parser/PostmanProcessor.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.processor.api.parser 2 | 3 | import cc.unitmesh.core.model.ApiCollection 4 | import cc.unitmesh.processor.api.base.ApiProcessor 5 | import cc.unitmesh.processor.api.postman.PostmanReader 6 | import java.io.File 7 | 8 | class PostmanProcessor(val file: File) : ApiProcessor { 9 | override fun convertApi(): List { 10 | val postmanReader = PostmanReader() 11 | val collection = postmanReader.readCollectionFile(file.absolutePath) 12 | val postmanParser = PostmanParser() 13 | 14 | return postmanParser.parse(collection) ?: emptyList() 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /processor/modules/api-processor/src/main/kotlin/cc/unitmesh/processor/api/postman/PostmanAuth.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.processor.api.postman 2 | 3 | import kotlinx.serialization.Serializable 4 | 5 | @Serializable 6 | class PostmanAuth { 7 | // auth type: "oauth2", 8 | var type: String? = null 9 | var bearer: List? = null 10 | var oauth2: List? = null 11 | 12 | fun format(): String { 13 | if (type == "bearer") { 14 | val string = bearer?.joinToString(",", transform = cc.unitmesh.processor.api.postman.PostmanVariable::format) 15 | return "Bearer $string" 16 | } 17 | 18 | if (type == "oauth2") { 19 | val string = oauth2?.joinToString(",", transform = cc.unitmesh.processor.api.postman.PostmanVariable::format) 20 | return "OAuth2 $string" 21 | } 22 | 23 | return "type $type" 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /processor/modules/api-processor/src/main/kotlin/cc/unitmesh/processor/api/postman/PostmanBody.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.processor.api.postman 2 | 3 | import kotlinx.serialization.Serializable 4 | 5 | @Serializable 6 | class PostmanBody { 7 | var mode: String? = null 8 | var file: PostmanFile? = null 9 | var raw: String? = null 10 | var urlencoded: List? = null 11 | var formdata: List? = null 12 | var options: PostmanOptions? = null 13 | } 14 | 15 | @Serializable 16 | class PostmanOptions { 17 | var raw: PostmanRaw? = null 18 | } 19 | 20 | @Serializable 21 | class PostmanRaw { 22 | var language: String? = null 23 | } 24 | 25 | @Serializable 26 | class PostmanFormData { 27 | var key: String? = null 28 | var value: String? = null 29 | var type: String? = null 30 | var disabled: Boolean? = null 31 | var description: String? = null 32 | } 33 | -------------------------------------------------------------------------------- /processor/modules/api-processor/src/main/kotlin/cc/unitmesh/processor/api/postman/PostmanCollection.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.processor.api.postman 2 | 3 | import kotlinx.serialization.Serializable 4 | 5 | @Serializable 6 | class PostmanCollection { 7 | var info: PostmanInfo? = null 8 | var item: List? = null 9 | var folderLookup: MutableMap = HashMap() 10 | var variable: List? = null 11 | var auth: cc.unitmesh.processor.api.postman.PostmanAuth? = null 12 | var event: List? = null 13 | 14 | fun init() { 15 | for (f in item!!) { 16 | folderLookup[f.name] = f 17 | } 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /processor/modules/api-processor/src/main/kotlin/cc/unitmesh/processor/api/postman/PostmanEnvValue.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.processor.api.postman 2 | 3 | class PostmanEnvValue { 4 | var key: String? = null 5 | var value: String? = null 6 | var type: String? = null 7 | var name: String? = null 8 | override fun toString(): String { 9 | return "[$key:$value]" 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /processor/modules/api-processor/src/main/kotlin/cc/unitmesh/processor/api/postman/PostmanEnvironment.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.processor.api.postman 2 | 3 | class PostmanEnvironment { 4 | var id: String? = null 5 | var name: String? = null 6 | var values: List? = null 7 | var timestamp: Long? = null 8 | var synced: Boolean? = null 9 | var lookup: MutableMap = HashMap() 10 | fun init() { 11 | for (`val` in values!!) { 12 | lookup[`val`.key] = `val` 13 | } 14 | } 15 | 16 | fun setEnvironmentVariable(key: String, value: String?) { 17 | val existingVar = lookup[key] 18 | if (existingVar != null) { 19 | // Update existing value if any 20 | existingVar.value = value 21 | } else { 22 | val newVar = PostmanEnvValue() 23 | newVar.key = key 24 | newVar.name = "RUNTIME-$key" 25 | newVar.type = "text" 26 | newVar.value = value 27 | lookup[key] = newVar 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /processor/modules/api-processor/src/main/kotlin/cc/unitmesh/processor/api/postman/PostmanEvent.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.processor.api.postman 2 | 3 | import kotlinx.serialization.Serializable 4 | 5 | @Serializable 6 | class PostmanEvent { 7 | var listen: String? = null 8 | var script: PostmanScript? = null 9 | } 10 | -------------------------------------------------------------------------------- /processor/modules/api-processor/src/main/kotlin/cc/unitmesh/processor/api/postman/PostmanFile.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.processor.api.postman 2 | 3 | import kotlinx.serialization.Serializable 4 | 5 | @Serializable 6 | class PostmanFile 7 | -------------------------------------------------------------------------------- /processor/modules/api-processor/src/main/kotlin/cc/unitmesh/processor/api/postman/PostmanFolder.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.processor.api.postman 2 | 3 | import kotlinx.serialization.Serializable 4 | 5 | @Serializable 6 | class PostmanFolder : PostmanItem() 7 | -------------------------------------------------------------------------------- /processor/modules/api-processor/src/main/kotlin/cc/unitmesh/processor/api/postman/PostmanHeader.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.processor.api.postman 2 | 3 | import kotlinx.serialization.Serializable 4 | 5 | @Serializable 6 | class PostmanHeader { 7 | var key: String? = null 8 | var value: String? = null 9 | var type: String? = null 10 | var description: String? = null 11 | var disabled: Boolean? = null 12 | } 13 | -------------------------------------------------------------------------------- /processor/modules/api-processor/src/main/kotlin/cc/unitmesh/processor/api/postman/PostmanInfo.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.processor.api.postman 2 | 3 | import kotlinx.serialization.Serializable 4 | 5 | @Serializable 6 | class PostmanInfo { 7 | var _postman_id: String? = null 8 | var name: String? = null 9 | var description: String? = null 10 | var schema: String? = null 11 | var _exporter_id: String? = null 12 | var _collection_link: String? = null 13 | } 14 | -------------------------------------------------------------------------------- /processor/modules/api-processor/src/main/kotlin/cc/unitmesh/processor/api/postman/PostmanItem.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.processor.api.postman 2 | 3 | import kotlinx.serialization.Serializable 4 | 5 | @Serializable 6 | open class PostmanItem { 7 | open val item: List? = null 8 | open var name: String? = null 9 | open var description: String? = null 10 | open var event: List? = null 11 | open var request: PostmanRequest? = null 12 | open var response: List? = null 13 | open var protocolProfileBehavior: PostmanProtocolProfileBehavior? = null 14 | } 15 | 16 | @Serializable 17 | class PostmanProtocolProfileBehavior { 18 | var disableBodyPruning: Boolean? = null 19 | } 20 | -------------------------------------------------------------------------------- /processor/modules/api-processor/src/main/kotlin/cc/unitmesh/processor/api/postman/PostmanQuery.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.processor.api.postman 2 | 3 | import kotlinx.serialization.Serializable 4 | 5 | @Serializable 6 | class PostmanQuery { 7 | var key: String? = null 8 | var value: String? = null 9 | var description: String? = null 10 | var disabled: Boolean? = null 11 | } 12 | -------------------------------------------------------------------------------- /processor/modules/api-processor/src/main/kotlin/cc/unitmesh/processor/api/postman/PostmanReader.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.processor.api.postman 2 | 3 | import com.fasterxml.jackson.core.JsonParseException 4 | import com.fasterxml.jackson.databind.DeserializationFeature 5 | import com.fasterxml.jackson.databind.JsonMappingException 6 | import com.fasterxml.jackson.databind.ObjectMapper 7 | import java.io.File 8 | import java.io.FileInputStream 9 | import java.io.IOException 10 | import java.io.InputStream 11 | 12 | class PostmanReader { 13 | var om: ObjectMapper = ObjectMapper() 14 | 15 | init { 16 | om.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) 17 | } 18 | 19 | @Throws(JsonParseException::class, JsonMappingException::class, IOException::class) 20 | fun readCollectionFileClasspath(fileOnClasspath: String): PostmanCollection { 21 | val fileName = fileOnClasspath.substring(fileOnClasspath.indexOf(":") + 1) 22 | val stream = Thread.currentThread().contextClassLoader.getResourceAsStream(fileName) 23 | val collection = om.readValue(stream, PostmanCollection::class.java) 24 | stream.close() 25 | return collection 26 | } 27 | 28 | @Throws(JsonParseException::class, JsonMappingException::class, IOException::class) 29 | fun readEnvironmentFileClasspath(fileOnClasspath: String): PostmanEnvironment { 30 | val fileName = fileOnClasspath.substring(fileOnClasspath.indexOf(":") + 1) 31 | val stream = Thread.currentThread().contextClassLoader.getResourceAsStream(fileName) 32 | val env = om.readValue(stream, PostmanEnvironment::class.java) 33 | stream.close() 34 | return env 35 | } 36 | 37 | @Throws(IOException::class) 38 | fun readCollectionFile(filePath: String): PostmanCollection { 39 | if (filePath.startsWith("classpath:")) { 40 | return readCollectionFileClasspath(filePath) 41 | } 42 | val stream: InputStream = FileInputStream(File(filePath)) 43 | val collection = om.readValue(stream, PostmanCollection::class.java) 44 | stream.close() 45 | return collection 46 | } 47 | 48 | @Throws(IOException::class) 49 | fun readEnvironmentFile(filePath: String?): PostmanEnvironment { 50 | if (filePath == null) { 51 | return PostmanEnvironment() 52 | } 53 | if (filePath.startsWith("classpath:")) { 54 | return readEnvironmentFileClasspath(filePath) 55 | } 56 | val stream: InputStream = FileInputStream(File(filePath)) 57 | val env = om.readValue(stream, PostmanEnvironment::class.java) 58 | stream.close() 59 | return env 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /processor/modules/api-processor/src/main/kotlin/cc/unitmesh/processor/api/postman/PostmanRequest.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.processor.api.postman 2 | 3 | import kotlinx.serialization.Serializable 4 | import java.net.URLEncoder 5 | import java.util.* 6 | 7 | @Serializable 8 | class PostmanRequest { 9 | var method: String? = null 10 | var header: List? = null 11 | var body: PostmanBody? = null 12 | var url: PostmanUrl? = null 13 | var description: String? = null 14 | var auth: cc.unitmesh.processor.api.postman.PostmanAuth? = null 15 | 16 | fun getData(`var`: PostmanVariables): String? { 17 | return if (body == null || body!!.mode == null) { 18 | "" 19 | } else { 20 | when (body!!.mode) { 21 | "raw" -> `var`.replace(body!!.raw) 22 | "urlencoded" -> urlFormEncodeData(`var`, body!!.urlencoded) 23 | else -> "" 24 | } 25 | } 26 | } 27 | 28 | fun urlFormEncodeData(`var`: PostmanVariables, formData: List?): String { 29 | var result = "" 30 | val i = 0 31 | for (encoded in formData!!) { 32 | result += encoded!!.key + "=" + URLEncoder.encode(`var`.replace(encoded.value)) 33 | if (i < formData.size - 1) { 34 | result += "&" 35 | } 36 | } 37 | return result 38 | } 39 | 40 | fun getUrl(`var`: PostmanVariables): String? { 41 | if (url == null) { 42 | return null 43 | } 44 | 45 | return `var`.replace(url!!.raw) 46 | } 47 | 48 | fun getHeaders(`var`: PostmanVariables): Map { 49 | val result: MutableMap = HashMap() 50 | if (header == null || header!!.isEmpty()) { 51 | return result 52 | } 53 | for (head in header!!) { 54 | if (head.key!!.uppercase(Locale.getDefault()) == PoyntHttpHeaders.REQUEST_ID_HEADER) { 55 | result[head.key!!.uppercase(Locale.getDefault())] = `var`.replace(head.value) 56 | } else { 57 | result[head.key] = `var`.replace(head.value) 58 | } 59 | } 60 | return result 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /processor/modules/api-processor/src/main/kotlin/cc/unitmesh/processor/api/postman/PostmanResponse.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.processor.api.postman 2 | 3 | import kotlinx.serialization.Serializable 4 | 5 | @Serializable 6 | class PostmanResponse( 7 | val _postman_previewlanguage: String? = null, 8 | val name: String? = null, 9 | val originalRequest: PostmanRequest? = null, 10 | val status: String? = null, 11 | val code: Int? = null, 12 | val header: List? = null, 13 | val cookie: List? = null, 14 | val body: String? = null, 15 | val urlencoded: List? = null, 16 | // val data: List, 17 | // val links: Links, 18 | // val meta: Meta 19 | ) 20 | -------------------------------------------------------------------------------- /processor/modules/api-processor/src/main/kotlin/cc/unitmesh/processor/api/postman/PostmanScript.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.processor.api.postman 2 | 3 | import kotlinx.serialization.Serializable 4 | 5 | @Serializable 6 | class PostmanScript { 7 | var id: String? = null 8 | var type: String? = null 9 | var exec: List? = null 10 | } 11 | -------------------------------------------------------------------------------- /processor/modules/api-processor/src/main/kotlin/cc/unitmesh/processor/api/postman/PostmanUrl.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.processor.api.postman 2 | 3 | import kotlinx.serialization.Serializable 4 | 5 | @Serializable 6 | class PostmanUrl { 7 | var raw: String? = null 8 | var host: List? = null 9 | var port: String? = null 10 | var path: List? = null 11 | var query: List? = null 12 | var variable: List? = null 13 | var protocol: String? = null 14 | } 15 | -------------------------------------------------------------------------------- /processor/modules/api-processor/src/main/kotlin/cc/unitmesh/processor/api/postman/PostmanUrlEncoded.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.processor.api.postman 2 | 3 | import kotlinx.serialization.Serializable 4 | 5 | @Serializable 6 | class PostmanUrlEncoded { 7 | var key: String? = null 8 | var value: String? = null 9 | var type: String? = null 10 | var description: String? = null 11 | var disabled: Boolean? = null 12 | } 13 | -------------------------------------------------------------------------------- /processor/modules/api-processor/src/main/kotlin/cc/unitmesh/processor/api/postman/PostmanVariable.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.processor.api.postman 2 | 3 | import kotlinx.serialization.Serializable 4 | 5 | @Serializable 6 | class PostmanVariable { 7 | var key: String? = null 8 | var value: String? = null 9 | var type: String? = null 10 | var description: String? = null 11 | fun format(): String { 12 | return "$key=$value" 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /processor/modules/api-processor/src/main/kotlin/cc/unitmesh/processor/api/postman/PostmanVariables.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.processor.api.postman 2 | 3 | import java.util.* 4 | import java.util.regex.Pattern 5 | 6 | class PostmanVariables(val env: PostmanEnvironment) { 7 | private val r = Random(1000) 8 | 9 | private fun getConstantVal(exp: String): String { 10 | return if (exp.equals(GUID, ignoreCase = true)) { 11 | UUID.randomUUID().toString() 12 | } else if (exp.equals(TIMESTAMP, ignoreCase = true)) { 13 | java.lang.Long.toString(System.currentTimeMillis() / 1000) 14 | } else if (exp.equals(RANDOMINT, ignoreCase = true)) { 15 | Integer.toString(r.nextInt(1000)) 16 | } else { 17 | throw IllegalArgumentException("Invalid POSTMAN dynamic variable $exp") 18 | } 19 | } 20 | 21 | private fun getVal(name: String): String? { 22 | if (name.startsWith("{{$")) { 23 | try { 24 | return getConstantVal(name) 25 | } catch (e: IllegalArgumentException) { 26 | // ignore 27 | } 28 | } 29 | val key = name.substring(2, name.length - 2).trim { it <= ' ' } 30 | val `val` = env.lookup[key] 31 | ?: // throw new IllegalArgumentException("Invalid dynamic variable: " + name); 32 | return "UNDEFINED" 33 | return `val`.value 34 | } 35 | 36 | /** 37 | * Replace all {{dynamic variable}} in orig string with values found in the 38 | * environment. If variable is not found, replace it with constant string 39 | * "UNDEFINED". 40 | * 41 | * @param orig - the original value 42 | * @return The new string with all dynamic variables replaced 43 | */ 44 | fun replace(orig: String?): String? { 45 | if (orig.isNullOrEmpty()) return orig 46 | // Get all the dynamic variables 47 | val allMatches: MutableList = ArrayList() 48 | val m = Pattern.compile(POSTMAN_EXP).matcher(orig) 49 | while (m.find()) { 50 | allMatches.add(m.group().trim { it <= ' ' }) 51 | } 52 | 53 | // TODO: this is not the most efficient way to do it 54 | // but it is the simplest in term of code and this is not 55 | // production code anyway. 56 | var result: String = orig 57 | for (`var` in allMatches) { 58 | val varVal = getVal(`var`) 59 | // System.out.println(var + " ==> " + varVal); 60 | // result = result.replace(`var` as CharSequence, varVal as CharSequence?) 61 | result = result.replace(`var`, varVal!!) 62 | } 63 | return result 64 | } 65 | 66 | companion object { 67 | const val POSTMAN_EXP = "\\{\\{[^\\}]+\\}\\}" 68 | const val GUID = "{{\$guid}}" 69 | const val TIMESTAMP = "{{\$timestamp}}" 70 | const val RANDOMINT = "{{\$randomInt}}" 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /processor/modules/api-processor/src/main/kotlin/cc/unitmesh/processor/api/postman/PoyntHttpHeaders.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.processor.api.postman 2 | 3 | object PoyntHttpHeaders { 4 | const val REQUEST_ID_HEADER = "POYNT-REQUEST-ID" 5 | } 6 | -------------------------------------------------------------------------------- /processor/modules/api-processor/src/main/kotlin/cc/unitmesh/processor/api/render/MarkdownTableRender.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.processor.api.render 2 | 3 | import cc.unitmesh.core.model.ApiCollection 4 | import cc.unitmesh.core.model.ApiTagOutput 5 | import cc.unitmesh.processor.api.base.ApiDetailRender 6 | 7 | class MarkdownTableRender : ApiDetailRender { 8 | override fun renderCollection(collection: ApiCollection): String { 9 | val result: MutableList = mutableListOf() 10 | 11 | // if a name starts with number and ".", return "" 12 | // for example, "3. Transfer state simulation" will return "" 13 | val name = collection.name 14 | if (name.matches(Regex("^\\d+\\..*"))) { 15 | return "" 16 | } 17 | 18 | if (collection.items.isEmpty()) { 19 | return "" 20 | } 21 | 22 | if (collection.name.isNotEmpty()) { 23 | result += listOf("## ${collection.name}\n") 24 | } 25 | 26 | if (collection.description.isNotEmpty()) { 27 | result += listOf("> ${collection.description}") 28 | } 29 | 30 | result += listOf("| API | Method | Description | Request | Response | Error Response |") 31 | result += listOf("| --- | --- | --- | --- | --- | --- |") 32 | collection.items.forEach { detail -> 33 | val api = detail.path 34 | val method = detail.method 35 | val description = detail.description 36 | val request = detail.request.toString() 37 | val response = detail.response.toString() 38 | val errorResponse = "400: {\"error\": String}" 39 | result += listOf("| $api | $method | $description | $request | $response | $errorResponse |") 40 | } 41 | 42 | return ApiTagOutput(result.joinToString("\n")).toString() 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /processor/modules/api-processor/src/main/kotlin/cc/unitmesh/processor/api/render/SimpleApiRender.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.processor.api.render 2 | 3 | import cc.unitmesh.core.model.ApiCollection 4 | import cc.unitmesh.core.model.ApiItem 5 | import cc.unitmesh.processor.api.base.ApiDetailRender 6 | 7 | class SimpleApiRender : ApiDetailRender { 8 | override fun renderCollection(collection: ApiCollection): String { 9 | val items = collection.items.joinToString("\n") { 10 | "${it.method} ${it.path} ${operationInformation(it)} " 11 | } 12 | 13 | return "${collection.name}\n$items" 14 | } 15 | 16 | private fun operationInformation(it: ApiItem): String { 17 | if (it.operationId.isEmpty()) return "" 18 | 19 | return " ${it.operationId}${ioParameters(it)}" 20 | } 21 | 22 | private fun ioParameters(details: ApiItem): String { 23 | val inputs = details.request.toString() 24 | val outputs = details.response.toString() 25 | if (inputs.isEmpty() && outputs.isEmpty()) return "()" 26 | if (inputs.isEmpty()) return "(): $outputs" 27 | if (outputs.isEmpty()) return "($inputs)" 28 | 29 | return "($inputs) : $outputs" 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /processor/modules/api-processor/src/main/kotlin/cc/unitmesh/processor/api/swagger/Swagger3Processor.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.processor.api.swagger 2 | 3 | import cc.unitmesh.core.model.ApiCollection 4 | import cc.unitmesh.core.model.ApiItem 5 | import cc.unitmesh.core.model.Parameter 6 | import cc.unitmesh.core.model.Request 7 | import cc.unitmesh.core.model.Response 8 | import cc.unitmesh.processor.api.base.ApiProcessor 9 | import io.swagger.v3.oas.models.OpenAPI 10 | import io.swagger.v3.oas.models.Operation 11 | import io.swagger.v3.oas.models.media.Schema 12 | import io.swagger.v3.oas.models.responses.ApiResponse 13 | import io.swagger.v3.parser.OpenAPIV3Parser 14 | import java.io.File 15 | 16 | class Swagger3Processor(private val api: OpenAPI) : ApiProcessor { 17 | private val apiSchemaMutableMap = api.components?.schemas 18 | 19 | override fun convertApi(): List { 20 | if (api.paths == null) return listOf() 21 | val allItems = mutableListOf() 22 | 23 | api.paths.forEach { (path, pathItem) -> 24 | pathItem.readOperationsMap().forEach { (method, operation) -> 25 | val apiItem = ApiItem( 26 | path = path, 27 | method = method.toString(), 28 | description = operation.description?.replace("\n", " ") ?: "", 29 | operationId = operation.operationId ?: "", 30 | tags = operation.tags ?: listOf(), 31 | request = convertRequest(operation), 32 | response = convertResponses(operation), 33 | ) 34 | 35 | allItems.add(apiItem) 36 | } 37 | } 38 | 39 | // group by tag 40 | val apiDetailsByTag = allItems.groupBy { it.tags.firstOrNull() ?: "" } 41 | return apiDetailsByTag.map { (tag, apiItems) -> 42 | ApiCollection(tag, "", apiItems) 43 | } 44 | } 45 | 46 | private fun convertResponses(operation: Operation): List { 47 | return operation.responses?.map { 48 | // use regex to get the status code 49 | val regex = Regex("([0-9]+)") 50 | val code = regex.find(it.key)?.value?.toInt() ?: 0 51 | val responseBody = handleResponse(it.value) ?: listOf() 52 | Response(code, responseBody) 53 | } ?: listOf() 54 | } 55 | 56 | private fun handleResponse(response: ApiResponse): List? { 57 | val content = response.content?.values 58 | val refName = content?.firstOrNull()?.schema?.`$ref` 59 | if (refName != null) { 60 | return getFromSchemaRef(refName) 61 | } 62 | 63 | val schema = content?.firstOrNull()?.schema 64 | if (schema != null) { 65 | return getFromSchemaItem(schema) 66 | } 67 | 68 | return null 69 | } 70 | 71 | private fun getFromSchemaItem(schema: Schema) = 72 | schema.properties?.map { (name, schema) -> 73 | Parameter( 74 | name = name, 75 | type = schema.type ?: "", 76 | ) 77 | } 78 | 79 | private fun getFromSchemaRef(refName: String): List? { 80 | val name = refName.split("/").last() 81 | val schema = apiSchemaMutableMap?.get(name) 82 | return schema?.properties?.map { (name, schema) -> 83 | Parameter( 84 | name = name, 85 | type = schema.type ?: "", 86 | ) 87 | } 88 | } 89 | 90 | private fun convertRequest(operation: Operation): Request { 91 | val parameters = operation.parameters?.map { 92 | Parameter( 93 | name = it.name ?: "", 94 | type = it.schema?.type ?: "", 95 | ) 96 | } 97 | 98 | val request = operation.requestBody?.content?.values?.flatMap { content -> 99 | content.schema?.properties?.map { (name, schema) -> 100 | Parameter( 101 | name = name, 102 | type = schema.type ?: "", 103 | ) 104 | } ?: listOf() 105 | } ?: listOf() 106 | 107 | return Request(parameters ?: listOf(), request) 108 | } 109 | 110 | companion object { 111 | val logger = org.slf4j.LoggerFactory.getLogger(Swagger3Processor::class.java)!! 112 | 113 | fun fromFile(file: File): OpenAPI? { 114 | try { 115 | return OpenAPIV3Parser().read(file.absolutePath) 116 | } catch (e: Exception) { 117 | logger.error("parse swagger file failed: ${file.absolutePath}") 118 | e.printStackTrace() 119 | } 120 | 121 | return null 122 | } 123 | } 124 | } 125 | -------------------------------------------------------------------------------- /processor/modules/api-processor/src/main/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 　　 3 | 4 | 5 | %-4relative [%thread] %-5level %logger{35} - %msg %n 6 | 7 | 8 | 9 | 10 | 11 | 12 | 　 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /processor/modules/api-processor/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /processor/modules/api-processor/src/test/kotlin/cc/unitmesh/processor/api/ApiProcessorDetectorTest.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.processor.api 2 | 3 | import cc.unitmesh.processor.api.parser.PostmanProcessor 4 | import cc.unitmesh.processor.api.swagger.Swagger3Processor 5 | import io.kotest.matchers.shouldBe 6 | import org.junit.jupiter.api.Test 7 | import java.io.File 8 | 9 | class ApiProcessorDetectorTest { 10 | 11 | @Test 12 | fun detectApiProcessor() { 13 | val file = File("src/test/resources/testsets/postman.json") 14 | val processor = ApiProcessorDetector.detectApiProcessor(file, true)!! 15 | processor.javaClass shouldBe PostmanProcessor::class.java 16 | 17 | // swagger-3.json 18 | val file2 = File("src/test/resources/testsets/swagger-3.json") 19 | val processor2 = ApiProcessorDetector.detectApiProcessor(file2, true)!! 20 | processor2.javaClass shouldBe Swagger3Processor::class.java 21 | 22 | // swagger-3.yaml 23 | val file3 = File("src/test/resources/testsets/swagger-3.yaml") 24 | val processor3 = ApiProcessorDetector.detectApiProcessor(file3, true)!! 25 | processor3.javaClass shouldBe Swagger3Processor::class.java 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /processor/modules/api-processor/src/test/kotlin/cc/unitmesh/processor/api/ModelSerializeTest.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.processor.api 2 | 3 | import cc.unitmesh.processor.api.postman.PostmanCollection 4 | import cc.unitmesh.processor.api.postman.PostmanReader 5 | import io.kotest.matchers.shouldBe 6 | import kotlinx.serialization.json.Json 7 | import org.junit.jupiter.api.Test 8 | import java.io.File 9 | 10 | class ModelSerializeTest { 11 | val file = javaClass.getResource("/openapi/CircleCI.postman_collection.json")!! 12 | private val text = file.readText() 13 | 14 | @Test 15 | fun should_success_get_info_name_when_give_a_collection_file() { 16 | // deserialize text to PostmanCollection 17 | val collection = Json.decodeFromString(PostmanCollection.serializer(), text) 18 | collection.info!!.name shouldBe "CircleCI" 19 | } 20 | 21 | @Test 22 | fun should_get_info_by_reader() { 23 | val postmanReader = PostmanReader() 24 | val collection = postmanReader.readCollectionFile(File(file.toURI()).absolutePath) 25 | 26 | collection.info!!.name shouldBe "CircleCI" 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /processor/modules/api-processor/src/test/kotlin/cc/unitmesh/processor/api/parser/PostmanParserTest.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.processor.api.parser 2 | 3 | import cc.unitmesh.processor.api.postman.PostmanReader 4 | import cc.unitmesh.processor.api.render.MarkdownTableRender 5 | import io.kotest.matchers.shouldBe 6 | import org.junit.jupiter.api.Test 7 | import java.io.File 8 | 9 | class PostmanParserTest { 10 | @Test 11 | fun should_print_out() { 12 | val file = javaClass.getResource("/openapi/CircleCI.postman_collection.json")!! 13 | val postmanReader = PostmanReader() 14 | val collection = postmanReader.readCollectionFile(File(file.toURI()).absolutePath) 15 | val postmanParser = PostmanParser() 16 | val listList = postmanParser.parse(collection)!! 17 | 18 | listList.size shouldBe 17 19 | listList[0].items.size shouldBe 2 20 | 21 | val output = MarkdownTableRender().render(listList) 22 | println(output) 23 | } 24 | 25 | @Test 26 | fun should_print_out_2() { 27 | val boxJson = javaClass.getResource("/openapi/Box.json")!! 28 | val postmanReader = PostmanReader() 29 | val collection = postmanReader.readCollectionFile(File(boxJson.toURI()).absolutePath) 30 | val postmanParser = PostmanParser() 31 | 32 | val listList = postmanParser.parse(collection)!! 33 | 34 | listList.size shouldBe 1 35 | listList[0].items.size shouldBe 5 36 | 37 | val output = MarkdownTableRender().render(listList) 38 | println(output) 39 | } 40 | 41 | @Test 42 | fun should_hande_description() { 43 | val boxJson = javaClass.getResource("/postman/Wechat-Simple.postman_collection.json")!! 44 | val postmanReader = PostmanReader() 45 | val collection = postmanReader.readCollectionFile(File(boxJson.toURI()).absolutePath) 46 | val postmanParser = PostmanParser() 47 | 48 | val listList = postmanParser.parse(collection)!! 49 | 50 | listList.size shouldBe 1 51 | listList[0].items.size shouldBe 10 52 | 53 | val output = MarkdownTableRender().render(listList) 54 | println(output) 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /processor/modules/api-processor/src/test/kotlin/cc/unitmesh/processor/api/render/SimpleApiRenderTest.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.processor.api.render 2 | 3 | import cc.unitmesh.processor.api.ApiProcessorDetector 4 | import org.junit.jupiter.api.Assertions.assertEquals 5 | import org.junit.jupiter.api.Test 6 | import java.io.File 7 | 8 | class SimpleApiRenderTest { 9 | @Test 10 | fun should_render_api_to_string() { 11 | val file2 = File("src/test/resources/testsets/swagger-3.yaml") 12 | val processor2 = ApiProcessorDetector.detectApiProcessor(file2, true)!! 13 | val apiDetails = processor2.convertApi() 14 | 15 | val render = SimpleApiRender() 16 | val result = render.render(apiDetails) 17 | 18 | val expected = 19 | """ 20 | Cashback campaign 21 | GET /cashback-campaigns/{campaignId} getCashbackCampaign(X-App-Token: string, campaignId: string) : [200: {campaignId: string, campaignName: string, startDate: string, endDate: string, status: string, remainingAmountInCents: integer}, 400: {errors: array}, 401: {errors: array}, 403: {errors: array}, 404: {errors: array}, 500: {errors: array}] 22 | 23 | Cashback 24 | GET /cashback-campaigns/{campaignId}/cashbacks getCashbackList(X-App-Token: string, campaignId: string, pageNumber: integer, pageSize: integer, fromDateTime: string, toDateTime: string, status: string) : [200: {cashbacks: array, totalElementCount: integer}, 400: {errors: array}, 401: {errors: array}, 403: {errors: array}, 404: {errors: array}, 500: {errors: array}] 25 | POST /cashback-campaigns/{campaignId}/cashbacks createCashback(X-App-Token: string, campaignId: string) : [201: {cashbackId: string, url: string, amountInCents: integer, createdDateTime: string, expiryDateTime: string, redeemedDateTime: string, status: string, referenceId: string, locationId: string, locationAddress: string}, 400: {errors: array}, 401: {errors: array}, 403: {errors: array}, 404: {errors: array}, 500: {errors: array}] 26 | GET /cashback-campaigns/{campaignId}/cashbacks/{cashbackId} getCashback(X-App-Token: string, campaignId: string, cashbackId: string) : [200: {cashbackId: string, url: string, amountInCents: integer, createdDateTime: string, expiryDateTime: string, redeemedDateTime: string, status: string, referenceId: string, locationId: string, locationAddress: string}, 400: {errors: array}, 401: {errors: array}, 403: {errors: array}, 404: {errors: array}, 500: {errors: array}] 27 | 28 | Cashback notification 29 | POST /cashback-subscriptions subscribeCashbackNotifications(X-App-Token: string) : [201: {subscriptionId: string}, 400: {errors: array}, 401: {errors: array}, 403: {errors: array}, 500: {errors: array}] 30 | DELETE /cashback-subscriptions deleteCashbackNotifications(X-App-Token: string) : [204: {}, 401: {errors: array}, 403: {errors: array}, 500: {errors: array}] 31 | """.trimIndent() 32 | 33 | assertEquals(expected, result) 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /processor/modules/plantuml-verifier/build.gradle.kts: -------------------------------------------------------------------------------- 1 | @Suppress("DSL_SCOPE_VIOLATION") 2 | plugins { 3 | alias(libs.plugins.kotlin.jvm) 4 | alias(libs.plugins.shadow) 5 | alias(libs.plugins.serialization) 6 | application 7 | } 8 | 9 | repositories { 10 | mavenCentral() 11 | } 12 | 13 | dependencies { 14 | implementation(projects.common.core) 15 | implementation(projects.common.verifier) 16 | 17 | implementation(libs.clikt) 18 | implementation(libs.serialization.json) 19 | 20 | // Logging 21 | implementation(libs.logging.slf4j.api) 22 | implementation(libs.logging.logback.classic) 23 | 24 | implementation(libs.plantuml) 25 | 26 | testImplementation(kotlin("test")) 27 | 28 | testImplementation(libs.bundles.test) 29 | } 30 | 31 | tasks.test { 32 | useJUnitPlatform() 33 | } 34 | 35 | kotlin { 36 | jvmToolchain(11) 37 | } 38 | 39 | application { 40 | mainClass.set("cc.unitmesh.processor.api.MainKt") 41 | } 42 | 43 | tasks { 44 | shadowJar { 45 | manifest { 46 | attributes(Pair("Main-Class", "cc.unitmesh.processor.api.MainKt")) 47 | } 48 | // minimize() 49 | dependencies { 50 | exclude(dependency("org.junit.jupiter:.*:.*")) 51 | exclude(dependency("org.junit:.*:.*")) 52 | exclude(dependency("junit:.*:.*")) 53 | } 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /processor/modules/plantuml-verifier/src/main/kotlin/cc/unitmesh/verifier/uml/Main.kt: -------------------------------------------------------------------------------- 1 | package cc.unitmesh.verifier.uml 2 | 3 | import cc.unitmesh.verifier.plantuml.PlantUmlParser 4 | import com.github.ajalt.clikt.core.CliktCommand 5 | import com.github.ajalt.clikt.parameters.arguments.argument 6 | import com.github.ajalt.clikt.parameters.arguments.default 7 | import com.github.ajalt.clikt.parameters.types.file 8 | import org.slf4j.Logger 9 | import org.slf4j.LoggerFactory 10 | import java.io.File 11 | import java.io.IOException 12 | 13 | fun main(args: Array) = PlantUmlVerifier() 14 | .main(args) 15 | 16 | val logger: Logger = LoggerFactory.getLogger(PlantUmlVerifier::class.java) 17 | 18 | class PlantUmlVerifier : CliktCommand() { 19 | private val sourceDir by argument().file().default(File("output", "domain")) 20 | 21 | override fun run() { 22 | logger.info("Unit Connector Started") 23 | File(sourceDir, "svg").mkdirs() 24 | // walkdir in source dir 25 | sourceDir.walkTopDown().forEach { 26 | if (it.isFile && it.extension == "puml") { 27 | logger.info("verifying ${it.absolutePath}") 28 | try { 29 | val isCorrect = PlantUmlParser(it).isCorrect() 30 | if (!isCorrect) { 31 | logger.info("failed to verify ${it.absolutePath}, will remove it") 32 | } 33 | } catch (e: IOException) { 34 | logger.error("failed to verify ${it.absolutePath}", e) 35 | } 36 | } 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /processor/modules/plantuml-verifier/src/main/resources/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 　　 3 | 4 | 5 | %-4relative [%thread] %-5level %logger{35} - %msg %n 6 | 7 | 8 | 9 | 10 | 11 | 　 12 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /processor/modules/plantuml-verifier/src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /processor/settings.gradle.kts: -------------------------------------------------------------------------------- 1 | @file:Suppress("UnstableApiUsage") 2 | 3 | enableFeaturePreview("STABLE_CONFIGURATION_CACHE") 4 | enableFeaturePreview("TYPESAFE_PROJECT_ACCESSORS") 5 | 6 | rootProject.name = "UnitProcessor" 7 | 8 | include( 9 | ":common:core", 10 | ":common:cli-core", 11 | ":common:verifier", 12 | ":common:prompter", 13 | 14 | ":examples:unit-demo", 15 | 16 | ":kotlin-modules:analysis", 17 | ":kotlin-modules:importer", 18 | 19 | ":java-modules:core-analysis", 20 | ":java-modules:codegen-processor", 21 | ":java-modules:spring-processor", 22 | ":java-modules:test-processor", 23 | 24 | ":modules:api-processor", 25 | ":modules:plantuml-verifier", 26 | ) 27 | -------------------------------------------------------------------------------- /prompter/prepare/.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | .idea 131 | _exec 132 | swagger-list.json 133 | swagger_output 134 | swagger_output.jsonl 135 | userstory_output 136 | userstory_to_api.jsonl 137 | test-api 138 | test_to_code.jsonl 139 | test_to_code 140 | test_to_code_output.jsonl 141 | test_to_code.zip 142 | test_lora.jsonl 143 | userstory_map 144 | *.zip 145 | *.jsonl 146 | userstory_detail 147 | repositories.json 148 | -------------------------------------------------------------------------------- /prompter/prepare/README.md: -------------------------------------------------------------------------------- 1 | # Prepare 2 | 3 | - [user-story.py](user-story.py)，调用 OpenAI 的用户故事拆解和用户故事生成。 4 | - [test-to-code.py](test-to-code.py) 测试代码与处理。 5 | - [swagger-user-story.py](swagger-user-story.py) Swagger 和用户故事相互转换。 6 | - [repository-to-prompt.py](repository-to-prompt.py)，调用 OpenAI 的代码生成，从 Repository 代码生成 Prompt。 7 | 8 | ## 数据生成 9 | 10 | 配置 OpenAI 的 API Key 11 | 12 | ```bash 13 | export OPENAI_API_KEY=sk-xxxx 14 | ``` 15 | 16 | ### 用户故事数据生成 17 | 18 | 生成用户故事任务 19 | 20 | ```bash 21 | # 生成用户故事任务 22 | python user-story.py create_user_tasks 23 | 24 | # 合并为 JSONL 25 | python user-story.py merge_created_user_story 26 | 27 | # 添加 instruction 28 | python user-story.py add_instruction 29 | 30 | # 创建用户故事 31 | python user-story.py create_user_story_detail 32 | 33 | # 合并为 JSONL 34 | python user-story.py merge_userstory_details 35 | ``` 36 | 37 | ### Swagger 数据生成 38 | 39 | ```bash 40 | # 根据 Swagger 生成用户故事 41 | python swagger-user-story.py swagger_to_userstory 42 | 43 | # 合并为 JSONL 44 | python swagger-user-story.py merge_swagger_output 45 | 46 | # 根据用户故事生成 Swagger 47 | python swagger-user-story.py userstory_to_swagger 48 | 49 | # 合并为 JSONL 50 | python swagger-user-story.py merge_api_output 51 | ``` 52 | 53 | ### 测试代码数据生成 54 | 55 | ```bash 56 | # 合并 Processor 输出为 JSONL 57 | python test-to-code.py merge_test_to_jsonl 58 | 59 | # 从测试生成代码 60 | python test-to-code.py generate_code_from_tests 61 | 62 | # 合并为 JSONL 63 | python test-to-code.py merge_test_output_to_jsonl 64 | 65 | # 生成微调所需要的 instruction 66 | python test-to-code.py generate_for_lora 67 | ``` 68 | 69 | ### Repository 代码数据生成 70 | 71 | ```bash 72 | # 生成 Repository 代码 73 | python repository-to-prompt.py process_prompt 74 | ``` 75 | 76 | ### 生成辅助代码 77 | 78 | 不需要 79 | -------------------------------------------------------------------------------- /prompter/prepare/prompts/create-user-story-name.md: -------------------------------------------------------------------------------- 1 | Design a User Story Mapping for ${domain} application based on your understanding. Here are the requirements: 2 | 3 | 1. Your user story map should include only user tasks to demonstrate how users will interact with the application. 4 | 2. Our user story map should be based on your understanding of the ${domain} application and its users, and should be designed to address their needs and pain points. 5 | 3. You may use any tools or formats you choose to create your user story map, but it should be easily shareable and understandable by stakeholders. 6 | 4. Your expression should be more concise and clear. 7 | 5. Your return should be like as follows: 8 | 9 | ``` 10 | User Tasks: 11 | 1. ... 12 | ``` 13 | -------------------------------------------------------------------------------- /prompter/prepare/prompts/create-user-story.md: -------------------------------------------------------------------------------- 1 | 为下面的需求编写用户故事：${domain} 应用的 ${story_name} 功能。 要求如下： 2 | 3 | 1. 必须要考虑尽可能考虑各种异常场景，添加更多的 AC。 4 | 2. 你的返回模板如下所示： 5 | 6 | ### 7 | 用户故事：可以选择宝贝出行服务 8 | 作为 莉莉妈 9 | 我想 在滴滴打车的手机客户端里选择宝贝出行服务 10 | 以便于 我能够带宝宝打车出行的时候打到有儿童座椅的车 11 | 12 | AC 1: 莉莉妈可以选择宝贝出行服务 13 | 假设 xxx 14 | 当 xxx 15 | 于是 xxx 16 | ### 17 | -------------------------------------------------------------------------------- /prompter/prepare/prompts/create_api_prompt.txt: -------------------------------------------------------------------------------- 1 | You are a software engineer and design a strict system now. Here are the requirements: 2 | 3 | 1. According follows user stories and follows format to design API. 4 | 2. Try you best to search corner case. 5 | 3. You only return the code, no explain. 6 | 7 | Your response examples： 8 | 9 | ``` 10 | User 11 | POST login(user: String, password: String):AuthToken /users/login Login with REST API 12 | POST register(registerInfo: RegisterInfo):User /users/register Register with REST API 13 | ``` 14 | 15 | The user stories are as follows: 16 | 17 | -------------------------------------------------------------------------------- /prompter/prepare/prompts/create_userstory_from_swagger.md: -------------------------------------------------------------------------------- 1 | You are a BA in a agile Team.Please according the input apis, to create the user stories of this software system in backend. 用户故事的模板如下： 2 | 3 | ### 4 | 用户故事：可以选择宝贝出行服务 5 | 作为 莉莉妈 6 | 我想 在滴滴打车的手机客户端里选择宝贝出行服务 7 | 以便于 我能够带宝宝打车出行的时候打到有儿童座椅的车 8 | 9 | AC 1: 莉莉妈可以选择宝贝出行服务 10 | 假设 xxx 11 | 当 xxx 12 | 于是 xxx 13 | ### 14 | 15 | -------------------------------------------------------------------------------- /prompter/prepare/prompts/test_to_code.md: -------------------------------------------------------------------------------- 1 | You are a programmer and implementation a method with TDD. Here are the requirements: 2 | 3 | 1. According follows class information and tests code to write a method. 4 | 2. Try you best to cover corner case. 5 | 3. You only return the code, no explain. 6 | -------------------------------------------------------------------------------- /prompter/prepare/repository-to-prompt.py: -------------------------------------------------------------------------------- 1 | """ 2 | python -m repository-to-prompt process_prompt 3 | """ 4 | import os 5 | 6 | import openai 7 | import json 8 | from concurrent.futures import ThreadPoolExecutor 9 | import fire 10 | 11 | 12 | def prompt_gpt35(prompt, value): 13 | response = openai.ChatCompletion.create( 14 | model="gpt-3.5-turbo", 15 | messages=[ 16 | {"role": "system", "content": prompt}, 17 | {"role": "user", "content": f'{value}'}, 18 | ] 19 | ) 20 | 21 | return response.choices[0]["message"]["content"].strip().replace("", "").replace("", "") 22 | 23 | 24 | def process_prompt(): 25 | # open repositories.json 26 | with open("repositories.json", "r") as f: 27 | data = json.load(f) 28 | 29 | with ThreadPoolExecutor(max_workers=1) as executor: 30 | futures = {executor.submit(process_repository, item, i) for i, item in enumerate(data)} 31 | 32 | 33 | prompt_text = "请编写用户故事，能覆盖下面的代码功能，要求：1. 描述业务并突出重点 2. 你返回的内容只有： 我想 xxx，以便于。 3. " \ 34 | "带上表和字段信息，示例：`我想查找用户（user）在某个时间段（receiptDate）内的物品清单（items），以便于进行统计和分析。`" 35 | 36 | 37 | def process_repository(item, i): 38 | output_file = f"repositories/repository-{item['id']}.json" 39 | if os.path.exists(output_file): 40 | print(f"skipping {item['id']}") 41 | return 42 | 43 | print("processing user story: ", i) 44 | # the input will be the output of the previous task 45 | output = prompt_gpt35(prompt_text, item['content']) 46 | translated_item = { 47 | "instruction": prompt_text, 48 | "input": item['content'], 49 | "output": output 50 | } 51 | save_item(translated_item, output_file) 52 | 53 | 54 | def save_item(item, file_name): 55 | with open(file_name, 'w') as f: 56 | json.dump(item, f, ensure_ascii=False, indent=4) 57 | 58 | 59 | def main(task, **kwargs): 60 | globals()[task](**kwargs) 61 | 62 | 63 | if __name__ == "__main__": 64 | fire.Fire(main) 65 | -------------------------------------------------------------------------------- /prompter/prepare/requirements.txt: -------------------------------------------------------------------------------- 1 | numpy 2 | rouge_score 3 | fire 4 | openai 5 | transformers>=4.26.1 6 | torch 7 | sentencepiece 8 | tokenizers==0.12.1 9 | wandb -------------------------------------------------------------------------------- /prompter/prepare/swagger-user-story.py: -------------------------------------------------------------------------------- 1 | """ 2 | python -m swagger-user-story swagger_to_userstory 3 | python -m swagger-user-story userstory_to_swagger 4 | 5 | """ 6 | import openai 7 | import json 8 | from concurrent.futures import ThreadPoolExecutor 9 | import os 10 | import re 11 | import fire 12 | 13 | from utils import json_to_jsonl 14 | 15 | 16 | def prompt_gpt35(prompt, value): 17 | response = openai.ChatCompletion.create( 18 | model="gpt-3.5-turbo", 19 | messages=[ 20 | {"role": "system", "content": prompt}, 21 | {"role": "user", "content": f'{value}'}, 22 | ] 23 | ) 24 | 25 | return response.choices[0]["message"]["content"].strip().replace("", "").replace("", "") 26 | 27 | 28 | def save_item(item, file_name): 29 | with open(file_name, 'w') as f: 30 | json.dump(item, f, ensure_ascii=False, indent=4) 31 | 32 | 33 | def process_swagger(item, i): 34 | create_user_story_prompt = open("prompts/create_userstory_from_swagger.md").read() + "\n" 35 | print("processing: ", i) 36 | output = prompt_gpt35(create_user_story_prompt, item['string']) 37 | translated_item = { 38 | "instruction": create_user_story_prompt, 39 | "input": item['string'], 40 | "output": output 41 | } 42 | save_item(translated_item, f"swagger_output/swagger{i}.json") 43 | 44 | 45 | def swagger_to_userstory(): 46 | with open('swagger-list.json', 'r') as f: 47 | data = json.load(f) 48 | 49 | with ThreadPoolExecutor(max_workers=1) as executor: 50 | futures = {executor.submit(process_swagger, item, i) for i, item in enumerate(data)} 51 | 52 | 53 | # merge swagger_output/*.json to one jsonl file 54 | def merge_swagger_output(): 55 | json_to_jsonl('swagger_output', 'swagger_output.jsonl') 56 | 57 | 58 | def merge_api_output(): 59 | json_to_jsonl('userstory_output', 'userstory_to_api.jsonl') 60 | 61 | 62 | def userstory_to_swagger(): 63 | data = [json.loads(l) for l in open('swagger_output.jsonl', "r")] 64 | 65 | with ThreadPoolExecutor(max_workers=1) as executor: 66 | futures = {executor.submit(process_userstory, item, i) for i, item in enumerate(data)} 67 | 68 | 69 | def process_userstory(item, i): 70 | # read create_api_prompt.txt.txt as the prompt 71 | create_api_prompt = open("prompts/create_api_prompt.txt").read() + "\n" 72 | 73 | print("processing user story: ", i) 74 | # the input will be the output of the previous task 75 | output = prompt_gpt35(create_api_prompt, item['output']) 76 | translated_item = { 77 | "instruction": create_api_prompt, 78 | "input": item['output'], 79 | "output": output 80 | } 81 | save_item(translated_item, f"userstory_output/userstory{i}.json") 82 | 83 | 84 | def main(task, **kwargs): 85 | if not os.path.exists('userstory_output'): 86 | os.makedirs('userstory_output') 87 | if not os.path.exists('swagger_output'): 88 | os.makedirs('swagger_output') 89 | globals()[task](**kwargs) 90 | 91 | 92 | if __name__ == "__main__": 93 | fire.Fire(main) 94 | -------------------------------------------------------------------------------- /prompter/prepare/test-to-code.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import openai 4 | import fire 5 | import tqdm 6 | import time 7 | 8 | from utils import json_to_jsonl 9 | 10 | jsonl_path = 'test_to_code.jsonl' 11 | output_dir = 'test_to_code' 12 | 13 | 14 | def merge_test_to_jsonl(): 15 | source = 'test-api' 16 | target = jsonl_path 17 | 18 | walkdir = os.walk(source) 19 | 20 | index = 0 21 | with open(target, 'w') as out_file: 22 | for root, dirs, files in walkdir: 23 | for file in files: 24 | if file.endswith('.json'): 25 | # format json to one line 26 | with open(os.path.join(root, file), 'r') as f: 27 | data = json.load(f) 28 | data['id'] = 'task_' + str(index) 29 | json.dump(data, out_file) 30 | out_file.write('\n') 31 | 32 | 33 | def merge_test_output_to_jsonl(): 34 | json_to_jsonl(output_dir, "test_to_code_output.jsonl", ".json") 35 | 36 | 37 | def generate_for_lora(): 38 | # open test_to_code_output.jsonl, the item is a dict, format is: 39 | # { testMethod, classInfo, code } 40 | # save to test_lora.jsonl, the format is: 41 | # { "instruction": "Write test for follow code", "input": code, "output": testMethod} 42 | with open("test_to_code_output.jsonl", 'r') as file: 43 | with open("test_lora.jsonl", 'w') as out_file: 44 | for line in file: 45 | data = json.loads(line) 46 | output = { 47 | "instruction": "Write test for follow code", 48 | "input": data['code'], 49 | "output": data['testMethod'] 50 | } 51 | json.dump(output, out_file) 52 | out_file.write('\n') 53 | 54 | 55 | def generate_code_from_tests(): 56 | tasks = [] 57 | with open(jsonl_path, 'r') as file: 58 | for line in file: 59 | tasks.append(json.loads(line)) 60 | 61 | print(f"Loaded {len(tasks)} tasks") 62 | 63 | os.makedirs(output_dir, exist_ok=True) 64 | 65 | # open test_code_code.md 66 | base_prompt = open("prompts/test_to_code.md").read() + "\n" 67 | 68 | idx = 1 69 | 70 | total = len(tasks) 71 | progress_bar = tqdm.tqdm(total=total) 72 | 73 | for task in tasks: 74 | # if output file exists, skip 75 | if os.path.exists(f"{output_dir}/{idx}.json"): 76 | idx = idx + 1 77 | progress_bar.update() 78 | continue 79 | 80 | prompt = f"{base_prompt}\n class information: ### {task['classInfo']} \n ### test code: ### {task['testMethod']} \n ###" 81 | 82 | try: 83 | response = openai.Completion.create( 84 | model="text-davinci-003", 85 | prompt=prompt, 86 | temperature=0, 87 | max_tokens=150, 88 | top_p=1.0, 89 | frequency_penalty=0.0, 90 | presence_penalty=0.0, 91 | stop=["\"\"\""] 92 | ) 93 | 94 | code = response['choices'][0]['text'] 95 | progress_bar.update() 96 | 97 | output = { 98 | "classInfo": task['classInfo'], 99 | "testMethod": task['testMethod'], 100 | "code": code 101 | } 102 | 103 | # write to file in test_to_code 104 | with open(f"{output_dir}/{idx}.json", 'w') as file: 105 | json.dump(output, file) 106 | 107 | # sleep_time = 3 108 | # time.sleep(sleep_time) 109 | idx = idx + 1 110 | except Exception as e: 111 | print(e) 112 | print("Error, sleeping for 5 minutes") 113 | time.sleep(300) 114 | continue 115 | 116 | 117 | def main(task, **kwargs): 118 | globals()[task](**kwargs) 119 | 120 | 121 | if __name__ == "__main__": 122 | fire.Fire(main) 123 | -------------------------------------------------------------------------------- /prompter/prepare/utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | 4 | 5 | def json_to_jsonl(source, target, ext='.json'): 6 | walkdir = os.walk(source) 7 | with open(target, 'w') as out_file: 8 | for root, dirs, files in walkdir: 9 | for file in files: 10 | if file.endswith(ext): 11 | # format json to one line 12 | with open(os.path.join(root, file), 'r') as f: 13 | data = json.load(f) 14 | json.dump(data, out_file) 15 | out_file.write('\n') 16 | -------------------------------------------------------------------------------- /train/README.md: -------------------------------------------------------------------------------- 1 | # 训练 Notebook 2 | 3 | - LLaMA 微调（Alpaca LoRA）：[alpaca-lora.ipynb](alpaca-lora.ipynb) 4 | - ChatGLM 微调（ChatGLM Tuning）：[chatglm-tuning.ipynb](chatglm-tuning.ipynb) 5 | --------------------------------------------------------------------------------