├── .DS_Store ├── .cache └── 41 │ └── cache.db ├── .env ├── .gitignore ├── .idea ├── .gitignore ├── Auto_Generate_Test_Cases.iml ├── inspectionProfiles │ ├── Project_Default.xml │ └── profiles_settings.xml ├── misc.xml ├── modules.xml └── vcs.xml ├── README.md ├── agent_results ├── quality_assurance_result.json ├── requirement_analyst_result.json ├── test_case_writer_result.json └── test_designer_result.json ├── docs ├── prd.md ├── system_design.json └── 智能停车场需求文档.pdf ├── requirements.txt ├── run_test_designer.py ├── search_eval ├── __pycache__ │ ├── annotation_tool.cpython-311.pyc │ ├── excel_processor.cpython-311.pyc │ ├── excel_utils.cpython-311.pyc │ ├── json_utils.cpython-311.pyc │ └── mrr_calculator.cpython-311.pyc ├── annotation_tool.py ├── dataset_evaluator.py ├── json_utils.py └── search_evaluation.jsonl ├── src ├── .DS_Store ├── .cache │ └── 41 │ │ └── cache.db ├── agents │ ├── __pycache__ │ │ ├── assistant.cpython-311.pyc │ │ ├── assistant.cpython-312.pyc │ │ ├── browser_use_agent.cpython-311.pyc │ │ ├── quality_assurance.cpython-311.pyc │ │ ├── quality_assurance.cpython-312.pyc │ │ ├── requirement_analyst.cpython-311.pyc │ │ ├── requirement_analyst.cpython-312.pyc │ │ ├── test_case_writer.cpython-311.pyc │ │ ├── test_case_writer.cpython-312.pyc │ │ ├── test_designer.cpython-311.pyc │ │ └── test_designer.cpython-312.pyc │ ├── assistant.py │ ├── browser_use_agent.py │ ├── quality_assurance.py │ ├── requirement_analyst.py │ ├── test_case_writer.py │ └── test_designer.py ├── logs │ └── ai_tester.log ├── main.py ├── models │ ├── __pycache__ │ │ ├── template.cpython-311.pyc │ │ ├── template.cpython-312.pyc │ │ ├── test_case.cpython-311.pyc │ │ ├── test_case.cpython-312-pytest-8.2.1.pyc │ │ └── test_case.cpython-312.pyc │ ├── template.py │ └── test_case.py ├── schemas │ ├── __pycache__ │ │ ├── communication.cpython-311.pyc │ │ └── communication.cpython-312.pyc │ └── communication.py ├── services │ ├── __pycache__ │ │ ├── document_processor.cpython-311.pyc │ │ ├── document_processor.cpython-312.pyc │ │ ├── export_service.cpython-311.pyc │ │ ├── export_service.cpython-312.pyc │ │ ├── test_case_generator.cpython-311.pyc │ │ ├── test_case_generator.cpython-312.pyc │ │ └── ui_auto_service.cpython-311.pyc │ ├── document_processor.py │ ├── export_service.py │ ├── test_case_generator.py │ └── ui_auto_service.py ├── templates │ ├── api_test_template.json │ ├── functional_test_template.json │ └── ui_auto_test_template.json └── utils │ ├── __pycache__ │ ├── agent_io.cpython-311.pyc │ ├── agent_io.cpython-312.pyc │ ├── cli_parser.cpython-311.pyc │ ├── cli_parser.cpython-312.pyc │ ├── config.cpython-312.pyc │ ├── env_loader.cpython-311.pyc │ ├── logger.cpython-311.pyc │ └── logger.cpython-312.pyc │ ├── agent_io.py │ ├── cli_parser.py │ ├── env_loader.py │ └── logger.py ├── template_config.json ├── test_case_writer_test.py ├── test_cases.xlxs.xlsx ├── test_improve_with_llm_direct.py ├── ui_test_results.xlsx └── ui_tst_case_example.json /.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tony3liu/Auto_Generate_Test_Cases/bcd100b08fac79262e2cb5c1ede32d95d3525e8d/.DS_Store -------------------------------------------------------------------------------- /.cache/41/cache.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tony3liu/Auto_Generate_Test_Cases/bcd100b08fac79262e2cb5c1ede32d95d3525e8d/.cache/41/cache.db -------------------------------------------------------------------------------- /.env: -------------------------------------------------------------------------------- 1 | AZURE_OPENAI_API_KEY="b" 2 | AZURE_OPENAI_BASE_URL="h" 3 | AZURE_OPENAI_MODEL="sus" 4 | AZURE_OPENAI_MODEL_VERSION="2023-07" 5 | # deepseek 火山 6 | # DS_BASE_URL='https://ark.cn-beijing.volce' 7 | # DS_API_KEY='dd159841c' 8 | # Deepseek 硅基 9 | DS_BASE_URL='https://api.siliconflow.cn' 10 | DS_API_KEY='sk-nivpqlzqrzboqitgslhfkhsijtjazeflxntdqcfugnzfjtc' 11 | # V3 12 | # DS_MODEL_V3='bot-2025' 13 | DS_MODEL_V3='Pro/deepseek-ai/DeepSeek-V3' 14 | # R1 15 | DS_MODEL_R1='bot-2025' 16 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | docs/* 2 | logs/ai_tester.log 3 | .~test_cases.xlsx 4 | .env* 5 | */.cache 6 | test_case.xlsx 7 | .gitignore 8 | ./docs/* 9 | ./agent_results 10 | .cache* 11 | src/.DS_Store* 12 | .venv* 13 | # 激活环境Set-ExecutionPolicy -ExecutionPolicy Bypass -Scope Process 14 | #.venv\Scripts\Activate.ps1 -------------------------------------------------------------------------------- /.idea/.gitignore: -------------------------------------------------------------------------------- 1 | # Default ignored files 2 | /shelf/ 3 | /workspace.xml 4 | # Editor-based HTTP Client requests 5 | /httpRequests/ 6 | # Datasource local storage ignored files 7 | /dataSources/ 8 | /dataSources.local.xml 9 | -------------------------------------------------------------------------------- /.idea/Auto_Generate_Test_Cases.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 14 | 15 | 20 | 21 | -------------------------------------------------------------------------------- /.idea/inspectionProfiles/Project_Default.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 136 | -------------------------------------------------------------------------------- /.idea/inspectionProfiles/profiles_settings.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 6 | -------------------------------------------------------------------------------- /.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 6 | 7 | -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # 自动化测试用例生成工具 2 | 3 | 基于Python开发的AI测试系统，利用AutoGen框架和多个AI代理自动分析需求并生成测试用例。 4 | 5 | ## 功能特点 6 | 7 | - 📝 基于AI的自动需求分析 8 | - 🤖 基于AutoGen框架的多代理系统 9 | - 📋 可配置的测试用例模板和格式 10 | - 📊 支持Excel格式导出 11 | - 🔄 支持多种文档格式（PDF、Word、Markdown、Text） 12 | - ⚙️ 可扩展的架构设计 13 | - 🌐 支持UI自动化测试执行 14 | - 🌐 有限支持搜索引擎的数据标注、指标评估和测试（在线评估的接口需要根据实际不同的搜索引擎适配json结构，使用者请自行开发） 15 | 16 | ## 版本说明 17 | - 📋 当前版本支持通过多线程并发进行测试用例生成、测试用例审查、测试用例改进 18 | - 📝 当前版本尚不支持接口测试用例生成，仅仅是加了入口，请使用功能测试 19 | - 🌐 已支持UI自动化测试执行，可通过JSON格式的测试用例文件执行自动化测试 20 | - 🌐 有限支持搜索引擎的数据标注、指标评估和测试（在线评估的接口需要根据实际不同的搜索引擎适配json结构，使用者请自行开发） 21 | 22 | ## 安装说明 23 | 24 | 1. 克隆代码仓库： 25 | ```bash 26 | git clone 27 | cd Auto_Generate_Test_Cases 28 | ``` 29 | 30 | 2. 创建并激活虚拟环境： 31 | ```bash 32 | python -m venv .venv 33 | source .venv/bin/activate # Windows系统使用: .venv\Scripts\activate 34 | ``` 35 | 36 | 3. 安装依赖： 37 | ```bash 38 | pip install -r requirements.txt 39 | ``` 40 | 41 | 4. 配置系统： 42 | - 复制配置文件模板： 43 | ```bash 44 | cp .env.example .env 45 | ``` 46 | - 在.env 中更新OpenAI API密钥和其他设置 47 | 48 | ## 使用方法 49 | 50 | 本工具支持通过命令行参数来控制测试用例的生成和执行： 51 | 52 | ```bash 53 | # 生成测试用例 54 | python src/main.py -d <需求文档路径> [-o <输出文件路径>] [-t <测试类型>] [-c <并发数：建议 10 以内>] 55 | 56 | # 执行UI自动化测试 57 | python src/main.py -i <测试用例文件路径> -t ui_auto -o <测试结果输出路径> 58 | ``` 59 | 60 | ### 命令行参数说明 61 | 62 | - `-d, --doc`：必需参数，指定需求文档的路径，支持PDF、Word、Markdown或Text格式 63 | - `-i, --input`：UI自动化测试时指定测试用例文件路径（JSON格式） 64 | - `-o, --output`：可选参数，指定测试用例输出文件路径，默认为"test_cases.xlsx" 65 | - `-c, --concurrency`：可选参数，指定并发数，默认为1 66 | - `-t, --type`：可选参数，指定测试类型，可选值： 67 | - `functional`：功能测试（默认值） 68 | - `api`：接口测试 69 | - `ui_auto`：UI自动化测试 70 | 71 | ### 使用示例 72 | 73 | 1. 生成功能测试用例： 74 | ```bash 75 | python src/main.py -d docs/需求文档.pdf -o 功能测试用例.xlsx -c 5 76 | # 或 77 | python src/main.py -d docs/需求文档.pdf -t functional -o 功能测试用例.xlsx -c 5 78 | ``` 79 | 80 | 2. 生成接口测试用例： 81 | ```bash 82 | python src/main.py -d docs/接口文档.md -t api -o 接口测试用例.xlsx -c 5 83 | ``` 84 | 85 | 3. 执行UI自动化测试： 86 | ```bash 87 | python src/main.py -i test_cases.json -t ui_auto -o test_results.xlsx 88 | ``` 89 | 90 | ### UI自动化测试用例格式 91 | 92 | UI自动化测试用例使用JSON格式，示例： 93 | 94 | ```json 95 | { 96 | "test_cases": [ 97 | { 98 | "id": "tc001", 99 | "title": "测试用例标题", 100 | "preconditions": [ 101 | "前置条件1", 102 | "前置条件2" 103 | ], 104 | "steps": [ 105 | "步骤1", 106 | "步骤2" 107 | ], 108 | "expected_results": [ 109 | "预期结果1", 110 | "预期结果2" 111 | ], 112 | "priority": "P0", 113 | "category": "功能测试", 114 | "description": "测试用例描述" 115 | } 116 | ] 117 | } 118 | ``` 119 | 120 | ### UI自动化测试结果 121 | 122 | UI自动化测试结果将导出为Excel文件，包含以下字段： 123 | - test_case_id：测试用例ID 124 | - title：测试用例标题 125 | - steps：测试步骤 126 | - expected_results：预期结果 127 | - actual_result：实际执行结果（passed/failed/warning） 128 | test_case_id title steps expected_results actual_result status execution_time 129 | TC-search-001 访问url并获取网站名称 ['浏览器访问https://www.xxx.com', '获取网站名称'] ['成功访问网站', '网站名称xxx'] Successfully accessed the website https://www.xxx.com and confirmed the website name is 'xxx' as expected. passed 2025-05-06 15:15:22 130 | 131 | ## 项目结构 132 | 133 | ``` 134 | / 135 | ├── .cache/ # 缓存目录 136 | ├── .env # 环境变量配置文件 137 | ├── agent_results/ # 代理执行结果存储目录 138 | │ ├── quality_assurance_result.json # 质量保证代理结果 139 | │ ├── requirement_analyst_result.json # 需求分析代理结果 140 | │ ├── test_case_writer_result.json # 测试用例编写代理结果 141 | │ └── test_designer_result.json # 测试设计代理结果 142 | ├── config.json # 全局配置文件 143 | ├── docs/ # 文档目录 144 | │ ├── prd.md # 产品需求文档 145 | │ ├── system_design.json # 系统设计文档 146 | │ └── 需求文档示例.pdf # 示例需求文档 147 | ├── logs/ # 日志目录 148 | ├── requirements.txt # 项目依赖 149 | ├── src/ # 源代码目录 150 | │ ├── agents/ # AutoGen框架的AI智能体 151 | │ │ ├── requirement_analyst.py # 需求分析智能体 152 | │ │ ├── test_designer.py # 测试设计智能体 153 | │ │ ├── test_case_writer.py # 测试用例编写智能体 154 | │ │ ├── quality_assurance.py # 质量保证智能体 155 | │ │ ├── browser_use_agent.py # UI自动化测试智能体 156 | │ │ └── assistant.py # 助手智能体 157 | │ ├── services/ # 核心服务 158 | │ │ ├── document_processor.py # 文档处理服务 159 | │ │ ├── test_case_generator.py # 测试用例生成服务 160 | │ │ ├── export_service.py # 导出服务 161 | │ │ └── ui_auto_service.py # UI自动化测试服务 162 | │ ├── models/ # 数据模型 163 | │ │ ├── test_case.py # 测试用例模型 164 | │ │ └── template.py # 模板模型 165 | │ ├── schemas/ # 数据结构模式 166 | │ │ └── communication.py # 通信数据结构 167 | │ ├── templates/ # 测试用例模板 168 | │ │ ├── api_test_template.json # API测试模板 169 | │ │ ├── functional_test_template.json # 功能测试模板 170 | │ │ └── ui_auto_test_template.json # UI自动化测试模板 171 | │ ├── utils/ # 工具类 172 | │ │ ├── logger.py # 日志工具 173 | │ │ ├── cli_parser.py # 命令行参数解析工具 174 | │ │ └── agent_io.py # 代理IO工具 175 | │ └── main.py # 应用程序入口 176 | └── template_config.json # 模板配置文件 177 | ``` 178 | 179 | ## 配置说明 180 | 181 | 系统通过config.json进行配置，主要配置项包括： 182 | 183 | - openai_api_key：OpenAI API密钥 184 | - log_level：日志级别（INFO、DEBUG等） 185 | - templates_dir：测试用例模板目录 186 | - output_dir：生成文件输出目录 187 | - 代理特定配置（模型、temperature等） 188 | 189 | ## 输出结果 190 | 191 | - 生成的测试用例将以Excel格式保存在指定的输出路径 192 | - 程序运行日志保存在logs目录下 193 | - 测试用例包含以下信息： 194 | - 测试用例ID 195 | - 测试场景 196 | - 前置条件 197 | - 测试步骤 198 | - 预期结果 199 | - 优先级 200 | - 备注 201 | 202 | ## 注意事项 203 | 204 | 1. UI自动化测试需要安装浏览器驱动 205 | 2. 测试用例文件必须符合指定的JSON格式 206 | 3. 建议使用虚拟环境运行项目 207 | 4. 确保已正确配置环境变量 208 | -------------------------------------------------------------------------------- /agent_results/requirement_analyst_result.json: -------------------------------------------------------------------------------- 1 | { 2 | "functional_requirements": [ 3 | "车辆进出管理：车牌识别、异常处理", 4 | "停车位引导：车位状态监测、动态导航", 5 | "收费管理：计费规则、支付方式", 6 | "用户管理：角色权限、用户注册", 7 | "数据统计与报表：实时数据、历史报表", 8 | "系统管理：设备监控、日志审计" 9 | ], 10 | "non_functional_requirements": [ 11 | "性能：支持高并发处理、响应时间要求", 12 | "安全性：数据加密、防御措施", 13 | "可靠性：高可用性、故障恢复时间", 14 | "兼容性：支持主流浏览器和移动端" 15 | ], 16 | "risk_areas": [ 17 | "车牌识别系统在恶劣天气或光线不足条件下的准确性", 18 | "高并发场景下的系统性能和稳定性", 19 | "支付过程中的数据泄露风险", 20 | "系统故障对停车场运营的潜在影响" 21 | ], 22 | "test_scenarios": [ 23 | { 24 | "id": "TS001", 25 | "description": "车辆进出管理功能测试", 26 | "test_cases": [ 27 | "测试车牌识别的准确性和响应时间", 28 | "测试异常处理流程的触发和记录" 29 | ] 30 | }, 31 | { 32 | "id": "TS002", 33 | "description": "停车位引导功能测试", 34 | "test_cases": [ 35 | "测试车位状态监测的准确性", 36 | "测试动态导航功能的有效性" 37 | ] 38 | }, 39 | { 40 | "id": "TS003", 41 | "description": "收费管理功能测试", 42 | "test_cases": [ 43 | "测试计费规则的正确性", 44 | "测试支付方式的可用性和安全性" 45 | ] 46 | }, 47 | { 48 | "id": "TS004", 49 | "description": "用户管理功能测试", 50 | "test_cases": [ 51 | "测试角色权限的正确配置", 52 | "测试用户注册流程的完整性和安全性" 53 | ] 54 | } 55 | ] 56 | } -------------------------------------------------------------------------------- /agent_results/test_designer_result.json: -------------------------------------------------------------------------------- 1 | { 2 | "test_approach": { 3 | "methodology": [ 4 | "功能测试方法：验证系统所有功能性需求的正确性和完整性，包括车辆进出管理、停车位引导、收费管理、用户管理、数据统计与报表、系统管理等模块的功能。", 5 | "性能测试方法：评估系统在高并发情况下的性能表现，确保车牌识别响应时间≤1秒，缴费流程≤5秒，并模拟500辆车同时进出的压力测试。", 6 | "安全测试方法：验证数据加密和防御措施的有效性，包括HTTPS传输、AES-256加密敏感信息，防SQL注入和DDoS攻击。", 7 | "兼容性测试方法：测试系统在主流浏览器（如Chrome、Safari、Edge）和移动端（Android/iOS）上的兼容性。", 8 | "可用性测试方法：评估系统的稳定性和故障恢复能力，确保系统可用性达到99.9%，故障恢复时间≤30分钟。" 9 | ], 10 | "tools": [ 11 | "Selenium：用于功能测试和兼容性测试。", 12 | "JMeter：用于性能测试和压力测试。", 13 | "OWASP ZAP：用于安全测试，识别和防御常见的安全威胁。", 14 | "Postman：用于接口测试，验证硬件接口和第三方支付接口的正确性。" 15 | ], 16 | "frameworks": [ 17 | "JUnit：用于Java编写的功能测试用例。", 18 | "PyTest：用于Python编写的性能测试用例。", 19 | "Cypress：用于前端界面的自动化测试。" 20 | ] 21 | }, 22 | "coverage_matrix": [ 23 | { 24 | "feature": "车辆进出管理：车牌识别、异常处理", 25 | "test_type": "功能测试、性能测试、安全测试" 26 | }, 27 | { 28 | "feature": "停车位引导：车位状态监测、动态导航", 29 | "test_type": "功能测试、性能测试" 30 | }, 31 | { 32 | "feature": "收费管理：计费规则、支付方式", 33 | "test_type": "功能测试、安全测试" 34 | }, 35 | { 36 | "feature": "用户管理：角色权限、用户注册", 37 | "test_type": "功能测试、安全测试" 38 | }, 39 | { 40 | "feature": "数据统计与报表：实时数据、历史报表", 41 | "test_type": "功能测试、性能测试" 42 | }, 43 | { 44 | "feature": "系统管理：设备监控、日志审计", 45 | "test_type": "功能测试、安全测试" 46 | }, 47 | { 48 | "feature": "性能：支持高并发处理、响应时间要求", 49 | "test_type": "性能测试" 50 | }, 51 | { 52 | "feature": "安全性：数据加密、防御措施", 53 | "test_type": "安全测试" 54 | }, 55 | { 56 | "feature": "可靠性：高可用性、故障恢复时间", 57 | "test_type": "可用性测试" 58 | }, 59 | { 60 | "feature": "兼容性：支持主流浏览器和移动端", 61 | "test_type": "兼容性测试" 62 | }, 63 | { 64 | "feature": "TS001：具体测试场景描述", 65 | "test_type": "功能测试、性能测试" 66 | }, 67 | { 68 | "feature": "车牌识别系统在恶劣天气或光线不足条件下的准确性", 69 | "test_type": "功能测试、性能测试" 70 | }, 71 | { 72 | "feature": "高并发场景下的系统性能和稳定性", 73 | "test_type": "性能测试" 74 | }, 75 | { 76 | "feature": "支付过程中的数据泄露风险", 77 | "test_type": "安全测试" 78 | }, 79 | { 80 | "feature": "系统故障对停车场运营的潜在影响", 81 | "test_type": "可用性测试" 82 | } 83 | ], 84 | "priorities": [ 85 | { 86 | "level": "P0", 87 | "description": "车辆进出管理：车牌识别、异常处理" 88 | }, 89 | { 90 | "level": "P1", 91 | "description": "收费管理：计费规则、支付方式" 92 | }, 93 | { 94 | "level": "P2", 95 | "description": "停车位引导：车位状态监测、动态导航" 96 | }, 97 | { 98 | "level": "P3", 99 | "description": "用户管理：角色权限、用户注册" 100 | }, 101 | { 102 | "level": "P4", 103 | "description": "数据统计与报表：实时数据、历史报表" 104 | } 105 | ], 106 | "resource_estimation": { 107 | "time": "4周", 108 | "personnel": "4名测试工程师，1名安全专家，1名性能测试专家", 109 | "tools": [ 110 | "Selenium", 111 | "JMeter", 112 | "OWASP ZAP", 113 | "Postman" 114 | ], 115 | "additional_resources": [ 116 | "测试服务器", 117 | "车牌识别测试设备", 118 | "模拟高并发环境的工具" 119 | ] 120 | } 121 | } -------------------------------------------------------------------------------- /docs/prd.md: -------------------------------------------------------------------------------- 1 | # AI软件测试员系统产品需求文档（PRD） 2 | 3 | ## 1. 项目信息 4 | 5 | - 项目名称：ai_test_case_generator 6 | - 编程语言：Python、React、JavaScript、Tailwind CSS 7 | - 框架：AutoGen 8 | 9 | ### 1.1 项目背景 10 | 11 | 随着软件开发规模和复杂度的不断增加，传统的人工测试用例编写方法已经难以满足快速迭代的需求。通过AI技术，特别是大语言模型和多智能体系统，我们可以显著提升测试用例生成的效率和质量。本项目旨在开发一个基于AutoGen框架的智能测试用例生成系统，帮助测试团队更高效地完成测试工作。 12 | 13 | ### 1.2 原始需求 14 | 15 | - 通过输入需求文档或文字信息进行需求分析 16 | - 使用AutoGen框架实现多智能体系统 17 | - 支持测试用例字段和格式的配置 18 | - 生成的测试用例需要通过Excel输出 19 | 20 | ## 2. 产品定义 21 | 22 | ### 2.1 产品目标 23 | 24 | 1. 提升效率：将测试用例生成时间减少70%，显著提高测试团队的工作效率 25 | 2. 保证质量：通过AI智能分析，确保测试用例覆盖率达到90%以上 26 | 3. 标准化输出：实现测试用例的标准化和结构化，提升测试文档的整体质量 27 | 28 | ### 2.2 用户故事 29 | 30 | 1. 作为一名测试工程师，我希望能够上传需求文档，系统自动分析并生成相应的测试用例，以节省手动编写测试用例的时间 31 | 2. 作为测试经理，我希望能够自定义测试用例的输出格式和字段，以适应不同项目的测试规范要求 32 | 3. 作为开发团队成员，我希望能够通过简单的文字描述获取功能测试建议，以在开发阶段提前发现潜在问题 33 | 4. 作为质量保证主管，我希望能够查看和管理已生成的测试用例，并能够进行导出和共享 34 | 35 | ### 2.3 竞品分析 36 | 37 | 1. Testim 38 | - 优势：强大的AI学习能力，支持自动化测试 39 | - 劣势：价格较高，配置复杂 40 | 41 | 2. Mabl 42 | - 优势：简单易用，界面友好 43 | - 劣势：功能相对单一，不支持深度定制 44 | 45 | 3. TestSigma 46 | - 优势：支持多平台测试，无代码自动化 47 | - 劣势：学习曲线较陡，文档不完善 48 | 49 | 4. GPTTest 50 | - 优势：基于GPT模型，自然语言理解能力强 51 | - 劣势：缺乏多智能体协作，输出格式单一 52 | 53 | 5. Functionize 54 | - 优势：AI驱动的测试自动化，维护成本低 55 | - 劣势：企业级定价，对小团队不友好 56 | 57 | ### 2.4 竞品象限图 58 | 59 | ```mermaid 60 | quadrantChart 61 | title "AI测试工具市场竞品分析" 62 | x-axis "易用性低" --> "易用性高" 63 | y-axis "功能基础" --> "功能丰富" 64 | quadrant-1 "市场领导者" 65 | quadrant-2 "功能待完善" 66 | quadrant-3 "待优化" 67 | quadrant-4 "潜力股" 68 | "Testim": [0.8, 0.9] 69 | "Mabl": [0.7, 0.6] 70 | "TestSigma": [0.5, 0.8] 71 | "GPTTest": [0.8, 0.4] 72 | "Functionize": [0.6, 0.85] 73 | "Our Product": [0.75, 0.7] 74 | ``` 75 | 76 | ## 3. 技术规格 77 | 78 | ### 3.1 需求分析 79 | 80 | #### 3.1.1 核心功能需求 81 | 1. 文档分析能力 82 | - 支持PDF、Word、Markdown等多种格式的需求文档输入 83 | - 支持直接输入文本描述 84 | - 自动提取关键功能点和测试要求 85 | 86 | 2. 多智能体系统 87 | - 基于AutoGen框架构建多个专业角色智能体 88 | - 包括需求分析师、测试设计师、质量审核员等角色 89 | - 实现智能体间的协作和交互 90 | 91 | 3. 测试用例生成 92 | - 支持功能测试、接口测试、性能测试等多种类型 93 | - 自动生成测试步骤和预期结果 94 | - 支持测试用例的优先级划分 95 | 96 | 4. 格式配置功能 97 | - 提供测试用例模板配置界面 98 | - 支持自定义字段添加和删除 99 | - 支持Excel格式的样式定制 100 | 101 | #### 3.1.2 非功能需求 102 | 1. 性能要求 103 | - 文档分析响应时间<30秒 104 | - 测试用例生成时间<60秒/100条 105 | - 支持并发处理多个项目 106 | 107 | 2. 可用性要求 108 | - 提供Web界面操作 109 | - 操作流程不超过3步 110 | - 支持批量处理 111 | 112 | 3. 安全性要求 113 | - 数据传输加密 114 | - 用户权限管理 115 | - 敏感信息脱敏 116 | 117 | ### 3.2 需求池（优先级划分） 118 | 119 | #### P0（必须实现） 120 | 1. 需求文档分析和关键信息提取 121 | 2. AutoGen多智能体系统框架搭建 122 | 3. 基础测试用例生成功能 123 | 4. Excel格式测试用例导出 124 | 5. 测试用例模板配置功能 125 | 126 | #### P1（应该实现） 127 | 1. 测试用例质量评估 128 | 2. 多种文档格式支持 129 | 3. 批量处理功能 130 | 4. 用户权限管理 131 | 5. 项目管理功能 132 | 133 | #### P2（可选实现） 134 | 1. 测试用例版本控制 135 | 2. 协作功能 136 | 3. 测试执行追踪 137 | 4. 数据统计和报表 138 | 5. API集成接口 139 | 140 | ### 3.3 UI设计草图 141 | 142 | 主界面布局将包含以下关键元素： 143 | 1. 左侧导航栏 144 | - 项目管理 145 | - 需求上传 146 | - 用例管理 147 | - 配置中心 148 | 149 | 2. 中央工作区 150 | - 文档预览/编辑区 151 | - 生成结果展示 152 | - 进度提示 153 | 154 | 3. 右侧配置面板 155 | - 模板选择 156 | - 字段配置 157 | - 导出选项 158 | 159 | ### 3.4 开放问题 160 | 161 | 1. AutoGen框架的具体智能体设计方案需要进一步细化 162 | 2. 测试用例生成的质量评估标准需要明确 163 | 3. 是否需要支持多语言测试用例生成 164 | 4. 私有化部署方案需要评估 165 | 5. 与现有测试管理工具的集成方案需要确定 166 | -------------------------------------------------------------------------------- /docs/system_design.json: -------------------------------------------------------------------------------- 1 | { 2 | "system_design": { 3 | "version": "1.0", 4 | "last_updated": "2024-02-19", 5 | "system_architecture": { 6 | "frontend": { 7 | "framework": "React + Tailwind CSS", 8 | "components": [ 9 | { 10 | "name": "项目管理模块", 11 | "features": ["项目创建", "项目列表", "项目详情"] 12 | }, 13 | { 14 | "name": "文档处理模块", 15 | "features": ["文档上传", "文档预览", "分析状态展示"] 16 | }, 17 | { 18 | "name": "测试用例管理模块", 19 | "features": ["用例列表", "用例编辑", "用例导出"] 20 | }, 21 | { 22 | "name": "配置中心模块", 23 | "features": ["模板管理", "字段配置", "导出设置"] 24 | } 25 | ] 26 | }, 27 | "backend": { 28 | "framework": "FastAPI", 29 | "components": [ 30 | { 31 | "name": "文档处理服务", 32 | "technologies": ["python-docx", "PyPDF2", "LangChain"] 33 | }, 34 | { 35 | "name": "智能体系统", 36 | "technologies": ["AutoGen", "OpenAI API"] 37 | }, 38 | { 39 | "name": "数据存储", 40 | "technologies": ["MongoDB", "Redis"] 41 | }, 42 | { 43 | "name": "导出服务", 44 | "technologies": ["openpyxl", "pandas"] 45 | } 46 | ] 47 | } 48 | }, 49 | "autogen_agents": { 50 | "agents": [ 51 | { 52 | "name": "RequirementAnalyst", 53 | "role": "需求分析师", 54 | "responsibilities": [ 55 | "解析需求文档", 56 | "提取功能点", 57 | "识别测试重点", 58 | "生成结构化需求" 59 | ], 60 | "skills": ["文档理解", "信息提取", "需求分析"] 61 | }, 62 | { 63 | "name": "TestDesigner", 64 | "role": "测试设计师", 65 | "responsibilities": [ 66 | "设计测试策略", 67 | "创建测试场景", 68 | "确定测试优先级" 69 | ], 70 | "skills": ["测试设计", "场景分析", "风险评估"] 71 | }, 72 | { 73 | "name": "TestCaseWriter", 74 | "role": "测试用例编写员", 75 | "responsibilities": [ 76 | "编写测试步骤", 77 | "设定预期结果", 78 | "生成完整用例" 79 | ], 80 | "skills": ["用例编写", "细节把控", "逻辑思维"] 81 | }, 82 | { 83 | "name": "QualityAssurance", 84 | "role": "质量保证员", 85 | "responsibilities": [ 86 | "审核测试用例", 87 | "评估覆盖率", 88 | "提供优化建议" 89 | ], 90 | "skills": ["质量控制", "测试评估", "优化分析"] 91 | }, 92 | { 93 | "name": "Assistant", 94 | "role": "协调助手", 95 | "responsibilities": [ 96 | "任务分配", 97 | "流程管理", 98 | "异常处理" 99 | ], 100 | "skills": ["任务协调", "流程控制", "问题处理"] 101 | } 102 | ], 103 | "communication_patterns": { 104 | "type": "chain", 105 | "flow": "RequirementAnalyst -> TestDesigner -> TestCaseWriter -> QualityAssurance", 106 | "coordinator": "Assistant" 107 | } 108 | }, 109 | "workflow": { 110 | "phases": [ 111 | { 112 | "name": "需求接收", 113 | "steps": [ 114 | "创建项目", 115 | "上传需求文档", 116 | "初始化配置" 117 | ] 118 | }, 119 | { 120 | "name": "需求分析", 121 | "steps": [ 122 | "文档解析", 123 | "关键信息提取", 124 | "需求结构化" 125 | ] 126 | }, 127 | { 128 | "name": "测试设计", 129 | "steps": [ 130 | "场景设计", 131 | "策略制定", 132 | "优先级划分" 133 | ] 134 | }, 135 | { 136 | "name": "用例生成", 137 | "steps": [ 138 | "用例编写", 139 | "数据填充", 140 | "格式化处理" 141 | ] 142 | }, 143 | { 144 | "name": "质量审核", 145 | "steps": [ 146 | "用例审核", 147 | "覆盖率检查", 148 | "反馈优化" 149 | ] 150 | }, 151 | { 152 | "name": "结果导出", 153 | "steps": [ 154 | "Excel生成", 155 | "格式优化", 156 | "文件导出" 157 | ] 158 | } 159 | ] 160 | }, 161 | "data_flow": { 162 | "components": [ 163 | { 164 | "name": "文档处理器", 165 | "input": ["原始文档"], 166 | "output": ["结构化文档", "关键信息"], 167 | "storage": "MongoDB.documents" 168 | }, 169 | { 170 | "name": "需求分析器", 171 | "input": ["结构化文档"], 172 | "output": ["需求分析结果", "测试重点"], 173 | "storage": "MongoDB.requirements" 174 | }, 175 | { 176 | "name": "测试设计器", 177 | "input": ["需求分析结果"], 178 | "output": ["测试场景", "测试策略"], 179 | "storage": "MongoDB.test_designs" 180 | }, 181 | { 182 | "name": "用例生成器", 183 | "input": ["测试场景", "模板配置"], 184 | "output": ["测试用例"], 185 | "storage": "MongoDB.test_cases" 186 | }, 187 | { 188 | "name": "质量控制器", 189 | "input": ["测试用例"], 190 | "output": ["审核结果", "优化建议"], 191 | "storage": "MongoDB.quality_reviews" 192 | }, 193 | { 194 | "name": "导出管理器", 195 | "input": ["测试用例", "导出配置"], 196 | "output": ["Excel文件"], 197 | "storage": "FileSystem" 198 | } 199 | ], 200 | "data_store": { 201 | "primary": "MongoDB", 202 | "cache": "Redis", 203 | "file_storage": "本地文件系统" 204 | } 205 | }, 206 | "api_interfaces": { 207 | "project_management": { 208 | "create_project": { 209 | "method": "POST", 210 | "path": "/api/projects", 211 | "params": { 212 | "name": "string", 213 | "description": "string" 214 | } 215 | }, 216 | "get_projects": { 217 | "method": "GET", 218 | "path": "/api/projects", 219 | "params": { 220 | "page": "integer", 221 | "size": "integer" 222 | } 223 | }, 224 | "get_project": { 225 | "method": "GET", 226 | "path": "/api/projects/{id}" 227 | }, 228 | "update_project": { 229 | "method": "PUT", 230 | "path": "/api/projects/{id}" 231 | }, 232 | "delete_project": { 233 | "method": "DELETE", 234 | "path": "/api/projects/{id}" 235 | } 236 | }, 237 | "document_management": { 238 | "upload_document": { 239 | "method": "POST", 240 | "path": "/api/documents/upload", 241 | "content_type": "multipart/form-data" 242 | }, 243 | "analyze_document": { 244 | "method": "POST", 245 | "path": "/api/documents/analyze/{id}" 246 | }, 247 | "get_analysis_result": { 248 | "method": "GET", 249 | "path": "/api/documents/analysis/{id}" 250 | } 251 | }, 252 | "test_case_management": { 253 | "generate_test_cases": { 254 | "method": "POST", 255 | "path": "/api/testcases/generate", 256 | "params": { 257 | "project_id": "string", 258 | "template_id": "string" 259 | } 260 | }, 261 | "get_test_cases": { 262 | "method": "GET", 263 | "path": "/api/testcases", 264 | "params": { 265 | "project_id": "string", 266 | "page": "integer", 267 | "size": "integer" 268 | } 269 | }, 270 | "update_test_case": { 271 | "method": "PUT", 272 | "path": "/api/testcases/{id}" 273 | }, 274 | "delete_test_case": { 275 | "method": "DELETE", 276 | "path": "/api/testcases/{id}" 277 | } 278 | }, 279 | "template_management": { 280 | "create_template": { 281 | "method": "POST", 282 | "path": "/api/templates" 283 | }, 284 | "get_templates": { 285 | "method": "GET", 286 | "path": "/api/templates" 287 | }, 288 | "update_template": { 289 | "method": "PUT", 290 | "path": "/api/templates/{id}" 291 | }, 292 | "delete_template": { 293 | "method": "DELETE", 294 | "path": "/api/templates/{id}" 295 | } 296 | }, 297 | "export_management": { 298 | "export_to_excel": { 299 | "method": "POST", 300 | "path": "/api/export/excel", 301 | "params": { 302 | "project_id": "string", 303 | "template_id": "string" 304 | } 305 | }, 306 | "get_export_status": { 307 | "method": "GET", 308 | "path": "/api/export/status/{id}" 309 | } 310 | } 311 | } 312 | } 313 | } -------------------------------------------------------------------------------- /docs/智能停车场需求文档.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tony3liu/Auto_Generate_Test_Cases/bcd100b08fac79262e2cb5c1ede32d95d3525e8d/docs/智能停车场需求文档.pdf -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | # Core dependencies 2 | python-dotenv>=1.0.0 3 | autogen 4 | openai>=1.0.0 # 添加 OpenAI 依赖 5 | asyncio>=3.4.3 6 | pydantic>=2.4.2 7 | fastapi>=0.104.0 8 | uvicorn>=0.23.2 9 | browser_use 10 | playwright 11 | 12 | # Document processing 13 | python-docx>=0.8.11 14 | PyPDF2>=3.0.1 15 | markdown>=3.4.3 16 | pandas>=2.1.1 17 | openpyxl>=3.1.2 18 | python-multipart>=0.0.6 19 | aiofiles>=23.2.1 20 | 21 | # Type hints and utilities 22 | typing>=3.7.4.3 23 | tenacity>=8.2.3 24 | numpy>=1.24.3 25 | matplotlib 26 | 27 | # Testing 28 | pytest>=7.4.2 29 | pytest-asyncio>=0.21.1 30 | pytest-cov>=4.1.0 31 | 32 | # Development tools 33 | black>=23.7.0 34 | isort>=5.12.0 35 | flake8>=6.1.0 -------------------------------------------------------------------------------- /run_test_designer.py: -------------------------------------------------------------------------------- 1 | # 从项目根目录运行测试设计器 2 | from src.agents.test_designer import TestDesignerAgent 3 | 4 | if __name__ == "__main__": 5 | # 创建测试设计代理 6 | designer = TestDesignerAgent() 7 | 8 | # 准备需求数据 9 | requirements = { 10 | "original_doc": "简本溯源-资质证照⼀键整理 ⼀、背景及⽬标 •背景：⽬前智爱产品缺少亮眼的、市场上独⼀份的、让⽤⼾有记忆点的功能模块；现有功能模块， 都可以在竞品中找到相似功能。交易并购、IPO往往会涉及对⽬标公司各类业务资质（最常⻅的如 营业执照，税务登记许可证、建筑业资质）的整理归纳。 •⽬标：运⽤ai识别、提取资质证照的内容，⾃动摘录成表格，并且借助溯源功能，⽅便⽤⼾快速校 对、修改。 ⼆、架构及流程 三、功能清单 序号端/系统功能模 块⼀级场景⼆级场景功能点功能说明 优先 级 1主要是 PC。h5 和⼩程序 ⽀持优先 级靠后简本溯 源⾮诉场景- 尽职调查资质证照清单 ⼀键整理⽂件上传 •⽀持pdf和图⽚（其他格 式的资质证照很少⻅，忽 略） •⽀持批量拖动⽂件或点击 批量⽂件上传 •⽀持拖动⽂件夹或点击⽂ 件夹上传（不好做就先不 做）（p2） •成功上传的⽂件有状态标 记，如⼩绿点，上传失败 有弹窗提⽰p0 2 p1 上传⽂件 修改(删 除）在开始任务前，⽀持即时删 除、补充上传⽂件，以上传的 ⽂件线上保持展⽰ 3 任务列表 •有最近任务列表的展⽰ •后台任务执⾏功能 （参考合同审查）p0 4 查看结果 •后台任务执⾏完毕后，可 点击查看整理结果（参考 合同审查） •新开标签⻚，并以在线⽂ 档形式展⽰整理结果（参 考合同审查） •⽀持多表格展⽰，每⼀份 ⽂件提取的内容对应⼀个 表格分开展⽰，⾃动分割 •点击提取内容，以原始⽂ 件的图⽚快照⽅式展现内 容对应的来源出处（⽀持 切换⽅向）p0 5 修改结果 •线上展⽰时，每⼀个表格 有单独的修改按钮，点击 后，可对该表格内容进⾏ 修改 •点击修改后，右侧⾯拉出 修改⻚⾯，修改⻚⾯分两 部分，左边是提取的表格 （表格的标题和对应的提 取内容都需要⽀持修 改），右边是原始⽂件的 在线溯源查看（pdf⽂件 要⽀持下划切换⻚⾯的功 能，图⽚要⽀持下划切换 图⽚）（⽀持切换⽅向）P0 6 下载结果 •点击下载，默认格式为 word输出，⽂件名默认 为“未命名_xx⽇年xx⽉ xx⽇”p0 批量合同智能 摘录（总结）⻅其他prd 关键事件图谱⻅其他prd 资⾦流⽔（敬 请期待）【放 上去，但是先 不开发】仅前端涉及展⽰ 历史沿⾰（敬 请期待）【放 上去，但是先 不开发】仅前端涉及展⽰ 裁判⽂书阅读 （敬请期待） 【放上去，但 是先不开发】仅前端涉及展⽰ ⾮诉场景- 公司合规（敬请期待） 【放上去，但 是先不开发】仅前端涉及展⽰ ⾮诉场景- 上市并购（敬请期待） 【放上去，但 是先不开发】仅前端涉及展⽰ 诉讼场景关键事件图谱⻅其他prd 微信聊天记录⻅其他prd 邮件沟通记录 （敬请期待）仅前端涉及展⽰ 裁判⽂书阅读 （敬请期待） 【放上去，但 是先不开发】仅前端涉及展⽰ 资⾦流⽔（敬 请期待）【放 上去，但是先 不开发】仅前端涉及展⽰ 通⽤证据分析 （敬请期待） 【放上去，但 是先不开发】仅前端涉及展⽰ 四、功能详情 1.⾸⻚展⽰ 2.上传⽂件 2.1点击⼆级场景后，切换到新的标签⻚，展⽰如下： 2.2⽂件上传动画展⽰（最多同时展⽰1 0个⽂件，⽀持下划拉动查看，上传成功 有标记打√，个别⽂件失败打x并弹窗提⽰） 3.查看（修改）整理结果+溯源 3.1点击查看后，默认效果如下： 资质证照类⽂件展⽰内容： •ai提取的字段（即上图左边的⼩标题） •ai提取字段对应的内容（即上图右边的提取内容） •每份⽂件对应⼀个表格，且表格⾃动⽣成、⾃动隔开 •线上和线下的表格样式需要保持⼀致（字体、字号、⾏间距、缩进、加粗、底纹） 3.2图⽚切⽚溯源 切⽚和提取内容的对应样式可以参考这个。 理想化是能够在提取的内容⾥的每⼀句话后⾯展⽰来源图⽚，若⼀句话由多个⽚段组合⽽成，则需展 ⽰多份照⽚，每个照⽚对应不同的数字编号，hover在数字编号上时，会⾃动展⽰图⽚。 如果⽬前不好做，直接展⽰完整⽂件（不是图⽚切⽚，⽀持下划） 3.3⽂件溯源及修改 点击修改后，右侧⾯拉出修改⻚⾯，修改⻚⾯分两部分，左边是提取的表格，右边是原始⽂件的在线 溯源查看（ 参考样式，注意原始⽂件保持在右边，不是下图的左边： •表格的标题和提取内容都⽀持修改 •修改这部分跟表格提取结果是对应的，不⽤原内容和修改后内容这种体现修改，就直接标题和提取 内容 •溯源：pdf⽂件要⽀持下划切换⻚⾯的功能，原始⽂件为图⽚的要⽀持下划切换图⽚ •⽀持溯源⽂件的⽅向转换以及放⼤缩⼩ 四、数据需求 序号数据模块数据名称数据说明 1总览 xxxxxxx xxxxxxxxxxxx 2访问数据 xxxxxxx xxxxxxx 3收⼊数据 xxxxxxx xxxxxxx 4留存数据 数据在管理后台数据中⼼的展⽰模块及统计说明 五、上线计划及清单 计划2025.1.14提测，2025.1.16上线 序号端/系统功能模块功能点 功能说明 优先级 1 H5 法律研究 xxxxxxx xxxxxxxxxxxx p0 2⼩程序⽂书起草 xxxxx xxxxxx p1 3 Web 智爱gpt xxxxxxx xxxxxxx p0 4管理后台数据中⼼ xxxxxxx xxxxxxx p0 六、其他 其他备注说明", 11 | "analysis_result": { 12 | "functional_requirements": ['支持PDF和图片格式的文件上传', '支持批量拖动文件或点击批量文件上传', '后台任务执行完毕后可以查看整理结果', '下载整理结果为Word格式输出'], 13 | "non_functional_requirements": ['上传文件后有状态标记和失败提示弹窗', '查看结果时支持多表格展示及在线文档形式展示', '通过AI识别提取资质证照内容并自动摘录成表格', '溯源功能支持在提取内容中展示来源图片'], 14 | "test_scenarios": [{'id': 'TS001', 'description': '测试文件上传功能，包括pdf和图片格式的单个及批量上传', 'test_cases': []}, {'id': 'TS002', 'description': '验证整理结果展示的正确性和多表格展示功能', 'test_cases': []}, {'id': 'TS003', 'description': '测试溯源功能中的来源图片展示是否准确', 'test_cases': []}, {'id': 'TS004', 'description': '检查下载结果的文件格式和命名是否符合要求', 'test_cases': []}], 15 | "risk_areas": ['文件上传失败可能导致用户体验不佳', 'AI识别提取的准确性可能影响整理结果的质量', '多表格展示可能存在样式不一致问题', '溯源功能的性能可能影响系统响应速度']} 16 | } 17 | 18 | # 生成测试策略 19 | test_strategy = designer.design(requirements) 20 | print(test_strategy) -------------------------------------------------------------------------------- /search_eval/__pycache__/annotation_tool.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tony3liu/Auto_Generate_Test_Cases/bcd100b08fac79262e2cb5c1ede32d95d3525e8d/search_eval/__pycache__/annotation_tool.cpython-311.pyc -------------------------------------------------------------------------------- /search_eval/__pycache__/excel_processor.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tony3liu/Auto_Generate_Test_Cases/bcd100b08fac79262e2cb5c1ede32d95d3525e8d/search_eval/__pycache__/excel_processor.cpython-311.pyc -------------------------------------------------------------------------------- /search_eval/__pycache__/excel_utils.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tony3liu/Auto_Generate_Test_Cases/bcd100b08fac79262e2cb5c1ede32d95d3525e8d/search_eval/__pycache__/excel_utils.cpython-311.pyc -------------------------------------------------------------------------------- /search_eval/__pycache__/json_utils.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tony3liu/Auto_Generate_Test_Cases/bcd100b08fac79262e2cb5c1ede32d95d3525e8d/search_eval/__pycache__/json_utils.cpython-311.pyc -------------------------------------------------------------------------------- /search_eval/__pycache__/mrr_calculator.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tony3liu/Auto_Generate_Test_Cases/bcd100b08fac79262e2cb5c1ede32d95d3525e8d/search_eval/__pycache__/mrr_calculator.cpython-311.pyc -------------------------------------------------------------------------------- /search_eval/annotation_tool.py: -------------------------------------------------------------------------------- 1 | import json 2 | import tkinter as tk 3 | from tkinter import ttk, messagebox 4 | from typing import Dict, List 5 | from pathlib import Path 6 | from json_utils import JsonUtils 7 | from datetime import datetime 8 | 9 | """ 10 | 小型评估数据集：50-100个查询 11 | 标准评估数据集：200-500个查询 12 | 大型评估数据集：1000+个查询 13 | 14 | 每个查询建议标注： 15 | - 相关文档：3-10个 16 | - 部分相关文档：5-15个 17 | - 不相关文档：10-20个 18 | 19 | 标注注意事项： 20 | - 确保查询多样性 21 | - 考虑不同难度级别 22 | - 包含边界情况 23 | """ 24 | 25 | class AnnotationTool: 26 | def __init__(self, jsonl_path: str): 27 | self.root = tk.Tk() 28 | self.root.title("搜索结果标注工具") 29 | self.root.geometry("800x600") 30 | 31 | self.jsonl_path = Path(jsonl_path) 32 | # 只加载未标注的数据 33 | self.data = [query for query in JsonUtils.load_jsonl(jsonl_path) 34 | if query.get('annotation_status') != 'completed'] 35 | 36 | if not self.data: 37 | messagebox.showinfo("提示", "没有需要标注的数据！") 38 | self.root.quit() 39 | return 40 | 41 | # 检查必要的字段 42 | required_fields = ['query_id', 'query_text', 'query_result'] 43 | for query in self.data: 44 | for field in required_fields: 45 | if field not in query: 46 | raise ValueError(f"JSONL文件缺少必要字段: {field}") 47 | 48 | # 添加relevant_docs字段（如果不存在） 49 | for query in self.data: 50 | if 'relevant_docs' not in query: 51 | query['relevant_docs'] = [] 52 | 53 | self.current_query_index = 0 54 | self._init_ui() 55 | self._load_next_query() 56 | 57 | def _init_ui(self): 58 | # 查询信息区域 59 | query_frame = ttk.LabelFrame(self.root, text="查询信息", padding="5") 60 | query_frame.pack(fill="x", padx=5, pady=5) 61 | 62 | # 查询ID 63 | ttk.Label(query_frame, text="查询ID:").pack(anchor="w") 64 | self.query_id_label = ttk.Label(query_frame, text="") 65 | self.query_id_label.pack(anchor="w") 66 | 67 | # 查询文本 68 | ttk.Label(query_frame, text="查询文本:").pack(anchor="w") 69 | self.query_text_label = ttk.Label(query_frame, text="") 70 | self.query_text_label.pack(anchor="w") 71 | 72 | # 搜索结果区域 73 | results_frame = ttk.LabelFrame(self.root, text="搜索结果", padding="5") 74 | results_frame.pack(fill="both", expand=True, padx=5, pady=5) 75 | 76 | # 创建树形视图 77 | columns = ("doc_id", "title", "content", "relevance") 78 | self.results_tree = ttk.Treeview(results_frame, columns=columns, show="headings") 79 | 80 | # 设置列标题 81 | self.results_tree.heading("doc_id", text="文档ID") 82 | self.results_tree.heading("title", text="标题") 83 | self.results_tree.heading("content", text="内容") 84 | self.results_tree.heading("relevance", text="相关性") 85 | 86 | # 设置列宽 87 | self.results_tree.column("doc_id", width=100) 88 | self.results_tree.column("title", width=200) 89 | self.results_tree.column("content", width=300) 90 | self.results_tree.column("relevance", width=100) 91 | 92 | # 添加滚动条 93 | scrollbar = ttk.Scrollbar(results_frame, orient="vertical", command=self.results_tree.yview) 94 | self.results_tree.configure(yscrollcommand=scrollbar.set) 95 | 96 | # 放置树形视图和滚动条 97 | self.results_tree.pack(side="left", fill="both", expand=True) 98 | scrollbar.pack(side="right", fill="y") 99 | 100 | # 添加相关性选择区域 101 | relevance_frame = ttk.LabelFrame(self.root, text="相关性标注", padding="5") 102 | relevance_frame.pack(fill="x", padx=5, pady=5) 103 | 104 | ttk.Label(relevance_frame, text="相关性说明：").pack(anchor="w") 105 | ttk.Label(relevance_frame, text="2 - 完全相关").pack(anchor="w") 106 | ttk.Label(relevance_frame, text="1 - 部分相关").pack(anchor="w") 107 | ttk.Label(relevance_frame, text="0 - 不相关").pack(anchor="w") 108 | 109 | # 控制区域 110 | control_frame = ttk.Frame(self.root) 111 | control_frame.pack(fill="x", padx=5, pady=5) 112 | 113 | ttk.Button(control_frame, text="保存标注", command=self._save_annotation).pack(side="left", padx=5) 114 | ttk.Button(control_frame, text="下一个查询", command=self._load_next_query).pack(side="left", padx=5) 115 | 116 | # 添加进度显示 117 | self.progress_label = ttk.Label(control_frame, text="") 118 | self.progress_label.pack(side="right", padx=5) 119 | 120 | # 绑定双击事件 121 | self.results_tree.bind('', self._on_double_click) 122 | 123 | def _on_double_click(self, event): 124 | """处理双击事件，用于修改相关性分数""" 125 | item = self.results_tree.selection()[0] 126 | current_values = self.results_tree.item(item)["values"] 127 | current_score = int(current_values[3]) 128 | 129 | # 循环切换相关性分数：0 -> 1 -> 2 -> 0 130 | new_score = (current_score + 1) % 3 131 | self.results_tree.item(item, values=(*current_values[:3], str(new_score))) 132 | 133 | def _load_next_query(self): 134 | """加载下一个待标注的查询""" 135 | if self.current_query_index < len(self.data): 136 | query = self.data[self.current_query_index] 137 | self.query_id_label.config(text=query['query_id']) 138 | self.query_text_label.config(text=query['query_text']) 139 | 140 | # 更新进度显示 141 | self.progress_label.config(text=f"进度: {self.current_query_index + 1}/{len(self.data)}") 142 | 143 | # 清空并加载搜索结果 144 | for item in self.results_tree.get_children(): 145 | self.results_tree.delete(item) 146 | 147 | try: 148 | search_results = query['query_result'] 149 | for result in search_results: 150 | # 检查是否已有标注 151 | relevance_score = '0' 152 | for doc in query.get('relevant_docs', []): 153 | if doc['doc_id'] == result['doc_id']: 154 | relevance_score = str(doc['relevance_score']) 155 | break 156 | 157 | self.results_tree.insert("", "end", values=( 158 | result.get('doc_id', ''), 159 | result.get('title', ''), 160 | result.get('content', ''), 161 | relevance_score 162 | )) 163 | except Exception as e: 164 | messagebox.showerror("错误", f"查询结果格式错误: {query['query_id']}") 165 | return 166 | else: 167 | messagebox.showinfo("完成", "所有查询都已标注完成！") 168 | self.root.quit() 169 | 170 | def _save_annotation(self): 171 | """保存当前查询的标注结果""" 172 | if self.current_query_index >= len(self.data): 173 | return 174 | 175 | query = self.data[self.current_query_index] 176 | relevant_docs = [] 177 | for item in self.results_tree.get_children(): 178 | values = self.results_tree.item(item)["values"] 179 | if values[3] != '0': # 只保存有相关性分数的文档 180 | relevant_docs.append({ 181 | "doc_id": values[0], 182 | "title": values[1], 183 | "relevance_score": int(values[3]) 184 | }) 185 | 186 | # 读取完整的数据集 187 | all_data = JsonUtils.load_jsonl(self.jsonl_path) 188 | 189 | # 更新当前查询的标注结果 190 | for item in all_data: 191 | if item['query_id'] == query['query_id']: 192 | item['relevant_docs'] = relevant_docs 193 | item['annotation_status'] = 'completed' 194 | item['updated_at'] = datetime.now().strftime('%Y-%m-%d %H:%M:%S') 195 | break 196 | 197 | # 保存更新后的完整数据集 198 | JsonUtils.save_jsonl(all_data, self.jsonl_path) 199 | 200 | messagebox.showinfo("成功", "标注已保存") 201 | 202 | # 移动到下一个查询 203 | self.current_query_index += 1 204 | self._load_next_query() 205 | 206 | def run(self): 207 | self.root.mainloop() 208 | 209 | def main(): 210 | # 使用示例 211 | jsonl_path = 'search_eval/search_evaluation.jsonl' 212 | tool = AnnotationTool(jsonl_path) 213 | tool.run() 214 | 215 | if __name__ == "__main__": 216 | main() -------------------------------------------------------------------------------- /search_eval/dataset_evaluator.py: -------------------------------------------------------------------------------- 1 | import json 2 | from typing import Dict, List, Optional, Tuple, Union 3 | from pathlib import Path 4 | from json_utils import JsonUtils 5 | from annotation_tool import AnnotationTool 6 | import numpy as np 7 | import matplotlib 8 | matplotlib.use('Agg') 9 | import matplotlib.pyplot as plt 10 | import os 11 | import requests 12 | 13 | class SearchEvaluator: 14 | def __init__(self, jsonl_path: str): 15 | self.jsonl_path = Path(jsonl_path) 16 | self.output_dir = Path('evaluation_results') 17 | self.output_dir.mkdir(exist_ok=True) 18 | 19 | def evaluate(self, 20 | k_values: List[int] = [1, 3, 5, 10], 21 | relevance_threshold: int = 1, 22 | output_path: Optional[str] = None, 23 | offline_mode: bool = False) -> Dict: 24 | """ 25 | 评估搜索系统性能 26 | 27 | Args: 28 | k_values: 评估的K值列表 29 | relevance_threshold: 判定为相关的最小相关性分数 30 | output_path: 评估结果保存路径 31 | offline_mode: 是否使用离线模式（使用已有搜索结果） 32 | 33 | Returns: 34 | Dict: 评估结果 35 | """ 36 | data = JsonUtils.load_jsonl(self.jsonl_path) 37 | needs_annotation = any('relevant_docs' not in query for query in data) 38 | if needs_annotation: 39 | print("启动标注工具进行标注...") 40 | annotation_tool = AnnotationTool(str(self.jsonl_path)) 41 | annotation_tool.run() 42 | data = JsonUtils.load_jsonl(self.jsonl_path) 43 | else: 44 | print("所有数据已完成标注，跳过标注步骤") 45 | 46 | # 准备评估数据 47 | queries = [] 48 | for query in data: 49 | query_result = query.get('query_result', []) 50 | if isinstance(query_result, str): 51 | try: 52 | query_result = json.loads(query_result) 53 | except json.JSONDecodeError: 54 | print(f"警告：查询 {query['query_id']} 的结果格式无效，跳过") 55 | continue 56 | 57 | relevant_docs = query.get('relevant_docs', []) 58 | if isinstance(relevant_docs, str): 59 | try: 60 | relevant_docs = json.loads(relevant_docs) 61 | except json.JSONDecodeError: 62 | print(f"警告：查询 {query['query_id']} 的相关文档格式无效，跳过") 63 | continue 64 | 65 | query_data = { 66 | 'query': query['query_text'], 67 | 'query_id': query['query_id'], 68 | 'search_results': query_result, 69 | 'relevant_docs': relevant_docs 70 | } 71 | queries.append(query_data) 72 | 73 | # 计算评估指标 74 | metrics = self._calculate_metrics(queries, k_values, relevance_threshold) 75 | 76 | results = { 77 | 'evaluation_params': { 78 | 'relevance_threshold': relevance_threshold, 79 | 'k_values': k_values, 80 | 'mode': 'offline' if offline_mode else 'online' 81 | }, 82 | 'metrics': metrics 83 | } 84 | 85 | if output_path: 86 | self._save_results(results, output_path) 87 | 88 | return results 89 | 90 | def _calculate_metrics(self, queries: List[Dict], k_values: List[int], relevance_threshold: int) -> Dict: 91 | """计算评估指标""" 92 | metrics = { 93 | 'map': 0.0, 94 | 'mrr': 0.0 95 | } 96 | 97 | # 初始化K值相关指标 98 | for k in k_values: 99 | metrics[f'precision@{k}'] = 0.0 100 | metrics[f'recall@{k}'] = 0.0 101 | metrics[f'f1@{k}'] = 0.0 102 | metrics[f'hit_rate@{k}'] = 0.0 # 添加hit rate@k 103 | 104 | total_queries = len(queries) 105 | if total_queries == 0: 106 | return metrics 107 | 108 | all_precision_points = [] 109 | all_recall_points = [] 110 | 111 | for query_data in queries: 112 | search_results = query_data['search_results'] 113 | relevant_docs = query_data['relevant_docs'] 114 | 115 | # 获取相关文档ID列表 116 | relevant_doc_ids = [doc['doc_id'] for doc in relevant_docs if doc['relevance_score'] >= relevance_threshold] 117 | 118 | # 计算MRR 119 | mrr = self._calculate_mrr(search_results, relevant_doc_ids) 120 | metrics['mrr'] += mrr 121 | 122 | # 计算AP并累加到MAP 123 | ap = self._calculate_average_precision(search_results, relevant_doc_ids) 124 | metrics['map'] += ap 125 | 126 | # 计算不同k值的指标 127 | for k in k_values: 128 | precision, recall, f1 = self._calculate_precision_recall_f1( 129 | search_results, relevant_doc_ids, k=k) 130 | metrics[f'precision@{k}'] += precision 131 | metrics[f'recall@{k}'] += recall 132 | metrics[f'f1@{k}'] += f1 133 | 134 | # 计算hit rate@k 135 | hit_rate = self._calculate_hit_rate(search_results, relevant_doc_ids, k=k) 136 | metrics[f'hit_rate@{k}'] += hit_rate 137 | 138 | # 计算P-R曲线点 139 | precision_points, recall_points = self._calculate_pr_curve_points( 140 | search_results, relevant_doc_ids) 141 | all_precision_points.append(precision_points) 142 | all_recall_points.append(recall_points) 143 | 144 | # 计算平均值 145 | for metric in metrics: 146 | metrics[metric] /= total_queries 147 | 148 | # 绘制平均P-R曲线 149 | if all_precision_points and all_recall_points: 150 | try: 151 | avg_precision_points = np.mean(np.array(all_precision_points), axis=0) 152 | avg_recall_points = np.mean(np.array(all_recall_points), axis=0) 153 | pr_curve_path = self.output_dir / 'pr_curve.png' 154 | self._plot_pr_curve(avg_precision_points.tolist(), avg_recall_points.tolist(), 155 | "Average Precision-Recall Curve", str(pr_curve_path)) 156 | print(f"\nP-R曲线已保存至: {pr_curve_path}") 157 | except Exception as e: 158 | print(f"\n警告：无法生成P-R曲线 - {str(e)}") 159 | 160 | return metrics 161 | 162 | def _calculate_mrr(self, search_results: List[Dict], 163 | relevant_doc_ids: List[str], 164 | doc_id_field: str = 'doc_id') -> float: 165 | """计算MRR值""" 166 | if not search_results or not relevant_doc_ids: 167 | return 0.0 168 | 169 | for rank, result in enumerate(search_results, 1): 170 | doc_id = result.get(doc_id_field) 171 | if doc_id in relevant_doc_ids: 172 | return 1.0 / rank 173 | 174 | return 0.0 175 | 176 | def _calculate_precision_recall_f1(self, search_results: List[Dict], 177 | relevant_doc_ids: List[str], 178 | doc_id_field: str = 'doc_id', 179 | k: Optional[int] = None) -> Tuple[float, float, float]: 180 | """计算Precision@K, Recall@K和F1 Score""" 181 | if not search_results or not relevant_doc_ids: 182 | return 0.0, 0.0, 0.0 183 | 184 | if k is not None: 185 | search_results = search_results[:k] 186 | 187 | retrieved_relevant = sum(1 for result in search_results 188 | if result.get(doc_id_field) in relevant_doc_ids) 189 | 190 | precision = retrieved_relevant / len(search_results) if search_results else 0.0 191 | recall = retrieved_relevant / len(relevant_doc_ids) if relevant_doc_ids else 0.0 192 | f1 = 2 * (precision * recall) / (precision + recall) if (precision + recall) > 0 else 0.0 193 | 194 | return precision, recall, f1 195 | 196 | def _calculate_hit_rate(self, search_results: List[Dict], 197 | relevant_doc_ids: List[str], 198 | doc_id_field: str = 'doc_id', 199 | k: Optional[int] = None) -> float: 200 | """计算命中率""" 201 | if not search_results or not relevant_doc_ids: 202 | return 0.0 203 | 204 | if k is not None: 205 | search_results = search_results[:k] 206 | 207 | for result in search_results: 208 | if result.get(doc_id_field) in relevant_doc_ids: 209 | return 1.0 210 | return 0.0 211 | 212 | def _calculate_pr_curve_points(self, search_results: List[Dict], 213 | relevant_doc_ids: List[str], 214 | doc_id_field: str = 'doc_id') -> Tuple[List[float], List[float]]: 215 | """计算P-R曲线的点""" 216 | precision_points = [] 217 | recall_points = [] 218 | 219 | for k in range(1, len(search_results) + 1): 220 | precision, recall, _ = self._calculate_precision_recall_f1( 221 | search_results, relevant_doc_ids, doc_id_field, k) 222 | precision_points.append(precision) 223 | recall_points.append(recall) 224 | 225 | return precision_points, recall_points 226 | 227 | def _plot_pr_curve(self, precision_points: List[float], 228 | recall_points: List[float], 229 | title: str = "Precision-Recall Curve", 230 | save_path: Optional[str] = None) -> None: 231 | """绘制P-R曲线""" 232 | try: 233 | plt.figure(figsize=(10, 6)) 234 | plt.plot(recall_points, precision_points, 'b-', label='P-R Curve') 235 | plt.xlabel('Recall') 236 | plt.ylabel('Precision') 237 | plt.title(title) 238 | plt.grid(True) 239 | plt.legend() 240 | 241 | if save_path: 242 | plt.savefig(save_path) 243 | plt.close() 244 | else: 245 | plt.show() 246 | except Exception as e: 247 | print(f"警告：无法绘制P-R曲线 - {str(e)}") 248 | plt.close() 249 | 250 | def _calculate_average_precision(self, search_results: List[Dict], 251 | relevant_doc_ids: List[str], 252 | doc_id_field: str = 'doc_id') -> float: 253 | """计算单个查询的Average Precision (AP)""" 254 | if not search_results or not relevant_doc_ids: 255 | return 0.0 256 | 257 | relevant_count = 0 258 | sum_precision = 0.0 259 | 260 | for rank, result in enumerate(search_results, 1): 261 | doc_id = result.get(doc_id_field) 262 | if doc_id in relevant_doc_ids: 263 | relevant_count += 1 264 | # 计算当前位置的precision 265 | precision = relevant_count / rank 266 | sum_precision += precision 267 | 268 | # AP = 所有相关文档位置的precision之和 / 相关文档总数 269 | ap = sum_precision / len(relevant_doc_ids) if relevant_doc_ids else 0.0 270 | return ap 271 | 272 | def _save_results(self, results: Dict, output_path: str): 273 | """保存评估结果""" 274 | JsonUtils.save_json(results, output_path) 275 | 276 | def main(input_path: str): 277 | print("欢迎使用搜索评估工具") 278 | print("-" * 50) 279 | 280 | # 获取JSONL文件路径 281 | while True: 282 | jsonl_path = input_path.strip() 283 | if not jsonl_path: 284 | print("路径不能为空，请重新输入") 285 | continue 286 | 287 | # 检查文件是否存在 288 | if not Path(jsonl_path).exists(): 289 | print(f"错误：文件 '{jsonl_path}' 不存在") 290 | continue 291 | 292 | # 检查文件扩展名 293 | if not jsonl_path.endswith('.jsonl'): 294 | print("错误：文件必须是JSONL格式（.jsonl）") 295 | continue 296 | 297 | try: 298 | # 尝试读取JSONL文件 299 | data = JsonUtils.load_jsonl(jsonl_path) 300 | 301 | # 检查必要的字段是否存在 302 | required_fields = ['query_id', 'query_text', 'query_result'] 303 | missing_fields = [] 304 | for query in data: 305 | for field in required_fields: 306 | if field not in query: 307 | missing_fields.append(field) 308 | if missing_fields: 309 | break 310 | if missing_fields: 311 | print(f"错误：JSONL文件缺少必要字段: {', '.join(missing_fields)}") 312 | continue 313 | 314 | break 315 | except Exception as e: 316 | print(f"错误：无法读取JSONL文件 - {str(e)}") 317 | print("请确保：") 318 | print("1. 文件是有效的JSONL文件") 319 | print("2. 文件没有被其他程序占用") 320 | print("3. 文件包含必要的字段：query_id, query_text, query_result") 321 | continue 322 | 323 | try: 324 | # 创建评估器 325 | evaluator = SearchEvaluator(jsonl_path) 326 | 327 | # 选择评估模式 328 | while True: 329 | print("\n请选择评估模式：") 330 | print("1. 在线评估模式（需要API获取查询结果）") 331 | print("2. 离线评估模式（使用已有查询结果）") 332 | mode = input("请输入选项（1或2）: ").strip() 333 | 334 | if mode not in ['1', '2']: 335 | print("无效的选项，请重新输入") 336 | continue 337 | 338 | offline_mode = mode == '2' 339 | break 340 | 341 | # 设置评估参数 342 | k_values = [1, 3, 5, 10] # 默认K值 343 | relevance_threshold = 1 # 默认相关性阈值 344 | 345 | # 在线模式 346 | if not offline_mode: 347 | api_url = "http://k8s-platform-cloudswa-fe77b476e2-b6ef1b9e2e540b4a/api/v1/intelligent/search" 348 | headers = {"Content-Type": "application/json", "X-TOKEN": "1"} 349 | 350 | # 获取没有检索结果的查询 351 | queries_without_results = [query for query in data if 'query_result' not in query or not query['query_result']] 352 | if queries_without_results: 353 | print(f"\n发现 {len(queries_without_results)} 个没有检索结果的查询，开始获取检索结果...") 354 | 355 | for query in queries_without_results: 356 | try: 357 | # 发送API请求 358 | request_data = { 359 | "q": query["query_text"], 360 | "count": 10, 361 | "offset": 0 362 | } 363 | response = requests.post(api_url, headers=headers, json=request_data) 364 | result = response.json() 365 | 366 | # 解析响应结果并适配到数据集格式 367 | if "webPages" in result and "value" in result["webPages"]: 368 | query_result = [] 369 | for item in result["webPages"]["value"]: 370 | query_result.append({ 371 | "doc_id": item.get("id", ""), 372 | "title": item.get("name", ""), 373 | "content": item.get("snippet", "") 374 | }) 375 | query["query_result"] = query_result 376 | print(f"成功获取查询 '{query['query_id']}' 的检索结果") 377 | else: 378 | print(f"警告：查询 '{query['query_id']}' 的响应格式不符合预期") 379 | except Exception as e: 380 | print(f"处理查询 '{query['query_id']}' 时出错: {str(e)}") 381 | 382 | # 保存更新后的数据 383 | JsonUtils.save_jsonl(data, jsonl_path) 384 | print("\n检索结果已更新到数据集") 385 | else: 386 | print("\n所有查询都有检索结果") 387 | 388 | # 设置输出路径 389 | output_path = './evaluation_results/evaluation_results.json' 390 | 391 | # 检查输出目录是否存在 392 | output_dir = Path(output_path).parent 393 | if not output_dir.exists(): 394 | try: 395 | output_dir.mkdir(parents=True) 396 | except Exception as e: 397 | print(f"错误：无法创建输出目录 - {str(e)}") 398 | return 399 | 400 | print("\n开始评估...") 401 | # 执行评估 402 | results = evaluator.evaluate( 403 | k_values=k_values, 404 | relevance_threshold=relevance_threshold, 405 | output_path=output_path, 406 | offline_mode=offline_mode 407 | ) 408 | 409 | # 打印评估结果 410 | print("\n评估完成！") 411 | print("-" * 50) 412 | print(f"评估模式: {'离线' if offline_mode else '在线'}") 413 | print("\n主要评估指标:") 414 | print(f"MAP: {results['metrics']['map']:.4f}") 415 | print(f"MRR: {results['metrics']['mrr']:.4f}") 416 | 417 | # 打印不同K值的指标 418 | print("\n不同K值的评估指标:") 419 | for k in k_values: 420 | print(f"\nK = {k}:") 421 | print(f"Precision@{k}: {results['metrics'][f'precision@{k}']:.4f}") 422 | print(f"Recall@{k}: {results['metrics'][f'recall@{k}']:.4f}") 423 | print(f"F1@{k}: {results['metrics'][f'f1@{k}']:.4f}") 424 | print(f"Hit Rate@{k}: {results['metrics'][f'hit_rate@{k}']:.4f}") 425 | 426 | print("\n详细评估结果已保存至:", output_path) 427 | 428 | except Exception as e: 429 | print(f"\n评估过程中出现错误: {str(e)}") 430 | return 431 | 432 | if __name__ == "__main__": 433 | input_path = 'search_eval/search_evaluation.jsonl' 434 | main(input_path) -------------------------------------------------------------------------------- /search_eval/json_utils.py: -------------------------------------------------------------------------------- 1 | import json 2 | from pathlib import Path 3 | from typing import List, Dict, Any, Optional 4 | import requests 5 | from datetime import datetime 6 | 7 | class JsonUtils: 8 | @staticmethod 9 | def load_jsonl(file_path: str) -> List[Dict[str, Any]]: 10 | """ 11 | 加载JSONL文件 12 | 13 | Args: 14 | file_path: JSONL文件路径 15 | 16 | Returns: 17 | List[Dict[str, Any]]: 加载的数据列表 18 | """ 19 | try: 20 | with open(file_path, 'r', encoding='utf-8') as f: 21 | return [json.loads(line) for line in f if line.strip()] 22 | except Exception as e: 23 | raise ValueError(f"无法读取JSONL文件: {str(e)}") 24 | 25 | @staticmethod 26 | def save_jsonl(data: List[Dict[str, Any]], file_path: str): 27 | """ 28 | 保存数据到JSONL文件 29 | 30 | Args: 31 | data: 要保存的数据列表 32 | file_path: 保存路径 33 | """ 34 | try: 35 | with open(file_path, 'w', encoding='utf-8') as f: 36 | for item in data: 37 | f.write(json.dumps(item, ensure_ascii=False) + '\n') 38 | except Exception as e: 39 | raise ValueError(f"保存JSONL文件失败: {str(e)}") 40 | 41 | @staticmethod 42 | def load_json(file_path: str) -> Dict[str, Any]: 43 | """ 44 | 加载JSON文件 45 | 46 | Args: 47 | file_path: JSON文件路径 48 | 49 | Returns: 50 | Dict[str, Any]: 加载的数据 51 | """ 52 | try: 53 | with open(file_path, 'r', encoding='utf-8') as f: 54 | return json.load(f) 55 | except Exception as e: 56 | raise ValueError(f"无法读取JSON文件: {str(e)}") 57 | 58 | @staticmethod 59 | def save_json(data: Dict[str, Any], file_path: str): 60 | """ 61 | 保存数据到JSON文件 62 | 63 | Args: 64 | data: 要保存的数据 65 | file_path: 保存路径 66 | """ 67 | try: 68 | with open(file_path, 'w', encoding='utf-8') as f: 69 | json.dump(data, f, ensure_ascii=False, indent=2) 70 | except Exception as e: 71 | raise ValueError(f"保存JSON文件失败: {str(e)}") 72 | 73 | @staticmethod 74 | def load_or_create_jsonl(jsonl_path: str) -> List[Dict]: 75 | """ 76 | 加载或创建JSONL文件 77 | 78 | Args: 79 | jsonl_path: JSONL文件路径 80 | 81 | Returns: 82 | List[Dict]: 数据列表 83 | """ 84 | try: 85 | path = Path(jsonl_path) 86 | if path.exists(): 87 | return JsonUtils.load_jsonl(jsonl_path) 88 | else: 89 | # 创建新的JSONL文件 90 | data = [] 91 | JsonUtils.save_jsonl(data, jsonl_path) 92 | return data 93 | except Exception as e: 94 | raise ValueError(f"无法处理JSONL文件: {str(e)}") 95 | 96 | @staticmethod 97 | def add_queries(jsonl_path: str, queries: List[Dict[str, str]]): 98 | """ 99 | 添加新的查询 100 | 101 | Args: 102 | jsonl_path: JSONL文件路径 103 | queries: 查询列表，每个查询包含query_id和query_text 104 | """ 105 | data = JsonUtils.load_jsonl(jsonl_path) 106 | for query in queries: 107 | new_query = { 108 | 'query_id': query['query_id'], 109 | 'query_text': query['query_text'], 110 | 'query_result': [], 111 | 'annotation_status': 'pending', 112 | 'relevant_docs': [], 113 | 'created_at': datetime.now().strftime('%Y-%m-%d %H:%M:%S'), 114 | 'updated_at': datetime.now().strftime('%Y-%m-%d %H:%M:%S') 115 | } 116 | data.append(new_query) 117 | 118 | # 保存更改 119 | JsonUtils.save_jsonl(data, jsonl_path) 120 | 121 | 122 | @staticmethod 123 | def update_annotation(jsonl_path: str, query_id: str, relevant_docs: List[Dict]): 124 | """ 125 | 更新标注结果 126 | 127 | Args: 128 | jsonl_path: JSONL文件路径 129 | query_id: 查询ID 130 | relevant_docs: 相关文档列表 131 | """ 132 | try: 133 | data = JsonUtils.load_jsonl(jsonl_path) 134 | # 查找对应的查询 135 | for query in data: 136 | if query['query_id'] == query_id: 137 | query['relevant_docs'] = relevant_docs 138 | JsonUtils.save_jsonl(data, jsonl_path) 139 | return 140 | 141 | raise ValueError(f"未找到查询ID: {query_id}") 142 | except Exception as e: 143 | raise ValueError(f"更新标注结果失败: {str(e)}") 144 | 145 | @staticmethod 146 | def get_pending_queries(jsonl_path: str) -> List[Dict]: 147 | """ 148 | 获取待处理的查询 149 | 150 | Args: 151 | jsonl_path: JSONL文件路径 152 | 153 | Returns: 154 | List[Dict]: 待处理的查询列表 155 | """ 156 | data = JsonUtils.load_jsonl(jsonl_path) 157 | return [query for query in data if query['annotation_status'] == 'pending'] 158 | 159 | @staticmethod 160 | def get_completed_queries(jsonl_path: str) -> List[Dict]: 161 | """ 162 | 获取已完成的查询 163 | 164 | Args: 165 | jsonl_path: JSONL文件路径 166 | 167 | Returns: 168 | List[Dict]: 已完成的查询列表 169 | """ 170 | data = JsonUtils.load_jsonl(jsonl_path) 171 | return [query for query in data if query['annotation_status'] == 'completed'] 172 | 173 | @staticmethod 174 | def export_dataset(jsonl_path: str, output_path: str): 175 | """ 176 | 导出数据集 177 | 178 | Args: 179 | jsonl_path: 源JSONL文件路径 180 | output_path: 输出文件路径 181 | """ 182 | try: 183 | data = JsonUtils.load_jsonl(jsonl_path) 184 | JsonUtils.save_jsonl(data, output_path) 185 | except Exception as e: 186 | raise ValueError(f"导出数据集失败: {str(e)}") 187 | 188 | def main(): 189 | # 使用示例 190 | jsonl_path = 'search_eval/search_evaluation.jsonl' 191 | 192 | # 添加示例查询 193 | queries = [ 194 | {'query_id': 'q1', 'query_text': 'Python异常处理最佳实践'}, 195 | {'query_id': 'q2', 'query_text': '机器学习模型评估方法'} 196 | ] 197 | JsonUtils.add_queries(jsonl_path, queries) 198 | 199 | # 搜索查询 200 | api_url = "YOUR_API_URL_HERE" 201 | request_body_template = { 202 | "query": "", 203 | # 其他API参数 204 | } 205 | JsonUtils.search_queries(jsonl_path, api_url, request_body_template) 206 | 207 | # 更新标注 208 | relevant_docs = [ 209 | { 210 | "doc_id": "doc1", 211 | "title": "Python异常处理完全指南", 212 | "relevance_score": 2 213 | } 214 | ] 215 | JsonUtils.update_annotation(jsonl_path, 'q1', relevant_docs) 216 | 217 | # 导出数据集 218 | JsonUtils.export_dataset(jsonl_path, 'search_eval/annotated_dataset.jsonl') 219 | 220 | if __name__ == "__main__": 221 | main() -------------------------------------------------------------------------------- /search_eval/search_evaluation.jsonl: -------------------------------------------------------------------------------- 1 | {"query_id": "q3", "query_text": "劳动手册和劳动法冲突怎么办？", "query_result": [{"doc_id": "https://api.bing.microsoft.com/api/v7/#WebPages.0", "title": "俄罗斯联邦劳动法典", "content": "在职工因患病、残疾、年老退休、进高等或中等专业学校、进修或根据法律可获得一定优惠和好处等理由提出要求解除劳动合同（契约）的，劳动手册中有关解雇的记载要说明这些理由 ..."}, {"doc_id": "https://api.bing.microsoft.com/api/v7/#WebPages.1", "title": "国际劳动法| Seyfarth Shaw LLP", "content": "合同、政策及程序：员工手册、聘书、劳动合同，以及工作手册、公司规章制度等。 裁员：包括个别裁员及大型跨国裁员计划。 外包管理：基于单一法域或者跨法域的大、小型 ..."}, {"doc_id": "https://api.bing.microsoft.com/api/v7/#WebPages.2", "title": "Reform Suggestions on Sample Labor Contracts in China", "content": "Apr 13, 2006 ... 条款只要不与《劳动合同法》的强制规定相冲突，则仍旧. 可以广泛运用 ... 《劳动合同和职工福利法律手册》，主编张晓，法律出版社，1995 年. 9 ..."}, {"doc_id": "https://api.bing.microsoft.com/api/v7/#WebPages.3", "title": "劳动法| HFGIP", "content": "... 冲突避免员工不当行为（骚扰、贿赂等） HFG的主要业务起草/修改劳动合同起草/修改公司劳动手册和政策起草/修改研发人员/发明所有权内部公司政策（保密、利益冲突、内部 ..."}, {"doc_id": "https://api.bing.microsoft.com/api/v7/#WebPages.4", "title": "用人单位知情权和劳动者隐私权的冲突与平衡| China Law Insight", "content": "Aug 1, 2014 ... 要解决用人单位知情权与劳动者隐私权的冲突，则应当对用人单位的知情权做 ... 我国《劳动法》第12条规定：“劳动者就业，不因民族、种族、性别 ..."}, {"doc_id": "https://api.bing.microsoft.com/api/v7/#WebPages.5", "title": "劳动与雇佣- 京都律师事务所", "content": "团队主张“提供可操作性的系统解决方案”，提倡“法律风险的提前防控”，致力于帮助用人单位与劳动者减少冲突、解决争议、构建和谐的劳资关系。 ... 首都机场集团公司法律顾问（劳动 ..."}, {"doc_id": "https://api.bing.microsoft.com/api/v7/#WebPages.6", "title": "盛美半导体设备（上海）有限公司", "content": "对于本合同的终止和解除的情况, 公司将根据《劳动法》、 《劳动合同法》和《员工手册》的相关规定处理。 ... 如果双方自劳动争议发生的30天内无法解决劳动争议, 双方 ..."}, {"doc_id": "https://api.bing.microsoft.com/api/v7/#WebPages.7", "title": "劳动者就医权与用人单位指定医院管理权的冲突与平衡- 律新社", "content": "《员工手册》的上述规定属于用人单位对长期病假人员的合理管理，并不违反法律规定。规章制度是用人单位组织劳动过程、进行劳动管理中不可或缺的环节，用人单位在合法合规的 ..."}, {"doc_id": "https://api.bing.microsoft.com/api/v7/#WebPages.8", "title": "政策手册", "content": "Apr 15, 2021 ... 联邦法律严格禁止子受方、计划管理者、承包商、计划工作人员和其他方面之间产. 生利益冲突。 “相关人”指的是州或一般当地政府单位或任何指定公共机构的 ..."}, {"doc_id": "https://api.bing.microsoft.com/api/v7/#WebPages.9", "title": "劳动法", "content": "... 劳动法冲突等提供法律意见和解决方案。 主要法律服务范围. 担任各类企业的劳动法律 ... 公司各类规章制度，员工手册的制定、审核，包括商业秘密保护制度，竞业限制 ..."}], "relevant_docs": [{"doc_id": "https://api.bing.microsoft.com/api/v7/#WebPages.0", "title": "俄罗斯联邦劳动法典", "relevance_score": 0}, {"doc_id": "https://api.bing.microsoft.com/api/v7/#WebPages.1", "title": "国际劳动法| Seyfarth Shaw LLP", "relevance_score": 0}, {"doc_id": "https://api.bing.microsoft.com/api/v7/#WebPages.2", "title": "Reform Suggestions on Sample Labor Contracts in China", "relevance_score": 0}, {"doc_id": "https://api.bing.microsoft.com/api/v7/#WebPages.3", "title": "劳动法| HFGIP", "relevance_score": 1}, {"doc_id": "https://api.bing.microsoft.com/api/v7/#WebPages.4", "title": "用人单位知情权和劳动者隐私权的冲突与平衡| China Law Insight", "relevance_score": 0}, {"doc_id": "https://api.bing.microsoft.com/api/v7/#WebPages.5", "title": "劳动与雇佣- 京都律师事务所", "relevance_score": 0}, {"doc_id": "https://api.bing.microsoft.com/api/v7/#WebPages.6", "title": "盛美半导体设备（上海）有限公司", "relevance_score": 0}, {"doc_id": "https://api.bing.microsoft.com/api/v7/#WebPages.7", "title": "劳动者就医权与用人单位指定医院管理权的冲突与平衡- 律新社", "relevance_score": 0}, {"doc_id": "https://api.bing.microsoft.com/api/v7/#WebPages.8", "title": "政策手册", "relevance_score": 0}, {"doc_id": "https://api.bing.microsoft.com/api/v7/#WebPages.9", "title": "劳动法", "relevance_score": 2}], "annotation_status": "completed", "updated_at": "2025-05-08 13:53:07"} 2 | {"query_id": "q4", "query_text": "新公司法有什么特点？", "query_result": [{"doc_id": "https://api.bing.microsoft.com/api/v7/#WebPages.0", "title": "中华人民共和国公司法_中国人大网", "content": "Dec 29, 2023 ... 第七十三条董事会的议事方式和表决程序，除本法有规定的外，由公司章程规定。 ... 两个以上公司合并设立一个新的公司为新设合并，合并各方解散。 第二 ..."}, {"doc_id": "https://api.bing.microsoft.com/api/v7/#WebPages.1", "title": "新公司法助力完善中国特色现代企业制度- 求是网", "content": "Jun 29, 2024 ... 新设有限责任公司的股东出资期限不得超过五年，新设股份有限公司成立前应足额缴纳出资。负有责任的董事、监事、高级管理人员对未履行核查催缴、股东抽逃 ..."}, {"doc_id": "https://api.bing.microsoft.com/api/v7/#WebPages.2", "title": "《公司法》修订解读与法律全文-广州市增城区人民政府门户网站", "content": "2023年12月29日，十四届全国人大常委会第七次会议修订通过《中华人民共和国公司法》，自2024年7月1日起施行。 修改公司法的背景意义是什么？修改的亮点有哪些？"}, {"doc_id": "https://api.bing.microsoft.com/api/v7/#WebPages.3", "title": "新《公司法》登记实务九十七问答", "content": "Jul 11, 2024 ... ②名称不含行业或者经营特点的企业满足新《办法》第二十条规定的 ... 答:(1)股东会的议事方式和表决程序,除《公司法》有规定的外,由公司章程规定。"}, {"doc_id": "https://api.bing.microsoft.com/api/v7/#WebPages.4", "title": "7月1日，新《公司法》生效！台山首发《章程》范本！免费下载→_ ...", "content": "Jul 1, 2024 ... ... 公司的运营机制，如何在遵守法律法规的前提下，制定符合自身特点的公司治理结构。此外，参考模板还提供了关于如何处理公司内部纠纷、如何保护股东权益 ..."}, {"doc_id": "https://api.bing.microsoft.com/api/v7/#WebPages.5", "title": "公司法新篇章：公司治理体系【新公司法50 问系列②】 - Lexology", "content": "Oct 18, 2024 ... 新《公司法》规定，上市公司的审计委员会有权审议批准承办公司审计业务的 ... 可能基于有限责任公司的封闭性特征及其所处的公司发展阶段[3] ..."}, {"doc_id": "https://api.bing.microsoft.com/api/v7/#WebPages.6", "title": "《公司法》修订，股东会、董事会、监事会、经理层职权有何新变化 ...", "content": "... 企业推动中国特色公司治理，有什么重要影响呢？ 我们今天进行一下对比分析，提供若干建议。 01股东会职权变化. 我们将现行《公司法》（2018年）的股东会职权要求，和新修改的《 ..."}, {"doc_id": "https://api.bing.microsoft.com/api/v7/#WebPages.7", "title": "新《公司法》背景下引入商业判断规则相关问题的探讨- 德恒探索- 德 ...", "content": "Nov 12, 2024 ... 经过长期的司法实践与学理论证，美国、日本、德国、澳大利亚的商业判断规则各有特点，而我国也在过往司法审判活动中尝试引入，但因缺乏明确的依据与指引，暴露 ..."}, {"doc_id": "https://api.bing.microsoft.com/api/v7/#WebPages.8", "title": "最高人民法院民二庭负责人就公司法时间效力的规定答记者问- 中华 ...", "content": "Jul 19, 2024 ... 公司法坚持以习近平新时代中国特色社会主义思想为指导，全面贯彻落实党 ... 答：此次公司法修订，条文变化很大，仅新增条文就有49个。《规定》曾 ..."}, {"doc_id": "https://api.bing.microsoft.com/api/v7/#WebPages.9", "title": "聚焦新《公司法》丨浅析有限公司章程对股权转让的限制- 金杜律师 ...", "content": "Dec 16, 2024 ... 我们认为，股东的股权转让权，是兼具自益权与共益权特征的复合型权利。一方面，股东通过转让股权获取投资收益，股权转让权有其自益权的面向；而另一方面，规范的 ..."}], "relevant_docs": [{"doc_id": "https://api.bing.microsoft.com/api/v7/#WebPages.0", "title": "中华人民共和国公司法_中国人大网", "relevance_score": 1}, {"doc_id": "https://api.bing.microsoft.com/api/v7/#WebPages.1", "title": "新公司法助力完善中国特色现代企业制度- 求是网", "relevance_score": 0}, {"doc_id": "https://api.bing.microsoft.com/api/v7/#WebPages.2", "title": "《公司法》修订解读与法律全文-广州市增城区人民政府门户网站", "relevance_score": 2}, {"doc_id": "https://api.bing.microsoft.com/api/v7/#WebPages.3", "title": "新《公司法》登记实务九十七问答", "relevance_score": 0}, {"doc_id": "https://api.bing.microsoft.com/api/v7/#WebPages.4", "title": "7月1日，新《公司法》生效！台山首发《章程》范本！免费下载→_ ...", "relevance_score": 0}, {"doc_id": "https://api.bing.microsoft.com/api/v7/#WebPages.5", "title": "公司法新篇章：公司治理体系【新公司法50 问系列②】 - Lexology", "relevance_score": 0}, {"doc_id": "https://api.bing.microsoft.com/api/v7/#WebPages.6", "title": "《公司法》修订，股东会、董事会、监事会、经理层职权有何新变化 ...", "relevance_score": 1}, {"doc_id": "https://api.bing.microsoft.com/api/v7/#WebPages.7", "title": "新《公司法》背景下引入商业判断规则相关问题的探讨- 德恒探索- 德 ...", "relevance_score": 0}, {"doc_id": "https://api.bing.microsoft.com/api/v7/#WebPages.8", "title": "最高人民法院民二庭负责人就公司法时间效力的规定答记者问- 中华 ...", "relevance_score": 0}, {"doc_id": "https://api.bing.microsoft.com/api/v7/#WebPages.9", "title": "聚焦新《公司法》丨浅析有限公司章程对股权转让的限制- 金杜律师 ...", "relevance_score": 1}], "annotation_status": "completed", "updated_at": "2025-05-08 13:53:30"} 3 | -------------------------------------------------------------------------------- /src/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tony3liu/Auto_Generate_Test_Cases/bcd100b08fac79262e2cb5c1ede32d95d3525e8d/src/.DS_Store -------------------------------------------------------------------------------- /src/.cache/41/cache.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tony3liu/Auto_Generate_Test_Cases/bcd100b08fac79262e2cb5c1ede32d95d3525e8d/src/.cache/41/cache.db -------------------------------------------------------------------------------- /src/agents/__pycache__/assistant.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tony3liu/Auto_Generate_Test_Cases/bcd100b08fac79262e2cb5c1ede32d95d3525e8d/src/agents/__pycache__/assistant.cpython-311.pyc -------------------------------------------------------------------------------- /src/agents/__pycache__/assistant.cpython-312.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tony3liu/Auto_Generate_Test_Cases/bcd100b08fac79262e2cb5c1ede32d95d3525e8d/src/agents/__pycache__/assistant.cpython-312.pyc -------------------------------------------------------------------------------- /src/agents/__pycache__/browser_use_agent.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tony3liu/Auto_Generate_Test_Cases/bcd100b08fac79262e2cb5c1ede32d95d3525e8d/src/agents/__pycache__/browser_use_agent.cpython-311.pyc -------------------------------------------------------------------------------- /src/agents/__pycache__/quality_assurance.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tony3liu/Auto_Generate_Test_Cases/bcd100b08fac79262e2cb5c1ede32d95d3525e8d/src/agents/__pycache__/quality_assurance.cpython-311.pyc -------------------------------------------------------------------------------- /src/agents/__pycache__/quality_assurance.cpython-312.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tony3liu/Auto_Generate_Test_Cases/bcd100b08fac79262e2cb5c1ede32d95d3525e8d/src/agents/__pycache__/quality_assurance.cpython-312.pyc -------------------------------------------------------------------------------- /src/agents/__pycache__/requirement_analyst.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tony3liu/Auto_Generate_Test_Cases/bcd100b08fac79262e2cb5c1ede32d95d3525e8d/src/agents/__pycache__/requirement_analyst.cpython-311.pyc -------------------------------------------------------------------------------- /src/agents/__pycache__/requirement_analyst.cpython-312.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tony3liu/Auto_Generate_Test_Cases/bcd100b08fac79262e2cb5c1ede32d95d3525e8d/src/agents/__pycache__/requirement_analyst.cpython-312.pyc -------------------------------------------------------------------------------- /src/agents/__pycache__/test_case_writer.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tony3liu/Auto_Generate_Test_Cases/bcd100b08fac79262e2cb5c1ede32d95d3525e8d/src/agents/__pycache__/test_case_writer.cpython-311.pyc -------------------------------------------------------------------------------- /src/agents/__pycache__/test_case_writer.cpython-312.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tony3liu/Auto_Generate_Test_Cases/bcd100b08fac79262e2cb5c1ede32d95d3525e8d/src/agents/__pycache__/test_case_writer.cpython-312.pyc -------------------------------------------------------------------------------- /src/agents/__pycache__/test_designer.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tony3liu/Auto_Generate_Test_Cases/bcd100b08fac79262e2cb5c1ede32d95d3525e8d/src/agents/__pycache__/test_designer.cpython-311.pyc -------------------------------------------------------------------------------- /src/agents/__pycache__/test_designer.cpython-312.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tony3liu/Auto_Generate_Test_Cases/bcd100b08fac79262e2cb5c1ede32d95d3525e8d/src/agents/__pycache__/test_designer.cpython-312.pyc -------------------------------------------------------------------------------- /src/agents/browser_use_agent.py: -------------------------------------------------------------------------------- 1 | from browser_use import Agent 2 | from langchain_openai import ChatOpenAI 3 | from pydantic import SecretStr 4 | import asyncio 5 | import sys 6 | import os 7 | import logging 8 | import json 9 | from dotenv import load_dotenv 10 | 11 | # 设置控制台编码为 UTF-8 12 | if sys.platform == 'win32': 13 | os.system('chcp 65001') 14 | sys.stdout.reconfigure(encoding='utf-8') 15 | sys.stderr.reconfigure(encoding='utf-8') 16 | 17 | # 配置日志，解决Windows控制台编码问题 18 | class UnicodeStreamHandler(logging.StreamHandler): 19 | def emit(self, record): 20 | try: 21 | msg = self.format(record) 22 | stream = self.stream 23 | if not isinstance(msg, str): 24 | msg = str(msg) 25 | stream.write(msg + self.terminator) 26 | self.flush() 27 | except UnicodeEncodeError: 28 | # 如果遇到编码错误，尝试使用GBK编码 29 | try: 30 | msg = self.format(record) 31 | stream = self.stream 32 | if not isinstance(msg, str): 33 | msg = str(msg) 34 | stream.write(msg.encode('gbk', errors='replace').decode('gbk') + self.terminator) 35 | self.flush() 36 | except Exception: 37 | self.handleError(record) 38 | 39 | 40 | # 添加项目根目录到 Python 路径 41 | sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) 42 | 43 | 44 | # 加载环境变量 45 | load_dotenv() 46 | # env_vars = load_env_variables() 47 | base_url = os.getenv('DS_BASE_URL') 48 | api_key = os.getenv('DS_API_KEY') 49 | model_v3 = os.getenv('DS_MODEL_V3') 50 | 51 | # 设置环境变量 52 | os.environ['OPENAI_API_KEY'] = api_key 53 | os.environ['OPENAI_BASE_URL'] = base_url 54 | # os.environ['OPENAI_API_KEY'] = env_vars['DS_API_KEY'] 55 | # os.environ['OPENAI_BASE_URL'] = env_vars['DS_BASE_URL'] 56 | 57 | llm = ChatOpenAI( 58 | base_url=base_url, 59 | model=model_v3, 60 | api_key=SecretStr(api_key), 61 | temperature=0.6, 62 | streaming=True, 63 | max_tokens=4096, 64 | request_timeout=60 65 | ) 66 | 67 | async def browser_use_agent(task): 68 | agent = Agent( 69 | task=task, 70 | # planner_llm='', # 规划模型，默认不启用，也可以使用较小的模型仅仅进行规划工作 71 | # use_vision=True, # 是否启用模型视觉理解 72 | # max_steps = 100, # 最大步数，默认100 73 | generate_gif = False, # 是否录制浏览器过程生成 GIF。为 True 时自动生成随机文件名；为字符串时将 GIF 存储到该路径 74 | llm=llm 75 | ) 76 | result = await agent.run() 77 | return result 78 | 79 | def read_test_cases(file_path): 80 | try: 81 | with open(file_path, 'r', encoding='utf-8') as f: 82 | data = json.load(f) 83 | return data.get('test_cases', []) 84 | except Exception as e: 85 | logging.error(f"读取测试用例文件失败: {str(e)}") 86 | return [] 87 | 88 | def build_task_prompt(test_case): 89 | # 只提取需要的字段 90 | title = test_case.get('title', '') 91 | steps = test_case.get('steps', []) 92 | expected_results = test_case.get('expected_results', []) 93 | 94 | # 构建任务提示 95 | prompt = f"测试用例标题: {title}\n\n" 96 | prompt += "测试步骤:\n" 97 | for i, step in enumerate(steps, 1): 98 | prompt += f"{i}. {step}\n" 99 | 100 | prompt += "\n预期结果:\n" 101 | for i, result in enumerate(expected_results, 1): 102 | prompt += f"{i}. {result}\n" 103 | 104 | return prompt 105 | 106 | 107 | if __name__ == "__main__": 108 | # 测试用例文件路径 109 | test_case_file_path = r'C:\\Users\\liut2\\Desktop\\Auto_Generate_Test_Cases\\ui_tst_case.json' 110 | 111 | # 读取测试用例 112 | test_cases = read_test_cases(test_case_file_path) 113 | 114 | # 遍历并执行每个测试用例 115 | for test_case in test_cases: 116 | extracted_case = { 117 | 'title': test_case.get('title', ''), 118 | 'steps': test_case.get('steps', []), 119 | 'expected_results': test_case.get('expected_results', []) 120 | } 121 | 122 | task_prompt = build_task_prompt(extracted_case) 123 | # print(f"\n执行测试用例: {test_case.get('id', '')}") 124 | # print(f"任务提示:\n{task_prompt}") 125 | 126 | # 执行测试用例并获取结果 127 | actual_results = asyncio.run(browser_use_agent(task_prompt)) 128 | 129 | # 断言 130 | final_result = actual_results.final_result() 131 | print(final_result) 132 | result = actual_results.is_successful() 133 | 134 | if result == True: 135 | print("✅ 测试通过") 136 | elif result == False: 137 | print(f"❌ 测试失败: {message}") 138 | else: # result == 'warning' 139 | print(f"⚠️ 警告: {message}") 140 | 141 | print("-" * 50) # 分隔线 142 | -------------------------------------------------------------------------------- /src/agents/quality_assurance.py: -------------------------------------------------------------------------------- 1 | # src/agents/quality_assurance.py 2 | import os 3 | import autogen 4 | from typing import Dict, List 5 | import logging 6 | from dotenv import load_dotenv 7 | from src.utils.agent_io import AgentIO 8 | load_dotenv() 9 | logger = logging.getLogger(__name__) 10 | 11 | # 使用 Azure OpenAI 配置 12 | gpt_api_key = os.getenv("AZURE_OPENAI_API_KEY") 13 | gpt_base_url = os.getenv("AZURE_OPENAI_BASE_URL") 14 | gpt_model = os.getenv("AZURE_OPENAI_MODEL") 15 | gpt_model_version = os.getenv("AZURE_OPENAI_MODEL_VERSION") 16 | #DS 17 | ds_api_key = os.getenv("DS_API_KEY") 18 | ds_base_url = os.getenv("DS_BASE_URL") 19 | ds_model_v3 = os.getenv("DS_MODEL_V3") 20 | ds_model_r1 = os.getenv("DS_MODEL_R1") 21 | 22 | class QualityAssuranceAgent: 23 | def __init__(self, concurrent_workers: int = 1): 24 | """初始化质量保证代理 25 | 26 | Args: 27 | concurrent_workers: 并发工作线程数，默认为1（不使用并发） 28 | """ 29 | self.config_list_gpt = [ 30 | { 31 | "model": gpt_model, 32 | "api_key": gpt_api_key, 33 | "base_url": gpt_base_url, 34 | "api_type": "azure", 35 | "api_version": gpt_model_version 36 | } 37 | ] 38 | 39 | self.config_list_ds_v3 = [ 40 | { 41 | "model": ds_model_v3, 42 | "api_key": ds_api_key, 43 | "base_url": ds_base_url, 44 | } 45 | ] 46 | 47 | self.config_list_ds_r1 = [ 48 | { 49 | "model": ds_model_r1, 50 | "api_key": ds_api_key, 51 | "base_url": ds_base_url, 52 | } 53 | ] 54 | 55 | # 设置并发工作线程数 56 | self.concurrent_workers = max(1, concurrent_workers) # 确保至少为1 57 | logger.info(f"质量保证代理初始化，并发工作线程数: {self.concurrent_workers}") 58 | 59 | # 初始化AgentIO用于保存和加载审查结果 60 | self.agent_io = AgentIO() 61 | 62 | # 初始化agent 63 | self.agent = autogen.AssistantAgent( 64 | name="quality_assurance", 65 | system_message="""你是一位专业的质量保证工程师，负责审查和改进测试用例。 66 | 你的职责是确保测试用例的完整性、清晰度、可执行性，并关注边界情况和错误场景。 67 | 68 | 在审查测试用例时，请重点关注以下方面并以JSON格式返回审查结果： 69 | { 70 | "review_comments": { 71 | "completeness": ["完整性相关的改进建议1", "完整性相关的改进建议2"], 72 | "clarity": ["清晰度相关的改进建议1", "清晰度相关的改进建议2"], 73 | "executability": ["可执行性相关的改进建议1", "可执行性相关的改进建议2"], 74 | "boundary_cases": ["边界情况相关的改进建议1", "边界情况相关的改进建议2"], 75 | "error_scenarios": ["错误场景相关的改进建议1", "错误场景相关的改进建议2"] 76 | } 77 | } 78 | 79 | 注意事项： 80 | 1. 必须严格按照上述JSON格式返回审查结果 81 | 2. 每个类别至少包含一条具体的改进建议 82 | 3. 所有建议必须清晰、具体、可执行 83 | 4. 不要返回任何JSON格式之外的文本内容""", 84 | llm_config={"config_list": self.config_list_ds_v3} 85 | ) 86 | 87 | # 添加last_review属性，用于跟踪最近的审查结果 88 | self.last_review = None 89 | 90 | # 尝试加载之前的审查结果 91 | self._load_last_review() 92 | 93 | def _load_last_review(self): 94 | """加载之前保存的审查结果""" 95 | try: 96 | result = self.agent_io.load_result("quality_assurance") 97 | if result: 98 | self.last_review = result 99 | logger.info("成功加载之前的质量审查结果") 100 | except Exception as e: 101 | logger.error(f"加载质量审查结果时出错: {str(e)}") 102 | 103 | def review(self, test_cases: List[Dict]) -> Dict: 104 | """审查和改进测试用例。 105 | 106 | 使用并发处理方式提高处理效率，并发数由concurrent_workers参数控制。 107 | """ 108 | try: 109 | # 验证输入参数 110 | if not test_cases or not isinstance(test_cases, list): 111 | logger.warning("输入的测试用例为空或格式不正确") 112 | return {"error": "输入的测试用例为空或格式不正确", "reviewed_cases": []} 113 | 114 | user_proxy = autogen.UserProxyAgent( 115 | name="user_proxy", 116 | system_message="测试用例提供者", 117 | human_input_mode="NEVER", 118 | code_execution_config={"use_docker": False} 119 | ) 120 | 121 | # 审查测试用例 122 | user_proxy.initiate_chat( 123 | self.agent, 124 | message=f"""请审查以下测试用例并提供改进建议： 125 | 126 | 测试用例: {test_cases} 127 | 128 | 检查以下方面： 129 | 1. 完整性 130 | 2. 清晰度 131 | 3. 可执行性 132 | 4. 边界情况 133 | 5. 错误场景""", 134 | max_turns=1 # 限制对话轮次为1，避免死循环 135 | ) 136 | 137 | # 获取审查反馈 138 | review_feedback = self.agent.last_message() 139 | 140 | # 确保反馈是字符串格式 141 | if not review_feedback: 142 | logger.warning("审查反馈为空") 143 | review_feedback = "" 144 | elif isinstance(review_feedback, dict): 145 | if 'content' in review_feedback: 146 | review_feedback = review_feedback['content'] 147 | else: 148 | logger.warning("无法从反馈字典中提取内容，使用空字符串") 149 | review_feedback = "" 150 | elif not isinstance(review_feedback, str): 151 | logger.warning(f"审查反馈格式不正确: {type(review_feedback)}，转换为字符串") 152 | review_feedback = str(review_feedback) 153 | 154 | # 提取反馈中的关键改进建议 155 | review_comments = self._extract_review_comments(review_feedback) 156 | 157 | # 使用并发方式处理测试用例 158 | if self.concurrent_workers > 1: 159 | logger.info(f"使用并发方式处理测试用例，并发数: {self.concurrent_workers}") 160 | reviewed_cases = self._process_review_concurrent(test_cases, review_feedback) 161 | else: 162 | logger.info("使用顺序方式处理测试用例") 163 | # 调用原有的处理方法 164 | reviewed_cases = self._process_review(test_cases, review_feedback) 165 | 166 | # 创建包含审查反馈和改进后测试用例的结果 167 | result = { 168 | "reviewed_cases": reviewed_cases, 169 | "review_comments": review_comments, 170 | "review_date": self._get_current_timestamp(), 171 | "review_status": "completed" 172 | } 173 | 174 | # 验证结果数据的完整性 175 | if not self._validate_result(result): 176 | logger.warning("审查结果数据不完整，可能影响后续处理") 177 | result["review_status"] = "incomplete" 178 | 179 | # 将审查结果保存到文件 180 | try: 181 | self.agent_io.save_result("quality_assurance", result) 182 | logger.info("质量审查结果已成功保存") 183 | except Exception as e: 184 | logger.error(f"保存质量审查结果时出错: {str(e)}") 185 | # 即使保存失败，仍然返回结果 186 | 187 | # 保存审查结果到last_review属性 188 | self.last_review = reviewed_cases 189 | logger.info(f"测试用例审查完成，共审查 {len(test_cases)} 个测试用例") 190 | 191 | # 清理临时批次文件 192 | self._delete_batch_files() 193 | 194 | return result 195 | 196 | except Exception as e: 197 | logger.error(f"测试用例审查错误: {str(e)}") 198 | error_result = { 199 | "error": str(e), 200 | "reviewed_cases": test_cases if isinstance(test_cases, list) else [], 201 | "review_comments": {}, 202 | "review_status": "error" 203 | } 204 | return error_result 205 | 206 | def _merge_feature_test_cases(self, batch_count: int) -> Dict: 207 | """合并多个批次的测试用例结果 208 | 209 | Args: 210 | batch_count: 批次数量 211 | 212 | Returns: 213 | 合并后的结果 214 | """ 215 | try: 216 | all_reviewed_cases = [] 217 | all_review_comments = { 218 | "completeness": [], 219 | "clarity": [], 220 | "executability": [], 221 | "boundary_cases": [], 222 | "error_scenarios": [] 223 | } 224 | 225 | # 加载并合并所有批次的结果 226 | for i in range(1, batch_count + 1): 227 | batch_result = self.agent_io.load_result(f"quality_assurance_batch_{i}") 228 | if batch_result and "reviewed_cases" in batch_result: 229 | all_reviewed_cases.extend(batch_result["reviewed_cases"]) 230 | 231 | # 合并评论 232 | if "review_comments" in batch_result: 233 | for category in all_review_comments.keys(): 234 | if category in batch_result["review_comments"]: 235 | all_review_comments[category].extend(batch_result["review_comments"][category]) 236 | 237 | # 去重评论 238 | for category in all_review_comments.keys(): 239 | all_review_comments[category] = list(set(all_review_comments[category])) 240 | 241 | # 创建合并结果 242 | merged_result = { 243 | "reviewed_cases": all_reviewed_cases, 244 | "review_comments": all_review_comments, 245 | "review_date": self._get_current_timestamp(), 246 | "review_status": "completed", 247 | "merged_from_batches": batch_count 248 | } 249 | 250 | # 保存合并结果 251 | self.agent_io.save_result("quality_assurance_merged", merged_result) 252 | logger.info(f"已合并{batch_count}个批次的测试用例结果，共{len(all_reviewed_cases)}个测试用例") 253 | 254 | return merged_result 255 | 256 | except Exception as e: 257 | logger.error(f"合并测试用例结果时出错: {str(e)}") 258 | return {"error": str(e), "review_status": "error"} 259 | 260 | def _extract_review_comments(self, feedback: str) -> Dict: 261 | """从字符串格式的反馈中提取结构化的审查评论。""" 262 | review_comments = { 263 | "completeness": [], 264 | "clarity": [], 265 | "executability": [], 266 | "boundary_cases": [], 267 | "error_scenarios": [] 268 | } 269 | 270 | if not feedback: 271 | return review_comments 272 | 273 | # 尝试解析JSON格式的反馈 274 | import json 275 | import re 276 | 277 | try: 278 | # 查找JSON内容 279 | json_match = re.search(r'\{[\s\S]*\}', feedback) 280 | if json_match: 281 | json_str = json_match.group(0) 282 | # 解析JSON 283 | parsed_feedback = json.loads(json_str) 284 | 285 | # 提取review_comments部分 286 | if 'review_comments' in parsed_feedback: 287 | return parsed_feedback['review_comments'] 288 | except Exception as e: 289 | logger.warning(f"JSON解析失败，将使用文本解析方式: {str(e)}") 290 | 291 | # 如果JSON解析失败，回退到文本解析方式 292 | feedback_sections = [line.strip() for line in feedback.split('\n') if line.strip()] 293 | current_section = None 294 | 295 | # 提取各个方面的改进建议 296 | for line in feedback_sections: 297 | # 识别章节标题 298 | section_mapping = { 299 | '1. 完整性': 'completeness', 300 | '2. 清晰度': 'clarity', 301 | '3. 可执行性': 'executability', 302 | '4. 边界情况': 'boundary_cases', 303 | '5. 错误场景': 'error_scenarios' 304 | } 305 | 306 | for title, section in section_mapping.items(): 307 | if title in line: 308 | current_section = section 309 | break 310 | 311 | # 提取建议内容 312 | if current_section and (line.startswith('-') or line.startswith('•')): 313 | content = line[1:].strip() 314 | if content: # 确保内容不为空 315 | review_comments[current_section].append(content) 316 | 317 | return review_comments 318 | 319 | def _get_current_timestamp(self) -> str: 320 | """获取当前时间戳""" 321 | from datetime import datetime 322 | return datetime.now().isoformat() 323 | 324 | def _validate_result(self, result: Dict) -> bool: 325 | """验证审查结果的完整性和有效性""" 326 | required_keys = ["reviewed_cases", "review_comments", "review_status"] 327 | if not all(key in result for key in required_keys): 328 | return False 329 | 330 | # 验证reviewed_cases是否为列表 331 | if not isinstance(result.get("reviewed_cases"), list): 332 | return False 333 | 334 | # 验证review_comments是否包含所有必要的类别 335 | comment_categories = ["completeness", "clarity", "executability", "boundary_cases", "error_scenarios"] 336 | if not all(category in result.get("review_comments", {}) for category in comment_categories): 337 | return False 338 | 339 | return True 340 | 341 | def _process_review_concurrent(self, original_cases: List[Dict], review_feedback) -> List[Dict]: 342 | """使用并发方式处理审查反馈并更新测试用例。 343 | 根据concurrent_workers参数控制并发数。 344 | """ 345 | if not original_cases: 346 | logger.warning("没有测试用例需要处理") 347 | return [] 348 | 349 | # 确定批次大小和批次数 350 | total_cases = len(original_cases) 351 | # 根据并发工作线程数确定批次数，每个工作线程至少处理一个批次 352 | num_batches = min(total_cases, self.concurrent_workers * 2) # 每个工作线程处理约2个批次 353 | batch_size = max(1, total_cases // num_batches) # 确保每批至少有1个用例 354 | 355 | batches = [original_cases[i:i+batch_size] for i in range(0, total_cases, batch_size)] 356 | logger.info(f"将{total_cases}个测试用例分成{len(batches)}批进行处理，每批约{batch_size}个用例，并发工作线程数: {self.concurrent_workers}") 357 | 358 | # 使用线程池并发处理测试用例 359 | all_reviewed_cases = [] 360 | import concurrent.futures 361 | 362 | # 定义批处理函数 363 | def process_batch(batch_index, batch_cases): 364 | logger.info(f"开始处理第{batch_index+1}批测试用例，共{len(batch_cases)}个") 365 | batch_reviewed_cases = [] 366 | for case in batch_cases: 367 | improved_case = self._improve_test_case(case, review_feedback) 368 | batch_reviewed_cases.append(improved_case) 369 | 370 | # 保存中间结果，防止因超时丢失数据 371 | temp_result = { 372 | "reviewed_cases": batch_reviewed_cases, # 只保存当前批次的结果，而不是累积结果 373 | "review_comments": self._extract_review_comments(review_feedback) if isinstance(review_feedback, str) else review_feedback, 374 | "review_date": self._get_current_timestamp(), 375 | "review_status": "in_progress", 376 | "batch_progress": f"{batch_index+1}/{len(batches)}" 377 | } 378 | try: 379 | self.agent_io.save_result(f"quality_assurance_batch_{batch_index+1}", temp_result) 380 | logger.info(f"已保存第{batch_index+1}批质量审查结果") 381 | except Exception as e: 382 | logger.error(f"保存第{batch_index+1}批质量审查结果时出错: {str(e)}") 383 | 384 | logger.info(f"第{batch_index+1}批测试用例处理完成") 385 | return batch_reviewed_cases 386 | 387 | # 使用线程池执行器并发处理批次 388 | with concurrent.futures.ThreadPoolExecutor(max_workers=self.concurrent_workers) as executor: 389 | # 提交所有批次任务 390 | future_to_batch = {executor.submit(process_batch, i, batch): i for i, batch in enumerate(batches)} 391 | 392 | # 收集结果 393 | for future in concurrent.futures.as_completed(future_to_batch): 394 | batch_index = future_to_batch[future] 395 | try: 396 | batch_result = future.result() 397 | all_reviewed_cases.extend(batch_result) 398 | logger.info(f"已合并第{batch_index+1}批测试用例结果") 399 | except Exception as e: 400 | logger.error(f"处理第{batch_index+1}批测试用例时出错: {str(e)}") 401 | 402 | logger.info(f"所有测试用例处理完成，共改进{len(all_reviewed_cases)}个测试用例") 403 | return all_reviewed_cases 404 | 405 | def _improve_test_case(self, test_case: Dict, feedback) -> Dict: 406 | """根据反馈改进测试用例。""" 407 | try: 408 | if not test_case: 409 | logger.warning("测试用例为空") 410 | return test_case 411 | 412 | if not feedback: 413 | logger.warning("反馈为空") 414 | return test_case 415 | 416 | # 创建改进后的测试用例副本 417 | improved_case = test_case.copy() 418 | 419 | # 检查feedback类型 420 | if isinstance(feedback, dict): 421 | # 如果是字典类型，尝试从content字段获取内容 422 | if 'content' in feedback: 423 | feedback = feedback['content'] 424 | else: 425 | logger.error(f"无法从字典中提取反馈内容: {feedback}") 426 | return test_case 427 | 428 | # 确保feedback是字符串类型 429 | if not isinstance(feedback, str): 430 | logger.error(f"反馈不是字符串类型: {type(feedback)}") 431 | return test_case 432 | 433 | # 解析反馈内容 434 | feedback_sections = [line.strip() for line in feedback.split('\n') if line.strip()] 435 | current_section = None 436 | improvements = { 437 | 'completeness': [], 438 | 'clarity': [], 439 | 'executability': [], 440 | 'boundary_cases': [], 441 | 'error_scenarios': [] 442 | } 443 | 444 | # 提取各个方面的改进建议 445 | for line in feedback_sections: 446 | # 识别章节标题 447 | section_mapping = { 448 | '1. 完整性': 'completeness', 449 | '2. 清晰度': 'clarity', 450 | '3. 可执行性': 'executability', 451 | '4. 边界情况': 'boundary_cases', 452 | '5. 错误场景': 'error_scenarios' 453 | } 454 | 455 | for title, section in section_mapping.items(): 456 | if title in line: 457 | current_section = section 458 | break 459 | 460 | # 提取建议内容 461 | if current_section and (line.startswith('-') or line.startswith('•')): 462 | content = line[1:].strip() 463 | if content: # 确保内容不为空 464 | improvements[current_section].append(content) 465 | 466 | # 根据反馈改进测试用例 467 | # 完整性改进 468 | if improvements['completeness']: 469 | required_fields = ['preconditions', 'steps', 'expected_results'] 470 | for field in required_fields: 471 | if field not in improved_case: 472 | improved_case[field] = [] 473 | elif not isinstance(improved_case[field], list): 474 | improved_case[field] = [improved_case[field]] 475 | 476 | # 清晰度改进 477 | if improvements['clarity']: 478 | # 确保标题清晰明确 479 | if 'title' in improved_case: 480 | improved_case['title'] = improved_case['title'].strip() if improved_case['title'] else '' 481 | # 确保步骤描述清晰 482 | if 'steps' in improved_case: 483 | improved_case['steps'] = [step.strip() for step in improved_case['steps'] if step] 484 | 485 | # 可执行性改进 486 | if improvements['executability']: 487 | steps = improved_case.get('steps', []) 488 | results = improved_case.get('expected_results', []) 489 | if steps: 490 | # 确保每个步骤都有对应的预期结果 491 | if len(steps) > len(results): 492 | results.extend(['待补充'] * (len(steps) - len(results))) 493 | improved_case['expected_results'] = results 494 | 495 | # 边界情况改进 496 | if improvements['boundary_cases']: 497 | boundary_conditions = improved_case.setdefault('boundary_conditions', []) 498 | # 去重并添加新的边界条件 499 | new_conditions = [cond for cond in improvements['boundary_cases'] 500 | if cond not in boundary_conditions] 501 | boundary_conditions.extend(new_conditions) 502 | 503 | # 错误场景改进 504 | if improvements['error_scenarios']: 505 | error_scenarios = improved_case.setdefault('error_scenarios', []) 506 | # 去重并添加新的错误场景 507 | new_scenarios = [scenario for scenario in improvements['error_scenarios'] 508 | if scenario not in error_scenarios] 509 | error_scenarios.extend(new_scenarios) 510 | 511 | # 验证改进后的测试用例 512 | if not self._validate_improvements(test_case, improved_case): 513 | logger.warning(f"测试用例改进可能导致数据丢失: {test_case.get('id', 'unknown')}") 514 | return test_case 515 | 516 | return improved_case 517 | 518 | except Exception as e: 519 | logger.error(f"改进测试用例错误: {str(e)}") 520 | return test_case 521 | 522 | def _validate_improvements(self, original: Dict, improved: Dict) -> bool: 523 | """验证改进是否保持测试用例的完整性。""" 524 | return all(key in improved for key in original.keys()) 525 | 526 | def _delete_batch_files(self) -> None: 527 | """删除质量审查过程中生成的临时批次文件。 528 | 在测试用例审查完成后调用此函数清理中间文件。 529 | """ 530 | try: 531 | import os 532 | import glob 533 | 534 | # 查找所有质量审查批次的临时文件 535 | pattern = os.path.join(self.agent_io.output_dir, "quality_assurance_batch_*_result.json") 536 | batch_files = glob.glob(pattern) 537 | 538 | # 删除找到的所有批次文件 539 | for file_path in batch_files: 540 | if os.path.exists(file_path): 541 | os.remove(file_path) 542 | logger.info(f"已删除临时质量审查批次文件: {file_path}") 543 | 544 | if batch_files: 545 | logger.info(f"所有临时质量审查批次文件已清理完毕，共删除 {len(batch_files)} 个文件") 546 | else: 547 | logger.info("未找到需要清理的临时质量审查批次文件") 548 | except Exception as e: 549 | logger.error(f"删除临时质量审查批次文件时出错: {str(e)}") 550 | 551 | def _process_review(self, original_cases: List[Dict], review_feedback) -> List[Dict]: 552 | """处理审查反馈并更新测试用例。 553 | 优化：将测试用例分批次进行改进，避免一次处理过多导致超时或输出不完整。 554 | """ 555 | if not original_cases: 556 | logger.warning("没有测试用例需要处理") 557 | return [] 558 | 559 | # 将测试用例分成3批进行处理，避免一次处理过多 560 | batch_size = max(1, len(original_cases) // 3) # 确保至少每批1个用例 561 | batches = [original_cases[i:i+batch_size] for i in range(0, len(original_cases), batch_size)] 562 | logger.info(f"将{len(original_cases)}个测试用例分成{len(batches)}批进行处理，每批约{batch_size}个用例") 563 | 564 | # 分批处理测试用例 565 | all_reviewed_cases = [] 566 | for i, batch in enumerate(batches): 567 | logger.info(f"开始处理第{i+1}批测试用例，共{len(batch)}个") 568 | batch_reviewed_cases = [] 569 | for case in batch: 570 | improved_case = self._improve_test_case(case, review_feedback) 571 | batch_reviewed_cases.append(improved_case) 572 | 573 | # 将当前批次的结果添加到总结果中 574 | all_reviewed_cases.extend(batch_reviewed_cases) 575 | logger.info(f"第{i+1}批测试用例处理完成") 576 | 577 | # 保存中间结果，防止因超时丢失数据 578 | temp_result = { 579 | "reviewed_cases": batch_reviewed_cases, # 只保存当前批次的结果，而不是累积结果 580 | "review_comments": self._extract_review_comments(review_feedback) if isinstance(review_feedback, str) else review_feedback, 581 | "review_date": self._get_current_timestamp(), 582 | "review_status": "in_progress", 583 | "batch_progress": f"{i+1}/{len(batches)}" 584 | } 585 | try: 586 | self.agent_io.save_result(f"quality_assurance_batch_{i+1}", temp_result) 587 | logger.info(f"已保存第{i+1}批质量审查结果") 588 | except Exception as e: 589 | logger.error(f"保存第{i+1}批质量审查结果时出错: {str(e)}") 590 | 591 | logger.info(f"所有测试用例处理完成，共改进{len(all_reviewed_cases)}个测试用例") 592 | return all_reviewed_cases -------------------------------------------------------------------------------- /src/agents/test_designer.py: -------------------------------------------------------------------------------- 1 | # src/agents/test_designer.py 2 | import os 3 | import ast 4 | import autogen 5 | from typing import Dict, List 6 | import logging 7 | from dotenv import load_dotenv 8 | from src.utils.agent_io import AgentIO 9 | load_dotenv() 10 | logger = logging.getLogger(__name__) 11 | 12 | # 使用 Azure OpenAI 配置 13 | gpt_api_key = os.getenv("AZURE_OPENAI_API_KEY") 14 | gpt_base_url = os.getenv("AZURE_OPENAI_BASE_URL") 15 | gpt_model = os.getenv("AZURE_OPENAI_MODEL") 16 | gpt_model_version = os.getenv("AZURE_OPENAI_MODEL_VERSION") 17 | #DS 18 | ds_api_key = os.getenv("DS_API_KEY") 19 | ds_base_url = os.getenv("DS_BASE_URL") 20 | ds_model_v3 = os.getenv("DS_MODEL_V3") 21 | ds_model_r1 = os.getenv("DS_MODEL_R1") 22 | 23 | class TestDesignerAgent: 24 | def __init__(self): 25 | self.config_list_gpt = [ 26 | { 27 | "model": gpt_model, 28 | "api_key": gpt_api_key, 29 | "base_url": gpt_base_url, 30 | "api_type": "azure", 31 | "api_version": gpt_model_version 32 | } 33 | ] 34 | 35 | self.config_list_ds_v3 = [ 36 | { 37 | "model": ds_model_v3, 38 | "api_key": ds_api_key, 39 | "base_url": ds_base_url, 40 | } 41 | ] 42 | 43 | self.config_list_ds_r1 = [ 44 | { 45 | "model": ds_model_r1, 46 | "api_key": ds_api_key, 47 | "base_url": ds_base_url, 48 | } 49 | ] 50 | 51 | # 初始化AgentIO用于保存和加载设计结果 52 | self.agent_io = AgentIO() 53 | 54 | self.agent = autogen.AssistantAgent( 55 | name="test_designer", 56 | system_message='''你是一位专业的测试设计师，擅长将需求转化为全面的测试策略。 57 | 58 | 你的主要职责包括： 59 | 1. 分析需求文档和需求分析结果 60 | 2. 设计全面的测试方法，包括但不限于： 61 | - 功能测试 62 | - 性能测试 63 | - 安全测试 64 | - 兼容性测试 65 | - 可用性测试 66 | 3. 创建详细的测试覆盖矩阵，确保所有功能点都被覆盖 67 | 4. 制定合理的测试优先级策略 68 | 5. 评估所需的测试资源 69 | 70 | 在提供测试策略时，你应该： 71 | - 确保测试方法与需求紧密对应 72 | - 提供具体的测试工具和框架建议 73 | - 考虑测试执行的可行性 74 | - 平衡测试覆盖率和资源限制 75 | 76 | 请按照以下 JSON 格式提供你的分析结果： 77 | { 78 | "test_approach": { 79 | "methodology": [ 80 | "测试方法1", 81 | "测试方法2" 82 | ], 83 | "tools": [ 84 | "工具1", 85 | "工具2" 86 | ], 87 | "frameworks": [ 88 | "框架1", 89 | "框架2" 90 | ] 91 | }, 92 | "coverage_matrix": [ 93 | { 94 | "feature": "功能点1", 95 | "test_type": "测试类型1" 96 | } 97 | ], 98 | "priorities": [ 99 | { 100 | "level": "P0", 101 | "description": "关键功能描述" 102 | } 103 | ], 104 | "resource_estimation": { 105 | "time": "预计时间", 106 | "personnel": "所需人员", 107 | "tools": ["所需工具1", "所需工具2"], 108 | "additional_resources": ["其他资源1", "其他资源2"] 109 | } 110 | } 111 | 112 | 注意： 113 | 1. 所有输出必须严格遵循上述 JSON 格式 114 | 2. 每个数组至少包含一个有效项 115 | 3. 所有文本必须使用双引号 116 | 4. JSON 必须是有效的且可解析的''', 117 | llm_config={"config_list": self.config_list_ds_v3} 118 | ) 119 | 120 | # 添加last_design属性，用于跟踪最近的设计结果 121 | self.last_design = None 122 | 123 | # 尝试加载之前的设计结果 124 | self._load_last_design() 125 | 126 | def _load_last_design(self): 127 | """加载之前保存的设计结果""" 128 | try: 129 | result = self.agent_io.load_result("test_designer") 130 | if result: 131 | self.last_design = result 132 | logger.info("成功加载之前的测试设计结果") 133 | except Exception as e: 134 | logger.error(f"加载测试设计结果时出错: {str(e)}") 135 | 136 | def design(self, requirements: Dict) -> Dict: 137 | """基于分析后的需求设计测试策略。 138 | 139 | Args: 140 | requirements: 包含原始需求文档和需求分析结果的字典 141 | - original_doc: 原始需求文档 142 | - analysis_result: 需求分析结果 143 | """ 144 | try: 145 | user_proxy = autogen.UserProxyAgent( 146 | name="user_proxy", 147 | system_message="需求提供者", 148 | human_input_mode="NEVER", 149 | code_execution_config={"use_docker": False} 150 | ) 151 | 152 | # 创建测试策略 153 | user_proxy.initiate_chat( 154 | self.agent, 155 | message=f"""基于以下需求创建详细的测试策略： 156 | 157 | 原始需求文档： 158 | {requirements.get('original_doc', '')} 159 | 160 | 需求分析结果： 161 | 功能需求：{requirements.get('analysis_result', {}).get('functional_requirements', [])} 162 | 非功能需求：{requirements.get('analysis_result', {}).get('non_functional_requirements', [])} 163 | 测试场景：{requirements.get('analysis_result', {}).get('test_scenarios', [])} 164 | 风险领域：{requirements.get('analysis_result', {}).get('risk_areas', [])} 165 | 166 | 请按照以下格式提供测试策略： 167 | 168 | 1. 测试方法 169 | - 功能测试方法 170 | - 安全测试方法 171 | - 兼容性测试方法 172 | - 可用性测试方法 173 | 174 | 2. 测试覆盖矩阵 175 | - 每个功能需求（不能简写，要如实描述需求） 176 | - 每个非功能需求的测试类型 177 | - 每个测试场景的覆盖方案 178 | - 风险领域的测试覆盖 179 | 180 | 3. 测试优先级 181 | P0：关键功能和高风险项 182 | P1：核心业务功能 183 | P2：重要但非核心功能 184 | P3：次要功能 185 | P4：低优先级功能 186 | 187 | 4. 资源估算 188 | - 预计所需时间 189 | - 所需人员配置 190 | - 测试工具清单 191 | - 其他资源需求 192 | 193 | 请直接提供分析结果，确保每个部分都有具体的内容和建议。""", 194 | max_turns=1 # 限制对话轮次为1，避免死循环 195 | ) 196 | 197 | # 处理代理的响应 198 | response = self.agent.last_message() 199 | if not response: 200 | logger.warning("测试设计代理返回空响应") 201 | return { 202 | "test_approach": { 203 | "methodology": [], 204 | "tools": [], 205 | "frameworks": [] 206 | }, 207 | "coverage_matrix": [], 208 | "priorities": [], 209 | "resource_estimation": { 210 | "time": None, 211 | "personnel": None, 212 | "tools": [], 213 | "additional_resources": [] 214 | } 215 | } 216 | 217 | # 确保响应是字符串类型 218 | response_str = str(response) if response else "" 219 | if not response_str.strip(): 220 | logger.warning("测试设计代理返回空响应") 221 | return { 222 | "test_approach": { 223 | "methodology": [], 224 | "tools": [], 225 | "frameworks": [] 226 | }, 227 | "coverage_matrix": [], 228 | "priorities": [], 229 | "resource_estimation": { 230 | "time": None, 231 | "personnel": None, 232 | "tools": [], 233 | "additional_resources": [] 234 | } 235 | } 236 | 237 | # 尝试解析JSON响应 238 | import json 239 | import re 240 | 241 | # 打印原始响应以便调试 242 | logger.info(f"AI响应内容: {response_str[:200]}...") # 只打印前200个字符避免日志过长 243 | 244 | # 尝试从响应中提取JSON部分 - 支持多种格式 245 | json_match = re.search(r'```(?:json)?\s*({\s*".*?})\s*```', response_str, re.DOTALL) 246 | if not json_match: 247 | # 尝试直接从响应中查找JSON对象 248 | json_match = re.search(r'({[\s\S]*"test_approach"[\s\S]*})', response_str) 249 | 250 | if json_match: 251 | try: 252 | # 提取JSON字符串并解析 253 | json_str = json_match.group(1) 254 | # 清理可能的格式问题 255 | json_str = json_str.strip() 256 | # print(f"json_str_type: {type(json_str)} ") 257 | json_str = re.sub(r'```json|```', '', json_str) 258 | json_str = json_str.replace(r'\n', '\n').replace(r'\"', '"') 259 | # print(f"content 的字符串内容：{json_str}") 260 | json_str_fix = re.sub(r"'content':\s*'(.*?)'", # 匹配content字段的值 261 | lambda m: "'content': '''{}'''".format(m.group(1).replace("'''", "\\'''")), # 转换为三重引号 262 | json_str, 263 | flags=re.DOTALL 264 | ) 265 | json_init_dict = ast.literal_eval(json_str_fix) 266 | # print(f"json_init_dict 类型：{type(json_init_dict)}") 267 | json_str = json.dumps(json_init_dict["content"]) 268 | test_strategy = json.loads(json_str) 269 | logger.info("成功从JSON响应中提取测试策略") 270 | 271 | # 确保test_strategy是字典类型 272 | if isinstance(test_strategy, str): 273 | test_strategy = json.loads(test_strategy) 274 | 275 | # 保存设计结果到last_design属性 276 | self.last_design = test_strategy 277 | 278 | # 将设计结果保存到文件 279 | self.agent_io.save_result("test_designer", test_strategy) 280 | 281 | logger.info("测试设计完成") 282 | 283 | return test_strategy 284 | 285 | except Exception as e: 286 | logger.error(f"测试设计错误: {str(e)}") 287 | # 发生异常时返回默认结构 288 | return { 289 | "test_approach": { 290 | "methodology": [], 291 | "tools": [], 292 | "frameworks": [] 293 | }, 294 | "coverage_matrix": [], 295 | "priorities": [], 296 | "resource_estimation": { 297 | "time": None, 298 | "personnel": None, 299 | "tools": [], 300 | "additional_resources": [] 301 | } 302 | } 303 | else: 304 | # 如果无法提取JSON，返回默认结构 305 | logger.warning("无法从响应中提取JSON格式的测试策略") 306 | return { 307 | "test_approach": { 308 | "methodology": [], 309 | "tools": [], 310 | "frameworks": [] 311 | }, 312 | "coverage_matrix": [], 313 | "priorities": [], 314 | "resource_estimation": { 315 | "time": None, 316 | "personnel": None, 317 | "tools": [], 318 | "additional_resources": [] 319 | } 320 | } 321 | 322 | except Exception as e: 323 | logger.error(f"测试设计过程中出错: {str(e)}") 324 | # 发生异常时返回默认结构 325 | return { 326 | "test_approach": { 327 | "methodology": [], 328 | "tools": [], 329 | "frameworks": [] 330 | }, 331 | "coverage_matrix": [], 332 | "priorities": [], 333 | "resource_estimation": { 334 | "time": None, 335 | "personnel": None, 336 | "tools": [], 337 | "additional_resources": [] 338 | } 339 | } 340 | 341 | def _extract_test_approach(self, message: str) -> Dict: 342 | """从代理消息中提取测试方法详情。""" 343 | test_approach = { 344 | 'methodology': [], 345 | 'tools': [], 346 | 'frameworks': [] 347 | } 348 | 349 | try: 350 | if not message: 351 | logger.warning("输入消息为空") 352 | return test_approach 353 | 354 | sections = message.split('\n') 355 | in_approach_section = False 356 | current_section = None 357 | 358 | for line in sections: 359 | try: 360 | line = line.strip() 361 | if not line: 362 | continue 363 | 364 | if '1. 测试方法' in line: 365 | in_approach_section = True 366 | continue 367 | elif '2. 测试覆盖矩阵' in line: 368 | break 369 | elif in_approach_section and not line.startswith('1.'): 370 | # 分类方法详情 371 | try: 372 | # 检查是否是新的方法类型标题 373 | if '功能测试' in line or '性能测试' in line or '安全测试' in line or '兼容性测试' in line or '可用性测试' in line: 374 | current_section = line.strip() 375 | test_approach['methodology'].append(current_section) 376 | elif line.startswith('-') or line.startswith('*'): 377 | # 提取具体的测试方法 378 | content = line.strip('- *').strip() 379 | if content: 380 | test_approach['methodology'].append(content) 381 | elif '工具' in line.lower(): 382 | # 提取工具信息 383 | if ':' in line or '：' in line: 384 | content = line.split(':', 1)[1].strip() if ':' in line else line.split('：', 1)[1].strip() 385 | test_approach['tools'].extend([t.strip() for t in content.split(',')]) 386 | elif '框架' in line.lower(): 387 | # 提取框架信息 388 | if ':' in line or '：' in line: 389 | content = line.split(':', 1)[1].strip() if ':' in line else line.split('：', 1)[1].strip() 390 | test_approach['frameworks'].extend([f.strip() for f in content.split(',')]) 391 | elif current_section and line.strip(): 392 | # 将其他内容添加到当前部分 393 | test_approach['methodology'].append(line.strip()) 394 | except IndexError as e: 395 | logger.error(f"解析方法详情时出错: {str(e)}，行内容: {line}") 396 | continue 397 | except Exception as e: 398 | logger.error(f"处理单行内容时出错: {str(e)}，行内容: {line}") 399 | continue 400 | 401 | # 去重并过滤空值 402 | test_approach['methodology'] = list(filter(None, set(test_approach['methodology']))) 403 | test_approach['tools'] = list(filter(None, set(test_approach['tools']))) 404 | test_approach['frameworks'] = list(filter(None, set(test_approach['frameworks']))) 405 | 406 | return test_approach 407 | except Exception as e: 408 | logger.error(f"提取测试方法错误: {str(e)}") 409 | return test_approach 410 | 411 | def _create_coverage_matrix(self, message: str) -> List[Dict]: 412 | """从代理消息中创建测试覆盖矩阵。""" 413 | coverage_matrix = [] 414 | 415 | try: 416 | if not message: 417 | logger.warning("输入消息为空") 418 | return coverage_matrix 419 | 420 | sections = message.split('\n') 421 | in_matrix_section = False 422 | current_feature = None 423 | 424 | for line in sections: 425 | try: 426 | line = line.strip() 427 | if not line: 428 | continue 429 | 430 | if '2. 测试覆盖矩阵' in line: 431 | in_matrix_section = True 432 | continue 433 | elif '3. 测试优先级' in line: 434 | break 435 | elif in_matrix_section and not line.startswith('2.'): 436 | try: 437 | # 识别功能及其测试覆盖 438 | if '|' in line: # 处理表格格式 439 | cells = [cell.strip() for cell in line.split('|')] 440 | cells = [cell for cell in cells if cell] # 移除空单元格 441 | 442 | if len(cells) >= 2: 443 | # 跳过表头和分隔行 444 | if not any(header in cells[0].lower() for header in ['需求类型', '用例编号', '-']): 445 | feature = cells[2] if len(cells) > 2 else cells[0] # 使用描述列或第一列 446 | test_cases = cells[-1] if len(cells) > 3 else '' # 使用最后一列作为测试用例 447 | 448 | if feature and test_cases: 449 | for test_case in test_cases.split(','): 450 | coverage_matrix.append({ 451 | 'feature': feature.strip(), 452 | 'test_type': test_case.strip() 453 | }) 454 | elif line.strip().endswith(':') or line.strip().endswith('：'): 455 | current_feature = line.strip().rstrip(':').rstrip('：').strip() 456 | elif current_feature and any(line.strip().startswith(marker) for marker in ['-', '•', '*', '>', '+']): 457 | test_type = line.strip()[1:].strip() 458 | if test_type: # 确保测试类型不为空 459 | coverage_matrix.append({ 460 | 'feature': current_feature, 461 | 'test_type': test_type 462 | }) 463 | elif line.strip() and not any(marker in line for marker in ['测试覆盖', '覆盖矩阵']): 464 | test_type = line.strip() 465 | if current_feature and test_type: # 确保特性和测试类型都不为空 466 | coverage_matrix.append({ 467 | 'feature': current_feature, 468 | 'test_type': test_type 469 | }) 470 | except Exception as e: 471 | logger.error(f"处理测试覆盖项时出错: {str(e)}，行内容: {line}") 472 | continue 473 | except Exception as e: 474 | logger.error(f"处理单行内容时出错: {str(e)}，行内容: {line}") 475 | continue 476 | 477 | # 去重并过滤空值 478 | unique_matrix = [] 479 | seen = set() 480 | for item in coverage_matrix: 481 | key = (item['feature'], item['test_type']) 482 | if key not in seen and item['feature'] and item['test_type']: 483 | seen.add(key) 484 | unique_matrix.append(item) 485 | 486 | return unique_matrix 487 | except Exception as e: 488 | logger.error(f"创建测试覆盖矩阵错误: {str(e)}") 489 | return coverage_matrix 490 | 491 | def _extract_priorities(self, message: str) -> List[Dict]: 492 | """从代理消息中提取测试优先级。""" 493 | priorities = [] 494 | try: 495 | if not message: 496 | logger.warning("输入消息为空") 497 | return priorities 498 | 499 | sections = message.split('\n') 500 | in_priorities_section = False 501 | 502 | for line in sections: 503 | try: 504 | line = line.strip() 505 | if not line: 506 | continue 507 | 508 | if '3. 测试优先级' in line: 509 | in_priorities_section = True 510 | continue 511 | elif '4. 资源估算' in line: 512 | break 513 | elif in_priorities_section and not line.startswith('3.'): 514 | try: 515 | # 解析优先级和描述 516 | if any(line.strip().lower().startswith(p) for p in ['p0', 'p1', 'p2', 'p3', 'p4']): 517 | if ':' in line or '：' in line: 518 | priority, description = (line.split(':', 1) if ':' in line else line.split('：', 1)) 519 | priority = priority.strip() 520 | description = description.strip() 521 | # 标准化优先级格式 522 | priority = f"P{priority[-1]}" if priority[-1].isdigit() else priority 523 | priorities.append({ 524 | 'level': priority.upper(), 525 | 'description': description 526 | }) 527 | except (IndexError, ValueError) as e: 528 | logger.error(f"解析优先级行时出错: {str(e)}，行内容: {line}") 529 | continue 530 | except Exception as e: 531 | logger.error(f"处理单行内容时出错: {str(e)}，行内容: {line}") 532 | continue 533 | 534 | return priorities 535 | except Exception as e: 536 | logger.error(f"提取优先级错误: {str(e)}") 537 | return priorities 538 | 539 | def _extract_resource_estimation(self, message: str) -> Dict: 540 | """从代理消息中提取资源估算。""" 541 | resource_estimation = { 542 | 'time': None, 543 | 'personnel': None, 544 | 'tools': [], 545 | 'additional_resources': [] 546 | } 547 | 548 | try: 549 | if not message: 550 | logger.warning("输入消息为空") 551 | return resource_estimation 552 | 553 | sections = message.split('\n') 554 | in_estimation_section = False 555 | 556 | for line in sections: 557 | try: 558 | line = line.strip() 559 | if not line: 560 | continue 561 | 562 | if '4. 资源估算' in line: 563 | in_estimation_section = True 564 | continue 565 | elif line.startswith('5.') or not line.strip(): 566 | break 567 | elif in_estimation_section and not line.startswith('4.'): 568 | try: 569 | # 解析资源详情 570 | if '时间:' in line.lower() or '时间：' in line: 571 | resource_estimation['time'] = line.split(':', 1)[1].strip() if ':' in line else line.split('：', 1)[1].strip() 572 | elif '人员:' in line.lower() or '人员：' in line: 573 | resource_estimation['personnel'] = line.split(':', 1)[1].strip() if ':' in line else line.split('：', 1)[1].strip() 574 | elif '工具:' in line.lower() or '工具：' in line: 575 | tools = line.split(':', 1)[1].strip() if ':' in line else line.split('：', 1)[1].strip() 576 | resource_estimation['tools'].append(tools) 577 | else: 578 | resource_estimation['additional_resources'].append(line.strip()) 579 | except IndexError as e: 580 | logger.error(f"解析资源详情时出错: {str(e)}，行内容: {line}") 581 | continue 582 | except Exception as e: 583 | logger.error(f"处理单行内容时出错: {str(e)}，行内容: {line}") 584 | continue 585 | 586 | return resource_estimation 587 | except Exception as e: 588 | logger.error(f"提取资源估算错误: {str(e)}") 589 | return resource_estimation -------------------------------------------------------------------------------- /src/main.py: -------------------------------------------------------------------------------- 1 | # src/main.py 2 | import sys 3 | import os 4 | sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) 5 | 6 | import asyncio 7 | import logging 8 | from typing import Dict, Optional 9 | from models.template import Template 10 | import json 11 | from src.utils.agent_io import AgentIO 12 | 13 | from agents.assistant import AssistantAgent 14 | from agents.requirement_analyst import RequirementAnalystAgent 15 | from agents.test_designer import TestDesignerAgent 16 | from agents.test_case_writer import TestCaseWriterAgent 17 | from agents.quality_assurance import QualityAssuranceAgent 18 | from services.document_processor import DocumentProcessor 19 | from services.test_case_generator import TestCaseGenerator 20 | from services.export_service import ExportService 21 | from services.ui_auto_service import UIAutoService 22 | from utils.logger import setup_logger 23 | # from utils.config import load_config 24 | 25 | # 把项目根目录添加到python路径（不添加，Windows环境可能会报错） 26 | sys.path.append(os.path.dirname(os.path.abspath(__file__))) 27 | 28 | logger = logging.getLogger(__name__) 29 | 30 | class AITestingSystem: 31 | def __init__(self, concurrent_workers: int = 1): 32 | setup_logger() 33 | # Initialize services 34 | self.doc_processor = DocumentProcessor() 35 | self.test_generator = TestCaseGenerator() 36 | self.export_service = ExportService() 37 | self.ui_auto_service = UIAutoService() 38 | 39 | # Initialize agents 40 | self.requirement_analyst = RequirementAnalystAgent() 41 | self.test_designer = TestDesignerAgent() 42 | self.test_case_writer = TestCaseWriterAgent(concurrent_workers=concurrent_workers) 43 | self.quality_assurance = QualityAssuranceAgent(concurrent_workers=concurrent_workers) 44 | self.assistant = AssistantAgent( 45 | [self.requirement_analyst, self.test_designer, 46 | self.test_case_writer, self.quality_assurance] 47 | ) 48 | 49 | async def process_requirements(self, 50 | doc_path: str, 51 | template_path: str, 52 | output_path: Optional[str] = None, 53 | test_type: str = "functional", 54 | input_path: Optional[str] = None) -> Dict: 55 | """Process requirements and generate test cases.""" 56 | try: 57 | # 如果是UI自动化测试，直接执行UI测试 58 | if test_type == "ui_auto": 59 | logger.info("开始执行UI自动化测试") 60 | # 使用input_path作为测试用例文件路径，如果未提供则使用doc_path 61 | test_case_path = input_path if input_path else doc_path 62 | result = await self.ui_auto_service.run_ui_tests(test_case_path, output_path) 63 | return result 64 | 65 | # 其他测试类型的处理逻辑保持不变 66 | # Process document 67 | doc_content = await self.doc_processor.process_document(doc_path) 68 | 69 | # 使用assistant协调工作流程，而不是直接调用各个代理 70 | task = { 71 | 'name': '测试用例生成', 72 | 'description': doc_content 73 | } 74 | 75 | logger.info("开始协调工作流程") 76 | try: 77 | result = await self.assistant.coordinate_workflow(task) 78 | logger.info(f"工作流程协调结果: {result}") 79 | except Exception as e: 80 | logger.error(f"工作流程协调错误: {str(e)}") 81 | return {'status': 'error', 'message': f'工作流程协调错误: {str(e)}'} 82 | 83 | # 如果需要修改，返回错误信息 84 | if result.get('status') == 'needs_revision': 85 | logger.error(f"需求分析结果需要调整: {result.get('message')}") 86 | return {'status': 'error', 'message': '需求分析结果需要调整'} 87 | 88 | # 从协调结果中获取各个阶段的结果 89 | requirements = None 90 | test_strategy = None 91 | test_cases = None 92 | reviewed_cases = None 93 | 94 | # 初始化AgentIO用于读取各个agent的结果 95 | agent_io = AgentIO() 96 | from agents.test_case_writer import TestCaseWriterAgent 97 | 98 | # 首先尝试从agent实例中获取结果 99 | for agent in self.assistant.agents: 100 | if isinstance(agent, RequirementAnalystAgent) and hasattr(agent, 'last_analysis'): 101 | requirements = agent.last_analysis 102 | elif isinstance(agent, TestDesignerAgent) and hasattr(agent, 'last_design'): 103 | test_strategy = agent.last_design 104 | elif isinstance(agent, TestCaseWriterAgent) and hasattr(agent, 'last_cases'): 105 | test_cases = agent.last_cases 106 | elif isinstance(agent, QualityAssuranceAgent) and hasattr(agent, 'last_review'): 107 | reviewed_cases = agent.last_review 108 | 109 | # 如果从agent实例中没有获取到结果，尝试从持久化存储中读取 110 | if not requirements: 111 | requirements = agent_io.load_result("requirement_analyst") 112 | logger.info("从持久化存储中加载需求分析结果") 113 | 114 | if not test_strategy: 115 | test_strategy = agent_io.load_result("test_designer") 116 | logger.info("从持久化存储中加载测试设计结果") 117 | 118 | # 从test_case_writer的持久化存储中加载最终的测试用例 119 | test_cases_data = agent_io.load_result("test_case_writer") 120 | logger.info("从test_case_writer的持久化存储中加载最终的测试用例") 121 | 122 | # 确保正确提取test_cases字段 123 | if isinstance(test_cases_data, dict) and 'test_cases' in test_cases_data: 124 | test_cases = test_cases_data['test_cases'] 125 | else: 126 | test_cases = test_cases_data 127 | 128 | # 如果没有获取到任何测试用例，返回错误 129 | if not test_cases: 130 | logger.error("没有生成任何测试用例") 131 | return {'status': 'error', 'message': '没有生成任何测试用例'} 132 | 133 | # Export test cases 134 | if output_path and test_cases: 135 | # 确保test_cases是列表类型 136 | if isinstance(test_cases, dict): 137 | test_cases = [test_cases] 138 | elif not isinstance(test_cases, list): 139 | logger.error("测试用例格式错误：必须是字典或字典列表") 140 | return {'status': 'error', 'message': '测试用例格式错误'} 141 | 142 | # 确保输出路径包含.xlsx扩展名 143 | if not output_path.endswith('.xlsx'): 144 | output_path = output_path + '.xlsx' 145 | 146 | # 如果template_path是路径，则从文件加载模板 147 | if isinstance(template_path, str): 148 | try: 149 | with open(template_path, 'r') as f: 150 | template_data = json.load(f) 151 | template = Template.from_dict(template_data) 152 | except Exception as e: 153 | logger.error(f"Error loading template: {str(e)}") 154 | # 使用默认模板 155 | template = Template( 156 | "Default Template", 157 | "Default test case template" 158 | ) 159 | else: 160 | # 假设template_path已经是Template对象 161 | template = template_path 162 | 163 | await self.export_service.export_to_excel( 164 | test_cases, 165 | template, 166 | output_path 167 | ) 168 | logger.info(f"测试用例已导出到 {output_path}") 169 | 170 | # 清理测试用例改进过程中生成的临时批次文件 171 | try: 172 | # 查找TestCaseWriterAgent实例并调用清理函数 173 | for agent in self.assistant.agents: 174 | if isinstance(agent, TestCaseWriterAgent): 175 | agent.delete_improved_batch_files() 176 | break 177 | else: 178 | # 如果在agents列表中没有找到，创建一个新实例并调用 179 | from src.agents.test_case_writer import TestCaseWriterAgent 180 | writer = TestCaseWriterAgent() 181 | writer.delete_improved_batch_files() 182 | logger.info("已清理测试用例改进过程中生成的临时批次文件") 183 | except Exception as e: 184 | logger.warning(f"清理临时批次文件时出错: {str(e)}") 185 | # 继续执行，不影响主流程 186 | 187 | return { 188 | "status": "success", 189 | "requirements": requirements, 190 | "test_strategy": test_strategy, 191 | "test_cases": test_cases, 192 | "workflow_result": result 193 | } 194 | 195 | except Exception as e: 196 | logger.error(f"Error processing requirements: {str(e)}") 197 | raise 198 | 199 | async def main(): 200 | # 使用命令行参数解析器获取参数 201 | from utils.cli_parser import get_cli_args 202 | try: 203 | args = get_cli_args() 204 | 205 | # 创建AITestingSystem实例，传入并发工作线程数 206 | system = AITestingSystem(concurrent_workers=args.concurrent_workers) 207 | result = await system.process_requirements( 208 | doc_path=args.doc_path, 209 | template_path=args.template_path, 210 | output_path=args.output_path, 211 | test_type=args.test_type, 212 | input_path=args.input_path 213 | ) 214 | 215 | if result.get('status') == 'success': 216 | print("测试执行成功！") 217 | if args.test_type == "ui_auto": 218 | print(f"总测试用例数: {result.get('total_cases', 0)}") 219 | print(f"通过用例数: {result.get('passed_cases', 0)}") 220 | print(f"测试结果已导出到: {args.output_path}") 221 | else: 222 | print(f"共生成 {len(result.get('test_cases', []))} 个测试用例") 223 | print(f"测试类型: {args.test_type}") 224 | if 'workflow_result' in result: 225 | print(f"工作流程状态: {result['workflow_result'].get('status', 'unknown')}") 226 | else: 227 | print(f"测试执行失败: {result.get('message', '未知错误')}") 228 | except Exception as e: 229 | print(f"程序执行错误: {str(e)}") 230 | print("使用方法示例:") 231 | print("1. 生成测试用例: python src/main.py -d docs/需求文档.pdf -t functional -o test_cases.xlsx") 232 | print("2. 执行UI测试: python src/main.py -i test_cases.json -t ui_auto -o test_results.xlsx") 233 | 234 | if __name__ == "__main__": 235 | asyncio.run(main()) -------------------------------------------------------------------------------- /src/models/__pycache__/template.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tony3liu/Auto_Generate_Test_Cases/bcd100b08fac79262e2cb5c1ede32d95d3525e8d/src/models/__pycache__/template.cpython-311.pyc -------------------------------------------------------------------------------- /src/models/__pycache__/template.cpython-312.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tony3liu/Auto_Generate_Test_Cases/bcd100b08fac79262e2cb5c1ede32d95d3525e8d/src/models/__pycache__/template.cpython-312.pyc -------------------------------------------------------------------------------- /src/models/__pycache__/test_case.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tony3liu/Auto_Generate_Test_Cases/bcd100b08fac79262e2cb5c1ede32d95d3525e8d/src/models/__pycache__/test_case.cpython-311.pyc -------------------------------------------------------------------------------- /src/models/__pycache__/test_case.cpython-312-pytest-8.2.1.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tony3liu/Auto_Generate_Test_Cases/bcd100b08fac79262e2cb5c1ede32d95d3525e8d/src/models/__pycache__/test_case.cpython-312-pytest-8.2.1.pyc -------------------------------------------------------------------------------- /src/models/__pycache__/test_case.cpython-312.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tony3liu/Auto_Generate_Test_Cases/bcd100b08fac79262e2cb5c1ede32d95d3525e8d/src/models/__pycache__/test_case.cpython-312.pyc -------------------------------------------------------------------------------- /src/models/template.py: -------------------------------------------------------------------------------- 1 | # src/models/template.py 2 | from typing import Dict, List 3 | from dataclasses import dataclass, field 4 | 5 | @dataclass 6 | class Template: 7 | """测试用例模板配置的模型。""" 8 | 9 | name: str 10 | description: str 11 | version: str = "1.0" 12 | custom_fields: List[str] = field(default_factory=list) 13 | column_widths: Dict[str, int] = field(default_factory=dict) 14 | conditional_formatting: List[Dict] = field(default_factory=list) 15 | 16 | def __post_init__(self): 17 | # 如果未提供列宽，则初始化默认列宽 18 | if not self.column_widths: 19 | self.column_widths = { 20 | 'ID': 10, 21 | 'Title': 50, 22 | 'Description': 100, 23 | 'Preconditions': 50, 24 | 'Steps': 100, 25 | 'Expected Results': 100, 26 | 'Priority': 15, 27 | 'Category': 20 28 | } 29 | 30 | def add_custom_field(self, field_name: str): 31 | """向模板添加自定义字段。 32 | 33 | Args: 34 | field_name: 要添加的字段名 35 | 36 | Raises: 37 | ValueError: 当字段名无效时 38 | """ 39 | if not isinstance(field_name, str): 40 | raise ValueError("字段名必须是字符串类型") 41 | 42 | if not field_name.strip(): 43 | raise ValueError("字段名不能为空") 44 | 45 | if field_name not in self.custom_fields: 46 | self.custom_fields.append(field_name) 47 | self.column_widths[field_name] = 30 # 默认宽度 48 | 49 | def remove_custom_field(self, field_name: str): 50 | """从模板中移除自定义字段。 51 | 52 | Args: 53 | field_name: 要移除的字段名 54 | 55 | Raises: 56 | ValueError: 当字段名无效时 57 | """ 58 | if not isinstance(field_name, str): 59 | raise ValueError("字段名必须是字符串类型") 60 | 61 | if not field_name.strip(): 62 | raise ValueError("字段名不能为空") 63 | 64 | if field_name in self.custom_fields: 65 | self.custom_fields.remove(field_name) 66 | self.column_widths.pop(field_name, None) 67 | 68 | def add_conditional_formatting(self, rule: Dict): 69 | """添加条件格式化规则。""" 70 | if self._validate_formatting_rule(rule): 71 | self.conditional_formatting.append(rule) 72 | 73 | def _validate_formatting_rule(self, rule: Dict) -> bool: 74 | """验证条件格式化规则。 75 | 76 | Args: 77 | rule: 包含格式化规则的字典 78 | 79 | Returns: 80 | bool: 规则验证是否通过 81 | 82 | Raises: 83 | ValueError: 当规则格式不正确时 84 | """ 85 | if not isinstance(rule, dict): 86 | raise ValueError("格式化规则必须是字典类型") 87 | 88 | required_keys = ['column', 'condition', 'format'] 89 | if not all(key in rule for key in required_keys): 90 | raise ValueError(f"格式化规则缺少必要的键: {', '.join(required_keys)}") 91 | 92 | if not isinstance(rule['column'], str) or not rule['column']: 93 | raise ValueError("column必须是非空字符串") 94 | 95 | if not isinstance(rule['condition'], str) or not rule['condition']: 96 | raise ValueError("condition必须是非空字符串") 97 | 98 | if not isinstance(rule['format'], str) or not rule['format']: 99 | raise ValueError("format必须是非空字符串") 100 | 101 | return True 102 | 103 | def to_dict(self) -> dict: 104 | """将模板转换为字典格式。""" 105 | return { 106 | 'name': self.name, 107 | 'description': self.description, 108 | 'version': self.version, 109 | 'custom_fields': self.custom_fields, 110 | 'column_widths': self.column_widths, 111 | 'conditional_formatting': self.conditional_formatting 112 | } 113 | 114 | @classmethod 115 | def from_dict(cls, data: dict) -> 'Template': 116 | """从字典创建模板。""" 117 | # 提供默认值，以防字典中缺少必要的键 118 | return cls( 119 | data.get('name', 'Default Template'), 120 | data.get('description', 'Default test case template'), 121 | version=data.get('version', '1.0'), 122 | custom_fields=data.get('custom_fields', []), 123 | column_widths=data.get('column_widths', {}), 124 | conditional_formatting=data.get('conditional_formatting', []) 125 | ) -------------------------------------------------------------------------------- /src/models/test_case.py: -------------------------------------------------------------------------------- 1 | # src/models/test_case.py 2 | import datetime 3 | from typing import List,Dict,Any,Optional 4 | from dataclasses import dataclass 5 | import uuid 6 | 7 | @dataclass 8 | class TestCase: 9 | """测试用例模型。""" 10 | 11 | title: str 12 | description: str 13 | preconditions: List[str] 14 | steps: List[str] 15 | expected_results: List[str] 16 | priority: str 17 | category: str 18 | test_data: Optional[Dict[str, Any]] = None 19 | 20 | def __post_init__(self): 21 | # 验证输入参数 22 | if not isinstance(self.title, str) or not self.title.strip(): 23 | raise ValueError("标题不能为空且必须是字符串类型") 24 | if not isinstance(self.description, str): 25 | raise ValueError("描述必须是字符串类型") 26 | if not isinstance(self.preconditions, list) or not all(isinstance(x, str) for x in self.preconditions): 27 | raise ValueError("前置条件必须是字符串列表") 28 | if not isinstance(self.steps, list) or not all(isinstance(x, str) for x in self.steps): 29 | raise ValueError("测试步骤必须是字符串列表") 30 | if not isinstance(self.expected_results, list) or not all(isinstance(x, str) for x in self.expected_results): 31 | raise ValueError("预期结果必须是字符串列表") 32 | if not isinstance(self.priority, str) or self.priority not in ["高", "中", "低"]: 33 | raise ValueError("优先级必须是'高'、'中'、'低'之一") 34 | if not isinstance(self.category, str) or not self.category.strip(): 35 | raise ValueError("类别不能为空且必须是字符串类型") 36 | if self.test_data is not None and not isinstance(self.test_data, dict): 37 | raise ValueError("测试数据必须是字典类型") 38 | 39 | # 初始化其他属性 40 | self.id = str(uuid.uuid4()) 41 | self.status = "Draft" 42 | self.created_at = datetime.datetime.now().isoformat() 43 | self.updated_at = self.created_at 44 | self.created_by = None 45 | self.last_updated_by = None 46 | 47 | def to_dict(self) -> dict: 48 | """将测试用例转换为字典格式。""" 49 | return { 50 | 'id': self.id, 51 | 'title': self.title, 52 | 'description': self.description, 53 | 'preconditions': self.preconditions, 54 | 'steps': self.steps, 55 | 'expected_results': self.expected_results, 56 | 'priority': self.priority, 57 | 'category': self.category, 58 | 'status': self.status, 59 | 'created_at': self.created_at, 60 | 'updated_at': self.updated_at, 61 | 'created_by': self.created_by, 62 | 'last_updated_by': self.last_updated_by 63 | } 64 | 65 | @classmethod 66 | def from_dict(cls, data: dict) -> 'TestCase': 67 | """从字典创建测试用例。""" 68 | instance = cls( 69 | title=data['title'], 70 | description=data['description'], 71 | preconditions=data['preconditions'], 72 | steps=data['steps'], 73 | expected_results=data['expected_results'], 74 | priority=data['priority'], 75 | category=data['category'] 76 | ) 77 | 78 | # 设置额外属性 79 | instance.id = data.get('id', instance.id) 80 | instance.status = data.get('status', instance.status) 81 | instance.created_at = data.get('created_at') 82 | instance.updated_at = data.get('updated_at') 83 | instance.created_by = data.get('created_by') 84 | instance.last_updated_by = data.get('last_updated_by') 85 | 86 | return instance -------------------------------------------------------------------------------- /src/schemas/__pycache__/communication.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tony3liu/Auto_Generate_Test_Cases/bcd100b08fac79262e2cb5c1ede32d95d3525e8d/src/schemas/__pycache__/communication.cpython-311.pyc -------------------------------------------------------------------------------- /src/schemas/__pycache__/communication.cpython-312.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tony3liu/Auto_Generate_Test_Cases/bcd100b08fac79262e2cb5c1ede32d95d3525e8d/src/schemas/__pycache__/communication.cpython-312.pyc -------------------------------------------------------------------------------- /src/schemas/communication.py: -------------------------------------------------------------------------------- 1 | # src/schemas/communication.py 2 | from typing import Dict, List, Optional 3 | from pydantic import BaseModel, Field 4 | 5 | class AgentMessage(BaseModel): 6 | """基础消息模型，所有agent间通信的消息都必须继承此类""" 7 | msg_type: str = Field(default="message", description="消息类型") 8 | version: str = Field(default="1.0", description="消息版本") 9 | timestamp: Optional[str] = Field(default=None, description="消息时间戳") 10 | 11 | class RequirementAnalysisRequest(AgentMessage): 12 | """需求分析请求消息""" 13 | doc_content: str = Field(..., description="需求文档内容") 14 | 15 | class TestScenario(BaseModel): 16 | """测试场景模型""" 17 | id: str = Field(..., description="场景ID") 18 | description: str = Field(..., description="场景描述") 19 | test_cases: List[str] = Field(default_factory=list, description="关联的测试用例ID列表") 20 | 21 | class RequirementAnalysisResponse(AgentMessage): 22 | """需求分析响应消息""" 23 | functional_requirements: List[str] = Field(default_factory=list, description="功能需求列表") 24 | non_functional_requirements: List[str] = Field(default_factory=list, description="非功能需求列表") 25 | test_scenarios: List[TestScenario] = Field(default_factory=list, description="测试场景列表") 26 | risk_areas: List[str] = Field(default_factory=list, description="风险领域列表") 27 | 28 | class TestDesignRequest(AgentMessage): 29 | """测试设计请求消息""" 30 | requirements: Dict = Field(..., description="需求分析结果") 31 | original_doc: Optional[str] = Field(default=None, description="原始需求文档") 32 | 33 | class TestDesignResponse(AgentMessage): 34 | """测试设计响应消息""" 35 | test_approach: Dict = Field(..., description="测试方法") 36 | coverage_matrix: List[Dict] = Field(..., description="测试覆盖矩阵") 37 | priorities: List[Dict] = Field(..., description="测试优先级") 38 | resource_estimation: Dict = Field(..., description="资源估算") 39 | 40 | class TestCaseWriteRequest(AgentMessage): 41 | """测试用例编写请求消息""" 42 | test_strategy: Dict = Field(..., description="测试策略") 43 | 44 | class TestCase(BaseModel): 45 | """测试用例模型""" 46 | id: str = Field(..., description="用例ID") 47 | title: str = Field(..., description="用例标题") 48 | preconditions: List[str] = Field(default_factory=list, description="前置条件") 49 | steps: List[str] = Field(..., description="测试步骤") 50 | expected_results: List[str] = Field(..., description="预期结果") 51 | priority: str = Field(..., description="优先级") 52 | category: str = Field(..., description="类别") 53 | 54 | class TestCaseWriteResponse(AgentMessage): 55 | """测试用例编写响应消息""" 56 | test_cases: List[TestCase] = Field(..., description="测试用例列表") 57 | 58 | class QualityAssuranceRequest(AgentMessage): 59 | """质量保证请求消息""" 60 | test_cases: List[Dict] = Field(..., description="待审查的测试用例列表") 61 | 62 | class QualityAssuranceResponse(AgentMessage): 63 | """质量保证响应消息""" 64 | reviewed_cases: List[Dict] = Field(..., description="审查后的测试用例列表") 65 | review_comments: List[str] = Field(default_factory=list, description="审查意见") 66 | 67 | class ErrorResponse(AgentMessage): 68 | """错误响应消息""" 69 | error_code: str = Field(..., description="错误代码") 70 | error_message: str = Field(..., description="错误信息") 71 | details: Optional[Dict] = Field(default=None, description="错误详情") -------------------------------------------------------------------------------- /src/services/__pycache__/document_processor.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tony3liu/Auto_Generate_Test_Cases/bcd100b08fac79262e2cb5c1ede32d95d3525e8d/src/services/__pycache__/document_processor.cpython-311.pyc -------------------------------------------------------------------------------- /src/services/__pycache__/document_processor.cpython-312.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tony3liu/Auto_Generate_Test_Cases/bcd100b08fac79262e2cb5c1ede32d95d3525e8d/src/services/__pycache__/document_processor.cpython-312.pyc -------------------------------------------------------------------------------- /src/services/__pycache__/export_service.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tony3liu/Auto_Generate_Test_Cases/bcd100b08fac79262e2cb5c1ede32d95d3525e8d/src/services/__pycache__/export_service.cpython-311.pyc -------------------------------------------------------------------------------- /src/services/__pycache__/export_service.cpython-312.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tony3liu/Auto_Generate_Test_Cases/bcd100b08fac79262e2cb5c1ede32d95d3525e8d/src/services/__pycache__/export_service.cpython-312.pyc -------------------------------------------------------------------------------- /src/services/__pycache__/test_case_generator.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tony3liu/Auto_Generate_Test_Cases/bcd100b08fac79262e2cb5c1ede32d95d3525e8d/src/services/__pycache__/test_case_generator.cpython-311.pyc -------------------------------------------------------------------------------- /src/services/__pycache__/test_case_generator.cpython-312.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tony3liu/Auto_Generate_Test_Cases/bcd100b08fac79262e2cb5c1ede32d95d3525e8d/src/services/__pycache__/test_case_generator.cpython-312.pyc -------------------------------------------------------------------------------- /src/services/__pycache__/ui_auto_service.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tony3liu/Auto_Generate_Test_Cases/bcd100b08fac79262e2cb5c1ede32d95d3525e8d/src/services/__pycache__/ui_auto_service.cpython-311.pyc -------------------------------------------------------------------------------- /src/services/document_processor.py: -------------------------------------------------------------------------------- 1 | # src/services/document_processor.py 2 | from pathlib import Path 3 | import logging 4 | from PyPDF2 import PdfReader 5 | from docx import Document 6 | import markdown 7 | 8 | logger = logging.getLogger(__name__) 9 | 10 | class DocumentProcessor: 11 | """用于处理不同类型输入文档的服务。""" 12 | 13 | SUPPORTED_FORMATS = {'.pdf', '.docx', '.md', '.txt'} 14 | 15 | async def process_document(self, doc_path: str) -> str: 16 | """处理输入文档并提取文本内容。""" 17 | try: 18 | path = Path(doc_path) 19 | if not path.exists(): 20 | raise FileNotFoundError(f"Document not found: {doc_path}") 21 | 22 | if path.suffix not in self.SUPPORTED_FORMATS: 23 | raise ValueError(f"Unsupported file format: {path.suffix}") 24 | 25 | content = self._extract_content(path) 26 | return self._preprocess_content(content) 27 | 28 | except Exception as e: 29 | logger.error(f"Error processing document {doc_path}: {str(e)}") 30 | raise 31 | 32 | def _extract_content(self, file_path: Path) -> str: 33 | """从不同文件格式中提取文本内容。""" 34 | if file_path.suffix == '.pdf': 35 | return self._extract_pdf(file_path) 36 | elif file_path.suffix == '.docx': 37 | return self._extract_docx(file_path) 38 | elif file_path.suffix == '.md': 39 | return self._extract_markdown(file_path) 40 | else: # .txt 41 | return self._extract_text(file_path) 42 | 43 | def _extract_pdf(self, file_path: Path) -> str: 44 | with open(file_path, 'rb') as file: 45 | reader = PdfReader(file) 46 | return ' '.join(page.extract_text() for page in reader.pages) 47 | 48 | def _extract_docx(self, file_path:Path) -> str: 49 | doc = Document(str(file_path)) 50 | return ' '.join(paragraph.text for paragraph in doc.paragraphs) 51 | 52 | def _extract_markdown(self, file_path: Path) -> str: 53 | with open(file_path, 'r', encoding='utf-8') as file: 54 | content = file.read() 55 | return markdown.markdown(content) 56 | 57 | def _extract_text(self, file_path: Path) -> str: 58 | with open(file_path, 'r', encoding='utf-8') as file: 59 | return file.read() 60 | 61 | def _preprocess_content(self, content: str) -> str: 62 | """预处理提取的内容以便更好地分析。""" 63 | # 删除多余的空白并规范化行尾 64 | content = ' '.join(content.split()) 65 | return content -------------------------------------------------------------------------------- /src/services/export_service.py: -------------------------------------------------------------------------------- 1 | # src/services/export_service.py 2 | from typing import List 3 | import pandas as pd 4 | from pathlib import Path 5 | import logging 6 | import os 7 | from models.test_case import TestCase 8 | from models.template import Template 9 | 10 | logger = logging.getLogger(__name__) 11 | 12 | class ExportService: 13 | """Service for exporting test cases to Excel format.""" 14 | 15 | def __init__(self): 16 | self.supported_formats = ['.xlsx'] 17 | self.max_file_size_mb = 50 # 最大文件大小限制(MB) 18 | 19 | async def export_to_excel(self, 20 | test_cases: List, 21 | template: Template, 22 | output_path: str) -> str: 23 | """Export test cases to Excel file using specified template.""" 24 | try: 25 | # 验证输出路径 26 | path = Path(output_path) 27 | self._validate_output_path(path) 28 | 29 | # 转换测试用例到DataFrame 30 | df = self._convert_to_dataframe(test_cases, template) 31 | 32 | # 应用模板样式 33 | styled_df = self._apply_template_styling(df, template) 34 | 35 | # 导出到Excel 36 | self._save_to_excel(styled_df, path, template) 37 | 38 | # 验证文件大小 39 | self._validate_file_size(path) 40 | 41 | return str(path) 42 | 43 | except Exception as e: 44 | logger.error(f"Error exporting test cases: {str(e)}") 45 | raise 46 | 47 | def _validate_output_path(self, path: Path): 48 | """验证输出路径的有效性""" 49 | # 确保路径有扩展名，如果没有则添加默认的.xlsx扩展名 50 | if not path.suffix: 51 | path = Path(str(path) + '.xlsx') 52 | logger.info(f"添加默认扩展名.xlsx到输出路径: {path}") 53 | 54 | # 检查文件格式 55 | if path.suffix not in self.supported_formats: 56 | logger.warning(f"不支持的导出格式: {path.suffix}，将使用.xlsx格式") 57 | # 替换不支持的扩展名为.xlsx 58 | path = Path(str(path.with_suffix('')) + '.xlsx') 59 | 60 | # 检查目录是否存在 61 | if not path.parent.exists(): 62 | raise ValueError(f"Output directory does not exist: {path.parent}") 63 | 64 | # 检查目录写入权限 65 | if not os.access(path.parent, os.W_OK): 66 | raise ValueError(f"No write permission for directory: {path.parent}") 67 | 68 | # 如果文件已存在，检查是否可写 69 | if path.exists() and not os.access(path, os.W_OK): 70 | raise ValueError(f"No write permission for file: {path}") 71 | 72 | def _validate_file_size(self, path: Path): 73 | """验证导出文件大小""" 74 | file_size_mb = path.stat().st_size / (1024 * 1024) 75 | if file_size_mb > self.max_file_size_mb: 76 | path.unlink() # 删除超大文件 77 | raise ValueError(f"Generated file size ({file_size_mb:.2f}MB) exceeds limit ({self.max_file_size_mb}MB)") 78 | 79 | def _convert_to_dataframe(self, 80 | test_cases: List, 81 | template: Template) -> pd.DataFrame: 82 | """Convert test cases to pandas DataFrame based on template.""" 83 | data = [] 84 | for test_case in test_cases: 85 | # 检查test_case是否为字典类型 86 | if isinstance(test_case, dict): 87 | # 如果是字典类型，直接使用字典的值 88 | row = { 89 | 'ID': test_case.get('id', ''), 90 | 'Title': test_case.get('title', ''), 91 | 'Description': test_case.get('description', ''), 92 | 'Preconditions': '\n'.join(test_case.get('preconditions', [])), 93 | 'Steps': '\n'.join(test_case.get('steps', [])), 94 | 'Expected Results': '\n'.join(test_case.get('expected_results', [])), 95 | 'Priority': test_case.get('priority', ''), 96 | 'Category': test_case.get('category', ''), 97 | 'Status': test_case.get('status', 'Draft'), 98 | 'Created At': test_case.get('created_at', ''), 99 | 'Updated At': test_case.get('updated_at', ''), 100 | 'Created By': test_case.get('created_by', ''), 101 | 'Last Updated By': test_case.get('last_updated_by', '') 102 | } 103 | else: 104 | # 如果是TestCase对象，使用对象的属性 105 | row = { 106 | 'ID': getattr(test_case, 'id', ''), 107 | 'Title': getattr(test_case, 'title', ''), 108 | 'Description': getattr(test_case, 'description', ''), 109 | 'Preconditions': '\n'.join(getattr(test_case, 'preconditions', [])), 110 | 'Steps': '\n'.join(getattr(test_case, 'steps', [])), 111 | 'Expected Results': '\n'.join(getattr(test_case, 'expected_results', [])), 112 | 'Priority': getattr(test_case, 'priority', ''), 113 | 'Category': getattr(test_case, 'category', ''), 114 | 'Status': getattr(test_case, 'status', 'Draft'), 115 | 'Created At': getattr(test_case, 'created_at', ''), 116 | 'Updated At': getattr(test_case, 'updated_at', ''), 117 | 'Created By': getattr(test_case, 'created_by', ''), 118 | 'Last Updated By': getattr(test_case, 'last_updated_by', '') 119 | } 120 | # 添加自定义字段 121 | for field in template.custom_fields: 122 | row[field] = getattr(test_case, field, '') 123 | data.append(row) 124 | 125 | return pd.DataFrame(data) 126 | 127 | def _apply_template_styling(self, 128 | df: pd.DataFrame, 129 | template: Template) -> pd.DataFrame: 130 | """Apply template styling to DataFrame.""" 131 | # 应用列宽 - 只转换为字符串类型，实际列宽在保存到Excel时应用 132 | for col, width in template.column_widths.items(): 133 | if col in df.columns: 134 | df[col] = df[col].astype(str) 135 | 136 | # 应用条件格式 137 | for rule in template.conditional_formatting: 138 | if rule['column'] in df.columns and 'condition' in rule: 139 | try: 140 | mask = df[rule['column']].str.contains(rule['condition'], na=False) 141 | 142 | # 应用格式 - 根据format字段的值应用不同的格式 143 | if 'format' in rule: 144 | format_type = rule['format'] 145 | if format_type == 'highlight': 146 | # 高亮显示 147 | df.loc[mask, rule['column']] = f"*** {df.loc[mask, rule['column']]} ***" 148 | elif format_type == 'prefix': 149 | # 添加前缀 150 | df.loc[mask, rule['column']] = f"! {df.loc[mask, rule['column']]}" 151 | elif format_type == 'uppercase': 152 | # 转换为大写 153 | df.loc[mask, rule['column']] = df.loc[mask, rule['column']].str.upper() 154 | else: 155 | # 默认格式 - 如果没有指定format 156 | df.loc[mask, rule['column']] = f"*** {df.loc[mask, rule['column']]} ***" 157 | except Exception as e: 158 | logger.warning(f"Error applying conditional format: {str(e)}") 159 | 160 | return df 161 | 162 | def _save_to_excel(self, df: pd.DataFrame, path: Path, template: Template | None = None): 163 | """Save DataFrame to Excel with formatting.""" 164 | with pd.ExcelWriter(path, engine='openpyxl') as writer: 165 | df.to_excel(writer, index=False, sheet_name='Test Cases') 166 | 167 | # 应用列宽 168 | worksheet = writer.sheets['Test Cases'] 169 | for idx, col in enumerate(df.columns): 170 | # 如果模板中定义了该列的宽度，则使用模板中的宽度 171 | if template and col in template.column_widths: 172 | worksheet.column_dimensions[chr(65 + idx)].width = template.column_widths[col] 173 | else: 174 | # 否则自动调整列宽 175 | max_length = max( 176 | df[col].astype(str).apply(len).max(), 177 | len(col) 178 | ) 179 | # 设置最小和最大列宽 180 | adjusted_width = min(max(max_length + 2, 10), 50) 181 | worksheet.column_dimensions[chr(65 + idx)].width = adjusted_width -------------------------------------------------------------------------------- /src/services/test_case_generator.py: -------------------------------------------------------------------------------- 1 | import json 2 | from typing import List, Dict, Any, Optional 3 | from datetime import datetime 4 | from models.test_case import TestCase 5 | from schemas.communication import TestScenario 6 | 7 | class TestCaseGenerator: 8 | def __init__(self, template_path: Optional[str] = None): 9 | self.template_path = template_path 10 | self.base_template = self._load_template() if template_path else {} 11 | 12 | def _load_template(self) -> Dict: 13 | """加载测试用例模板配置""" 14 | if self.template_path is None: 15 | return {} 16 | try: 17 | with open(file=self.template_path, mode='r', encoding='utf-8') as f: 18 | return json.load(f) 19 | except FileNotFoundError: 20 | return {} 21 | 22 | def generate_test_cases(self, test_strategy: Dict[str, Any]) -> List[TestCase]: 23 | """基于测试策略生成测试用例 24 | 25 | Args: 26 | test_strategy: 包含测试策略的字典，应包含以下字段： 27 | - scenarios: 测试场景列表 28 | - test_types: 测试类型配置 29 | - priorities: 优先级定义 30 | - validation_rules: 验证规则 31 | 32 | Returns: 33 | List[TestCase]: 生成的测试用例列表 34 | """ 35 | test_cases = [] 36 | scenarios = test_strategy.get('scenarios', []) 37 | test_types = test_strategy.get('test_types', {}) 38 | priorities = test_strategy.get('priorities', {}) 39 | validation_rules = test_strategy.get('validation_rules', {}) 40 | 41 | for scenario in scenarios: 42 | # 根据场景类型生成对应的测试用例 43 | scenario_type = scenario.get('type') 44 | if scenario_type in test_types: 45 | test_case = self._create_test_case( 46 | scenario=scenario, 47 | test_type=test_types[scenario_type], 48 | priorities=priorities, 49 | validation_rules=validation_rules 50 | ) 51 | if test_case: 52 | test_cases.append(test_case) 53 | 54 | return test_cases 55 | 56 | def _create_test_case(self, 57 | scenario: Dict[str, Any], 58 | test_type: Dict[str, Any], 59 | priorities: Dict[str, Any], 60 | validation_rules: Dict[str, Any]) -> Optional[TestCase]: 61 | """创建单个测试用例 62 | 63 | Args: 64 | scenario: 测试场景信息 65 | test_type: 测试类型配置 66 | priorities: 优先级定义 67 | validation_rules: 验证规则 68 | 69 | Returns: 70 | Optional[TestCase]: 生成的测试用例，如果生成失败则返回None 71 | """ 72 | try: 73 | # 获取测试用例基本信息 74 | title = f"{test_type.get('name', '')} - {scenario.get('description', '')}" 75 | priority = self._determine_priority(scenario, priorities) 76 | category = test_type.get('category', '功能测试') 77 | 78 | # 生成测试步骤和预期结果 79 | steps = self._generate_steps(test_type, scenario) 80 | expected_results = self._generate_expected_results(test_type, scenario, validation_rules) 81 | 82 | # 创建测试用例 83 | test_case = TestCase( 84 | title=title, 85 | description=scenario.get('description', ''), 86 | preconditions=scenario.get('preconditions', []), 87 | steps=steps, 88 | expected_results=expected_results, 89 | priority=priority, 90 | category=category 91 | ) 92 | 93 | # 添加额外的测试数据 94 | test_case.test_data = self._generate_test_data(test_type, scenario) 95 | 96 | return test_case 97 | except Exception as e: 98 | print(f"创建测试用例失败: {str(e)}") 99 | return None 100 | 101 | def _determine_priority(self, scenario: Dict[str, Any], priorities: Dict[str, Any]) -> str: 102 | """根据场景和优先级定义确定测试用例优先级""" 103 | scenario_priority = scenario.get('priority') 104 | if scenario_priority in priorities: 105 | return priorities[scenario_priority].get('level', '中') 106 | return '中' 107 | 108 | def _generate_steps(self, test_type: Dict[str, Any], scenario: Dict[str, Any]) -> List[str]: 109 | """生成测试步骤""" 110 | base_steps = test_type.get('base_steps', []) 111 | scenario_steps = scenario.get('steps', []) 112 | return base_steps + scenario_steps 113 | 114 | def _generate_expected_results(self, 115 | test_type: Dict[str, Any], 116 | scenario: Dict[str, Any], 117 | validation_rules: Dict[str, Any]) -> List[str]: 118 | """生成预期结果""" 119 | base_results = test_type.get('base_expected_results', []) 120 | scenario_results = scenario.get('expected_results', []) 121 | 122 | # 添加验证规则相关的预期结果 123 | if validation_rules: 124 | rule_results = self._generate_validation_rule_results(test_type, validation_rules) 125 | return base_results + scenario_results + rule_results 126 | 127 | return base_results + scenario_results 128 | 129 | def _generate_validation_rule_results(self, 130 | test_type: Dict[str, Any], 131 | validation_rules: Dict[str, Any]) -> List[str]: 132 | """根据验证规则生成预期结果""" 133 | results = [] 134 | type_rules = validation_rules.get(test_type.get('name', ''), {}) 135 | 136 | for rule_name, rule_value in type_rules.items(): 137 | if isinstance(rule_value, dict): 138 | threshold = rule_value.get('threshold') 139 | if threshold is not None: 140 | results.append(f"{rule_name}应达到{threshold}") 141 | elif isinstance(rule_value, (int, float)): 142 | results.append(f"{rule_name}应达到{rule_value}") 143 | 144 | return results 145 | 146 | def _generate_test_data(self, test_type: Dict[str, Any], scenario: Dict[str, Any]) -> Dict[str, Any]: 147 | """生成测试数据""" 148 | test_data = {} 149 | 150 | # 合并测试类型的基础数据和场景特定数据 151 | base_data = test_type.get('test_data', {}) 152 | scenario_data = scenario.get('test_data', {}) 153 | 154 | test_data.update(base_data) 155 | test_data.update(scenario_data) 156 | 157 | return test_data -------------------------------------------------------------------------------- /src/services/ui_auto_service.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from datetime import datetime 3 | import pandas as pd 4 | from src.agents.browser_use_agent import browser_use_agent, read_test_cases 5 | 6 | logger = logging.getLogger(__name__) 7 | 8 | class UIAutoService: 9 | def __init__(self, concurrent_workers: int = 1): 10 | self.test_cases = [] 11 | self.test_results = [] 12 | 13 | async def run_ui_tests(self, input_path: str, output_path: str): 14 | """运行UI自动化测试并将结果导出到Excel 15 | 16 | Args: 17 | input_path: 测试用例文件路径，从cli的--input参数获取 18 | output_path: 测试结果输出路径 19 | """ 20 | try: 21 | # 读取测试用例 22 | logger.info(f"从 {input_path} 读取测试用例") 23 | self.test_cases = read_test_cases(input_path) 24 | 25 | if not self.test_cases: 26 | logger.error("未找到测试用例") 27 | return { 28 | "status": "error", 29 | "message": "未找到测试用例" 30 | } 31 | 32 | # 执行每个测试用例 33 | for test_case in self.test_cases: 34 | result = await self._execute_test_case(test_case) 35 | self.test_results.append(result) 36 | 37 | # 导出结果到Excel 38 | await self._export_to_excel(output_path) 39 | 40 | return { 41 | "status": "success", 42 | "message": f"UI自动化测试完成，结果已导出到: {output_path}", 43 | "total_cases": len(self.test_cases), 44 | "passed_cases": sum(1 for r in self.test_results if r["status"] == "passed") 45 | } 46 | 47 | except Exception as e: 48 | logger.error(f"UI自动化测试执行失败: {str(e)}") 49 | return { 50 | "status": "error", 51 | "message": f"UI自动化测试执行失败: {str(e)}" 52 | } 53 | 54 | async def _execute_test_case(self, test_case): 55 | """执行单个测试用例,测试用例是json形式""" 56 | try: 57 | # 构建任务提示 58 | task_prompt = self._build_task_prompt(test_case) 59 | 60 | #执行测试 61 | actual_results = await browser_use_agent(task_prompt) 62 | 63 | # 获取测试结果 64 | final_result = actual_results.final_result() 65 | is_successful = actual_results.is_successful() 66 | 67 | return { 68 | "test_case_id": test_case.get("id", ""), 69 | "title": test_case.get("title", ""), 70 | "steps": test_case.get("steps", []), 71 | "expected_results": test_case.get("expected_results", []), 72 | "actual_result": final_result, 73 | "status": "passed" if is_successful == True else "failed" if is_successful == False else "warning", 74 | "execution_time": datetime.now().strftime("%Y-%m-%d %H:%M:%S") 75 | } 76 | 77 | except Exception as e: 78 | logger.error(f"测试用例执行失败: {str(e)}") 79 | return { 80 | "test_case_id": test_case.get("id", ""), 81 | "title": test_case.get("title", ""), 82 | "steps": test_case.get("steps", []), 83 | "expected_results": test_case.get("expected_results", []), 84 | "actual_result": str(e), 85 | "status": "error", 86 | "execution_time": datetime.now().strftime("%Y-%m-%d %H:%M:%S") 87 | } 88 | 89 | def _build_task_prompt(self, test_case): 90 | """构建任务提示""" 91 | title = test_case.get("title", "") 92 | steps = test_case.get("steps", []) 93 | expected_results = test_case.get("expected_results", []) 94 | 95 | prompt = f"测试用例标题: {title}\n\n" 96 | prompt += "测试步骤:\n" 97 | for i, step in enumerate(steps, 1): 98 | prompt += f"{i}. {step}\n" 99 | 100 | prompt += "\n预期结果:\n" 101 | for i, result in enumerate(expected_results, 1): 102 | prompt += f"{i}. {result}\n" 103 | 104 | return prompt 105 | 106 | async def _export_to_excel(self, output_path: str): 107 | """导出测试结果到Excel""" 108 | try: 109 | # 确保输出路径以.xlsx结尾 110 | if not output_path.endswith('.xlsx'): 111 | output_path = output_path + '.xlsx' 112 | 113 | # 创建DataFrame 114 | df = pd.DataFrame(self.test_results) 115 | 116 | # 重新排列列顺序 117 | columns = [ 118 | "test_case_id", 119 | "title", 120 | "steps", 121 | "expected_results", 122 | "actual_result", 123 | "status", 124 | "execution_time" 125 | ] 126 | df = df[columns] 127 | 128 | # 导出到Excel 129 | df.to_excel(output_path, index=False, engine='openpyxl') 130 | logger.info(f"测试结果已导出到: {output_path}") 131 | 132 | except Exception as e: 133 | logger.error(f"导出Excel失败: {str(e)}") 134 | raise -------------------------------------------------------------------------------- /src/templates/api_test_template.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "API测试模板", 3 | "description": "用于REST API接口测试的标准模板", 4 | "version": "1.0", 5 | "custom_fields": [ 6 | "API路径", 7 | "请求方法", 8 | "请求头", 9 | "请求参数", 10 | "响应码", 11 | "响应时间" 12 | ], 13 | "column_widths": { 14 | "ID": 10, 15 | "Title": 50, 16 | "Description": 100, 17 | "API路径": 60, 18 | "请求方法": 20, 19 | "请求头": 80, 20 | "请求参数": 100, 21 | "响应码": 15, 22 | "响应时间": 20, 23 | "Preconditions": 80, 24 | "Steps": 120, 25 | "Expected Results": 120, 26 | "Priority": 15, 27 | "Category": 20 28 | }, 29 | "conditional_formatting": [ 30 | { 31 | "column": "响应码", 32 | "condition": "^5\\d{2}$", 33 | "format": "red" 34 | }, 35 | { 36 | "column": "响应码", 37 | "condition": "^4\\d{2}$", 38 | "format": "yellow" 39 | }, 40 | { 41 | "column": "响应时间", 42 | "condition": ">1000", 43 | "format": "orange" 44 | }, 45 | { 46 | "column": "Priority", 47 | "condition": "High", 48 | "format": "bold_red" 49 | } 50 | ] 51 | } -------------------------------------------------------------------------------- /src/templates/functional_test_template.json: -------------------------------------------------------------------------------- 1 | { 2 | "test_scenarios": [ 3 | { 4 | "id": "FUNC-001", 5 | "description": "文件格式验证测试", 6 | "parameters": { 7 | "supported_formats": ["pdf", "jpg", "png"], 8 | "max_batch_size": 50, 9 | "invalid_formats": ["exe", "bat"] 10 | } 11 | }, 12 | { 13 | "id": "AI-001", 14 | "description": "资质内容提取准确性验证", 15 | "validation_criteria": { 16 | "accuracy_threshold": 0.98, 17 | "allowed_formats": ["营业执照", "身份证"] 18 | } 19 | }, 20 | { 21 | "id": "TRACE-001", 22 | "description": "溯源功能响应验证", 23 | "performance_requirements": { 24 | "max_response_time": 2, 25 | "concurrent_users": 100 26 | } 27 | } 28 | ], 29 | "common_parameters": { 30 | "test_environment": { 31 | "browser": "Chrome 120+", 32 | "resolution": "1920x1080" 33 | }, 34 | "data_requirements": { 35 | "sample_size": 1000, 36 | "data_distribution": { 37 | "pdf": 40, 38 | "jpg": 35, 39 | "png": 25 40 | } 41 | } 42 | } 43 | } -------------------------------------------------------------------------------- /src/templates/ui_auto_test_template.json: -------------------------------------------------------------------------------- 1 | { 2 | "test_cases": [ 3 | { 4 | "id": "tc_xxx", 5 | "title": "测试用例标题，描述测试的主要目的", 6 | "preconditions": [ 7 | "前置条件1，例如：系统环境要求", 8 | "前置条件2，例如：数据准备要求" 9 | ], 10 | "steps": [ 11 | "步骤1：具体的操作步骤", 12 | "步骤2：具体的操作步骤" 13 | ], 14 | "expected_results": [ 15 | "预期结果1：期望的系统响应或结果", 16 | "预期结果2：期望的系统响应或结果" 17 | ], 18 | "priority": "P0/P1/P2", 19 | "category": "功能测试/性能测试/安全测试等", 20 | "description": "测试用例的详细描述，可选字段" 21 | } 22 | ] 23 | } -------------------------------------------------------------------------------- /src/utils/__pycache__/agent_io.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tony3liu/Auto_Generate_Test_Cases/bcd100b08fac79262e2cb5c1ede32d95d3525e8d/src/utils/__pycache__/agent_io.cpython-311.pyc -------------------------------------------------------------------------------- /src/utils/__pycache__/agent_io.cpython-312.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tony3liu/Auto_Generate_Test_Cases/bcd100b08fac79262e2cb5c1ede32d95d3525e8d/src/utils/__pycache__/agent_io.cpython-312.pyc -------------------------------------------------------------------------------- /src/utils/__pycache__/cli_parser.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tony3liu/Auto_Generate_Test_Cases/bcd100b08fac79262e2cb5c1ede32d95d3525e8d/src/utils/__pycache__/cli_parser.cpython-311.pyc -------------------------------------------------------------------------------- /src/utils/__pycache__/cli_parser.cpython-312.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tony3liu/Auto_Generate_Test_Cases/bcd100b08fac79262e2cb5c1ede32d95d3525e8d/src/utils/__pycache__/cli_parser.cpython-312.pyc -------------------------------------------------------------------------------- /src/utils/__pycache__/config.cpython-312.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tony3liu/Auto_Generate_Test_Cases/bcd100b08fac79262e2cb5c1ede32d95d3525e8d/src/utils/__pycache__/config.cpython-312.pyc -------------------------------------------------------------------------------- /src/utils/__pycache__/env_loader.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tony3liu/Auto_Generate_Test_Cases/bcd100b08fac79262e2cb5c1ede32d95d3525e8d/src/utils/__pycache__/env_loader.cpython-311.pyc -------------------------------------------------------------------------------- /src/utils/__pycache__/logger.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tony3liu/Auto_Generate_Test_Cases/bcd100b08fac79262e2cb5c1ede32d95d3525e8d/src/utils/__pycache__/logger.cpython-311.pyc -------------------------------------------------------------------------------- /src/utils/__pycache__/logger.cpython-312.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tony3liu/Auto_Generate_Test_Cases/bcd100b08fac79262e2cb5c1ede32d95d3525e8d/src/utils/__pycache__/logger.cpython-312.pyc -------------------------------------------------------------------------------- /src/utils/agent_io.py: -------------------------------------------------------------------------------- 1 | # src/utils/agent_io.py 2 | import json 3 | import os 4 | import logging 5 | from typing import Dict, Any, Optional 6 | 7 | logger = logging.getLogger(__name__) 8 | 9 | class AgentIO: 10 | """处理Agent结果的序列化和反序列化 11 | 12 | 这个类提供了保存和加载Agent执行结果的功能，使各个Agent之间的数据流转更加清晰。 13 | 每个Agent的结果将被保存为单独的JSON文件，并可以在需要时被其他Agent读取。 14 | """ 15 | 16 | def __init__(self, output_dir: str = "agent_results"): 17 | """初始化AgentIO 18 | 19 | Args: 20 | output_dir: 保存Agent结果的目录，默认为'agent_results' 21 | """ 22 | self.output_dir = output_dir 23 | # 确保输出目录存在 24 | os.makedirs(output_dir, exist_ok=True) 25 | 26 | def save_result(self, agent_name: str, result: Dict[str, Any]) -> str: 27 | """将Agent的执行结果保存为JSON文件 28 | 29 | Args: 30 | agent_name: Agent的名称，用于生成文件名 31 | result: 要保存的结果数据 32 | 33 | Returns: 34 | 保存的文件路径 35 | """ 36 | file_path = os.path.join(self.output_dir, f"{agent_name}_result.json") 37 | try: 38 | # 处理Pydantic模型对象的序列化 39 | def pydantic_encoder(obj): 40 | if hasattr(obj, 'dict') and callable(getattr(obj, 'dict')): 41 | return obj.dict() 42 | if hasattr(obj, 'model_dump') and callable(getattr(obj, 'model_dump')): 43 | return obj.model_dump() 44 | raise TypeError(f"Object of type {type(obj)} is not JSON serializable") 45 | 46 | with open(file_path, 'w', encoding='utf-8') as f: 47 | json.dump(result, f, ensure_ascii=False, indent=2, default=pydantic_encoder) 48 | logger.info(f"已保存{agent_name}的执行结果到{file_path}") 49 | return file_path 50 | except Exception as e: 51 | logger.error(f"保存{agent_name}结果时出错: {str(e)}") 52 | raise 53 | 54 | def load_result(self, agent_name: str) -> Optional[Dict[str, Any]]: 55 | """加载指定Agent的执行结果 56 | 57 | Args: 58 | agent_name: Agent的名称，用于查找文件 59 | 60 | Returns: 61 | 加载的结果数据，如果文件不存在则返回None 62 | """ 63 | file_path = os.path.join(self.output_dir, f"{agent_name}_result.json") 64 | if not os.path.exists(file_path): 65 | logger.warning(f"找不到{agent_name}的结果文件: {file_path}") 66 | return None 67 | 68 | try: 69 | with open(file_path, 'r', encoding='utf-8') as f: 70 | result = json.load(f) 71 | logger.info(f"已加载{agent_name}的执行结果") 72 | return result 73 | except Exception as e: 74 | logger.error(f"加载{agent_name}结果时出错: {str(e)}") 75 | return None -------------------------------------------------------------------------------- /src/utils/cli_parser.py: -------------------------------------------------------------------------------- 1 | # src/utils/cli_parser.py 2 | import argparse 3 | import os 4 | from pathlib import Path 5 | import logging 6 | 7 | logger = logging.getLogger(__name__) 8 | 9 | class CLIParser: 10 | """命令行参数解析器，用于处理用户输入的命令行参数。""" 11 | 12 | def __init__(self): 13 | self.parser = argparse.ArgumentParser( 14 | description="自动化测试用例生成工具", 15 | formatter_class=argparse.ArgumentDefaultsHelpFormatter 16 | ) 17 | self._setup_arguments() 18 | 19 | def _setup_arguments(self): 20 | """设置命令行参数""" 21 | # 添加文档路径参数 22 | self.parser.add_argument( 23 | "-d", "--doc", 24 | dest="doc_path", 25 | help="需求文档路径或测试用例文件路径", 26 | type=str 27 | ) 28 | 29 | # 添加输入文件参数（用于UI自动化测试） 30 | self.parser.add_argument( 31 | "-i", "--input", 32 | dest="input_path", 33 | help="测试用例文件路径（用于UI自动化测试）", 34 | type=str 35 | ) 36 | 37 | # 添加输出路径参数 38 | self.parser.add_argument( 39 | "-o", "--output", 40 | dest="output_path", 41 | help="测试用例输出路径", 42 | type=str, 43 | default="test_cases.xlsx" 44 | ) 45 | 46 | # 添加测试类型参数 47 | self.parser.add_argument( 48 | "-t", "--type", 49 | dest="test_type", 50 | help="测试类型：functional(功能测试)、api(接口测试) 或 ui_auto(UI自动化测试)", 51 | type=str, 52 | choices=["functional", "api", "ui_auto"], 53 | default="functional" 54 | ) 55 | 56 | # 添加并发数参数 57 | self.parser.add_argument( 58 | "-c", "--concurrent", 59 | dest="concurrent_workers", 60 | help="并发工作线程数，用于提高测试用例生成和审查效率", 61 | type=int, 62 | default=1 63 | ) 64 | 65 | def parse_args(self): 66 | """解析命令行参数""" 67 | args = self.parser.parse_args() 68 | 69 | # 验证至少提供了一个输入参数 70 | if not args.doc_path and not args.input_path: 71 | logger.error("必须提供至少一个输入参数(-d/--doc 或 -i/--input)") 72 | raise ValueError("必须提供至少一个输入参数(-d/--doc 或 -i/--input)") 73 | 74 | # 如果是UI自动化测试，使用input_path作为测试用例文件路径 75 | if args.test_type == "ui_auto": 76 | if not args.input_path: 77 | args.input_path = args.doc_path 78 | if not os.path.exists(args.input_path): 79 | logger.error(f"测试用例文件不存在: {args.input_path}") 80 | raise ValueError(f"测试用例文件不存在: {args.input_path}") 81 | else: 82 | # 验证文档路径 83 | if args.doc_path and not os.path.exists(args.doc_path): 84 | logger.error(f"文档路径不存在: {args.doc_path}") 85 | raise ValueError(f"文档路径不存在: {args.doc_path}") 86 | 87 | # 根据测试类型选择模板 88 | template_dir = Path(__file__).parent.parent / "templates" 89 | 90 | if args.test_type == "functional": 91 | args.template_path = str(template_dir / "functional_test_template.json") 92 | logger.info(f"使用功能测试模板: {args.template_path}") 93 | elif args.test_type == "api": 94 | args.template_path = str(template_dir / "api_test_template.json") 95 | logger.info(f"使用接口测试模板: {args.template_path}") 96 | else: # ui_auto 97 | args.template_path = str(template_dir / "ui_auto_test_template.json") 98 | logger.info(f"使用UI自动化测试模板: {args.template_path}") 99 | 100 | return args 101 | 102 | def get_cli_args(): 103 | """获取命令行参数的便捷函数""" 104 | parser = CLIParser() 105 | return parser.parse_args() -------------------------------------------------------------------------------- /src/utils/env_loader.py: -------------------------------------------------------------------------------- 1 | from dotenv import load_dotenv 2 | import os 3 | 4 | def load_env_variables(): 5 | """ 6 | 加载环境变量 7 | 返回: dict 8 | - DS_BASE_URL: 基础URL 9 | - DS_API_KEY: API密钥 10 | - DS_MODEL_V3: 模型版本 11 | - OPENAI_API_KEY: OpenAI API密钥 12 | """ 13 | # 加载.env文件 14 | load_dotenv() 15 | 16 | # 获取环境变量 17 | env_vars = { 18 | 'DS_BASE_URL': os.getenv('DS_BASE_URL'), 19 | 'DS_API_KEY': os.getenv('DS_API_KEY'), 20 | 'DS_MODEL_V3': os.getenv('DS_MODEL_V3'), 21 | 'OPENAI_API_KEY': os.getenv('DS_API_KEY') # 使用相同的API密钥 22 | } 23 | 24 | # 检查是否所有必要的环境变量都已设置 25 | missing_vars = [key for key, value in env_vars.items() if value is None] 26 | if missing_vars: 27 | raise ValueError(f"缺少必要的环境变量: {', '.join(missing_vars)}") 28 | 29 | return env_vars -------------------------------------------------------------------------------- /src/utils/logger.py: -------------------------------------------------------------------------------- 1 | # src/utils/logger.py 2 | import logging 3 | import sys 4 | from pathlib import Path 5 | from logging.handlers import RotatingFileHandler 6 | 7 | def setup_logger(log_level: str = "INFO", log_file: str = "ai_tester.log"): 8 | """Configure logging for the application.""" 9 | # Create logs directory if it doesn't exist 10 | log_dir = Path("logs") 11 | log_dir.mkdir(exist_ok=True) 12 | log_path = log_dir / log_file 13 | 14 | # Set log level 15 | numeric_level = getattr(logging, log_level.upper(), logging.INFO) 16 | 17 | # Configure logging format 18 | log_format = logging.Formatter( 19 | '%(asctime)s - %(name)s - %(levelname)s - %(message)s', 20 | datefmt='%Y-%m-%d %H:%M:%S' 21 | ) 22 | 23 | # Configure file handler with rotation 24 | file_handler = RotatingFileHandler( 25 | log_path, 26 | maxBytes=10*1024*1024, # 10MB 27 | backupCount=5 28 | ) 29 | file_handler.setFormatter(log_format) 30 | 31 | # Configure console handler 32 | console_handler = logging.StreamHandler(sys.stdout) 33 | console_handler.setFormatter(log_format) 34 | 35 | # Set up root logger 36 | root_logger = logging.getLogger() 37 | root_logger.setLevel(numeric_level) 38 | root_logger.addHandler(file_handler) 39 | root_logger.addHandler(console_handler) 40 | 41 | # Log initial setup message 42 | logging.info(f"Logging configured. Level: {log_level}, File: {log_path}") -------------------------------------------------------------------------------- /template_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "description": "A versatile Python project template for general-purpose development. Features proper package structure, dependency management with pip, and virtual environment setup. Includes testing framework configuration, logging setup, and configuration management.", 3 | "required_fields": [], 4 | "required_files": [ 5 | "main.py", 6 | "requirements.txt" 7 | ], 8 | "lang": "Python", 9 | "framework": "Python", 10 | "style": "python_template", 11 | "scene": "default_python_project" 12 | } -------------------------------------------------------------------------------- /test_case_writer_test.py: -------------------------------------------------------------------------------- 1 | # test_case_writer_test.py 2 | import os 3 | import json 4 | import logging 5 | from dotenv import load_dotenv 6 | from src.agents.test_case_writer import TestCaseWriterAgent 7 | 8 | # 配置日志 9 | logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') 10 | logger = logging.getLogger(__name__) 11 | 12 | # 加载环境变量 13 | load_dotenv() 14 | 15 | def test_generate_feature_test_cases(): 16 | """测试为特定功能点生成测试用例的功能""" 17 | # 初始化测试用例编写者代理 18 | writer = TestCaseWriterAgent() 19 | 20 | # 模拟车牌识别功能点的测试覆盖项 21 | feature = "车牌识别" 22 | feature_items = [ 23 | { 24 | "feature": "车牌识别", 25 | "test_type": "功能测试" 26 | } 27 | ] 28 | 29 | # 模拟优先级定义 30 | priorities = [ 31 | {"level": "P0", "description": "核心功能，必须测试"} 32 | ] 33 | 34 | # 模拟测试方法 35 | test_approach = { 36 | "测试方法": ["黑盒测试", "功能测试"] 37 | } 38 | 39 | # 调用测试用例生成方法 40 | test_cases = writer._generate_feature_test_cases( 41 | feature=feature, 42 | feature_items=feature_items, 43 | priorities=priorities, 44 | test_approach=test_approach 45 | ) 46 | 47 | # 输出结果 48 | logger.info(f"生成的测试用例数量: {len(test_cases)}") 49 | if test_cases: 50 | logger.info(f"第一个测试用例: {json.dumps(test_cases[0], ensure_ascii=False, indent=2)}") 51 | else: 52 | logger.error("未能生成任何测试用例") 53 | 54 | return test_cases 55 | 56 | def main(): 57 | logger.info("开始测试测试用例生成功能") 58 | test_cases = test_generate_feature_test_cases() 59 | 60 | if test_cases: 61 | logger.info("测试成功: 成功生成测试用例") 62 | else: 63 | logger.error("测试失败: 未能生成测试用例") 64 | 65 | if __name__ == "__main__": 66 | main() -------------------------------------------------------------------------------- /test_cases.xlxs.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tony3liu/Auto_Generate_Test_Cases/bcd100b08fac79262e2cb5c1ede32d95d3525e8d/test_cases.xlxs.xlsx -------------------------------------------------------------------------------- /test_improve_with_llm_direct.py: -------------------------------------------------------------------------------- 1 | # test_improve_with_llm_direct.py 2 | import json 3 | import os 4 | import sys 5 | import logging 6 | from typing import List, Dict 7 | 8 | # 配置日志 9 | logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') 10 | logger = logging.getLogger(__name__) 11 | 12 | # 添加项目根目录到系统路径 13 | sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) 14 | 15 | # 导入TestCaseWriterAgent 16 | from src.agents.test_case_writer import TestCaseWriterAgent 17 | 18 | def main(): 19 | """使用test_case_writer_result.json测试_improve_with_llm函数""" 20 | try: 21 | # 初始化测试用例编写者代理 22 | writer = TestCaseWriterAgent() 23 | 24 | # 加载测试用例数据 25 | test_cases_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 26 | 'agent_results', 'test_case_writer_result.json') 27 | 28 | logger.info(f"加载测试用例数据: {test_cases_path}") 29 | with open(test_cases_path, 'r', encoding='utf-8') as f: 30 | test_data = json.load(f) 31 | test_cases = test_data.get('test_cases', []) 32 | 33 | # 确保至少有一些测试用例 34 | if len(test_cases) == 0: 35 | logger.error("测试数据中没有测试用例") 36 | return 37 | 38 | logger.info(f"成功加载 {len(test_cases)} 个测试用例") 39 | 40 | # 准备质量审查反馈 41 | feedback = """质量审查反馈： 42 | 1. 完整性： 43 | - 部分测试用例缺少详细的前置条件 44 | - 有些测试用例的步骤描述不够详细 45 | 46 | 2. 清晰度： 47 | - 测试用例标题应更具体，明确测试目标 48 | - 步骤描述应更加清晰，避免歧义 49 | 50 | 3. 可执行性： 51 | - 确保每个步骤都有对应的预期结果 52 | - 添加具体的测试数据示例 53 | 54 | 4. 边界情况： 55 | - 缺少对边界值的测试 56 | - 应考虑极端情况下的系统行为 57 | 58 | 5. 错误场景： 59 | - 缺少对错误处理的测试 60 | - 应添加异常路径测试""" 61 | 62 | # 使用前5个测试用例进行测试，避免处理过多数据 63 | test_sample = test_cases[:5] 64 | logger.info(f"使用前 {len(test_sample)} 个测试用例进行测试") 65 | 66 | # 调用_improve_with_llm函数 67 | logger.info("开始调用_improve_with_llm函数...") 68 | improved_cases = writer._improve_with_llm(test_sample, feedback) 69 | 70 | # 检查结果 71 | if not improved_cases: 72 | logger.warning("_improve_with_llm函数返回空结果") 73 | return 74 | 75 | logger.info(f"成功获取 {len(improved_cases)} 个改进后的测试用例") 76 | 77 | # 输出改进前后的对比 78 | logger.info("\n改进前后的测试用例对比:") 79 | for i, (original, improved) in enumerate(zip(test_sample, improved_cases[:len(test_sample)])): 80 | logger.info(f"\n测试用例 {i+1}:") 81 | logger.info(f"原始ID: {original.get('id', 'N/A')} -> 改进后ID: {improved.get('id', 'N/A')}") 82 | logger.info(f"原始标题: {original.get('title', 'N/A')}") 83 | logger.info(f"改进后标题: {improved.get('title', 'N/A')}") 84 | 85 | # 比较前置条件数量 86 | orig_precond = original.get('preconditions', []) 87 | impr_precond = improved.get('preconditions', []) 88 | logger.info(f"前置条件: {len(orig_precond)} -> {len(impr_precond)}") 89 | 90 | # 比较步骤数量 91 | orig_steps = original.get('steps', []) 92 | impr_steps = improved.get('steps', []) 93 | logger.info(f"步骤数量: {len(orig_steps)} -> {len(impr_steps)}") 94 | 95 | # 比较预期结果数量 96 | orig_results = original.get('expected_results', []) 97 | impr_results = improved.get('expected_results', []) 98 | logger.info(f"预期结果数量: {len(orig_results)} -> {len(impr_results)}") 99 | 100 | # 检查是否有新增的测试用例 101 | if len(improved_cases) > len(test_sample): 102 | logger.info(f"\n新增了 {len(improved_cases) - len(test_sample)} 个测试用例:") 103 | for i, new_case in enumerate(improved_cases[len(test_sample):]): 104 | logger.info(f"新增测试用例 {i+1}: {new_case.get('id', 'N/A')} - {new_case.get('title', 'N/A')}") 105 | 106 | # 保存改进后的测试用例到文件 107 | output_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 108 | 'agent_results', 'test_case_writer_improved_direct_result.json') 109 | with open(output_path, 'w', encoding='utf-8') as f: 110 | json.dump({"test_cases": improved_cases}, f, ensure_ascii=False, indent=2) 111 | 112 | logger.info(f"改进后的测试用例已保存到: {output_path}") 113 | 114 | except Exception as e: 115 | logger.error(f"测试过程中发生错误: {str(e)}") 116 | 117 | if __name__ == '__main__': 118 | main() -------------------------------------------------------------------------------- /ui_test_results.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tony3liu/Auto_Generate_Test_Cases/bcd100b08fac79262e2cb5c1ede32d95d3525e8d/ui_test_results.xlsx -------------------------------------------------------------------------------- /ui_tst_case_example.json: -------------------------------------------------------------------------------- 1 | { 2 | "test_cases": [ 3 | { 4 | "id": "TC-search-001", 5 | "title": "访问url并获取网站名称", 6 | "preconditions": [ 7 | "网络正常" 8 | ], 9 | "steps": [ 10 | "浏览器访问https://www.google.com", 11 | "获取网站名称" 12 | ], 13 | "expected_results": [ 14 | "成功访问网站", 15 | "网站名称google或谷歌" 16 | ], 17 | "priority": "P0", 18 | "category": "功能测试", 19 | "description": "" 20 | } 21 | ] 22 | } --------------------------------------------------------------------------------