├── prompts.log ├── src ├── tools │ ├── index.ts │ └── ExampleTools.ts ├── compression │ ├── index.ts │ └── ContextCompressor.ts ├── agents │ ├── index.ts │ ├── MessageBus.ts │ ├── ToolManager.ts │ └── OrchestratorAgent.ts ├── core │ ├── interfaces │ │ ├── index.ts │ │ ├── IContextEngine.ts │ │ ├── IAgent.ts │ │ ├── IMessageBus.ts │ │ └── ITool.ts │ └── types.ts ├── engines │ ├── index.ts │ ├── KeywordRAGEngine.ts │ ├── GraphRAGEngineClient.ts │ ├── HybridRetriever.ts │ └── DocumentRAGEngine.ts ├── index.ts ├── __tests__ │ ├── engines │ │ ├── KeywordRAGEngine.test.ts │ │ ├── HybridRetriever.test.ts │ │ └── DocumentRAGEngine.test.ts │ └── agents │ │ ├── MessageBus.test.ts │ │ └── ToolManager.test.ts ├── examples │ └── demo.ts └── server.ts ├── diagrams ├── output │ └── png │ │ ├── Sentient Framework - Data Flow.png │ │ ├── Sentient Framework - System Context.png │ │ ├── Sentient Framework - Container Diagram.png │ │ ├── Sentient Framework - Deployment Diagram.png │ │ ├── Sentient Framework - Query Flow Sequence.png │ │ ├── Sentient Framework - Agentic Core Components.png │ │ ├── Sentient Framework - Service Layer Components.png │ │ └── Sentient Framework - Context Engine Layer Components.png ├── _config.yml ├── README.md ├── 01-context-diagram.puml ├── 03-component-diagram-service-layer.puml ├── generate-images.sh ├── 08-data-flow-diagram.puml ├── 06-sequence-diagram-query-flow.puml ├── 04-component-diagram-agentic-core.puml ├── 02-container-diagram.puml ├── 05-component-diagram-context-engine.puml ├── 07-deployment-diagram.puml ├── QUICKSTART.md ├── INDEX.md └── index.html ├── env.example ├── .gitignore ├── vitest.config.ts ├── tsconfig.json ├── package.json ├── .github └── workflows │ └── deploy-diagrams.yml ├── QUICKSTART.md └── IMPLEMENTATION.md /prompts.log: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/tools/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Example tool implementations for the Sentient Framework 3 | */ 4 | 5 | export * from './ExampleTools.js'; 6 | 7 | -------------------------------------------------------------------------------- /src/compression/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Context Compression module for the Sentient Framework 3 | */ 4 | 5 | export * from './ContextCompressor.js'; 6 | 7 | -------------------------------------------------------------------------------- /diagrams/output/png/Sentient Framework - Data Flow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/phodal/codelotus-poc/master/diagrams/output/png/Sentient Framework - Data Flow.png -------------------------------------------------------------------------------- /diagrams/output/png/Sentient Framework - System Context.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/phodal/codelotus-poc/master/diagrams/output/png/Sentient Framework - System Context.png -------------------------------------------------------------------------------- /diagrams/output/png/Sentient Framework - Container Diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/phodal/codelotus-poc/master/diagrams/output/png/Sentient Framework - Container Diagram.png -------------------------------------------------------------------------------- /diagrams/output/png/Sentient Framework - Deployment Diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/phodal/codelotus-poc/master/diagrams/output/png/Sentient Framework - Deployment Diagram.png -------------------------------------------------------------------------------- /diagrams/output/png/Sentient Framework - Query Flow Sequence.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/phodal/codelotus-poc/master/diagrams/output/png/Sentient Framework - Query Flow Sequence.png -------------------------------------------------------------------------------- /diagrams/output/png/Sentient Framework - Agentic Core Components.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/phodal/codelotus-poc/master/diagrams/output/png/Sentient Framework - Agentic Core Components.png -------------------------------------------------------------------------------- /diagrams/output/png/Sentient Framework - Service Layer Components.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/phodal/codelotus-poc/master/diagrams/output/png/Sentient Framework - Service Layer Components.png -------------------------------------------------------------------------------- /diagrams/output/png/Sentient Framework - Context Engine Layer Components.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/phodal/codelotus-poc/master/diagrams/output/png/Sentient Framework - Context Engine Layer Components.png -------------------------------------------------------------------------------- /src/agents/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Agentic Core implementations for the Sentient Framework 3 | */ 4 | 5 | export * from './MessageBus.js'; 6 | export * from './ToolManager.js'; 7 | export * from './OrchestratorAgent.js'; 8 | 9 | -------------------------------------------------------------------------------- /diagrams/_config.yml: -------------------------------------------------------------------------------- 1 | # GitHub Pages Configuration 2 | title: Sentient Framework - Architecture Diagrams 3 | description: C4 + PlantUML Architecture Diagrams for Advanced Context Engineering System 4 | theme: jekyll-theme-cayman 5 | 6 | -------------------------------------------------------------------------------- /src/core/interfaces/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Core interfaces for the Sentient Framework 3 | */ 4 | 5 | export * from './IAgent.js'; 6 | export * from './IContextEngine.js'; 7 | export * from './IMessageBus.js'; 8 | export * from './ITool.js'; 9 | 10 | -------------------------------------------------------------------------------- /src/engines/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Context Engine implementations for the Sentient Framework 3 | */ 4 | 5 | export * from './DocumentRAGEngine.js'; 6 | export * from './GraphRAGEngineClient.js'; 7 | export * from './KeywordRAGEngine.js'; 8 | export * from './HybridRetriever.js'; 9 | 10 | -------------------------------------------------------------------------------- /env.example: -------------------------------------------------------------------------------- 1 | # OpenAI API Configuration 2 | OPENAI_API_KEY=your_openai_api_key_here 3 | 4 | # Google Gemini API Configuration 5 | GOOGLE_API_KEY=your_google_api_key_here 6 | 7 | # Server Configuration 8 | PORT=3000 9 | 10 | # Graph RAG Service 11 | GRAPH_RAG_SERVICE_URL=http://localhost:3001 12 | 13 | # Vector Store Configuration 14 | VECTOR_STORE_TYPE=memory 15 | # For production, use: weaviate, pinecone, chroma 16 | 17 | # Context Compression Settings 18 | COMPRESSION_ENABLED=true 19 | COMPRESSION_THRESHOLD=0.7 20 | 21 | # Logging 22 | LOG_LEVEL=info 23 | 24 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Dependencies 2 | node_modules/ 3 | package-lock.json 4 | yarn.lock 5 | pnpm-lock.yaml 6 | 7 | # Build output 8 | dist/ 9 | *.tsbuildinfo 10 | 11 | # Environment variables 12 | .env 13 | .env.local 14 | .env.*.local 15 | 16 | # IDE 17 | .vscode/ 18 | .idea/ 19 | *.swp 20 | *.swo 21 | *~ 22 | 23 | # OS 24 | .DS_Store 25 | Thumbs.db 26 | 27 | # Logs 28 | logs/ 29 | *.log 30 | npm-debug.log* 31 | 32 | # Test coverage 33 | coverage/ 34 | 35 | # Temporary files 36 | tmp/ 37 | temp/ 38 | *.tmp 39 | 40 | # Cache 41 | .cache/ 42 | .repomap.tags.cache.v1/ 43 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Sentient Framework - Main Export 3 | * An Architectural Blueprint for Advanced Context Engineering 4 | */ 5 | 6 | // Core Types and Interfaces 7 | export * from './core/types.js'; 8 | export * from './core/interfaces/index.js'; 9 | 10 | // RAG Engines 11 | export * from './engines/index.js'; 12 | 13 | // Context Compression 14 | export * from './compression/index.js'; 15 | 16 | // Agentic Core 17 | export * from './agents/index.js'; 18 | 19 | // Tools 20 | export * from './tools/index.js'; 21 | 22 | // Version 23 | export const VERSION = '0.1.0'; 24 | 25 | -------------------------------------------------------------------------------- /vitest.config.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig } from 'vitest/config'; 2 | 3 | export default defineConfig({ 4 | test: { 5 | globals: true, 6 | environment: 'node', 7 | coverage: { 8 | provider: 'v8', 9 | reporter: ['text', 'json', 'html'], 10 | exclude: [ 11 | 'node_modules/', 12 | 'dist/', 13 | 'src/__tests__/', 14 | 'src/examples/', 15 | '**/*.config.ts', 16 | '**/*.d.ts' 17 | ] 18 | }, 19 | testTimeout: 30000, // 30 seconds for tests involving API calls 20 | hookTimeout: 30000 21 | } 22 | }); 23 | 24 | -------------------------------------------------------------------------------- /src/core/interfaces/IContextEngine.ts: -------------------------------------------------------------------------------- 1 | import { Document } from "@langchain/core/documents"; 2 | 3 | /** 4 | * Represents a context retrieval engine. 5 | * Any component responsible for fetching relevant documents based on a query 6 | * must implement this interface. 7 | */ 8 | export interface IContextEngine { 9 | /** 10 | * Retrieves a list of relevant documents for a given query. 11 | * @param query The user's query string. 12 | * @returns A promise that resolves to an array of LangChain Document objects. 13 | */ 14 | retrieve(query: string): Promise; 15 | } 16 | 17 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES2022", 4 | "module": "NodeNext", 5 | "moduleResolution": "NodeNext", 6 | "lib": ["ES2022"], 7 | "rootDir": "./src", 8 | "outDir": "./dist", 9 | "esModuleInterop": true, 10 | "forceConsistentCasingInFileNames": true, 11 | "strict": true, 12 | "skipLibCheck": true, 13 | "declaration": true, 14 | "declarationMap": true, 15 | "sourceMap": true, 16 | "resolveJsonModule": true, 17 | "allowSyntheticDefaultImports": true, 18 | "types": ["node"] 19 | }, 20 | "include": ["src/**/*"], 21 | "exclude": ["node_modules", "dist", "**/*.test.ts"] 22 | } 23 | 24 | -------------------------------------------------------------------------------- /src/core/interfaces/IAgent.ts: -------------------------------------------------------------------------------- 1 | import { SentientPayload } from "../types.js"; 2 | 3 | /** 4 | * Represents an autonomous agent capable of processing a task. 5 | * Each agent takes the central SentientPayload, performs its specialized function, 6 | * and returns the modified payload. 7 | */ 8 | export interface IAgent { 9 | /** 10 | * The unique identifier for this agent 11 | */ 12 | readonly id: string; 13 | 14 | /** 15 | * The human-readable name of this agent 16 | */ 17 | readonly name: string; 18 | 19 | /** 20 | * Processes the SentientPayload to perform a specific task. 21 | * @param payload The current state of the request, including query, history, and context. 22 | * @returns A promise that resolves to the updated SentientPayload. 23 | */ 24 | process(payload: SentientPayload): Promise; 25 | } 26 | 27 | -------------------------------------------------------------------------------- /src/core/interfaces/IMessageBus.ts: -------------------------------------------------------------------------------- 1 | import { A2AMessage } from "../types.js"; 2 | 3 | /** 4 | * Represents the message bus for inter-agent communication, 5 | * abstracting the A2A protocol. 6 | */ 7 | export interface IMessageBus { 8 | /** 9 | * Sends a message to another agent. 10 | * @param message An A2A-compliant message object, containing target agent, task, and content. 11 | * @returns A promise that resolves when the message has been successfully dispatched. 12 | */ 13 | send(message: A2AMessage): Promise; 14 | 15 | /** 16 | * Subscribes to messages for a specific agent 17 | * @param agentId The agent ID to subscribe to 18 | * @param handler The callback function to handle incoming messages 19 | */ 20 | subscribe(agentId: string, handler: (message: A2AMessage) => Promise): void; 21 | 22 | /** 23 | * Unsubscribes from messages for a specific agent 24 | * @param agentId The agent ID to unsubscribe from 25 | */ 26 | unsubscribe(agentId: string): void; 27 | } 28 | 29 | -------------------------------------------------------------------------------- /src/core/interfaces/ITool.ts: -------------------------------------------------------------------------------- 1 | import { z, ZodObject, ZodRawShape } from "zod"; 2 | 3 | /** 4 | * Represents an external tool that an agent can execute. 5 | * It defines the tool's identity, purpose, input schema, and execution logic. 6 | */ 7 | export interface ITool { 8 | /** 9 | * A unique, programmatic name for the tool. 10 | * Should be in snake_case. 11 | */ 12 | readonly name: string; 13 | 14 | /** 15 | * A detailed description of what the tool does, its parameters, and when it should be used. 16 | * This is critical for the LLM to make correct decisions. 17 | */ 18 | readonly description: string; 19 | 20 | /** 21 | * A Zod schema defining the input parameters for the tool. 22 | */ 23 | readonly schema: ZodObject; 24 | 25 | /** 26 | * The function that executes the tool's logic. 27 | * @param args The validated input arguments, conforming to the schema. 28 | * @returns A promise that resolves to the tool's output, typically a string or JSON object. 29 | */ 30 | execute(args: z.infer>): Promise; 31 | } 32 | 33 | -------------------------------------------------------------------------------- /diagrams/README.md: -------------------------------------------------------------------------------- 1 | # 🎨 Sentient Framework - Architecture Diagrams 2 | 3 | > C4 + PlantUML 交互式架构图 4 | 5 | ## 🌐 在线查看 6 | 7 | 访问部署的网站：**[https://phodal.github.io/codelotus/](https://phodal.github.io/codelotus/)** 8 | 9 | （部署后将此链接替换为你的实际地址） 10 | 11 | ## 📐 架构图列表 12 | 13 | 本目录包含 8 个完整的架构图： 14 | 15 | 1. **系统上下文图** - 展示系统与外部世界的交互 16 | 2. **容器图** - 展示三层架构和主要服务 17 | 3. **服务层组件图** - MCP & A2A 服务实现细节 18 | 4. **代理核心组件图** - Orchestrator 和多 Agent 系统 19 | 5. **上下文引擎组件图** - 多模态 RAG 引擎详解 20 | 6. **查询流程时序图** - 完整的请求生命周期 21 | 7. **部署架构图** - Kubernetes 生产环境 22 | 8. **数据流图** - SentientPayload 状态演化 23 | 24 | ## 🚀 快速开始 25 | 26 | ### 本地查看 27 | 28 | ```bash 29 | # 方式 1：直接打开 HTML 30 | open index.html 31 | 32 | # 方式 2：启动本地服务器 33 | python3 -m http.server 8000 34 | # 访问 http://localhost:8000 35 | ``` 36 | 37 | ### 生成图片 38 | 39 | ```bash 40 | ./generate-images.sh 41 | ``` 42 | 43 | 图片将生成到： 44 | - `output/png/` - PNG 格式 45 | - `output/svg/` - SVG 格式（推荐） 46 | 47 | ## 📚 文档 48 | 49 | - **[index.html](./index.html)** - 交互式查看器（推荐） 50 | - **[QUICKSTART.md](./QUICKSTART.md)** - 详细的快速开始指南 51 | - **[INDEX.md](./INDEX.md)** - 完整的文档索引 52 | 53 | ## 🛠️ 文件说明 54 | 55 | - `*.puml` - PlantUML 源文件 56 | - `index.html` - 交互式 Web 查看器 57 | - `generate-images.sh` - 图片生成脚本 58 | - `output/` - 生成的图片（自动生成） 59 | 60 | ## 🔄 自动部署 61 | 62 | 每次推送 `.puml` 文件到 master 分支，GitHub Actions 会自动： 63 | 1. 生成所有图片（PNG + SVG） 64 | 2. 部署到 GitHub Pages 65 | 3. 更新在线网站 66 | 67 | 查看 [部署指南](../README-DEPLOYMENT.md) 了解更多。 68 | 69 | ## 📖 相关资源 70 | 71 | - [PlantUML 官方文档](https://plantuml.com/) 72 | - [C4 模型](https://c4model.com/) 73 | - [原始设计文档](../README.md) 74 | 75 | --- 76 | 77 | **开始探索架构吧！** 🎉 78 | -------------------------------------------------------------------------------- /diagrams/01-context-diagram.puml: -------------------------------------------------------------------------------- 1 | @startuml Sentient Framework - System Context 2 | !include https://raw.githubusercontent.com/plantuml-stdlib/C4-PlantUML/master/C4_Context.puml 3 | 4 | LAYOUT_WITH_LEGEND() 5 | 6 | title System Context Diagram for Sentient Framework 7 | 8 | Person(user, "User", "Developer or AI application user who needs context-aware AI capabilities") 9 | Person(admin, "System Administrator", "Manages and monitors the Sentient system") 10 | 11 | System(sentient, "Sentient Framework", "Advanced Context Engineering System that provides multi-modal retrieval, agentic orchestration, and context optimization") 12 | 13 | System_Ext(llm_service, "LLM Services", "External LLM providers (OpenAI, Gemini)") 14 | System_Ext(vector_db, "Vector Database", "Persistent vector storage (Weaviate, Pinecone, Chroma)") 15 | System_Ext(code_repo, "Code Repositories", "Git repositories to be analyzed") 16 | System_Ext(doc_storage, "Document Storage", "PDFs, documentation, and knowledge bases") 17 | System_Ext(external_agents, "External AI Agents", "Third-party agents communicating via A2A protocol") 18 | 19 | Rel(user, sentient, "Submits queries, receives AI-generated responses", "HTTP/HTTPS") 20 | Rel(admin, sentient, "Monitors, configures", "Admin API") 21 | 22 | Rel(sentient, llm_service, "Sends prompts, receives completions", "HTTPS/API") 23 | Rel(sentient, vector_db, "Stores/retrieves embeddings", "API") 24 | Rel(sentient, code_repo, "Analyzes code structure", "Git/File System") 25 | Rel(sentient, doc_storage, "Ingests documents", "File System/API") 26 | Rel(sentient, external_agents, "Collaborates via A2A protocol", "HTTP/JSON") 27 | Rel(external_agents, sentient, "Delegates tasks, discovers tools via MCP", "HTTP/JSON") 28 | 29 | @enduml 30 | 31 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "sentient-framework", 3 | "version": "0.1.0", 4 | "description": "An Architectural Blueprint for Advanced Context Engineering", 5 | "type": "module", 6 | "main": "dist/index.js", 7 | "types": "dist/index.d.ts", 8 | "scripts": { 9 | "build": "tsc", 10 | "dev": "tsx", 11 | "start": "node dist/server.js", 12 | "server": "tsx src/server.ts", 13 | "demo": "tsx src/examples/demo.ts", 14 | "test": "vitest run", 15 | "test:watch": "vitest", 16 | "test:coverage": "vitest run --coverage", 17 | "lint": "eslint src --ext ts", 18 | "format": "prettier --write \"src/**/*.ts\"" 19 | }, 20 | "keywords": [ 21 | "ai", 22 | "rag", 23 | "langchain", 24 | "context-engineering", 25 | "multi-agent", 26 | "mcp", 27 | "a2a" 28 | ], 29 | "author": "", 30 | "license": "MIT", 31 | "dependencies": { 32 | "@langchain/core": "^0.3.29", 33 | "@langchain/community": "^0.3.20", 34 | "@langchain/openai": "^0.3.17", 35 | "@langchain/textsplitters": "^0.1.0", 36 | "@google/generative-ai": "^0.21.0", 37 | "zod": "^3.24.1", 38 | "express": "^4.21.2", 39 | "axios": "^1.7.9", 40 | "pdf-parse": "^1.1.1", 41 | "langchain": "^0.3.7", 42 | "dotenv": "^16.4.7", 43 | "uuid": "^11.0.3", 44 | "cors": "^2.8.5" 45 | }, 46 | "devDependencies": { 47 | "@types/node": "^22.10.2", 48 | "@types/express": "^5.0.0", 49 | "@types/pdf-parse": "^1.1.4", 50 | "@types/uuid": "^10.0.0", 51 | "@types/cors": "^2.8.17", 52 | "typescript": "^5.7.2", 53 | "tsx": "^4.19.2", 54 | "vitest": "^2.1.8", 55 | "eslint": "^9.17.0", 56 | "prettier": "^3.4.2" 57 | }, 58 | "engines": { 59 | "node": ">=18.0.0" 60 | } 61 | } 62 | 63 | -------------------------------------------------------------------------------- /diagrams/03-component-diagram-service-layer.puml: -------------------------------------------------------------------------------- 1 | @startuml Sentient Framework - Service Layer Components 2 | !include https://raw.githubusercontent.com/plantuml-stdlib/C4-PlantUML/master/C4_Component.puml 3 | 4 | LAYOUT_WITH_LEGEND() 5 | 6 | title Component Diagram - Service Layer 7 | 8 | Container(external, "External Clients", "Various", "Users, external agents, and applications") 9 | 10 | Container_Boundary(service_layer, "Service Layer") { 11 | Component(mcp_endpoint, "MCP Endpoint", "Express.js Route", "GET /mcp/tools - Returns tool manifest") 12 | Component(mcp_executor, "MCP Tool Executor", "TypeScript", "Executes tool calls from MCP clients") 13 | Component(tool_registry, "Tool Registry", "TypeScript Map", "Maintains registry of available tools") 14 | 15 | Component(a2a_endpoint, "A2A Endpoint", "Express.js Route", "POST /a2a/task - Receives agent tasks") 16 | Component(a2a_router, "A2A Task Router", "TypeScript", "Routes tasks to appropriate agents") 17 | Component(agent_card_manager, "Agent Card Manager", "TypeScript", "Manages agent metadata and discovery") 18 | } 19 | 20 | Container(agentic_core, "Agentic Core", "TypeScript", "Orchestrator and specialized agents") 21 | Container(tool_impl, "Tool Implementations", "TypeScript", "Actual tool logic (e.g., RepoMapper)") 22 | 23 | Rel(external, mcp_endpoint, "Discovers tools", "HTTP GET") 24 | Rel(external, mcp_executor, "Invokes tools", "HTTP POST") 25 | Rel(external, a2a_endpoint, "Submits tasks", "HTTP POST") 26 | 27 | Rel(mcp_endpoint, tool_registry, "Queries available tools", "") 28 | Rel(mcp_executor, tool_registry, "Looks up tool", "") 29 | Rel(mcp_executor, tool_impl, "Executes", "Function call") 30 | 31 | Rel(a2a_endpoint, a2a_router, "Passes task", "") 32 | Rel(a2a_router, agent_card_manager, "Resolves agent", "") 33 | Rel(a2a_router, agentic_core, "Delegates to agent", "Function call") 34 | 35 | Rel(tool_registry, tool_impl, "References", "") 36 | 37 | @enduml 38 | 39 | -------------------------------------------------------------------------------- /.github/workflows/deploy-diagrams.yml: -------------------------------------------------------------------------------- 1 | name: Generate and Deploy Diagrams 2 | 3 | on: 4 | push: 5 | branches: [ master, main ] 6 | paths: 7 | - 'diagrams/**/*.puml' 8 | - 'diagrams/index.html' 9 | - '.github/workflows/deploy-diagrams.yml' 10 | workflow_dispatch: 11 | 12 | permissions: 13 | contents: read 14 | pages: write 15 | id-token: write 16 | 17 | concurrency: 18 | group: "pages" 19 | cancel-in-progress: false 20 | 21 | jobs: 22 | build: 23 | runs-on: ubuntu-latest 24 | steps: 25 | - name: Checkout 26 | uses: actions/checkout@v4 27 | 28 | - name: Setup Pages 29 | uses: actions/configure-pages@v4 30 | 31 | - name: Setup Java 32 | uses: actions/setup-java@v4 33 | with: 34 | distribution: 'temurin' 35 | java-version: '17' 36 | 37 | - name: Install GraphViz 38 | run: | 39 | sudo apt-get update 40 | sudo apt-get install -y graphviz 41 | 42 | - name: Download Latest PlantUML 43 | run: | 44 | wget https://github.com/plantuml/plantuml/releases/download/v1.2025.8/plantuml-1.2025.8.jar -O plantuml.jar 45 | echo "PlantUML downloaded" 46 | java -jar plantuml.jar -version 47 | 48 | - name: Generate Diagrams 49 | run: | 50 | cd diagrams 51 | mkdir -p output/png output/svg 52 | echo "Generating diagrams with PlantUML 1.2025.8..." 53 | for file in *.puml; do 54 | if [ -f "$file" ]; then 55 | echo "Processing $file..." 56 | java -jar ../plantuml.jar -tpng "$file" -o "output/png" || echo "Warning: PNG generation failed for $file" 57 | java -jar ../plantuml.jar -tsvg "$file" -o "output/svg" || echo "Warning: SVG generation failed for $file" 58 | fi 59 | done 60 | echo "Diagram generation complete!" 61 | 62 | - name: Copy files for deployment 63 | run: | 64 | mkdir -p _site 65 | cp -r diagrams/* _site/ 66 | 67 | - name: Upload artifact 68 | uses: actions/upload-pages-artifact@v3 69 | with: 70 | path: '_site' 71 | 72 | deploy: 73 | environment: 74 | name: github-pages 75 | url: ${{ steps.deployment.outputs.page_url }} 76 | runs-on: ubuntu-latest 77 | needs: build 78 | steps: 79 | - name: Deploy to GitHub Pages 80 | id: deployment 81 | uses: actions/deploy-pages@v4 82 | -------------------------------------------------------------------------------- /diagrams/generate-images.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Sentient Framework - Diagram Image Generator 4 | # This script generates PNG and SVG images from all PlantUML diagrams 5 | 6 | echo "🎨 Sentient Framework - Generating Architecture Diagrams..." 7 | echo "" 8 | 9 | # Check if PlantUML is installed 10 | if ! command -v plantuml &> /dev/null 11 | then 12 | echo "❌ PlantUML is not installed!" 13 | echo "" 14 | echo "Please install PlantUML:" 15 | echo " macOS: brew install plantuml" 16 | echo " Ubuntu: sudo apt-get install plantuml" 17 | echo " Manual: Download from https://plantuml.com/download" 18 | echo "" 19 | exit 1 20 | fi 21 | 22 | # Detect script location and find diagrams directory 23 | SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 24 | DIAGRAMS_DIR="" 25 | 26 | # Check if we're in the diagrams directory 27 | if [ -f "$SCRIPT_DIR/01-context-diagram.puml" ]; then 28 | DIAGRAMS_DIR="$SCRIPT_DIR" 29 | echo "📂 Running from diagrams directory" 30 | # Check if diagrams is a subdirectory 31 | elif [ -d "$SCRIPT_DIR/diagrams" ]; then 32 | DIAGRAMS_DIR="$SCRIPT_DIR/diagrams" 33 | echo "📂 Running from project root, using diagrams/ subdirectory" 34 | else 35 | echo "❌ Error: Cannot find .puml files!" 36 | echo "Please run this script from:" 37 | echo " - The diagrams/ directory, or" 38 | echo " - The project root directory" 39 | exit 1 40 | fi 41 | 42 | echo "📍 Diagrams location: $DIAGRAMS_DIR" 43 | echo "" 44 | 45 | # Change to diagrams directory 46 | cd "$DIAGRAMS_DIR" || exit 1 47 | 48 | # Create output directories 49 | mkdir -p output/png 50 | mkdir -p output/svg 51 | 52 | # Counter for diagrams 53 | count=0 54 | 55 | # Generate images for all .puml files 56 | for file in *.puml; do 57 | if [ -f "$file" ]; then 58 | filename=$(basename "$file" .puml) 59 | echo "📐 Processing: $filename" 60 | 61 | # Generate PNG 62 | if plantuml -tpng "$file" -o "output/png" 2>&1; then 63 | echo " ✅ PNG generated: output/png/${filename}.png" 64 | else 65 | echo " ⚠️ PNG generation failed for $file" 66 | fi 67 | 68 | # Generate SVG 69 | if plantuml -tsvg "$file" -o "output/svg" 2>&1; then 70 | echo " ✅ SVG generated: output/svg/${filename}.svg" 71 | else 72 | echo " ⚠️ SVG generation failed for $file" 73 | fi 74 | 75 | echo "" 76 | ((count++)) 77 | fi 78 | done 79 | 80 | echo "🎉 Complete! Generated images for $count diagrams" 81 | echo "" 82 | echo "📁 Output locations:" 83 | echo " PNG: $DIAGRAMS_DIR/output/png/" 84 | echo " SVG: $DIAGRAMS_DIR/output/svg/" 85 | echo "" 86 | echo "💡 Tip: SVG files are vector graphics and scale better for documentation" 87 | 88 | -------------------------------------------------------------------------------- /src/core/types.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Core type definitions for the Sentient Framework 3 | */ 4 | 5 | /** 6 | * Represents a single message turn in the conversation history 7 | */ 8 | export interface MessageTurn { 9 | role: 'user' | 'assistant' | 'system'; 10 | content: string; 11 | timestamp: Date; 12 | } 13 | 14 | /** 15 | * Represents a fragment of retrieved context 16 | */ 17 | export interface ContextFragment { 18 | content: string; 19 | source: string; 20 | score: number; 21 | engine: 'DocumentRAG' | 'GraphRAG' | 'KeywordRAG'; 22 | metadata?: Record; 23 | } 24 | 25 | /** 26 | * Represents the state of the agentic workflow 27 | */ 28 | export interface AgenticState { 29 | tasks: Task[]; 30 | currentTask?: string; 31 | dependencies: Map; 32 | intermediateResults: Map; 33 | } 34 | 35 | /** 36 | * Represents a single task in the workflow 37 | */ 38 | export interface Task { 39 | id: string; 40 | description: string; 41 | status: 'pending' | 'in_progress' | 'completed' | 'failed'; 42 | assignedAgent?: string; 43 | dependencies?: string[]; 44 | result?: any; 45 | } 46 | 47 | /** 48 | * The central data object passed between layers and agents 49 | */ 50 | export interface SentientPayload { 51 | query: string; 52 | sessionId: string; 53 | history: MessageTurn[]; 54 | state: AgenticState; 55 | contextFragments: ContextFragment[]; 56 | compressedContext?: string; 57 | finalOutput?: string; 58 | metadata: { 59 | startTime: Date; 60 | endTime?: Date; 61 | performanceMetrics?: Record; 62 | traceId: string; 63 | }; 64 | } 65 | 66 | /** 67 | * A2A Protocol message structure 68 | */ 69 | export interface A2AMessage { 70 | id: string; 71 | taskId: string; 72 | targetAgentId: string; 73 | sourceAgentId: string; 74 | type: 'task' | 'status' | 'result' | 'error'; 75 | payload: any; 76 | timestamp: Date; 77 | } 78 | 79 | /** 80 | * A2A Task structure 81 | */ 82 | export interface A2ATask { 83 | id: string; 84 | type: string; 85 | description: string; 86 | status: 'submitted' | 'working' | 'completed' | 'failed'; 87 | targetAgentId: string; 88 | payload: SentientPayload; 89 | artifacts?: any[]; 90 | } 91 | 92 | /** 93 | * Agent Card metadata (A2A Protocol) 94 | */ 95 | export interface AgentCard { 96 | id: string; 97 | name: string; 98 | description: string; 99 | version: string; 100 | capabilities: string[]; 101 | endpointUrl: string; 102 | authentication?: { 103 | type: string; 104 | required: boolean; 105 | }; 106 | } 107 | 108 | /** 109 | * Tool execution result 110 | */ 111 | export interface ToolResult { 112 | success: boolean; 113 | data?: any; 114 | error?: string; 115 | executionTime: number; 116 | } 117 | 118 | -------------------------------------------------------------------------------- /diagrams/08-data-flow-diagram.puml: -------------------------------------------------------------------------------- 1 | @startuml Sentient Framework - Data Flow 2 | !theme plain 3 | 4 | title SentientPayload Data Flow Through System Layers 5 | 6 | participant "User" as User 7 | participant "Service Layer" as Service 8 | participant "Orchestrator\nAgent" as Orchestrator 9 | database "SentientPayload\nState" as Payload 10 | participant "Message Bus" as Bus 11 | participant "Specialized\nAgents" as Agents 12 | participant "Context\nEngines" as Engines 13 | participant "Context\nCompressor" as Compressor 14 | participant "LLM" as LLM 15 | 16 | User -> Service: HTTP Request\n{"query": "..."} 17 | activate Service 18 | 19 | Service -> Payload: **Create**\n{\l query: "...",\l sessionId: "uuid",\l history: [...],\l state: {tasks: []},\l contextFragments: [],\l compressedContext: null,\l finalOutput: null,\l metadata: {timestamp, ...}\l} 20 | activate Payload 21 | 22 | Service -> Orchestrator: Pass Payload 23 | activate Orchestrator 24 | 25 | Orchestrator -> Payload: **Update state**\n{\l state: {\l tasks: [\l {id: 1, status: "pending", ...},\l {id: 2, status: "pending", ...}\l ]\l }\l} 26 | Payload --> Orchestrator: Updated Payload 27 | 28 | == Task Execution Loop == 29 | 30 | loop For each task 31 | Orchestrator -> Bus: Delegate task\n(Payload + task_id) 32 | activate Bus 33 | Bus -> Agents: Route to specialized agent 34 | activate Agents 35 | 36 | Agents -> Engines: retrieve(query) 37 | activate Engines 38 | Engines --> Agents: Document[] 39 | deactivate Engines 40 | 41 | Agents -> Payload: **Append contextFragments**\n{\l contextFragments: [\l ...existing,\l {\l content: "...",\l source: "file.py",\l score: 0.92,\l engine: "GraphRAG"\l }\l ]\l} 42 | 43 | Agents -> Payload: **Update task status**\n{\l state: {\l tasks: [\l {id: 1, status: "completed", result: ...}\l ]\l }\l} 44 | 45 | Agents --> Bus: Return Payload 46 | deactivate Agents 47 | Bus --> Orchestrator: Updated Payload 48 | deactivate Bus 49 | end 50 | 51 | == Context Optimization == 52 | 53 | Orchestrator -> Compressor: Pass contextFragments 54 | activate Compressor 55 | 56 | Compressor -> LLM: Classify each sentence\n(parallel requests) 57 | LLM --> Compressor: Binary classifications 58 | 59 | Compressor -> Payload: **Set compressedContext**\n{\l compressedContext: "Dense, relevant text..."\l} 60 | deactivate Compressor 61 | 62 | Payload --> Orchestrator: Updated Payload 63 | 64 | == Final Generation == 65 | 66 | Orchestrator -> LLM: Generate response\n(query + compressedContext) 67 | LLM --> Orchestrator: Generated text 68 | 69 | Orchestrator -> Payload: **Set finalOutput**\n{\l finalOutput: "...",\l metadata: {\l ...existing,\l performance: {\l total_time_ms: 2341,\l compression_ratio: 0.23,\l tokens_saved: 1842\l }\l }\l} 70 | 71 | Orchestrator --> Service: Return final Payload 72 | deactivate Orchestrator 73 | deactivate Payload 74 | 75 | Service -> User: HTTP Response\n{\l "response": "...",\l "metadata": {...}\l} 76 | deactivate Service 77 | 78 | @enduml 79 | 80 | -------------------------------------------------------------------------------- /diagrams/06-sequence-diagram-query-flow.puml: -------------------------------------------------------------------------------- 1 | @startuml Sentient Framework - Query Flow Sequence 2 | !theme plain 3 | 4 | title Query Lifecycle: "Analyze RepoMapper codebase, identify bottlenecks, suggest optimizations" 5 | 6 | actor User 7 | participant "A2A Service" as A2A 8 | participant "Orchestrator\nAgent" as Orchestrator 9 | participant "Task\nDecomposer" as Decomposer 10 | participant "Message Bus" as Bus 11 | participant "Coding\nAgent" as CodingAgent 12 | participant "Doc\nAgent" as DocAgent 13 | participant "Graph RAG" as GraphRAG 14 | participant "Document RAG" as DocRAG 15 | participant "Context\nCompressor" as Compressor 16 | participant "LLM\n(Gemini)" as LLM 17 | 18 | User -> A2A: Submit complex query 19 | activate A2A 20 | 21 | A2A -> Orchestrator: Create SentientPayload\nwith query 22 | activate Orchestrator 23 | 24 | Orchestrator -> Decomposer: Analyze query 25 | activate Decomposer 26 | Decomposer -> LLM: "Break down this task" 27 | LLM --> Decomposer: Task breakdown 28 | Decomposer --> Orchestrator: [Task1: Get repo structure,\nTask2: Find ranking algorithm,\nTask3: Analyze bottlenecks,\nTask4: Get README,\nTask5: Synthesize] 29 | deactivate Decomposer 30 | 31 | == Parallel Retrieval Phase == 32 | 33 | par Task 1 & 2: Code Analysis 34 | Orchestrator -> Bus: Delegate Task1,2\nto CodingAgent 35 | activate Bus 36 | Bus -> CodingAgent: A2A Message 37 | activate CodingAgent 38 | CodingAgent -> GraphRAG: retrieve("/path/to/RepoMapper") 39 | activate GraphRAG 40 | GraphRAG --> CodingAgent: Repo map:\n- repomap_class.py (rank: 10.8)\n- utils.py (rank: 0.23) 41 | deactivate GraphRAG 42 | CodingAgent -> Bus: Add to contextFragments 43 | deactivate CodingAgent 44 | Bus --> Orchestrator: Results 45 | deactivate Bus 46 | 47 | else Task 4: Documentation Retrieval 48 | Orchestrator -> Bus: Delegate Task4\nto DocAgent 49 | activate Bus 50 | Bus -> DocAgent: A2A Message 51 | activate DocAgent 52 | DocAgent -> DocRAG: retrieve("performance optimization") 53 | activate DocRAG 54 | DocRAG --> DocAgent: README sections 55 | deactivate DocRAG 56 | DocAgent -> Bus: Add to contextFragments 57 | deactivate DocAgent 58 | Bus --> Orchestrator: Results 59 | deactivate Bus 60 | end 61 | 62 | == Tool Use for Task 3 == 63 | 64 | Orchestrator -> CodingAgent: Use profiler tool\nvia ToolManager 65 | activate CodingAgent 66 | CodingAgent -> LLM: functionCall(tool: "cProfile",\nargs: {file: "importance.py"}) 67 | LLM --> CodingAgent: Profiler output 68 | CodingAgent --> Orchestrator: Add to contextFragments 69 | deactivate CodingAgent 70 | 71 | == Context Compression & Synthesis == 72 | 73 | Orchestrator -> Compressor: Compress all contextFragments 74 | activate Compressor 75 | loop For each sentence in fragments 76 | Compressor -> LLM: "Is this sentence relevant?" 77 | LLM --> Compressor: Yes/No 78 | end 79 | Compressor --> Orchestrator: compressedContext 80 | deactivate Compressor 81 | 82 | Orchestrator -> LLM: Final prompt:\n"Based on compressed context,\nsuggest optimizations" 83 | LLM --> Orchestrator: finalOutput:\nCode suggestions +\nREADME updates 84 | 85 | Orchestrator --> A2A: Return SentientPayload\nwith finalOutput 86 | deactivate Orchestrator 87 | 88 | A2A --> User: AI-generated response 89 | deactivate A2A 90 | 91 | @enduml 92 | 93 | -------------------------------------------------------------------------------- /src/__tests__/engines/KeywordRAGEngine.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, it, expect, beforeEach } from 'vitest'; 2 | import { KeywordRAGEngine } from '../../engines/KeywordRAGEngine.js'; 3 | import { Document } from '@langchain/core/documents'; 4 | 5 | describe('KeywordRAGEngine', () => { 6 | let engine: KeywordRAGEngine; 7 | 8 | beforeEach(() => { 9 | engine = new KeywordRAGEngine(); 10 | }); 11 | 12 | describe('indexDocuments', () => { 13 | it('should index documents successfully', async () => { 14 | const docs = [ 15 | new Document({ 16 | pageContent: 'TypeScript is a typed superset of JavaScript.', 17 | metadata: { source: 'typescript.txt' } 18 | }), 19 | new Document({ 20 | pageContent: 'BM25 is a ranking function used in information retrieval.', 21 | metadata: { source: 'bm25.txt' } 22 | }) 23 | ]; 24 | 25 | await engine.indexDocuments(docs); 26 | 27 | expect(true).toBe(true); // Should not throw 28 | }); 29 | }); 30 | 31 | describe('retrieve', () => { 32 | it('should retrieve documents with exact keyword matches', async () => { 33 | const docs = [ 34 | new Document({ 35 | pageContent: 'The API_KEY environment variable is required.', 36 | metadata: { source: 'config.txt' } 37 | }), 38 | new Document({ 39 | pageContent: 'TypeScript provides type safety.', 40 | metadata: { source: 'typescript.txt' } 41 | }), 42 | new Document({ 43 | pageContent: 'The configuration uses environment variables.', 44 | metadata: { source: 'env.txt' } 45 | }) 46 | ]; 47 | 48 | await engine.indexDocuments(docs); 49 | 50 | const results = await engine.retrieve('API_KEY', 2); 51 | 52 | expect(results.length).toBeGreaterThan(0); 53 | expect(results[0].pageContent).toContain('API_KEY'); 54 | expect(results[0].metadata.engine).toBe('KeywordRAG'); 55 | }); 56 | 57 | it('should return empty array when no documents indexed', async () => { 58 | const results = await engine.retrieve('test query'); 59 | 60 | expect(results).toEqual([]); 61 | }); 62 | 63 | it('should score documents using BM25', async () => { 64 | const docs = [ 65 | new Document({ 66 | pageContent: 'context context context', 67 | metadata: { source: 'repeat.txt' } 68 | }), 69 | new Document({ 70 | pageContent: 'context engineering framework', 71 | metadata: { source: 'relevant.txt' } 72 | }) 73 | ]; 74 | 75 | await engine.indexDocuments(docs); 76 | 77 | const results = await engine.retrieve('context engineering', 2); 78 | 79 | expect(results.length).toBeGreaterThan(0); 80 | expect(results[0].metadata.score).toBeDefined(); 81 | }); 82 | }); 83 | 84 | describe('clear', () => { 85 | it('should clear the index', async () => { 86 | const docs = [ 87 | new Document({ 88 | pageContent: 'Test document', 89 | metadata: { source: 'test.txt' } 90 | }) 91 | ]; 92 | 93 | await engine.indexDocuments(docs); 94 | engine.clear(); 95 | 96 | const results = await engine.retrieve('test'); 97 | expect(results).toEqual([]); 98 | }); 99 | }); 100 | }); 101 | 102 | -------------------------------------------------------------------------------- /diagrams/04-component-diagram-agentic-core.puml: -------------------------------------------------------------------------------- 1 | @startuml Sentient Framework - Agentic Core Components 2 | !include https://raw.githubusercontent.com/plantuml-stdlib/C4-PlantUML/master/C4_Component.puml 3 | 4 | LAYOUT_WITH_LEGEND() 5 | 6 | title Component Diagram - Agentic Core 7 | 8 | Container(service_layer, "Service Layer", "Express.js", "MCP & A2A services") 9 | 10 | Container_Boundary(agentic_core, "Agentic Core") { 11 | Component(orchestrator, "Orchestrator Agent", "TypeScript Class", "Receives requests, decomposes tasks, synthesizes results") 12 | Component(task_decomposer, "Task Decomposer", "LLM-powered", "Breaks complex queries into subtasks with dependencies") 13 | Component(task_scheduler, "Task Scheduler", "TypeScript", "Manages task execution order based on dependencies") 14 | 15 | Component(message_bus, "Message Bus", "TypeScript/A2A", "Central message routing for inter-agent communication") 16 | Component(message_queue, "Message Queue", "In-memory", "Buffers messages between agents") 17 | 18 | Component(tool_manager, "Tool Manager", "TypeScript", "Manages tool registration and execution") 19 | Component(tool_executor, "Tool Executor", "TypeScript", "Safely executes tool functions with validated arguments") 20 | Component(schema_validator, "Schema Validator", "Zod", "Validates tool inputs against schemas") 21 | 22 | Component(coding_agent, "Coding Agent", "TypeScript/Gemini", "Code analysis, refactoring, generation") 23 | Component(retrieval_agent, "Retrieval Agent", "TypeScript", "Information gathering specialist") 24 | Component(doc_agent, "Documentation Agent", "TypeScript/Gemini", "Document processing specialist") 25 | 26 | Component(payload_manager, "Payload Manager", "TypeScript", "Manages SentientPayload lifecycle and state") 27 | } 28 | 29 | Container(context_engines, "Context Engine Layer", "TypeScript", "RAG engines and retrievers") 30 | Container(llm_service, "LLM Service", "Gemini API", "Language model for reasoning") 31 | 32 | Rel(service_layer, orchestrator, "Routes initial request", "SentientPayload") 33 | 34 | Rel(orchestrator, task_decomposer, "Analyzes query", "") 35 | Rel(task_decomposer, llm_service, "Requests task breakdown", "API call") 36 | Rel(orchestrator, task_scheduler, "Submits subtasks", "") 37 | 38 | Rel(orchestrator, message_bus, "Delegates subtasks", "A2A Message") 39 | Rel(message_bus, message_queue, "Enqueues", "") 40 | Rel(message_queue, coding_agent, "Delivers task", "") 41 | Rel(message_queue, retrieval_agent, "Delivers task", "") 42 | Rel(message_queue, doc_agent, "Delivers task", "") 43 | 44 | Rel(coding_agent, message_bus, "Sends results", "A2A Message") 45 | Rel(retrieval_agent, message_bus, "Sends results", "A2A Message") 46 | Rel(doc_agent, message_bus, "Sends results", "A2A Message") 47 | Rel(message_bus, orchestrator, "Returns results", "") 48 | 49 | Rel(orchestrator, tool_manager, "Invokes tools", "") 50 | Rel(coding_agent, tool_manager, "Uses tools", "") 51 | Rel(tool_manager, schema_validator, "Validates args", "") 52 | Rel(tool_manager, tool_executor, "Executes validated call", "") 53 | 54 | Rel(retrieval_agent, context_engines, "Queries RAG engines", "API call") 55 | 56 | Rel(orchestrator, payload_manager, "Updates payload state", "") 57 | Rel(payload_manager, orchestrator, "Provides current state", "") 58 | 59 | Rel(orchestrator, llm_service, "Final synthesis", "API call") 60 | Rel(coding_agent, llm_service, "Code generation", "API call") 61 | Rel(doc_agent, llm_service, "Doc generation", "API call") 62 | 63 | @enduml 64 | 65 | -------------------------------------------------------------------------------- /diagrams/02-container-diagram.puml: -------------------------------------------------------------------------------- 1 | @startuml Sentient Framework - Container Diagram 2 | !include https://raw.githubusercontent.com/plantuml-stdlib/C4-PlantUML/master/C4_Container.puml 3 | 4 | LAYOUT_WITH_LEGEND() 5 | 6 | title Container Diagram for Sentient Framework 7 | 8 | Person(user, "User", "Developer or AI application") 9 | 10 | System_Boundary(sentient, "Sentient Framework") { 11 | Container(mcp_service, "MCP Service", "Express.js/TypeScript", "Model Context Protocol server exposing tools and capabilities") 12 | Container(a2a_service, "A2A Service", "Express.js/TypeScript", "Agent-to-Agent protocol server for inter-agent communication") 13 | 14 | Container(orchestrator, "Orchestrator Agent", "TypeScript/Gemini", "Main coordinator that decomposes tasks and orchestrates workflow") 15 | Container(tool_manager, "Tool Manager", "TypeScript", "Manages tool lifecycle and execution") 16 | Container(message_bus, "Message Bus", "TypeScript/A2A", "Routes messages between agents") 17 | 18 | Container(coding_agent, "Coding Agent", "TypeScript/Gemini", "Specialized in code analysis and generation") 19 | Container(retrieval_agent, "Retrieval Agent", "TypeScript", "Expert in information gathering across engines") 20 | Container(doc_agent, "Documentation Agent", "TypeScript/Gemini", "Processes prose and semi-structured documents") 21 | 22 | Container(doc_rag, "Document RAG Engine", "LangChain.js", "Semantic vector search for unstructured text") 23 | Container(graph_rag, "Graph RAG Engine", "tree-sitter/networkx", "Structural analysis for codebases") 24 | Container(keyword_rag, "Keyword RAG Engine", "BM25/TF-IDF", "Lexical search for exact matches") 25 | Container(hybrid_retriever, "Hybrid Retriever", "TypeScript/RRF", "Fuses results from all RAG engines using Reciprocal Rank Fusion") 26 | 27 | Container(compressor, "Context Compressor", "TypeScript/EXIT", "Intelligent extractive context compression") 28 | } 29 | 30 | System_Ext(llm, "LLM Service", "Gemini/OpenAI") 31 | System_Ext(vector_store, "Vector Store", "MemoryVectorStore/Weaviate") 32 | ContainerDb(cache, "Cache Storage", "File System/Redis", "Stores parsed symbols and graph ranks") 33 | 34 | Rel(user, mcp_service, "Discovers tools", "HTTP/MCP") 35 | Rel(user, a2a_service, "Submits tasks", "HTTP/A2A") 36 | 37 | Rel(a2a_service, orchestrator, "Routes requests", "Function call") 38 | Rel(mcp_service, tool_manager, "Exposes tools", "MCP protocol") 39 | 40 | Rel(orchestrator, message_bus, "Delegates subtasks", "A2A message") 41 | Rel(message_bus, coding_agent, "Routes coding tasks", "A2A message") 42 | Rel(message_bus, retrieval_agent, "Routes retrieval tasks", "A2A message") 43 | Rel(message_bus, doc_agent, "Routes documentation tasks", "A2A message") 44 | 45 | Rel(orchestrator, tool_manager, "Invokes tools", "Function call") 46 | Rel(coding_agent, tool_manager, "Uses tools", "Function call") 47 | 48 | Rel(retrieval_agent, hybrid_retriever, "Queries", "API call") 49 | Rel(hybrid_retriever, doc_rag, "Semantic search", "API call") 50 | Rel(hybrid_retriever, graph_rag, "Structural search", "API call") 51 | Rel(hybrid_retriever, keyword_rag, "Lexical search", "API call") 52 | 53 | Rel(orchestrator, compressor, "Compresses context", "API call") 54 | Rel(compressor, llm, "Relevance classification", "HTTPS") 55 | 56 | Rel(doc_rag, vector_store, "Stores/retrieves vectors", "API") 57 | Rel(graph_rag, cache, "Reads/writes cache", "File I/O") 58 | 59 | Rel(orchestrator, llm, "Final generation", "HTTPS") 60 | Rel(coding_agent, llm, "Code generation", "HTTPS") 61 | Rel(doc_agent, llm, "Doc generation", "HTTPS") 62 | 63 | @enduml 64 | 65 | -------------------------------------------------------------------------------- /src/agents/MessageBus.ts: -------------------------------------------------------------------------------- 1 | import { IMessageBus } from "../core/interfaces/IMessageBus.js"; 2 | import { A2AMessage } from "../core/types.js"; 3 | 4 | /** 5 | * In-memory message bus implementation for inter-agent communication 6 | * Implements the A2A protocol for message routing 7 | */ 8 | export class MessageBus implements IMessageBus { 9 | private subscribers: Map Promise)[]>; 10 | private messageQueue: A2AMessage[]; 11 | private processing: boolean; 12 | 13 | constructor() { 14 | this.subscribers = new Map(); 15 | this.messageQueue = []; 16 | this.processing = false; 17 | } 18 | 19 | /** 20 | * Sends a message to the target agent 21 | */ 22 | async send(message: A2AMessage): Promise { 23 | console.log(`[MessageBus] Routing message ${message.id} to agent ${message.targetAgentId}`); 24 | 25 | const handlers = this.subscribers.get(message.targetAgentId); 26 | 27 | if (!handlers || handlers.length === 0) { 28 | console.warn(`[MessageBus] No handlers registered for agent: ${message.targetAgentId}`); 29 | // Queue the message for later delivery 30 | this.messageQueue.push(message); 31 | return; 32 | } 33 | 34 | // Deliver to all registered handlers 35 | await Promise.all(handlers.map(handler => handler(message))); 36 | } 37 | 38 | /** 39 | * Subscribes to messages for a specific agent 40 | */ 41 | subscribe(agentId: string, handler: (message: A2AMessage) => Promise): void { 42 | if (!this.subscribers.has(agentId)) { 43 | this.subscribers.set(agentId, []); 44 | } 45 | 46 | this.subscribers.get(agentId)!.push(handler); 47 | console.log(`[MessageBus] Agent ${agentId} subscribed to message bus`); 48 | 49 | // Process any queued messages for this agent 50 | this.processQueuedMessages(agentId); 51 | } 52 | 53 | /** 54 | * Unsubscribes from messages 55 | */ 56 | unsubscribe(agentId: string): void { 57 | this.subscribers.delete(agentId); 58 | console.log(`[MessageBus] Agent ${agentId} unsubscribed from message bus`); 59 | } 60 | 61 | /** 62 | * Processes queued messages for a newly subscribed agent 63 | */ 64 | private async processQueuedMessages(agentId: string): Promise { 65 | if (this.processing) return; 66 | 67 | this.processing = true; 68 | const pendingMessages = this.messageQueue.filter(msg => msg.targetAgentId === agentId); 69 | 70 | for (const message of pendingMessages) { 71 | await this.send(message); 72 | // Remove from queue 73 | const index = this.messageQueue.indexOf(message); 74 | if (index > -1) { 75 | this.messageQueue.splice(index, 1); 76 | } 77 | } 78 | 79 | this.processing = false; 80 | } 81 | 82 | /** 83 | * Broadcasts a message to all subscribers 84 | */ 85 | async broadcast(message: Omit): Promise { 86 | const agentIds = Array.from(this.subscribers.keys()); 87 | 88 | await Promise.all( 89 | agentIds.map(agentId => 90 | this.send({ ...message, targetAgentId: agentId } as A2AMessage) 91 | ) 92 | ); 93 | } 94 | 95 | /** 96 | * Gets statistics about the message bus 97 | */ 98 | getStats(): { 99 | subscribers: number; 100 | queuedMessages: number; 101 | subscriberIds: string[]; 102 | } { 103 | return { 104 | subscribers: this.subscribers.size, 105 | queuedMessages: this.messageQueue.length, 106 | subscriberIds: Array.from(this.subscribers.keys()) 107 | }; 108 | } 109 | } 110 | 111 | -------------------------------------------------------------------------------- /src/__tests__/engines/HybridRetriever.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, it, expect, beforeEach } from 'vitest'; 2 | import { HybridRetriever } from '../../engines/HybridRetriever.js'; 3 | import { DocumentRAGEngine } from '../../engines/DocumentRAGEngine.js'; 4 | import { KeywordRAGEngine } from '../../engines/KeywordRAGEngine.js'; 5 | import { Document } from '@langchain/core/documents'; 6 | 7 | describe('HybridRetriever', () => { 8 | let docEngine: DocumentRAGEngine; 9 | let keywordEngine: KeywordRAGEngine; 10 | let hybridRetriever: HybridRetriever; 11 | 12 | beforeEach(async () => { 13 | docEngine = new DocumentRAGEngine(); 14 | keywordEngine = new KeywordRAGEngine(); 15 | 16 | const sampleDocs = [ 17 | new Document({ 18 | pageContent: 'Sentient Framework combines semantic and lexical search.', 19 | metadata: { source: 'intro.txt' } 20 | }), 21 | new Document({ 22 | pageContent: 'Reciprocal Rank Fusion (RRF) merges results from multiple retrievers.', 23 | metadata: { source: 'rrf.txt' } 24 | }), 25 | new Document({ 26 | pageContent: 'Vector embeddings enable semantic similarity search.', 27 | metadata: { source: 'embeddings.txt' } 28 | }) 29 | ]; 30 | 31 | await docEngine.ingestDocuments(sampleDocs); 32 | await keywordEngine.indexDocuments(sampleDocs); 33 | 34 | hybridRetriever = new HybridRetriever([docEngine, keywordEngine]); 35 | }); 36 | 37 | describe('retrieve', () => { 38 | it('should fuse results from multiple engines', async () => { 39 | const results = await hybridRetriever.retrieve('semantic search', 3); 40 | 41 | expect(results.length).toBeGreaterThan(0); 42 | expect(results.length).toBeLessThanOrEqual(3); 43 | expect(results[0].metadata.engine).toBe('HybridRetriever'); 44 | expect(results[0].metadata.rrfScore).toBeDefined(); 45 | }); 46 | 47 | it('should include fusion metadata', async () => { 48 | const results = await hybridRetriever.retrieve('RRF', 2); 49 | 50 | expect(results[0].metadata.fusedFrom).toBeDefined(); 51 | expect(Array.isArray(results[0].metadata.fusedFrom)).toBe(true); 52 | }); 53 | 54 | it('should handle custom weights', async () => { 55 | const weightedRetriever = new HybridRetriever( 56 | [docEngine, keywordEngine], 57 | 60, 58 | [2, 1] // Favor docEngine 59 | ); 60 | 61 | const results = await weightedRetriever.retrieve('search', 2); 62 | 63 | expect(results.length).toBeGreaterThan(0); 64 | expect(results[0].metadata.rrfScore).toBeDefined(); 65 | }); 66 | }); 67 | 68 | describe('retrieveWithDetails', () => { 69 | it('should return fusion details', async () => { 70 | const result = await hybridRetriever.retrieveWithDetails('semantic', 3); 71 | 72 | expect(result.documents.length).toBeGreaterThan(0); 73 | expect(result.fusionDetails).toBeDefined(); 74 | expect(result.fusionDetails.totalCandidates).toBeGreaterThan(0); 75 | expect(result.fusionDetails.engineResults).toHaveLength(2); 76 | expect(result.fusionDetails.fusionTime).toBeGreaterThan(0); 77 | }); 78 | }); 79 | 80 | describe('error handling', () => { 81 | it('should handle engine failures gracefully', async () => { 82 | const failingEngine = { 83 | retrieve: async () => { 84 | throw new Error('Engine failed'); 85 | } 86 | }; 87 | 88 | const retriever = new HybridRetriever([docEngine, failingEngine as any]); 89 | 90 | const results = await retriever.retrieve('test', 2); 91 | 92 | // Should still return results from working engine 93 | expect(results).toBeDefined(); 94 | }); 95 | }); 96 | }); 97 | 98 | -------------------------------------------------------------------------------- /src/__tests__/engines/DocumentRAGEngine.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, it, expect, beforeEach } from 'vitest'; 2 | import { DocumentRAGEngine } from '../../engines/DocumentRAGEngine.js'; 3 | import { Document } from '@langchain/core/documents'; 4 | 5 | describe('DocumentRAGEngine', () => { 6 | let engine: DocumentRAGEngine; 7 | 8 | beforeEach(() => { 9 | engine = new DocumentRAGEngine(); 10 | }); 11 | 12 | describe('ingestDocuments', () => { 13 | it('should ingest documents successfully', async () => { 14 | const docs = [ 15 | new Document({ 16 | pageContent: 'The Sentient Framework is for context engineering.', 17 | metadata: { source: 'test1.txt' } 18 | }), 19 | new Document({ 20 | pageContent: 'Multi-modal retrieval combines semantic and lexical search.', 21 | metadata: { source: 'test2.txt' } 22 | }) 23 | ]; 24 | 25 | await engine.ingestDocuments(docs); 26 | 27 | // Should not throw 28 | expect(true).toBe(true); 29 | }); 30 | }); 31 | 32 | describe('retrieve', () => { 33 | it('should retrieve relevant documents', async () => { 34 | const docs = [ 35 | new Document({ 36 | pageContent: 'The Sentient Framework is an advanced context engineering system.', 37 | metadata: { source: 'intro.txt' } 38 | }), 39 | new Document({ 40 | pageContent: 'Document RAG uses vector embeddings for semantic search.', 41 | metadata: { source: 'rag.txt' } 42 | }), 43 | new Document({ 44 | pageContent: 'Python is a programming language.', 45 | metadata: { source: 'python.txt' } 46 | }) 47 | ]; 48 | 49 | await engine.ingestDocuments(docs); 50 | 51 | const results = await engine.retrieve('What is Sentient Framework?', 2); 52 | 53 | expect(results.length).toBeGreaterThan(0); 54 | expect(results.length).toBeLessThanOrEqual(2); 55 | expect(results[0]).toHaveProperty('pageContent'); 56 | expect(results[0]).toHaveProperty('metadata'); 57 | expect(results[0].metadata.engine).toBe('DocumentRAG'); 58 | }); 59 | 60 | it('should throw error when vector store not initialized', async () => { 61 | await expect(engine.retrieve('test query')).rejects.toThrow(); 62 | }); 63 | }); 64 | 65 | describe('retrieveWithMMR', () => { 66 | it('should retrieve diverse documents using MMR', async () => { 67 | const docs = [ 68 | new Document({ 69 | pageContent: 'Context engineering is important. Context engineering helps AI.', 70 | metadata: { source: 'context1.txt' } 71 | }), 72 | new Document({ 73 | pageContent: 'RAG systems retrieve relevant information for LLMs.', 74 | metadata: { source: 'rag.txt' } 75 | }), 76 | new Document({ 77 | pageContent: 'Context engineering is critical. Context engineering matters.', 78 | metadata: { source: 'context2.txt' } 79 | }) 80 | ]; 81 | 82 | await engine.ingestDocuments(docs); 83 | 84 | const results = await engine.retrieveWithMMR('context engineering', 2); 85 | 86 | expect(results.length).toBeGreaterThan(0); 87 | expect(results[0].metadata.retrievalMethod).toBe('MMR'); 88 | }); 89 | }); 90 | 91 | describe('clear', () => { 92 | it('should clear the vector store', async () => { 93 | const docs = [ 94 | new Document({ 95 | pageContent: 'Test document', 96 | metadata: { source: 'test.txt' } 97 | }) 98 | ]; 99 | 100 | await engine.ingestDocuments(docs); 101 | engine.clear(); 102 | 103 | await expect(engine.retrieve('test')).rejects.toThrow(); 104 | }); 105 | }); 106 | }); 107 | 108 | -------------------------------------------------------------------------------- /diagrams/05-component-diagram-context-engine.puml: -------------------------------------------------------------------------------- 1 | @startuml Sentient Framework - Context Engine Layer Components 2 | !include https://raw.githubusercontent.com/plantuml-stdlib/C4-PlantUML/master/C4_Component.puml 3 | 4 | LAYOUT_WITH_LEGEND() 5 | 6 | title Component Diagram - Context Engine Layer 7 | 8 | Container(agentic_core, "Agentic Core", "TypeScript", "Orchestrator and agents") 9 | 10 | Container_Boundary(context_layer, "Context Engine Layer") { 11 | Component(hybrid_retriever, "Hybrid Retriever", "TypeScript", "Implements IContextEngine, orchestrates multi-modal retrieval") 12 | Component(rrf_ranker, "RRF Re-ranker", "TypeScript", "Reciprocal Rank Fusion algorithm for merging results") 13 | 14 | Component(doc_rag, "Document RAG Engine", "LangChain.js", "Semantic vector search") 15 | Component(doc_loader_factory, "Document Loader Factory", "TypeScript", "Creates appropriate loaders based on file type") 16 | Component(pdf_loader, "PDF Loader", "LangChain/pdf-parse", "Loads PDF documents") 17 | Component(web_loader, "Web Loader", "CheerioWebBaseLoader", "Scrapes web content") 18 | Component(text_splitter, "Text Splitter", "RecursiveCharacterTextSplitter", "Splits documents into semantic chunks") 19 | Component(embedder, "Embedder", "OpenAIEmbeddings", "Converts text to vectors") 20 | Component(vector_store, "Vector Store", "MemoryVectorStore/Weaviate", "Stores and searches vectors") 21 | Component(doc_retriever, "Document Retriever", "LangChain Retriever", "Queries vector store with MMR") 22 | 23 | Component(graph_rag, "Graph RAG Engine", "TypeScript Client", "Structural code analysis") 24 | Component(graph_service_client, "RepoMapper Service Client", "HTTP/MCP Client", "Communicates with external service") 25 | 26 | Component(keyword_rag, "Keyword RAG Engine", "TypeScript", "Lexical search") 27 | Component(bm25_index, "BM25 Index", "TF-IDF/BM25", "Inverted index for keyword matching") 28 | 29 | Component(context_compressor, "Context Compressor", "EXIT-inspired", "Extractive compression module") 30 | Component(sentence_tokenizer, "Sentence Tokenizer", "TypeScript", "Decomposes text into sentences") 31 | Component(relevance_classifier, "Relevance Classifier", "LLM-powered", "Context-aware sentence classification") 32 | Component(doc_reassembler, "Document Reassembler", "TypeScript", "Rebuilds compressed context") 33 | } 34 | 35 | ContainerDb(vector_db, "Vector Database", "Persistent storage", "Weaviate/Pinecone/Chroma") 36 | System_Ext(repomapper_service, "RepoMapper Microservice", "tree-sitter/networkx/Python") 37 | System_Ext(llm_fast, "Fast LLM", "Gemini Flash") 38 | 39 | Rel(agentic_core, hybrid_retriever, "Queries for context", "IContextEngine.retrieve()") 40 | 41 | Rel(hybrid_retriever, doc_rag, "Semantic retrieval", "") 42 | Rel(hybrid_retriever, graph_rag, "Structural retrieval", "") 43 | Rel(hybrid_retriever, keyword_rag, "Lexical retrieval", "") 44 | Rel(hybrid_retriever, rrf_ranker, "Fuses results", "") 45 | 46 | Rel(doc_rag, doc_loader_factory, "Gets loader", "") 47 | Rel(doc_loader_factory, pdf_loader, "Creates", "") 48 | Rel(doc_loader_factory, web_loader, "Creates", "") 49 | Rel(pdf_loader, text_splitter, "Passes documents", "") 50 | Rel(text_splitter, embedder, "Passes chunks", "") 51 | Rel(embedder, vector_store, "Stores embeddings", "") 52 | Rel(vector_store, vector_db, "Persists", "Optional") 53 | Rel(doc_rag, doc_retriever, "Uses", "") 54 | Rel(doc_retriever, vector_store, "Queries", "") 55 | 56 | Rel(graph_rag, graph_service_client, "Requests repo map", "") 57 | Rel(graph_service_client, repomapper_service, "HTTP/MCP call", "POST /repomap") 58 | 59 | Rel(keyword_rag, bm25_index, "Searches", "") 60 | 61 | Rel(agentic_core, context_compressor, "Compresses context", "") 62 | Rel(context_compressor, sentence_tokenizer, "Decomposes", "") 63 | Rel(context_compressor, relevance_classifier, "Classifies", "") 64 | Rel(relevance_classifier, llm_fast, "Binary classification", "API call") 65 | Rel(context_compressor, doc_reassembler, "Assembles result", "") 66 | 67 | @enduml 68 | 69 | -------------------------------------------------------------------------------- /diagrams/07-deployment-diagram.puml: -------------------------------------------------------------------------------- 1 | @startuml Sentient Framework - Deployment Diagram 2 | !include https://raw.githubusercontent.com/plantuml-stdlib/C4-PlantUML/master/C4_Deployment.puml 3 | 4 | LAYOUT_WITH_LEGEND() 5 | 6 | title Deployment Diagram - Production Architecture 7 | 8 | Deployment_Node(client_zone, "Client Zone", "Internet") { 9 | Deployment_Node(browser, "Web Browser", "Chrome/Firefox") { 10 | Container(web_app, "Web Application", "React/TypeScript", "User interface") 11 | } 12 | 13 | Deployment_Node(external_agent_sys, "External Agent System", "Cloud") { 14 | Container(third_party_agent, "Third-party Agent", "Various", "External AI agent") 15 | } 16 | } 17 | 18 | Deployment_Node(cloud, "Cloud Platform", "AWS/GCP/Azure") { 19 | Deployment_Node(k8s_cluster, "Kubernetes Cluster", "k8s 1.28+") { 20 | 21 | Deployment_Node(service_pods, "Service Layer Pods", "Docker") { 22 | Container(mcp_service, "MCP Service", "Express.js/Node 20", "Tool discovery endpoint") 23 | Container(a2a_service, "A2A Service", "Express.js/Node 20", "Agent communication endpoint") 24 | } 25 | 26 | Deployment_Node(agent_pods, "Agent Pods", "Docker") { 27 | Container(orchestrator_pod, "Orchestrator Agent", "TypeScript/Node 20", "Main coordinator") 28 | Container(specialized_agents, "Specialized Agents", "TypeScript/Node 20", "Coding, Retrieval, Doc agents") 29 | } 30 | 31 | Deployment_Node(engine_pods, "RAG Engine Pods", "Docker") { 32 | Container(doc_rag_pod, "Document RAG", "LangChain.js/Node 20", "Semantic search engine") 33 | Container(keyword_rag_pod, "Keyword RAG", "TypeScript/Node 20", "Lexical search engine") 34 | } 35 | 36 | Deployment_Node(load_balancer, "Load Balancer", "Ingress/NGINX") { 37 | Container(ingress, "Ingress Controller", "NGINX", "Routes external traffic") 38 | } 39 | } 40 | 41 | Deployment_Node(microservice_zone, "Microservice Zone", "Separate VPC") { 42 | Deployment_Node(graph_service, "Graph RAG Service", "Docker/Python") { 43 | Container(repomapper, "RepoMapper Service", "Python/FastAPI", "Code analysis service") 44 | ContainerDb(parse_cache, "Parse Cache", "File System", ".repomap.tags.cache.v1/") 45 | } 46 | } 47 | 48 | Deployment_Node(data_tier, "Data Tier", "Managed Services") { 49 | ContainerDb(vector_db, "Vector Database", "Weaviate Cluster", "Persistent embeddings") 50 | ContainerDb(redis, "Redis Cache", "ElastiCache", "Session and result cache") 51 | ContainerDb(postgres, "Metadata DB", "RDS PostgreSQL", "Agent cards, tool registry") 52 | } 53 | } 54 | 55 | Deployment_Node(external_services, "External Services", "Internet") { 56 | System_Ext(openai, "OpenAI API", "Embeddings & LLM") 57 | System_Ext(gemini, "Google Gemini API", "LLM & Function Calling") 58 | System_Ext(git_repos, "Git Repositories", "GitHub/GitLab") 59 | } 60 | 61 | Rel(web_app, ingress, "HTTPS", "443") 62 | Rel(third_party_agent, ingress, "HTTPS", "443") 63 | 64 | Rel(ingress, mcp_service, "Routes /mcp/*", "HTTP") 65 | Rel(ingress, a2a_service, "Routes /a2a/*", "HTTP") 66 | 67 | Rel(mcp_service, orchestrator_pod, "Internal", "gRPC/HTTP") 68 | Rel(a2a_service, orchestrator_pod, "Internal", "gRPC/HTTP") 69 | 70 | Rel(orchestrator_pod, specialized_agents, "Message passing", "Internal") 71 | Rel(specialized_agents, doc_rag_pod, "Queries", "Internal") 72 | Rel(specialized_agents, keyword_rag_pod, "Queries", "Internal") 73 | 74 | Rel(orchestrator_pod, repomapper, "HTTP/MCP", "HTTPS") 75 | Rel(repomapper, parse_cache, "R/W", "File I/O") 76 | Rel(repomapper, git_repos, "Clone/Pull", "Git protocol") 77 | 78 | Rel(doc_rag_pod, vector_db, "Store/Query", "gRPC") 79 | Rel(orchestrator_pod, redis, "Cache payload", "Redis protocol") 80 | Rel(mcp_service, postgres, "Query tools", "PostgreSQL") 81 | Rel(a2a_service, postgres, "Query agents", "PostgreSQL") 82 | 83 | Rel(orchestrator_pod, gemini, "API calls", "HTTPS") 84 | Rel(specialized_agents, gemini, "API calls", "HTTPS") 85 | Rel(doc_rag_pod, openai, "Embeddings", "HTTPS") 86 | 87 | @enduml 88 | 89 | -------------------------------------------------------------------------------- /src/engines/KeywordRAGEngine.ts: -------------------------------------------------------------------------------- 1 | import { Document } from "@langchain/core/documents"; 2 | import { IContextEngine } from "../core/interfaces/IContextEngine.js"; 3 | 4 | /** 5 | * Keyword RAG Engine for lexical search using BM25-like scoring 6 | * Provides high-precision search for exact terms, acronyms, and identifiers 7 | */ 8 | export class KeywordRAGEngine implements IContextEngine { 9 | private documents: Document[] = []; 10 | private invertedIndex: Map> = new Map(); 11 | 12 | constructor() {} 13 | 14 | /** 15 | * Indexes documents for keyword search 16 | * @param documents Documents to index 17 | */ 18 | public async indexDocuments(documents: Document[]): Promise { 19 | console.log(`[KeywordRAG] Indexing ${documents.length} documents`); 20 | 21 | this.documents = documents; 22 | this.invertedIndex.clear(); 23 | 24 | documents.forEach((doc, docIndex) => { 25 | const tokens = this.tokenize(doc.pageContent); 26 | tokens.forEach(token => { 27 | if (!this.invertedIndex.has(token)) { 28 | this.invertedIndex.set(token, new Set()); 29 | } 30 | this.invertedIndex.get(token)!.add(docIndex); 31 | }); 32 | }); 33 | 34 | console.log(`[KeywordRAG] Indexed ${this.invertedIndex.size} unique terms`); 35 | } 36 | 37 | /** 38 | * Retrieves documents matching keywords using BM25-inspired scoring 39 | * @param query The search query 40 | * @param k Number of documents to retrieve 41 | * @returns Ranked documents 42 | */ 43 | public async retrieve(query: string, k: number = 4): Promise { 44 | if (this.documents.length === 0) { 45 | console.warn("[KeywordRAG] No documents indexed"); 46 | return []; 47 | } 48 | 49 | console.log(`[KeywordRAG] Searching for: "${query}"`); 50 | 51 | const queryTokens = this.tokenize(query); 52 | const scores = new Map(); 53 | 54 | // Calculate BM25-inspired scores 55 | const avgDocLength = this.documents.reduce((sum, doc) => 56 | sum + this.tokenize(doc.pageContent).length, 0 57 | ) / this.documents.length; 58 | 59 | queryTokens.forEach(token => { 60 | const docIndices = this.invertedIndex.get(token); 61 | if (!docIndices) return; 62 | 63 | const idf = Math.log((this.documents.length - docIndices.size + 0.5) / (docIndices.size + 0.5) + 1); 64 | 65 | docIndices.forEach(docIndex => { 66 | const doc = this.documents[docIndex]; 67 | const docTokens = this.tokenize(doc.pageContent); 68 | const tf = docTokens.filter(t => t === token).length; 69 | const docLength = docTokens.length; 70 | 71 | // BM25 parameters 72 | const k1 = 1.5; 73 | const b = 0.75; 74 | 75 | const score = idf * (tf * (k1 + 1)) / 76 | (tf + k1 * (1 - b + b * (docLength / avgDocLength))); 77 | 78 | scores.set(docIndex, (scores.get(docIndex) || 0) + score); 79 | }); 80 | }); 81 | 82 | // Sort and return top k 83 | const rankedDocs = Array.from(scores.entries()) 84 | .sort((a, b) => b[1] - a[1]) 85 | .slice(0, k) 86 | .map(([docIndex, score]) => { 87 | const doc = this.documents[docIndex]; 88 | return new Document({ 89 | pageContent: doc.pageContent, 90 | metadata: { 91 | ...doc.metadata, 92 | engine: 'KeywordRAG', 93 | score, 94 | retrievalTime: new Date().toISOString() 95 | } 96 | }); 97 | }); 98 | 99 | console.log(`[KeywordRAG] Found ${rankedDocs.length} relevant documents`); 100 | return rankedDocs; 101 | } 102 | 103 | /** 104 | * Simple tokenizer 105 | * In production, use a more sophisticated tokenizer 106 | */ 107 | private tokenize(text: string): string[] { 108 | return text 109 | .toLowerCase() 110 | .replace(/[^\w\s]/g, ' ') 111 | .split(/\s+/) 112 | .filter(token => token.length > 0); 113 | } 114 | 115 | /** 116 | * Clears the index 117 | */ 118 | public clear(): void { 119 | this.documents = []; 120 | this.invertedIndex.clear(); 121 | console.log('[KeywordRAG] Index cleared'); 122 | } 123 | } 124 | 125 | -------------------------------------------------------------------------------- /src/__tests__/agents/MessageBus.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, it, expect, beforeEach, vi } from 'vitest'; 2 | import { MessageBus } from '../../agents/MessageBus.js'; 3 | import { A2AMessage } from '../../core/types.js'; 4 | 5 | describe('MessageBus', () => { 6 | let messageBus: MessageBus; 7 | 8 | beforeEach(() => { 9 | messageBus = new MessageBus(); 10 | }); 11 | 12 | describe('send and subscribe', () => { 13 | it('should deliver message to subscribed agent', async () => { 14 | const handler = vi.fn(); 15 | messageBus.subscribe('agent1', handler); 16 | 17 | const message: A2AMessage = { 18 | id: 'msg1', 19 | taskId: 'task1', 20 | targetAgentId: 'agent1', 21 | sourceAgentId: 'orchestrator', 22 | type: 'task', 23 | payload: { test: 'data' }, 24 | timestamp: new Date() 25 | }; 26 | 27 | await messageBus.send(message); 28 | 29 | expect(handler).toHaveBeenCalledWith(message); 30 | }); 31 | 32 | it('should queue messages for unsubscribed agents', async () => { 33 | const message: A2AMessage = { 34 | id: 'msg1', 35 | taskId: 'task1', 36 | targetAgentId: 'agent1', 37 | sourceAgentId: 'orchestrator', 38 | type: 'task', 39 | payload: { test: 'data' }, 40 | timestamp: new Date() 41 | }; 42 | 43 | // Send before subscription 44 | await messageBus.send(message); 45 | 46 | const stats = messageBus.getStats(); 47 | expect(stats.queuedMessages).toBe(1); 48 | }); 49 | 50 | it('should deliver queued messages on subscription', async () => { 51 | const message: A2AMessage = { 52 | id: 'msg1', 53 | taskId: 'task1', 54 | targetAgentId: 'agent1', 55 | sourceAgentId: 'orchestrator', 56 | type: 'task', 57 | payload: { test: 'data' }, 58 | timestamp: new Date() 59 | }; 60 | 61 | // Send before subscription 62 | await messageBus.send(message); 63 | 64 | const handler = vi.fn(); 65 | messageBus.subscribe('agent1', handler); 66 | 67 | // Wait for async processing 68 | await new Promise(resolve => setTimeout(resolve, 100)); 69 | 70 | expect(handler).toHaveBeenCalled(); 71 | }); 72 | 73 | it('should support multiple handlers for same agent', async () => { 74 | const handler1 = vi.fn(); 75 | const handler2 = vi.fn(); 76 | 77 | messageBus.subscribe('agent1', handler1); 78 | messageBus.subscribe('agent1', handler2); 79 | 80 | const message: A2AMessage = { 81 | id: 'msg1', 82 | taskId: 'task1', 83 | targetAgentId: 'agent1', 84 | sourceAgentId: 'orchestrator', 85 | type: 'task', 86 | payload: {}, 87 | timestamp: new Date() 88 | }; 89 | 90 | await messageBus.send(message); 91 | 92 | expect(handler1).toHaveBeenCalled(); 93 | expect(handler2).toHaveBeenCalled(); 94 | }); 95 | }); 96 | 97 | describe('unsubscribe', () => { 98 | it('should remove agent subscription', async () => { 99 | const handler = vi.fn(); 100 | messageBus.subscribe('agent1', handler); 101 | messageBus.unsubscribe('agent1'); 102 | 103 | const message: A2AMessage = { 104 | id: 'msg1', 105 | taskId: 'task1', 106 | targetAgentId: 'agent1', 107 | sourceAgentId: 'orchestrator', 108 | type: 'task', 109 | payload: {}, 110 | timestamp: new Date() 111 | }; 112 | 113 | await messageBus.send(message); 114 | 115 | expect(handler).not.toHaveBeenCalled(); 116 | }); 117 | }); 118 | 119 | describe('broadcast', () => { 120 | it('should send message to all subscribers', async () => { 121 | const handler1 = vi.fn(); 122 | const handler2 = vi.fn(); 123 | const handler3 = vi.fn(); 124 | 125 | messageBus.subscribe('agent1', handler1); 126 | messageBus.subscribe('agent2', handler2); 127 | messageBus.subscribe('agent3', handler3); 128 | 129 | await messageBus.broadcast({ 130 | id: 'broadcast1', 131 | taskId: 'task1', 132 | sourceAgentId: 'orchestrator', 133 | type: 'status', 134 | payload: { status: 'ready' }, 135 | timestamp: new Date() 136 | }); 137 | 138 | expect(handler1).toHaveBeenCalled(); 139 | expect(handler2).toHaveBeenCalled(); 140 | expect(handler3).toHaveBeenCalled(); 141 | }); 142 | }); 143 | 144 | describe('getStats', () => { 145 | it('should return correct statistics', () => { 146 | messageBus.subscribe('agent1', vi.fn()); 147 | messageBus.subscribe('agent2', vi.fn()); 148 | 149 | const stats = messageBus.getStats(); 150 | 151 | expect(stats.subscribers).toBe(2); 152 | expect(stats.subscriberIds).toContain('agent1'); 153 | expect(stats.subscriberIds).toContain('agent2'); 154 | }); 155 | 156 | it('should track queued messages', async () => { 157 | await messageBus.send({ 158 | id: 'msg1', 159 | taskId: 'task1', 160 | targetAgentId: 'unsubscribed', 161 | sourceAgentId: 'orchestrator', 162 | type: 'task', 163 | payload: {}, 164 | timestamp: new Date() 165 | }); 166 | 167 | const stats = messageBus.getStats(); 168 | expect(stats.queuedMessages).toBe(1); 169 | }); 170 | }); 171 | }); 172 | 173 | -------------------------------------------------------------------------------- /src/tools/ExampleTools.ts: -------------------------------------------------------------------------------- 1 | import { ITool } from "../core/interfaces/ITool.js"; 2 | import { z } from "zod"; 3 | 4 | /** 5 | * Example tool: Calculator 6 | */ 7 | export class CalculatorTool implements ITool<{ 8 | operation: z.ZodEnum<["add", "subtract", "multiply", "divide"]>; 9 | a: z.ZodNumber; 10 | b: z.ZodNumber; 11 | }> { 12 | readonly name = "calculator"; 13 | readonly description = "Performs basic arithmetic operations. Supports add, subtract, multiply, and divide."; 14 | readonly schema = z.object({ 15 | operation: z.enum(["add", "subtract", "multiply", "divide"]).describe("The arithmetic operation to perform"), 16 | a: z.number().describe("First number"), 17 | b: z.number().describe("Second number") 18 | }); 19 | 20 | async execute(args: z.infer): Promise { 21 | const { operation, a, b } = args; 22 | 23 | switch (operation) { 24 | case "add": 25 | return { result: a + b, operation }; 26 | case "subtract": 27 | return { result: a - b, operation }; 28 | case "multiply": 29 | return { result: a * b, operation }; 30 | case "divide": 31 | if (b === 0) throw new Error("Division by zero"); 32 | return { result: a / b, operation }; 33 | default: 34 | throw new Error(`Unknown operation: ${operation}`); 35 | } 36 | } 37 | } 38 | 39 | /** 40 | * Example tool: Web Search (simulated) 41 | */ 42 | export class WebSearchTool implements ITool<{ 43 | query: z.ZodString; 44 | maxResults: z.ZodOptional; 45 | }> { 46 | readonly name = "web_search"; 47 | readonly description = "Searches the web for information. Returns a list of relevant results (simulated)."; 48 | readonly schema = z.object({ 49 | query: z.string().describe("The search query"), 50 | maxResults: z.number().optional().describe("Maximum number of results to return (default: 5)") 51 | }); 52 | 53 | async execute(args: z.infer): Promise { 54 | const { query, maxResults = 5 } = args; 55 | 56 | // Simulate web search results 57 | const results = Array.from({ length: Math.min(maxResults, 5) }, (_, i) => ({ 58 | title: `Result ${i + 1} for "${query}"`, 59 | url: `https://example.com/result-${i + 1}`, 60 | snippet: `This is a simulated search result about ${query}. It contains relevant information.`, 61 | relevanceScore: 0.9 - (i * 0.1) 62 | })); 63 | 64 | return { 65 | query, 66 | results, 67 | totalResults: results.length 68 | }; 69 | } 70 | } 71 | 72 | /** 73 | * Example tool: Code Analyzer 74 | */ 75 | export class CodeAnalyzerTool implements ITool<{ 76 | code: z.ZodString; 77 | language: z.ZodOptional; 78 | }> { 79 | readonly name = "code_analyzer"; 80 | readonly description = "Analyzes code and provides insights about complexity, style, and potential issues."; 81 | readonly schema = z.object({ 82 | code: z.string().describe("The code to analyze"), 83 | language: z.string().optional().describe("Programming language (default: auto-detect)") 84 | }); 85 | 86 | async execute(args: z.infer): Promise { 87 | const { code, language = "javascript" } = args; 88 | 89 | // Simple analysis 90 | const lines = code.split('\n').length; 91 | const functions = (code.match(/function\s+\w+/g) || []).length; 92 | const classes = (code.match(/class\s+\w+/g) || []).length; 93 | const complexity = this.calculateComplexity(code); 94 | 95 | return { 96 | language, 97 | metrics: { 98 | lines, 99 | functions, 100 | classes, 101 | complexity 102 | }, 103 | insights: [ 104 | lines > 100 ? "Consider breaking this into smaller modules" : "Code length is reasonable", 105 | complexity > 20 ? "High complexity detected - consider refactoring" : "Complexity is acceptable" 106 | ] 107 | }; 108 | } 109 | 110 | private calculateComplexity(code: string): number { 111 | // Simplified cyclomatic complexity 112 | const conditionals = (code.match(/\b(if|else|for|while|switch|case)\b/g) || []).length; 113 | return 1 + conditionals; 114 | } 115 | } 116 | 117 | /** 118 | * Example tool: Document Summarizer 119 | */ 120 | export class DocumentSummarizerTool implements ITool<{ 121 | text: z.ZodString; 122 | maxLength: z.ZodOptional; 123 | }> { 124 | readonly name = "document_summarizer"; 125 | readonly description = "Summarizes long documents into concise summaries."; 126 | readonly schema = z.object({ 127 | text: z.string().describe("The text to summarize"), 128 | maxLength: z.number().optional().describe("Maximum length of summary in words (default: 100)") 129 | }); 130 | 131 | async execute(args: z.infer): Promise { 132 | const { text, maxLength = 100 } = args; 133 | 134 | // Simple extractive summarization 135 | const sentences = text.split(/[.!?]+/).filter(s => s.trim().length > 0); 136 | const words = text.split(/\s+/).length; 137 | 138 | // Take first few sentences as summary 139 | const summaryLength = Math.min(3, sentences.length); 140 | const summary = sentences.slice(0, summaryLength).join('. ') + '.'; 141 | 142 | return { 143 | originalLength: words, 144 | summaryLength: summary.split(/\s+/).length, 145 | summary, 146 | compressionRatio: (summary.split(/\s+/).length / words).toFixed(2) 147 | }; 148 | } 149 | } 150 | 151 | -------------------------------------------------------------------------------- /src/engines/GraphRAGEngineClient.ts: -------------------------------------------------------------------------------- 1 | import { Document } from "@langchain/core/documents"; 2 | import { IContextEngine } from "../core/interfaces/IContextEngine.js"; 3 | import axios, { AxiosInstance } from "axios"; 4 | 5 | /** 6 | * Graph RAG Engine Client for structural code analysis 7 | * Communicates with the standalone RepoMapper microservice 8 | * Inspired by Aider's Repo Map feature 9 | */ 10 | export class GraphRAGEngineClient implements IContextEngine { 11 | private readonly serviceUrl: string; 12 | private readonly axiosClient: AxiosInstance; 13 | 14 | constructor(serviceUrl: string = "http://localhost:3001") { 15 | this.serviceUrl = serviceUrl; 16 | this.axiosClient = axios.create({ 17 | baseURL: serviceUrl, 18 | timeout: 30000, 19 | headers: { 20 | 'Content-Type': 'application/json' 21 | } 22 | }); 23 | } 24 | 25 | /** 26 | * Retrieves a repository map for structural code analysis 27 | * @param query Expected to be the path to the repository 28 | * @returns Array containing a single Document with the repo map 29 | */ 30 | public async retrieve(query: string): Promise { 31 | try { 32 | const repoPath = query; 33 | 34 | console.log(`[GraphRAG] Requesting repo map for: ${repoPath}`); 35 | 36 | // In a real implementation, this would call the MCP/HTTP endpoint 37 | // For now, we'll simulate the response 38 | let repoMapContent: string; 39 | 40 | try { 41 | const response = await this.axiosClient.post('/repomap', { 42 | project_root: repoPath, 43 | map_tokens: 1024 44 | }); 45 | repoMapContent = response.data.repoMap || response.data; 46 | } catch (error) { 47 | console.warn('[GraphRAG] Service unavailable, using simulated response'); 48 | repoMapContent = this.getSimulatedRepoMap(repoPath); 49 | } 50 | 51 | const repoMapDoc = new Document({ 52 | pageContent: repoMapContent, 53 | metadata: { 54 | source: `GraphRAG:${repoPath}`, 55 | engine: 'GraphRAG', 56 | retrievalTime: new Date().toISOString(), 57 | type: 'repository_map' 58 | }, 59 | }); 60 | 61 | return [repoMapDoc]; 62 | } catch (error) { 63 | console.error("[GraphRAG] Error retrieving from GraphRAG service:", error); 64 | 65 | // Return a fallback document 66 | return [new Document({ 67 | pageContent: `# Repository structure analysis unavailable for: ${query}\nError: ${error}`, 68 | metadata: { 69 | source: `GraphRAG:${query}`, 70 | engine: 'GraphRAG', 71 | error: true 72 | } 73 | })]; 74 | } 75 | } 76 | 77 | /** 78 | * Analyzes specific files in a repository 79 | * @param repoPath Path to the repository 80 | * @param filePaths Specific files to analyze 81 | * @returns Document with focused analysis 82 | */ 83 | public async analyzeFiles(repoPath: string, filePaths: string[]): Promise { 84 | try { 85 | console.log(`[GraphRAG] Analyzing specific files in: ${repoPath}`); 86 | 87 | const response = await this.axiosClient.post('/analyze', { 88 | project_root: repoPath, 89 | files: filePaths 90 | }); 91 | 92 | const analysisContent = response.data.analysis || JSON.stringify(response.data, null, 2); 93 | 94 | return [new Document({ 95 | pageContent: analysisContent, 96 | metadata: { 97 | source: `GraphRAG:${repoPath}`, 98 | engine: 'GraphRAG', 99 | retrievalTime: new Date().toISOString(), 100 | type: 'file_analysis', 101 | analyzedFiles: filePaths 102 | } 103 | })]; 104 | } catch (error) { 105 | console.error("[GraphRAG] Error analyzing files:", error); 106 | return []; 107 | } 108 | } 109 | 110 | /** 111 | * Gets a simulated repo map for demonstration purposes 112 | */ 113 | private getSimulatedRepoMap(repoPath: string): string { 114 | return `# Repository Map: ${repoPath} 115 | ## Architecture Overview (Generated by tree-sitter + PageRank) 116 | 117 | src/core/ 118 | interfaces/ 119 | IContextEngine.ts 120 | (Rank: 8.5) - Core interface for all retrieval engines 121 | - interface IContextEngine 122 | - retrieve(query: string): Promise 123 | 124 | types.ts 125 | (Rank: 9.2) - Central type definitions 126 | - interface SentientPayload 127 | - interface ContextFragment 128 | - interface AgenticState 129 | 130 | src/engines/ 131 | DocumentRAGEngine.ts 132 | (Rank: 7.8) - Semantic document retrieval 133 | - class DocumentRAGEngine implements IContextEngine 134 | - ingestPdf(filePath: string): Promise 135 | - retrieve(query: string): Promise 136 | 137 | GraphRAGEngineClient.ts 138 | (Rank: 6.5) - Code structure analysis client 139 | - class GraphRAGEngineClient implements IContextEngine 140 | 141 | src/agents/ 142 | OrchestratorAgent.ts 143 | (Rank: 10.0) - Main orchestration logic 144 | - class OrchestratorAgent implements IAgent 145 | - process(payload: SentientPayload): Promise 146 | 147 | ## Key Dependencies 148 | - DocumentRAGEngine -> IContextEngine 149 | - OrchestratorAgent -> IAgent, IMessageBus 150 | - All engines implement IContextEngine for polymorphism 151 | `; 152 | } 153 | 154 | /** 155 | * Health check for the GraphRAG service 156 | */ 157 | public async healthCheck(): Promise { 158 | try { 159 | const response = await this.axiosClient.get('/health'); 160 | return response.status === 200; 161 | } catch { 162 | return false; 163 | } 164 | } 165 | } 166 | 167 | -------------------------------------------------------------------------------- /src/engines/HybridRetriever.ts: -------------------------------------------------------------------------------- 1 | import { Document } from "@langchain/core/documents"; 2 | import { IContextEngine } from "../core/interfaces/IContextEngine.js"; 3 | 4 | /** 5 | * Hybrid Retriever that combines multiple retrieval engines using 6 | * Reciprocal Rank Fusion (RRF) for optimal results 7 | * 8 | * Fuses results from Document RAG (semantic), Graph RAG (structural), 9 | * and Keyword RAG (lexical) engines 10 | */ 11 | export class HybridRetriever implements IContextEngine { 12 | private readonly retrievers: IContextEngine[]; 13 | private readonly kConstant: number; 14 | private readonly weights: number[]; 15 | 16 | /** 17 | * @param retrievers Array of context engines to combine 18 | * @param kConstant RRF constant (default: 60) 19 | * @param weights Optional weights for each retriever (default: equal weights) 20 | */ 21 | constructor( 22 | retrievers: IContextEngine[], 23 | kConstant: number = 60, 24 | weights?: number[] 25 | ) { 26 | this.retrievers = retrievers; 27 | this.kConstant = kConstant; 28 | this.weights = weights || Array(retrievers.length).fill(1); 29 | 30 | if (this.weights.length !== retrievers.length) { 31 | throw new Error("Number of weights must match number of retrievers"); 32 | } 33 | } 34 | 35 | /** 36 | * Retrieves documents using Reciprocal Rank Fusion 37 | * @param query The search query 38 | * @param k Number of final documents to return 39 | * @returns Fused and re-ranked documents 40 | */ 41 | public async retrieve(query: string, k: number = 4): Promise { 42 | console.log(`[HybridRetriever] Retrieving from ${this.retrievers.length} engines`); 43 | 44 | // Run all retrievers in parallel 45 | const startTime = Date.now(); 46 | const allResults = await Promise.all( 47 | this.retrievers.map(async (retriever, index) => { 48 | try { 49 | const results = await retriever.retrieve(query, k); 50 | console.log(`[HybridRetriever] Engine ${index} returned ${results.length} documents`); 51 | return results; 52 | } catch (error) { 53 | console.error(`[HybridRetriever] Engine ${index} failed:`, error); 54 | return []; 55 | } 56 | }) 57 | ); 58 | 59 | // Calculate RRF scores 60 | const docScores = new Map(); 61 | const docStore = new Map(); 62 | 63 | allResults.forEach((rankedList, retrieverIndex) => { 64 | const weight = this.weights[retrieverIndex]; 65 | 66 | rankedList.forEach((doc, rank) => { 67 | // Use content + source as unique key 68 | const docKey = this.getDocumentKey(doc); 69 | 70 | if (!docStore.has(docKey)) { 71 | docStore.set(docKey, doc); 72 | docScores.set(docKey, 0); 73 | } 74 | 75 | // RRF formula: score = weight * (1 / (k + rank)) 76 | const rrfScore = weight / (this.kConstant + rank + 1); 77 | const currentScore = docScores.get(docKey) || 0; 78 | docScores.set(docKey, currentScore + rrfScore); 79 | }); 80 | }); 81 | 82 | // Sort by RRF score and return top k 83 | const sortedDocs = Array.from(docScores.entries()) 84 | .sort((a, b) => b[1] - a[1]) 85 | .slice(0, k) 86 | .map(([key, score]) => { 87 | const doc = docStore.get(key)!; 88 | return new Document({ 89 | pageContent: doc.pageContent, 90 | metadata: { 91 | ...doc.metadata, 92 | engine: 'HybridRetriever', 93 | rrfScore: score, 94 | fusedFrom: this.getFusedEngines(key, allResults), 95 | retrievalTime: new Date().toISOString() 96 | } 97 | }); 98 | }); 99 | 100 | const elapsedTime = Date.now() - startTime; 101 | console.log(`[HybridRetriever] Fused ${sortedDocs.length} documents in ${elapsedTime}ms`); 102 | 103 | return sortedDocs; 104 | } 105 | 106 | /** 107 | * Creates a unique key for a document 108 | */ 109 | private getDocumentKey(doc: Document): string { 110 | // Use first 100 chars + source as key to handle similar but not identical content 111 | const contentPreview = doc.pageContent.substring(0, 100); 112 | const source = doc.metadata?.source || 'unknown'; 113 | return `${source}:${contentPreview}`; 114 | } 115 | 116 | /** 117 | * Identifies which engines contributed to this document 118 | */ 119 | private getFusedEngines(docKey: string, allResults: Document[][]): string[] { 120 | const engines = new Set(); 121 | 122 | allResults.forEach(results => { 123 | results.forEach(doc => { 124 | if (this.getDocumentKey(doc) === docKey) { 125 | const engine = doc.metadata?.engine || 'unknown'; 126 | engines.add(engine); 127 | } 128 | }); 129 | }); 130 | 131 | return Array.from(engines); 132 | } 133 | 134 | /** 135 | * Retrieves documents with detailed fusion metadata for debugging 136 | */ 137 | public async retrieveWithDetails(query: string, k: number = 4): Promise<{ 138 | documents: Document[]; 139 | fusionDetails: { 140 | totalCandidates: number; 141 | engineResults: number[]; 142 | fusionTime: number; 143 | }; 144 | }> { 145 | const startTime = Date.now(); 146 | 147 | const allResults = await Promise.all( 148 | this.retrievers.map(retriever => retriever.retrieve(query, k)) 149 | ); 150 | 151 | const documents = await this.retrieve(query, k); 152 | 153 | return { 154 | documents, 155 | fusionDetails: { 156 | totalCandidates: allResults.reduce((sum, results) => sum + results.length, 0), 157 | engineResults: allResults.map(results => results.length), 158 | fusionTime: Date.now() - startTime 159 | } 160 | }; 161 | } 162 | } 163 | 164 | -------------------------------------------------------------------------------- /src/engines/DocumentRAGEngine.ts: -------------------------------------------------------------------------------- 1 | import { Document } from "@langchain/core/documents"; 2 | import { OpenAIEmbeddings } from "@langchain/openai"; 3 | import { RecursiveCharacterTextSplitter } from "@langchain/textsplitters"; 4 | import { MemoryVectorStore } from "langchain/vectorstores/memory"; 5 | import { PDFLoader } from "@langchain/community/document_loaders/fs/pdf"; 6 | import { CheerioWebBaseLoader } from "@langchain/community/document_loaders/web/cheerio"; 7 | import { IContextEngine } from "../core/interfaces/IContextEngine.js"; 8 | 9 | /** 10 | * Document RAG Engine for semantic retrieval from unstructured documents 11 | * Supports PDF, web pages, and text files using LangChain.js 12 | */ 13 | export class DocumentRAGEngine implements IContextEngine { 14 | private vectorStore: MemoryVectorStore | null = null; 15 | private embeddings: OpenAIEmbeddings; 16 | private textSplitter: RecursiveCharacterTextSplitter; 17 | 18 | constructor( 19 | embeddingsModel: string = "text-embedding-3-small", 20 | chunkSize: number = 1000, 21 | chunkOverlap: number = 200 22 | ) { 23 | // Ensure OPENAI_API_KEY is set in environment variables 24 | this.embeddings = new OpenAIEmbeddings({ model: embeddingsModel }); 25 | 26 | this.textSplitter = new RecursiveCharacterTextSplitter({ 27 | chunkSize, 28 | chunkOverlap, 29 | }); 30 | } 31 | 32 | /** 33 | * Ingests and indexes a PDF file. 34 | * In production, this would be part of a separate, persistent ingestion pipeline. 35 | * @param filePath Path to the PDF file. 36 | */ 37 | public async ingestPdf(filePath: string): Promise { 38 | console.log(`[DocumentRAG] Ingesting PDF from: ${filePath}`); 39 | 40 | const loader = new PDFLoader(filePath); 41 | const rawDocs = await loader.load(); 42 | 43 | const splitDocs = await this.textSplitter.splitDocuments(rawDocs); 44 | 45 | if (this.vectorStore === null) { 46 | this.vectorStore = await MemoryVectorStore.fromDocuments( 47 | splitDocs, 48 | this.embeddings 49 | ); 50 | } else { 51 | await this.vectorStore.addDocuments(splitDocs); 52 | } 53 | 54 | console.log(`[DocumentRAG] Successfully indexed ${splitDocs.length} chunks from PDF`); 55 | } 56 | 57 | /** 58 | * Ingests and indexes a web page 59 | * @param url URL of the web page to ingest 60 | */ 61 | public async ingestWebPage(url: string): Promise { 62 | console.log(`[DocumentRAG] Ingesting web page from: ${url}`); 63 | 64 | const loader = new CheerioWebBaseLoader(url); 65 | const rawDocs = await loader.load(); 66 | 67 | const splitDocs = await this.textSplitter.splitDocuments(rawDocs); 68 | 69 | if (this.vectorStore === null) { 70 | this.vectorStore = await MemoryVectorStore.fromDocuments( 71 | splitDocs, 72 | this.embeddings 73 | ); 74 | } else { 75 | await this.vectorStore.addDocuments(splitDocs); 76 | } 77 | 78 | console.log(`[DocumentRAG] Successfully indexed ${splitDocs.length} chunks from web page`); 79 | } 80 | 81 | /** 82 | * Ingests raw text documents 83 | * @param documents Array of Document objects to ingest 84 | */ 85 | public async ingestDocuments(documents: Document[]): Promise { 86 | console.log(`[DocumentRAG] Ingesting ${documents.length} documents`); 87 | 88 | const splitDocs = await this.textSplitter.splitDocuments(documents); 89 | 90 | if (this.vectorStore === null) { 91 | this.vectorStore = await MemoryVectorStore.fromDocuments( 92 | splitDocs, 93 | this.embeddings 94 | ); 95 | } else { 96 | await this.vectorStore.addDocuments(splitDocs); 97 | } 98 | 99 | console.log(`[DocumentRAG] Successfully indexed ${splitDocs.length} chunks`); 100 | } 101 | 102 | /** 103 | * Retrieves relevant documents for a query using semantic search 104 | * @param query The search query 105 | * @param k Number of documents to retrieve (default: 4) 106 | * @returns Array of relevant documents 107 | */ 108 | public async retrieve(query: string, k: number = 4): Promise { 109 | if (!this.vectorStore) { 110 | throw new Error("Vector store not initialized. Ingest documents first."); 111 | } 112 | 113 | console.log(`[DocumentRAG] Retrieving ${k} documents for query: "${query.substring(0, 50)}..."`); 114 | 115 | const retriever = this.vectorStore.asRetriever(k); 116 | const documents = await retriever.invoke(query); 117 | 118 | // Add engine metadata to documents 119 | documents.forEach(doc => { 120 | doc.metadata = { 121 | ...doc.metadata, 122 | engine: 'DocumentRAG', 123 | retrievalTime: new Date().toISOString() 124 | }; 125 | }); 126 | 127 | return documents; 128 | } 129 | 130 | /** 131 | * Retrieves documents using Maximal Marginal Relevance (MMR) 132 | * This ensures diversity in the retrieved documents 133 | * @param query The search query 134 | * @param k Number of documents to retrieve 135 | * @param fetchK Number of documents to initially fetch 136 | * @param lambda Diversity parameter (0 = max diversity, 1 = max relevance) 137 | * @returns Array of relevant and diverse documents 138 | */ 139 | public async retrieveWithMMR( 140 | query: string, 141 | k: number = 4, 142 | fetchK: number = 20, 143 | lambda: number = 0.5 144 | ): Promise { 145 | if (!this.vectorStore) { 146 | throw new Error("Vector store not initialized. Ingest documents first."); 147 | } 148 | 149 | console.log(`[DocumentRAG] Retrieving ${k} documents with MMR`); 150 | 151 | const documents = await this.vectorStore.maxMarginalRelevanceSearch( 152 | query, 153 | { k, fetchK, lambda } 154 | ); 155 | 156 | // Add engine metadata 157 | documents.forEach(doc => { 158 | doc.metadata = { 159 | ...doc.metadata, 160 | engine: 'DocumentRAG', 161 | retrievalMethod: 'MMR', 162 | retrievalTime: new Date().toISOString() 163 | }; 164 | }); 165 | 166 | return documents; 167 | } 168 | 169 | /** 170 | * Clears the vector store 171 | */ 172 | public clear(): void { 173 | this.vectorStore = null; 174 | console.log('[DocumentRAG] Vector store cleared'); 175 | } 176 | } 177 | 178 | -------------------------------------------------------------------------------- /src/__tests__/agents/ToolManager.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, it, expect, beforeEach } from 'vitest'; 2 | import { ToolManager } from '../../agents/ToolManager.js'; 3 | import { CalculatorTool, WebSearchTool } from '../../tools/ExampleTools.js'; 4 | 5 | describe('ToolManager', () => { 6 | let toolManager: ToolManager; 7 | 8 | beforeEach(() => { 9 | toolManager = new ToolManager(); 10 | }); 11 | 12 | describe('registerTool', () => { 13 | it('should register a tool successfully', () => { 14 | const calculator = new CalculatorTool(); 15 | toolManager.registerTool(calculator); 16 | 17 | const tool = toolManager.getTool('calculator'); 18 | expect(tool).toBeDefined(); 19 | expect(tool?.name).toBe('calculator'); 20 | }); 21 | 22 | it('should overwrite existing tool with warning', () => { 23 | const calculator1 = new CalculatorTool(); 24 | const calculator2 = new CalculatorTool(); 25 | 26 | toolManager.registerTool(calculator1); 27 | toolManager.registerTool(calculator2); 28 | 29 | const tools = toolManager.listTools(); 30 | expect(tools).toHaveLength(1); 31 | }); 32 | }); 33 | 34 | describe('registerTools', () => { 35 | it('should register multiple tools at once', () => { 36 | toolManager.registerTools([ 37 | new CalculatorTool(), 38 | new WebSearchTool() 39 | ]); 40 | 41 | const tools = toolManager.listTools(); 42 | expect(tools).toHaveLength(2); 43 | expect(tools).toContain('calculator'); 44 | expect(tools).toContain('web_search'); 45 | }); 46 | }); 47 | 48 | describe('executeTool', () => { 49 | beforeEach(() => { 50 | toolManager.registerTool(new CalculatorTool()); 51 | }); 52 | 53 | it('should execute tool with valid arguments', async () => { 54 | const result = await toolManager.executeTool('calculator', { 55 | operation: 'add', 56 | a: 5, 57 | b: 3 58 | }); 59 | 60 | expect(result.success).toBe(true); 61 | expect(result.data.result).toBe(8); 62 | expect(result.executionTime).toBeGreaterThan(0); 63 | }); 64 | 65 | it('should return error for non-existent tool', async () => { 66 | const result = await toolManager.executeTool('nonexistent', {}); 67 | 68 | expect(result.success).toBe(false); 69 | expect(result.error).toContain('not found'); 70 | }); 71 | 72 | it('should return error for invalid arguments', async () => { 73 | const result = await toolManager.executeTool('calculator', { 74 | operation: 'add', 75 | a: 'invalid', // Should be number 76 | b: 3 77 | }); 78 | 79 | expect(result.success).toBe(false); 80 | expect(result.error).toBeDefined(); 81 | }); 82 | 83 | it('should handle tool execution errors', async () => { 84 | const result = await toolManager.executeTool('calculator', { 85 | operation: 'divide', 86 | a: 10, 87 | b: 0 88 | }); 89 | 90 | expect(result.success).toBe(false); 91 | expect(result.error).toContain('Division by zero'); 92 | }); 93 | }); 94 | 95 | describe('getToolDefinitions', () => { 96 | it('should return MCP-compatible tool definitions', () => { 97 | toolManager.registerTool(new CalculatorTool()); 98 | 99 | const definitions = toolManager.getToolDefinitions(); 100 | 101 | expect(definitions).toHaveLength(1); 102 | expect(definitions[0].name).toBe('calculator'); 103 | expect(definitions[0].description).toBeDefined(); 104 | expect(definitions[0].schema).toBeDefined(); 105 | expect(definitions[0].schema.type).toBe('object'); 106 | }); 107 | }); 108 | 109 | describe('getToolStats', () => { 110 | beforeEach(() => { 111 | toolManager.registerTool(new CalculatorTool()); 112 | }); 113 | 114 | it('should return null for tool with no executions', () => { 115 | const stats = toolManager.getToolStats('calculator'); 116 | expect(stats).toBeNull(); 117 | }); 118 | 119 | it('should track execution statistics', async () => { 120 | await toolManager.executeTool('calculator', { 121 | operation: 'add', 122 | a: 1, 123 | b: 2 124 | }); 125 | 126 | await toolManager.executeTool('calculator', { 127 | operation: 'multiply', 128 | a: 3, 129 | b: 4 130 | }); 131 | 132 | const stats = toolManager.getToolStats('calculator'); 133 | 134 | expect(stats).toBeDefined(); 135 | expect(stats!.totalExecutions).toBe(2); 136 | expect(stats!.successRate).toBe(1); 137 | expect(stats!.averageExecutionTime).toBeGreaterThan(0); 138 | }); 139 | 140 | it('should calculate success rate correctly', async () => { 141 | // Successful execution 142 | await toolManager.executeTool('calculator', { 143 | operation: 'add', 144 | a: 1, 145 | b: 2 146 | }); 147 | 148 | // Failed execution 149 | await toolManager.executeTool('calculator', { 150 | operation: 'divide', 151 | a: 1, 152 | b: 0 153 | }); 154 | 155 | const stats = toolManager.getToolStats('calculator'); 156 | 157 | expect(stats!.totalExecutions).toBe(2); 158 | expect(stats!.successRate).toBe(0.5); 159 | }); 160 | }); 161 | 162 | describe('unregisterTool', () => { 163 | it('should unregister a tool', () => { 164 | toolManager.registerTool(new CalculatorTool()); 165 | 166 | const removed = toolManager.unregisterTool('calculator'); 167 | expect(removed).toBe(true); 168 | 169 | const tool = toolManager.getTool('calculator'); 170 | expect(tool).toBeUndefined(); 171 | }); 172 | 173 | it('should return false for non-existent tool', () => { 174 | const removed = toolManager.unregisterTool('nonexistent'); 175 | expect(removed).toBe(false); 176 | }); 177 | }); 178 | 179 | describe('clear', () => { 180 | it('should clear all tools and history', async () => { 181 | toolManager.registerTool(new CalculatorTool()); 182 | await toolManager.executeTool('calculator', { 183 | operation: 'add', 184 | a: 1, 185 | b: 2 186 | }); 187 | 188 | toolManager.clear(); 189 | 190 | expect(toolManager.listTools()).toHaveLength(0); 191 | expect(toolManager.getToolStats('calculator')).toBeNull(); 192 | }); 193 | }); 194 | }); 195 | 196 | -------------------------------------------------------------------------------- /diagrams/QUICKSTART.md: -------------------------------------------------------------------------------- 1 | # 🚀 Quick Start Guide - Sentient Framework Architecture Diagrams 2 | 3 | 欢迎！这是一个快速指南，帮助你查看和使用 Sentient Framework 的 C4 架构图。 4 | 5 | ## 📋 目录 6 | 7 | 1. [快速查看](#快速查看) 8 | 2. [生成图片](#生成图片) 9 | 3. [在线查看](#在线查看) 10 | 4. [集成到文档](#集成到文档) 11 | 5. [修改和定制](#修改和定制) 12 | 13 | --- 14 | 15 | ## 🎯 快速查看 16 | 17 | ### 方法 1：使用浏览器查看器（推荐） 18 | 19 | 直接打开 HTML 查看器，无需安装任何工具： 20 | 21 | ```bash 22 | # 直接在浏览器中打开 23 | open viewer.html 24 | 25 | # 或者使用 Python 启动本地服务器 26 | cd diagrams 27 | python3 -m http.server 8000 28 | # 然后访问: http://localhost:8000/viewer.html 29 | ``` 30 | 31 | ### 方法 2：VS Code 插件 32 | 33 | 1. 安装 PlantUML 插件： 34 | ``` 35 | 在 VS Code 中搜索并安装：PlantUML (作者: jebbs) 36 | ``` 37 | 38 | 2. 打开任意 `.puml` 文件 39 | 40 | 3. 按下快捷键查看预览： 41 | - macOS: `Option + D` 42 | - Windows/Linux: `Alt + D` 43 | 44 | ### 方法 3：在线 PlantUML 编辑器 45 | 46 | 1. 访问 [PlantUML Online Editor](https://www.plantuml.com/plantuml/uml/) 47 | 2. 打开任意 `.puml` 文件 48 | 3. 复制内容到在线编辑器 49 | 4. 立即查看渲染结果 50 | 51 | --- 52 | 53 | ## 🖼️ 生成图片 54 | 55 | ### 使用提供的脚本（推荐） 56 | 57 | ```bash 58 | # 运行生成脚本 59 | ./generate-images.sh 60 | ``` 61 | 62 | 这会生成： 63 | - `output/png/` - PNG 格式图片 64 | - `output/svg/` - SVG 格式图片（推荐用于文档） 65 | 66 | ### 手动生成 67 | 68 | #### 安装 PlantUML 69 | 70 | **macOS:** 71 | ```bash 72 | brew install plantuml 73 | ``` 74 | 75 | **Ubuntu/Debian:** 76 | ```bash 77 | sudo apt-get install plantuml 78 | ``` 79 | 80 | **Windows:** 81 | 下载 JAR 文件：https://plantuml.com/download 82 | 83 | #### 生成单个图表 84 | 85 | ```bash 86 | # 生成 PNG 87 | plantuml 01-context-diagram.puml 88 | 89 | # 生成 SVG（矢量图，推荐） 90 | plantuml -tsvg 01-context-diagram.puml 91 | 92 | # 指定输出目录 93 | plantuml -o output/png 01-context-diagram.puml 94 | ``` 95 | 96 | #### 批量生成所有图表 97 | 98 | ```bash 99 | # 生成所有 PNG 100 | for file in *.puml; do plantuml "$file"; done 101 | 102 | # 生成所有 SVG 103 | for file in *.puml; do plantuml -tsvg "$file"; done 104 | ``` 105 | 106 | --- 107 | 108 | ## 🌐 在线查看 109 | 110 | ### PlantUML Server（Docker） 111 | 112 | 启动本地 PlantUML 服务器： 113 | 114 | ```bash 115 | # 使用 Docker 启动服务器 116 | docker run -d -p 8080:8080 plantuml/plantuml-server:jetty 117 | 118 | # 访问 119 | open http://localhost:8080 120 | ``` 121 | 122 | ### 直接在 GitHub 查看 123 | 124 | 如果你的仓库托管在 GitHub，可以使用 GitHub 的渲染服务： 125 | 126 | ```markdown 127 | ![Architecture](http://www.plantuml.com/plantuml/proxy?src=https://raw.githubusercontent.com/your-username/your-repo/main/diagrams/01-context-diagram.puml) 128 | ``` 129 | 130 | --- 131 | 132 | ## 📚 集成到文档 133 | 134 | ### Markdown 文档 135 | 136 | ```markdown 137 | # 系统架构 138 | 139 | ## 系统上下文图 140 | 141 | ![System Context](./output/svg/01-context-diagram.svg) 142 | 143 | ## 容器图 144 | 145 | ![Container Diagram](./output/svg/02-container-diagram.svg) 146 | ``` 147 | 148 | ### Wiki 或 Confluence 149 | 150 | 1. 生成 PNG 或 SVG 文件 151 | 2. 上传图片到 Wiki 152 | 3. 在页面中引用 153 | 154 | ### 演示文稿（PowerPoint/Keynote） 155 | 156 | 1. 生成 PNG 文件（高分辨率）： 157 | ```bash 158 | plantuml -tpng -DPLANTUML_LIMIT_SIZE=8192 01-context-diagram.puml 159 | ``` 160 | 2. 将 PNG 插入演示文稿 161 | 162 | --- 163 | 164 | ## ✏️ 修改和定制 165 | 166 | ### 基本修改 167 | 168 | 每个 `.puml` 文件都是纯文本，可以直接编辑： 169 | 170 | ```plantuml 171 | ' 添加新的组件 172 | Component(new_component, "New Component", "TypeScript", "Description") 173 | 174 | ' 添加关系 175 | Rel(orchestrator, new_component, "Uses", "API call") 176 | 177 | ' 修改颜色 178 | Component(my_component, "My Component", "TypeScript", "Description", $tags="important") 179 | ``` 180 | 181 | ### 常见定制场景 182 | 183 | #### 1. 添加新的 Agent 184 | 185 | 编辑 `04-component-diagram-agentic-core.puml`： 186 | 187 | ```plantuml 188 | Component(my_new_agent, "My New Agent", "TypeScript/Gemini", "Handles special tasks") 189 | Rel(message_bus, my_new_agent, "Routes tasks", "A2A Message") 190 | Rel(my_new_agent, message_bus, "Sends results", "") 191 | ``` 192 | 193 | #### 2. 添加新的 RAG Engine 194 | 195 | 编辑 `05-component-diagram-context-engine.puml`： 196 | 197 | ```plantuml 198 | Component(my_rag, "My RAG Engine", "Custom", "Special retrieval strategy") 199 | Rel(hybrid_retriever, my_rag, "Queries", "") 200 | ``` 201 | 202 | #### 3. 修改部署架构 203 | 204 | 编辑 `07-deployment-diagram.puml` 添加新的部署节点。 205 | 206 | ### 主题和样式 207 | 208 | PlantUML 支持多种主题： 209 | 210 | ```plantuml 211 | ' 在文件开头添加 212 | !theme bluegray 213 | ' 或 214 | !theme aws-orange 215 | ' 或 216 | !theme plain 217 | ``` 218 | 219 | 可用主题：https://plantuml.com/theme 220 | 221 | --- 222 | 223 | ## 🔧 故障排查 224 | 225 | ### 问题 1：PlantUML 命令不存在 226 | 227 | **解决方案：** 228 | ```bash 229 | # 检查是否安装 230 | which plantuml 231 | 232 | # 如果没有，使用 Java jar 直接运行 233 | java -jar plantuml.jar diagram.puml 234 | ``` 235 | 236 | ### 问题 2：图表太大，内存不足 237 | 238 | **解决方案：** 239 | ```bash 240 | # 增加内存限制 241 | java -Xmx2048m -jar plantuml.jar diagram.puml 242 | 243 | # 或设置环境变量 244 | export PLANTUML_LIMIT_SIZE=8192 245 | ``` 246 | 247 | ### 问题 3：C4 标准库加载失败 248 | 249 | **解决方案：** 250 | ```plantuml 251 | ' 使用本地副本或 GitHub 镜像 252 | !include https://raw.githubusercontent.com/plantuml-stdlib/C4-PlantUML/master/C4_Context.puml 253 | ``` 254 | 255 | ### 问题 4：中文显示为方框 256 | 257 | **解决方案：** 258 | ```plantuml 259 | ' 在文件开头指定字体 260 | skinparam defaultFontName "Microsoft YaHei" 261 | ' 或 262 | skinparam defaultFontName "PingFang SC" 263 | ``` 264 | 265 | --- 266 | 267 | ## 📖 推荐资源 268 | 269 | ### PlantUML 学习资源 270 | 271 | - [官方文档](https://plantuml.com/) 272 | - [C4 模型介绍](https://c4model.com/) 273 | - [C4-PlantUML GitHub](https://github.com/plantuml-stdlib/C4-PlantUML) 274 | 275 | ### 示例和模板 276 | 277 | - [Real World PlantUML](https://real-world-plantuml.com/) 278 | - [PlantUML Gallery](https://plantuml.com/gallery) 279 | - [C4 Examples](https://github.com/plantuml-stdlib/C4-PlantUML/tree/master/samples) 280 | 281 | --- 282 | 283 | ## 🎯 下一步 284 | 285 | 1. **查看所有 8 个图表** 286 | - 从系统上下文图开始 287 | - 逐层深入到组件细节 288 | 289 | 2. **理解架构层次** 290 | - Service Layer（服务层） 291 | - Agentic Core（代理核心） 292 | - Context Engine Layer（上下文引擎层） 293 | 294 | 3. **根据需求定制** 295 | - 添加你的特定组件 296 | - 调整为你的部署环境 297 | - 生成适合你团队的文档 298 | 299 | 4. **分享和协作** 300 | - 将图表集成到团队文档 301 | - 用于架构评审和设计讨论 302 | - 作为新成员培训材料 303 | 304 | --- 305 | 306 | ## 💡 小贴士 307 | 308 | - **SVG 优于 PNG**：矢量图在缩放时不会失真 309 | - **保持源文件版本控制**：`.puml` 文件易于 diff 和协作 310 | - **定期更新**：代码变更时同步更新架构图 311 | - **使用注释**：在 `.puml` 文件中添加注释说明设计决策 312 | 313 | --- 314 | 315 | ## 🤝 贡献 316 | 317 | 发现问题或有改进建议？ 318 | - 修改对应的 `.puml` 文件 319 | - 提交 Pull Request 320 | - 在 Issue 中讨论架构改进 321 | 322 | --- 323 | 324 | ## 📄 许可 325 | 326 | 这些架构图是 Sentient Framework 项目的一部分，遵循项目的开源许可。 327 | 328 | --- 329 | 330 | **Happy Diagramming! 🎨** 331 | 332 | -------------------------------------------------------------------------------- /QUICKSTART.md: -------------------------------------------------------------------------------- 1 | # 🚀 Sentient Framework - Quick Start Guide 2 | 3 | Get up and running with Sentient Framework in 5 minutes! 4 | 5 | ## Prerequisites 6 | 7 | - **Node.js** >= 18.0.0 8 | - **npm** or **yarn** or **pnpm** 9 | - **OpenAI API Key** (for embeddings) 10 | - **Google Gemini API Key** (for orchestration and compression) 11 | 12 | ## Step 1: Installation 13 | 14 | ```bash 15 | # Install dependencies 16 | npm install 17 | 18 | # Or if you're starting fresh 19 | git clone https://github.com/phodal/codelotus.git 20 | cd codelotus 21 | npm install 22 | ``` 23 | 24 | ## Step 2: Configuration 25 | 26 | ```bash 27 | # Copy environment template 28 | cp env.example .env 29 | 30 | # Edit .env with your API keys 31 | ``` 32 | 33 | Add your API keys to `.env`: 34 | ```env 35 | OPENAI_API_KEY=sk-your-openai-key-here 36 | GOOGLE_API_KEY=your-google-api-key-here 37 | PORT=3000 38 | ``` 39 | 40 | ### Getting API Keys 41 | 42 | **OpenAI API Key:** 43 | 1. Go to https://platform.openai.com/api-keys 44 | 2. Create a new API key 45 | 3. Copy it to your `.env` file 46 | 47 | **Google Gemini API Key:** 48 | 1. Go to https://makersuite.google.com/app/apikey 49 | 2. Create a new API key 50 | 3. Copy it to your `.env` file 51 | 52 | ## Step 3: Run Tests and Demo 53 | 54 | ### Run Tests 55 | 56 | ```bash 57 | # Run all tests 58 | npm test 59 | 60 | # Run tests with coverage 61 | npm run test:coverage 62 | 63 | # Watch mode for development 64 | npm run test:watch 65 | ``` 66 | 67 | ### Run Demo 68 | 69 | ```bash 70 | npm run demo 71 | ``` 72 | 73 | This runs the complete end-to-end demo: 74 | - Multi-modal RAG retrieval 75 | - Context compression 76 | - Agent orchestration 77 | - Final response synthesis 78 | 79 | ## Step 4: Start the Server 80 | 81 | ```bash 82 | npm run server 83 | ``` 84 | 85 | The server will start on `http://localhost:3000` 86 | 87 | ### Test the API 88 | 89 | **List available tools (MCP):** 90 | ```bash 91 | curl http://localhost:3000/mcp/tools 92 | ``` 93 | 94 | **Execute a tool:** 95 | ```bash 96 | curl -X POST http://localhost:3000/mcp/tools/calculator/execute \ 97 | -H "Content-Type: application/json" \ 98 | -d '{"operation": "add", "a": 5, "b": 3}' 99 | ``` 100 | 101 | **List registered agents (A2A):** 102 | ```bash 103 | curl http://localhost:3000/a2a/agents 104 | ``` 105 | 106 | **Health check:** 107 | ```bash 108 | curl http://localhost:3000/health 109 | ``` 110 | 111 | ## Step 5: Use in Your Code 112 | 113 | ### Basic Usage 114 | 115 | ```typescript 116 | import { DocumentRAGEngine } from 'sentient-framework'; 117 | import { Document } from '@langchain/core/documents'; 118 | 119 | async function myApp() { 120 | // Initialize engine 121 | const engine = new DocumentRAGEngine(); 122 | 123 | // Ingest documents 124 | await engine.ingestDocuments([ 125 | new Document({ 126 | pageContent: "Your content here", 127 | metadata: { source: "doc1.txt" } 128 | }) 129 | ]); 130 | 131 | // Retrieve 132 | const results = await engine.retrieve("your query", 4); 133 | console.log(results); 134 | } 135 | ``` 136 | 137 | ### Complete Workflow 138 | 139 | ```typescript 140 | import { 141 | DocumentRAGEngine, 142 | KeywordRAGEngine, 143 | HybridRetriever, 144 | ContextCompressor, 145 | OrchestratorAgent, 146 | MessageBus, 147 | ToolManager 148 | } from 'sentient-framework'; 149 | 150 | async function advancedWorkflow() { 151 | // 1. Set up RAG 152 | const docEngine = new DocumentRAGEngine(); 153 | const keywordEngine = new KeywordRAGEngine(); 154 | const hybrid = new HybridRetriever([docEngine, keywordEngine]); 155 | 156 | // 2. Set up compression 157 | const compressor = new ContextCompressor(); 158 | 159 | // 3. Set up agents 160 | const messageBus = new MessageBus(); 161 | const toolManager = new ToolManager(); 162 | const orchestrator = new OrchestratorAgent(messageBus, toolManager); 163 | 164 | // 4. Process query 165 | const docs = await hybrid.retrieve("your query"); 166 | const compressed = await compressor.compress("your query", docs); 167 | 168 | // 5. Use orchestrator 169 | const payload = createPayload("your query", compressed); 170 | const result = await orchestrator.process(payload); 171 | 172 | console.log(result.finalOutput); 173 | } 174 | ``` 175 | 176 | ## Troubleshooting 177 | 178 | ### Common Issues 179 | 180 | **"API key not found" error:** 181 | - Ensure `.env` file exists in the project root 182 | - Check that API keys are set correctly 183 | - Restart your terminal/IDE after setting environment variables 184 | 185 | **Import errors:** 186 | - Run `npm run build` to compile TypeScript 187 | - Check that all dependencies are installed: `npm install` 188 | 189 | **Memory errors with large documents:** 190 | - Use `MemoryVectorStore` for small datasets 191 | - For production, use persistent stores (Weaviate, Pinecone, Chroma) 192 | 193 | ## Next Steps 194 | 195 | 1. **Read the Architecture**: See [README.md](./README.md) for comprehensive documentation 196 | 2. **View Diagrams**: See [diagrams/](./diagrams/) for interactive architecture diagrams 197 | 3. **Explore Examples**: Check [src/examples/](./src/examples/) for more use cases 198 | 4. **Build Custom Tools**: See [IMPLEMENTATION.md](./IMPLEMENTATION.md#adding-a-custom-tool) 199 | 5. **Deploy to Production**: See deployment guides in architecture docs 200 | 201 | ## Useful Commands 202 | 203 | ```bash 204 | # Development 205 | npm run dev src/your-file.ts # Run TypeScript file directly 206 | npm run build # Build to dist/ 207 | npm start # Run built server 208 | 209 | # Examples 210 | npm run example:basic # Basic RAG 211 | npm run example:hybrid # Hybrid retrieval 212 | npm run example:compression # Context compression 213 | npm run example:tools # Tool usage 214 | npm run example:full # Full workflow 215 | 216 | # Testing & Quality 217 | npm test # Run tests 218 | npm run lint # Lint code 219 | npm run format # Format code 220 | 221 | # Server 222 | npm run server # Start MCP & A2A services 223 | ``` 224 | 225 | ## Resources 226 | 227 | - 📚 [Full Documentation](./README.md) 228 | - 🎨 [Architecture Diagrams](./diagrams/) 229 | - 💻 [Implementation Guide](./IMPLEMENTATION.md) 230 | - 🤝 [Contributing Guide](./CONTRIBUTING.md) 231 | - 🔗 [GitHub Repository](https://github.com/phodal/codelotus) 232 | 233 | ## Getting Help 234 | 235 | - **Issues**: Open an issue on GitHub 236 | - **Discussions**: Use GitHub Discussions 237 | - **Documentation**: Check README.md and diagrams 238 | 239 | --- 240 | 241 | **Happy Building! 🚀** 242 | 243 | If you find this framework useful, please give it a ⭐ on GitHub! 244 | 245 | -------------------------------------------------------------------------------- /src/agents/ToolManager.ts: -------------------------------------------------------------------------------- 1 | import { ITool } from "../core/interfaces/ITool.js"; 2 | import { ToolResult } from "../core/types.js"; 3 | import { ZodRawShape } from "zod"; 4 | 5 | /** 6 | * Tool Manager for registering, discovering, and executing tools 7 | * Integrates with MCP (Model Context Protocol) for standardized tool exposure 8 | */ 9 | export class ToolManager { 10 | private tools: Map>; 11 | private executionHistory: Map; 12 | 13 | constructor() { 14 | this.tools = new Map(); 15 | this.executionHistory = new Map(); 16 | } 17 | 18 | /** 19 | * Registers a new tool 20 | */ 21 | registerTool(tool: ITool): void { 22 | if (this.tools.has(tool.name)) { 23 | console.warn(`[ToolManager] Tool ${tool.name} already registered, overwriting`); 24 | } 25 | 26 | this.tools.set(tool.name, tool); 27 | console.log(`[ToolManager] Registered tool: ${tool.name}`); 28 | } 29 | 30 | /** 31 | * Registers multiple tools at once 32 | */ 33 | registerTools(tools: ITool[]): void { 34 | tools.forEach(tool => this.registerTool(tool)); 35 | } 36 | 37 | /** 38 | * Unregisters a tool 39 | */ 40 | unregisterTool(toolName: string): boolean { 41 | const removed = this.tools.delete(toolName); 42 | if (removed) { 43 | console.log(`[ToolManager] Unregistered tool: ${toolName}`); 44 | } 45 | return removed; 46 | } 47 | 48 | /** 49 | * Gets a tool by name 50 | */ 51 | getTool(toolName: string): ITool | undefined { 52 | return this.tools.get(toolName); 53 | } 54 | 55 | /** 56 | * Lists all registered tools 57 | */ 58 | listTools(): string[] { 59 | return Array.from(this.tools.keys()); 60 | } 61 | 62 | /** 63 | * Gets tool definitions in MCP format 64 | */ 65 | getToolDefinitions(): Array<{ 66 | name: string; 67 | description: string; 68 | schema: any; 69 | }> { 70 | return Array.from(this.tools.values()).map(tool => ({ 71 | name: tool.name, 72 | description: tool.description, 73 | schema: this.zodToJsonSchema(tool.schema) 74 | })); 75 | } 76 | 77 | /** 78 | * Executes a tool by name with given arguments 79 | */ 80 | async executeTool(toolName: string, args: any): Promise { 81 | const startTime = Date.now(); 82 | const tool = this.tools.get(toolName); 83 | 84 | if (!tool) { 85 | const error = `Tool not found: ${toolName}`; 86 | console.error(`[ToolManager] ${error}`); 87 | return { 88 | success: false, 89 | error, 90 | executionTime: Date.now() - startTime 91 | }; 92 | } 93 | 94 | try { 95 | console.log(`[ToolManager] Executing tool: ${toolName}`); 96 | 97 | // Validate arguments against schema 98 | const validatedArgs = tool.schema.parse(args); 99 | 100 | // Execute the tool 101 | const data = await tool.execute(validatedArgs); 102 | 103 | const result: ToolResult = { 104 | success: true, 105 | data, 106 | executionTime: Date.now() - startTime 107 | }; 108 | 109 | // Store execution history 110 | this.recordExecution(toolName, result); 111 | 112 | console.log(`[ToolManager] Tool ${toolName} executed successfully in ${result.executionTime}ms`); 113 | return result; 114 | 115 | } catch (error) { 116 | const errorMessage = error instanceof Error ? error.message : String(error); 117 | console.error(`[ToolManager] Tool execution failed:`, errorMessage); 118 | 119 | const result: ToolResult = { 120 | success: false, 121 | error: errorMessage, 122 | executionTime: Date.now() - startTime 123 | }; 124 | 125 | this.recordExecution(toolName, result); 126 | return result; 127 | } 128 | } 129 | 130 | /** 131 | * Records tool execution for history 132 | */ 133 | private recordExecution(toolName: string, result: ToolResult): void { 134 | if (!this.executionHistory.has(toolName)) { 135 | this.executionHistory.set(toolName, []); 136 | } 137 | 138 | const history = this.executionHistory.get(toolName)!; 139 | history.push(result); 140 | 141 | // Keep only last 100 executions per tool 142 | if (history.length > 100) { 143 | history.shift(); 144 | } 145 | } 146 | 147 | /** 148 | * Gets execution history for a tool 149 | */ 150 | getExecutionHistory(toolName: string): ToolResult[] { 151 | return this.executionHistory.get(toolName) || []; 152 | } 153 | 154 | /** 155 | * Gets execution statistics for a tool 156 | */ 157 | getToolStats(toolName: string): { 158 | totalExecutions: number; 159 | successRate: number; 160 | averageExecutionTime: number; 161 | } | null { 162 | const history = this.executionHistory.get(toolName); 163 | if (!history || history.length === 0) { 164 | return null; 165 | } 166 | 167 | const totalExecutions = history.length; 168 | const successfulExecutions = history.filter(r => r.success).length; 169 | const successRate = successfulExecutions / totalExecutions; 170 | const averageExecutionTime = history.reduce((sum, r) => sum + r.executionTime, 0) / totalExecutions; 171 | 172 | return { 173 | totalExecutions, 174 | successRate, 175 | averageExecutionTime 176 | }; 177 | } 178 | 179 | /** 180 | * Converts Zod schema to JSON Schema for MCP compatibility 181 | */ 182 | private zodToJsonSchema(zodSchema: any): any { 183 | // Simple conversion - in production, use a library like zod-to-json-schema 184 | try { 185 | const shape = zodSchema._def.shape(); 186 | const properties: any = {}; 187 | const required: string[] = []; 188 | 189 | Object.entries(shape).forEach(([key, value]: [string, any]) => { 190 | properties[key] = { 191 | type: this.getZodType(value), 192 | description: value._def.description || '' 193 | }; 194 | 195 | if (!value.isOptional()) { 196 | required.push(key); 197 | } 198 | }); 199 | 200 | return { 201 | type: 'object', 202 | properties, 203 | required 204 | }; 205 | } catch (error) { 206 | console.error('[ToolManager] Error converting Zod schema:', error); 207 | return { type: 'object' }; 208 | } 209 | } 210 | 211 | /** 212 | * Maps Zod types to JSON Schema types 213 | */ 214 | private getZodType(zodType: any): string { 215 | const typeName = zodType._def.typeName; 216 | 217 | const typeMap: Record = { 218 | 'ZodString': 'string', 219 | 'ZodNumber': 'number', 220 | 'ZodBoolean': 'boolean', 221 | 'ZodArray': 'array', 222 | 'ZodObject': 'object', 223 | 'ZodNull': 'null' 224 | }; 225 | 226 | return typeMap[typeName] || 'string'; 227 | } 228 | 229 | /** 230 | * Clears all tools and history 231 | */ 232 | clear(): void { 233 | this.tools.clear(); 234 | this.executionHistory.clear(); 235 | console.log('[ToolManager] All tools and history cleared'); 236 | } 237 | } 238 | 239 | -------------------------------------------------------------------------------- /src/examples/demo.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Sentient Framework Demo 3 | * Complete end-to-end workflow demonstration 4 | */ 5 | 6 | import { DocumentRAGEngine } from '../engines/DocumentRAGEngine.js'; 7 | import { GraphRAGEngineClient } from '../engines/GraphRAGEngineClient.js'; 8 | import { KeywordRAGEngine } from '../engines/KeywordRAGEngine.js'; 9 | import { HybridRetriever } from '../engines/HybridRetriever.js'; 10 | import { ContextCompressor } from '../compression/ContextCompressor.js'; 11 | import { OrchestratorAgent } from '../agents/OrchestratorAgent.js'; 12 | import { MessageBus } from '../agents/MessageBus.js'; 13 | import { ToolManager } from '../agents/ToolManager.js'; 14 | import { CalculatorTool, WebSearchTool } from '../tools/ExampleTools.js'; 15 | import { SentientPayload } from '../core/types.js'; 16 | import { Document } from '@langchain/core/documents'; 17 | import * as dotenv from 'dotenv'; 18 | import { v4 as uuidv4 } from 'uuid'; 19 | 20 | dotenv.config(); 21 | 22 | async function main() { 23 | console.log('╔══════════════════════════════════════════════════════════════╗'); 24 | console.log('║ ║'); 25 | console.log('║ 🚀 Sentient Framework - Full Workflow Demo ║'); 26 | console.log('║ ║'); 27 | console.log('╚══════════════════════════════════════════════════════════════╝\n'); 28 | 29 | // ======================================== 30 | // Step 1: Initialize Core Components 31 | // ======================================== 32 | console.log('[Step 1] Initializing core components...\n'); 33 | 34 | const messageBus = new MessageBus(); 35 | const toolManager = new ToolManager(); 36 | 37 | // Register tools 38 | toolManager.registerTools([ 39 | new CalculatorTool(), 40 | new WebSearchTool() 41 | ]); 42 | 43 | const orchestrator = new OrchestratorAgent( 44 | messageBus, 45 | toolManager 46 | ); 47 | 48 | // ======================================== 49 | // Step 2: Initialize RAG Engines 50 | // ======================================== 51 | console.log('[Step 2] Initializing RAG engines...\n'); 52 | 53 | const docEngine = new DocumentRAGEngine(); 54 | const keywordEngine = new KeywordRAGEngine(); 55 | const graphEngine = new GraphRAGEngineClient(); 56 | 57 | // Sample knowledge base 58 | const knowledgeBase = [ 59 | new Document({ 60 | pageContent: ` 61 | The Sentient Framework is an advanced system for context engineering. 62 | It uses three types of RAG: Document RAG for semantic search, 63 | Graph RAG for code analysis, and Keyword RAG for lexical matching. 64 | These are combined using Reciprocal Rank Fusion (RRF). 65 | `, 66 | metadata: { source: 'sentient_overview.txt', type: 'documentation' } 67 | }), 68 | new Document({ 69 | pageContent: ` 70 | Document RAG uses OpenAI embeddings with vector stores like MemoryVectorStore. 71 | It supports multiple formats: PDF, web pages, and text files. 72 | The RecursiveCharacterTextSplitter creates optimal chunks with overlap. 73 | Retrieval can use standard similarity search or MMR for diversity. 74 | `, 75 | metadata: { source: 'document_rag_guide.txt', type: 'documentation' } 76 | }), 77 | new Document({ 78 | pageContent: ` 79 | The OrchestratorAgent coordinates the entire workflow. 80 | It decomposes queries into tasks, delegates to specialized agents, 81 | and synthesizes results. It uses the A2A protocol for communication 82 | and the MCP protocol for tool integration. 83 | `, 84 | metadata: { source: 'orchestrator_guide.txt', type: 'documentation' } 85 | }), 86 | new Document({ 87 | pageContent: ` 88 | Context compression uses EXIT-inspired techniques. 89 | Each sentence is evaluated for relevance within its document context. 90 | This achieves 70-90% compression while retaining essential information. 91 | The compressor uses Gemini Flash for fast classification. 92 | `, 93 | metadata: { source: 'compression_guide.txt', type: 'documentation' } 94 | }) 95 | ]; 96 | 97 | // Ingest into engines 98 | await docEngine.ingestDocuments(knowledgeBase); 99 | await keywordEngine.indexDocuments(knowledgeBase); 100 | 101 | const hybridRetriever = new HybridRetriever([ 102 | docEngine, 103 | keywordEngine 104 | ]); 105 | 106 | // ======================================== 107 | // Step 3: Initialize Context Compressor 108 | // ======================================== 109 | console.log('[Step 3] Initializing context compressor...\n'); 110 | 111 | const compressor = new ContextCompressor(); 112 | 113 | // ======================================== 114 | // Step 4: Create Sentient Payload 115 | // ======================================== 116 | console.log('[Step 4] Creating request payload...\n'); 117 | 118 | const userQuery = "How does the Sentient Framework combine different RAG techniques, and what role does context compression play?"; 119 | 120 | console.log(`User Query: "${userQuery}"\n`); 121 | 122 | const payload: SentientPayload = { 123 | query: userQuery, 124 | sessionId: uuidv4(), 125 | history: [], 126 | state: { 127 | tasks: [], 128 | dependencies: new Map(), 129 | intermediateResults: new Map() 130 | }, 131 | contextFragments: [], 132 | metadata: { 133 | startTime: new Date(), 134 | traceId: uuidv4() 135 | } 136 | }; 137 | 138 | // ======================================== 139 | // Step 5: Retrieve Context 140 | // ======================================== 141 | console.log('[Step 5] Retrieving relevant context...\n'); 142 | 143 | const retrievedDocs = await hybridRetriever.retrieve(userQuery, 4); 144 | 145 | console.log(`Retrieved ${retrievedDocs.length} documents from hybrid retriever:`); 146 | retrievedDocs.forEach((doc, i) => { 147 | console.log(` [${i + 1}] ${doc.metadata.source} (RRF: ${doc.metadata.rrfScore?.toFixed(4)})`); 148 | }); 149 | console.log(); 150 | 151 | // Add to payload 152 | payload.contextFragments = retrievedDocs.map(doc => ({ 153 | content: doc.pageContent, 154 | source: doc.metadata.source as string, 155 | score: doc.metadata.rrfScore || 0, 156 | engine: doc.metadata.engine as any, 157 | metadata: doc.metadata 158 | })); 159 | 160 | // ======================================== 161 | // Step 6: Compress Context 162 | // ======================================== 163 | console.log('[Step 6] Compressing context...\n'); 164 | 165 | const compressionResult = await compressor.compressWithMetrics( 166 | userQuery, 167 | retrievedDocs 168 | ); 169 | 170 | console.log('Compression metrics:'); 171 | console.log(` Original: ${compressionResult.metrics.originalLength} chars`); 172 | console.log(` Compressed: ${compressionResult.metrics.compressedLength} chars`); 173 | console.log(` Ratio: ${(compressionResult.metrics.compressionRatio * 100).toFixed(1)}%`); 174 | console.log(` Sentences: ${compressionResult.metrics.sentencesRetained}/${compressionResult.metrics.sentencesTotal}`); 175 | console.log(); 176 | 177 | payload.compressedContext = compressionResult.compressedContext; 178 | 179 | // ======================================== 180 | // Step 7: Orchestrator Processing 181 | // ======================================== 182 | console.log('[Step 7] Orchestrator processing...\n'); 183 | 184 | const finalPayload = await orchestrator.process(payload); 185 | 186 | // ======================================== 187 | // Step 8: Display Results 188 | // ======================================== 189 | console.log('\n╔══════════════════════════════════════════════════════════════╗'); 190 | console.log('║ Final Response ║'); 191 | console.log('╚══════════════════════════════════════════════════════════════╝\n'); 192 | 193 | console.log(finalPayload.finalOutput); 194 | 195 | // ======================================== 196 | // Step 9: Performance Metrics 197 | // ======================================== 198 | console.log('\n╔══════════════════════════════════════════════════════════════╗'); 199 | console.log('║ Performance Metrics ║'); 200 | console.log('╚══════════════════════════════════════════════════════════════╝\n'); 201 | 202 | const totalTime = finalPayload.metadata.endTime 203 | ? finalPayload.metadata.endTime.getTime() - finalPayload.metadata.startTime.getTime() 204 | : 0; 205 | 206 | console.log(`Total execution time: ${totalTime}ms`); 207 | console.log(`Tasks executed: ${finalPayload.state.tasks.length}`); 208 | console.log(`Context fragments: ${finalPayload.contextFragments.length}`); 209 | console.log(`Compression ratio: ${(compressionResult.metrics.compressionRatio * 100).toFixed(1)}%`); 210 | console.log(`Trace ID: ${finalPayload.metadata.traceId}`); 211 | 212 | console.log('\n═══════════════════════════════════════════════════════════════'); 213 | console.log(' ✨ Workflow Complete ✨ '); 214 | console.log('═══════════════════════════════════════════════════════════════\n'); 215 | } 216 | 217 | main().catch(error => { 218 | console.error('Error running workflow:', error); 219 | process.exit(1); 220 | }); 221 | 222 | -------------------------------------------------------------------------------- /diagrams/INDEX.md: -------------------------------------------------------------------------------- 1 | # 📐 Sentient Framework - Architecture Diagrams Index 2 | 3 | > 基于 README.md 的完整 C4 + PlantUML 架构图体系 4 | 5 | ## 📦 已创建文件清单 6 | 7 | ### 🎨 核心架构图（8个） 8 | 9 | | 文件 | 类型 | 层级 | 描述 | 10 | |------|------|------|------| 11 | | `01-context-diagram.puml` | C4 Context | L1 | 系统上下文图：展示系统与外部实体的关系 | 12 | | `02-container-diagram.puml` | C4 Container | L2 | 容器图：展示三层架构的主要服务 | 13 | | `03-component-diagram-service-layer.puml` | C4 Component | L3 | 服务层组件：MCP & A2A 服务实现 | 14 | | `04-component-diagram-agentic-core.puml` | C4 Component | L3 | 代理核心组件：Orchestrator & 专门 Agent | 15 | | `05-component-diagram-context-engine.puml` | C4 Component | L3 | 上下文引擎组件：多模态 RAG 引擎 | 16 | | `06-sequence-diagram-query-flow.puml` | Sequence | Dynamic | 查询流程时序图：完整的请求生命周期 | 17 | | `07-deployment-diagram.puml` | Deployment | Physical | 部署架构图：Kubernetes 生产环境 | 18 | | `08-data-flow-diagram.puml` | Sequence | Dynamic | 数据流图：SentientPayload 状态演化 | 19 | 20 | ### 📚 文档文件 21 | 22 | | 文件 | 用途 | 23 | |------|------| 24 | | `README.md` | 详细的架构图说明文档（3000+ 字） | 25 | | `QUICKSTART.md` | 快速开始指南（中文） | 26 | | `INDEX.md` | 本文件：项目总览和使用指引 | 27 | 28 | ### 🛠️ 工具文件 29 | 30 | | 文件 | 用途 | 31 | |------|------| 32 | | `generate-images.sh` | 一键生成所有图表的 PNG/SVG 脚本 | 33 | | `viewer.html` | 交互式 HTML 查看器（无需安装工具） | 34 | 35 | --- 36 | 37 | ## 🚀 快速开始 38 | 39 | ### 方式 1：浏览器查看（最简单） 40 | 41 | ```bash 42 | # 直接打开 HTML 查看器 43 | open viewer.html 44 | ``` 45 | 46 | ### 方式 2：生成图片 47 | 48 | ```bash 49 | # 一键生成所有图表 50 | chmod +x generate-images.sh 51 | ./generate-images.sh 52 | 53 | # 查看生成的图片 54 | ls output/png/ 55 | ls output/svg/ 56 | ``` 57 | 58 | ### 方式 3：在线编辑 59 | 60 | 1. 打开 https://www.plantuml.com/plantuml/uml/ 61 | 2. 复制任意 `.puml` 文件内容 62 | 3. 粘贴到在线编辑器 63 | 4. 实时预览和编辑 64 | 65 | --- 66 | 67 | ## 📊 架构图层次结构 68 | 69 | ``` 70 | Sentient Framework Architecture 71 | │ 72 | ├─ Level 1: System Context 73 | │ └─ 01-context-diagram.puml 74 | │ └─ 展示系统边界、外部用户和外部系统 75 | │ 76 | ├─ Level 2: Container 77 | │ └─ 02-container-diagram.puml 78 | │ ├─ Service Layer (MCP, A2A) 79 | │ ├─ Agentic Core (Orchestrator, Agents, Message Bus) 80 | │ └─ Context Engine Layer (RAG Engines, Compressor) 81 | │ 82 | ├─ Level 3: Component 83 | │ ├─ 03-component-diagram-service-layer.puml 84 | │ │ ├─ MCP Endpoint & Tool Registry 85 | │ │ └─ A2A Endpoint & Agent Card Manager 86 | │ │ 87 | │ ├─ 04-component-diagram-agentic-core.puml 88 | │ │ ├─ Task Decomposer & Scheduler 89 | │ │ ├─ Message Bus & Queue 90 | │ │ ├─ Tool Manager & Executor 91 | │ │ └─ Specialized Agents (Coding, Retrieval, Doc) 92 | │ │ 93 | │ └─ 05-component-diagram-context-engine.puml 94 | │ ├─ Document RAG (LangChain, Vector Store) 95 | │ ├─ Graph RAG (RepoMapper, tree-sitter) 96 | │ ├─ Keyword RAG (BM25) 97 | │ ├─ Hybrid Retriever (RRF) 98 | │ └─ Context Compressor (EXIT) 99 | │ 100 | ├─ Dynamic Diagrams 101 | │ ├─ 06-sequence-diagram-query-flow.puml 102 | │ │ └─ 展示复杂查询的完整生命周期 103 | │ │ 104 | │ └─ 08-data-flow-diagram.puml 105 | │ └─ 展示 SentientPayload 在各层的状态变化 106 | │ 107 | └─ Physical Diagrams 108 | └─ 07-deployment-diagram.puml 109 | └─ 展示 Kubernetes 生产部署架构 110 | ``` 111 | 112 | --- 113 | 114 | ## 🎯 每个图表的核心价值 115 | 116 | ### 1️⃣ 系统上下文图 117 | **适用场景：** 118 | - 向高层管理人员介绍系统定位 119 | - 新员工入职培训 120 | - 系统集成规划 121 | 122 | **关键信息：** 123 | - 系统边界清晰可见 124 | - 外部依赖一目了然 125 | - 交互协议明确标注 126 | 127 | --- 128 | 129 | ### 2️⃣ 容器图 130 | **适用场景：** 131 | - 技术架构评审 132 | - 技术选型讨论 133 | - 团队分工规划 134 | 135 | **关键信息：** 136 | - 三层架构清晰可见 137 | - 每个容器的技术栈明确 138 | - 容器间通信方式清楚 139 | 140 | --- 141 | 142 | ### 3️⃣ 服务层组件图 143 | **适用场景：** 144 | - API 设计评审 145 | - 协议实现讨论 146 | - 接口文档编写 147 | 148 | **关键信息：** 149 | - MCP 协议实现细节 150 | - A2A 协议路由逻辑 151 | - 工具注册机制 152 | 153 | --- 154 | 155 | ### 4️⃣ 代理核心组件图 156 | **适用场景：** 157 | - Agent 系统设计 158 | - 任务编排逻辑讨论 159 | - 工具管理机制设计 160 | 161 | **关键信息：** 162 | - Orchestrator 内部机制 163 | - Agent 间通信方式 164 | - 工具执行流程 165 | 166 | --- 167 | 168 | ### 5️⃣ 上下文引擎组件图 169 | **适用场景：** 170 | - RAG 系统优化 171 | - 检索策略选择 172 | - 上下文压缩调优 173 | 174 | **关键信息：** 175 | - 三种 RAG 引擎的实现细节 176 | - 混合检索和重排序机制 177 | - 上下文压缩管道 178 | 179 | --- 180 | 181 | ### 6️⃣ 查询流程时序图 182 | **适用场景：** 183 | - 性能瓶颈分析 184 | - 流程优化讨论 185 | - Bug 调试和追踪 186 | 187 | **关键信息：** 188 | - 完整的请求生命周期 189 | - 并行执行的部分 190 | - 各组件交互时序 191 | 192 | --- 193 | 194 | ### 7️⃣ 部署架构图 195 | **适用场景：** 196 | - 生产环境规划 197 | - DevOps 配置 198 | - 扩容和容灾设计 199 | 200 | **关键信息：** 201 | - Kubernetes 集群组织 202 | - 数据存储方案 203 | - 负载均衡策略 204 | 205 | --- 206 | 207 | ### 8️⃣ 数据流图 208 | **适用场景：** 209 | - 数据模型设计 210 | - 状态管理讨论 211 | - API 响应格式设计 212 | 213 | **关键信息：** 214 | - SentientPayload 结构演化 215 | - 各阶段数据增量 216 | - 元数据收集点 217 | 218 | --- 219 | 220 | ## 📖 文档映射 221 | 222 | | README.md 章节 | 对应图表 | 223 | |----------------|----------| 224 | | Chapter 1: Core Pillars | 图 1, 2 | 225 | | Chapter 2: Data Flow | 图 8 | 226 | | Chapter 3: Document RAG | 图 5 | 227 | | Chapter 4: Graph RAG | 图 5 | 228 | | Chapter 5: Hybrid Retrieval | 图 5 | 229 | | Chapter 6: MCP & Tools | 图 3, 4 | 230 | | Chapter 7: Multi-Agent | 图 2, 4, 6 | 231 | | Chapter 8: Context Compression | 图 5 | 232 | | Chapter 9-12: Implementation | 图 4, 5 | 233 | | Chapter 13-14: Deployment | 图 7 | 234 | 235 | --- 236 | 237 | ## 🛠️ 使用工具推荐 238 | 239 | ### 本地编辑 240 | 241 | | 工具 | 优点 | 安装 | 242 | |------|------|------| 243 | | **VS Code + PlantUML 插件** | 实时预览、语法高亮 | `ext install plantuml` | 244 | | **IntelliJ IDEA + PlantUML** | 智能补全、重构支持 | 插件市场搜索 "PlantUML" | 245 | | **Sublime Text + PlantUML** | 轻量级、快速启动 | Package Control 安装 | 246 | 247 | ### 在线工具 248 | 249 | | 工具 | 特点 | 链接 | 250 | |------|------|------| 251 | | **PlantUML Online** | 官方编辑器、功能完整 | [plantuml.com](https://plantuml.com/plantuml) | 252 | | **PlantText** | 简洁界面、快速预览 | [planttext.com](https://www.planttext.com/) | 253 | | **Gravizo** | 支持 Markdown 嵌入 | [gravizo.com](http://www.gravizo.com/) | 254 | 255 | ### CI/CD 集成 256 | 257 | ```yaml 258 | # GitHub Actions 示例 259 | name: Generate Diagrams 260 | 261 | on: [push] 262 | 263 | jobs: 264 | generate: 265 | runs-on: ubuntu-latest 266 | steps: 267 | - uses: actions/checkout@v2 268 | - name: Generate PlantUML 269 | uses: cloudbees/plantuml-github-action@master 270 | with: 271 | args: -v -tsvg diagrams/*.puml 272 | ``` 273 | 274 | --- 275 | 276 | ## 🎨 定制和扩展 277 | 278 | ### 添加新的架构视图 279 | 280 | 根据 C4 模型，你可以继续创建： 281 | 282 | 1. **代码级视图（Level 4）** 283 | - 类图、接口图 284 | - 文件名：`09-class-diagram-*.puml` 285 | 286 | 2. **更多动态视图** 287 | - 状态机图 288 | - 活动图 289 | 290 | 3. **领域视图** 291 | - 特定业务流程 292 | - 错误处理流程 293 | 294 | ### 自定义样式主题 295 | 296 | 创建 `theme.puml`： 297 | 298 | ```plantuml 299 | ' 自定义颜色 300 | !define PRIMARY_COLOR #667eea 301 | !define SECONDARY_COLOR #764ba2 302 | !define SUCCESS_COLOR #48bb78 303 | 304 | ' 自定义字体 305 | skinparam defaultFontName "SF Pro Display" 306 | skinparam defaultFontSize 12 307 | 308 | ' 组件样式 309 | skinparam component { 310 | BackgroundColor PRIMARY_COLOR 311 | BorderColor SECONDARY_COLOR 312 | } 313 | ``` 314 | 315 | 在其他图表中引用： 316 | ```plantuml 317 | !include theme.puml 318 | ``` 319 | 320 | --- 321 | 322 | ## 📈 最佳实践 323 | 324 | ### ✅ DO 325 | 326 | - **保持图表简洁**：每个图表关注一个层面 327 | - **使用一致的命名**：与代码库保持同步 328 | - **添加详细注释**：解释设计决策 329 | - **定期更新**：代码变更后同步更新 330 | - **版本控制**：将 `.puml` 文件纳入 Git 331 | 332 | ### ❌ DON'T 333 | 334 | - **避免过度细节**：不要在高层图中显示实现细节 335 | - **避免重复信息**：不同层次图表应互补而非重复 336 | - **避免手动绘图**：使用代码生成更易维护 337 | - **避免孤立图表**：确保与实际代码保持一致 338 | 339 | --- 340 | 341 | ## 🤔 常见问题 342 | 343 | ### Q1: 如何选择查看哪个图表？ 344 | 345 | **A:** 根据你的角色和目标： 346 | 347 | - **产品经理/业务人员** → 图 1（系统上下文） 348 | - **架构师/技术 Leader** → 图 2（容器图） 349 | - **后端开发工程师** → 图 3-5（组件图） 350 | - **DevOps/运维** → 图 7（部署图） 351 | - **全栈开发/调试** → 图 6, 8（时序图、数据流图） 352 | 353 | ### Q2: 图表太复杂看不懂怎么办？ 354 | 355 | **A:** 按顺序阅读： 356 | 1. 先看图 1 了解整体 357 | 2. 再看图 2 了解内部结构 358 | 3. 选择你关心的子系统深入到图 3-5 359 | 4. 需要了解运行时行为时看图 6, 8 360 | 361 | ### Q3: 如何在我的项目中复用这些图表？ 362 | 363 | **A:** 364 | 1. 复制对应的 `.puml` 文件 365 | 2. 修改组件名称和描述 366 | 3. 调整关系和交互 367 | 4. 生成适合你项目的图表 368 | 369 | ### Q4: 图表更新后如何通知团队？ 370 | 371 | **A:** 372 | 1. 在 Pull Request 中包含图表变更 373 | 2. 在团队会议中演示新架构 374 | 3. 更新 Wiki 中的嵌入图片 375 | 4. 在 Slack/钉钉发布更新通知 376 | 377 | --- 378 | 379 | ## 🔗 相关资源 380 | 381 | ### 官方文档 382 | - [PlantUML 官方网站](https://plantuml.com/) 383 | - [C4 模型官网](https://c4model.com/) 384 | - [C4-PlantUML GitHub](https://github.com/plantuml-stdlib/C4-PlantUML) 385 | 386 | ### 社区资源 387 | - [Awesome PlantUML](https://github.com/qjebbs/awesome-plantuml) 388 | - [Real World PlantUML](https://real-world-plantuml.com/) 389 | - [PlantUML 中文教程](https://plantuml.com/zh/) 390 | 391 | ### 相关工具 392 | - [Structurizr](https://structurizr.com/) - C4 模型的另一种实现 393 | - [Mermaid.js](https://mermaid-js.github.io/) - 另一种图表即代码工具 394 | - [draw.io](https://draw.io/) - 传统图表工具 395 | 396 | --- 397 | 398 | ## 📊 统计信息 399 | 400 | ``` 401 | 总文件数： 12 个 402 | 核心图表： 8 个 403 | 文档页面： 3 个 404 | 工具脚本： 2 个 405 | 406 | 覆盖层次： 407 | - C4 Level 1: 1 个（系统上下文） 408 | - C4 Level 2: 1 个（容器） 409 | - C4 Level 3: 3 个（组件） 410 | - 动态视图： 2 个（时序、数据流） 411 | - 物理视图： 1 个（部署） 412 | 413 | 技术栈标注： 414 | - TypeScript/Node.js 415 | - LangChain.js 416 | - Google Gemini 417 | - Kubernetes 418 | - Vector Databases 419 | - MCP & A2A Protocols 420 | ``` 421 | 422 | --- 423 | 424 | ## 🎯 下一步行动 425 | 426 | ### 立即开始 427 | 1. 📖 阅读 `QUICKSTART.md` 428 | 2. 🌐 打开 `viewer.html` 浏览所有图表 429 | 3. 🎨 使用 `generate-images.sh` 生成图片 430 | 431 | ### 深入学习 432 | 1. 📚 对照 `README.md` 理解每个架构决策 433 | 2. 🔍 从图 1 开始，逐层深入到图 5 434 | 3. 🚀 查看图 6、8 理解运行时行为 435 | 436 | ### 实践应用 437 | 1. ✏️ 根据你的需求修改图表 438 | 2. 📤 集成到你的文档系统 439 | 3. 🤝 与团队分享和讨论 440 | 441 | --- 442 | 443 | ## 📝 反馈与改进 444 | 445 | 发现问题或有改进建议？ 446 | 447 | 1. **报告问题**：在 Issue 中描述问题 448 | 2. **提出改进**：提交 Pull Request 449 | 3. **分享经验**：在讨论区交流使用心得 450 | 451 | --- 452 | 453 | **祝你使用愉快！如有任何问题，请参考 QUICKSTART.md 或联系团队。** 🚀 454 | 455 | -------------------------------------------------------------------------------- /IMPLEMENTATION.md: -------------------------------------------------------------------------------- 1 | # Sentient Framework - TypeScript Implementation 2 | 3 | > **An Architectural Blueprint for Advanced Context Engineering** 4 | 5 | This is the official TypeScript implementation of the Sentient Framework, based on the comprehensive architectural documentation in `README.md`. 6 | 7 | ## 🎯 Overview 8 | 9 | Sentient is a production-ready framework that combines: 10 | - **Multi-Modal Retrieval**: Document RAG, Graph RAG, and Keyword RAG 11 | - **Agentic Orchestration**: Task decomposition, delegation, and synthesis 12 | - **Context Optimization**: EXIT-inspired intelligent compression 13 | - **Standardized Protocols**: MCP for tools, A2A for agents 14 | 15 | ## 🏗️ Architecture 16 | 17 | ``` 18 | src/ 19 | ├── core/ # Core interfaces and types 20 | │ ├── interfaces/ # IContextEngine, ITool, IAgent, IMessageBus 21 | │ └── types.ts # SentientPayload, A2AMessage, etc. 22 | │ 23 | ├── engines/ # RAG Engine implementations 24 | │ ├── DocumentRAGEngine.ts # Semantic search with LangChain 25 | │ ├── GraphRAGEngineClient.ts # Code structure analysis 26 | │ ├── KeywordRAGEngine.ts # BM25 lexical search 27 | │ └── HybridRetriever.ts # RRF fusion 28 | │ 29 | ├── compression/ # Context compression 30 | │ └── ContextCompressor.ts # EXIT-inspired sentence classification 31 | │ 32 | ├── agents/ # Agentic Core 33 | │ ├── OrchestratorAgent.ts # Main coordinator 34 | │ ├── ToolManager.ts # MCP tool management 35 | │ └── MessageBus.ts # A2A message routing 36 | │ 37 | ├── tools/ # Example tools 38 | │ └── ExampleTools.ts # Calculator, WebSearch, CodeAnalyzer, etc. 39 | │ 40 | ├── examples/ # Usage examples 41 | │ ├── basic_rag.ts 42 | │ ├── hybrid_retrieval.ts 43 | │ ├── context_compression.ts 44 | │ ├── tool_usage.ts 45 | │ └── full_workflow.ts 46 | │ 47 | └── server.ts # MCP & A2A service endpoints 48 | ``` 49 | 50 | ## 🚀 Quick Start 51 | 52 | ### Installation 53 | 54 | ```bash 55 | # Install dependencies 56 | npm install 57 | 58 | # Copy environment template 59 | cp .env.example .env 60 | 61 | # Edit .env with your API keys 62 | # Required: OPENAI_API_KEY, GOOGLE_API_KEY 63 | ``` 64 | 65 | ### Running Tests 66 | 67 | ```bash 68 | # Run all tests 69 | npm test 70 | 71 | # Run tests in watch mode 72 | npm run test:watch 73 | 74 | # Run tests with coverage 75 | npm run test:coverage 76 | 77 | # Run demo 78 | npm run demo 79 | ``` 80 | 81 | ### Start the Server 82 | 83 | ```bash 84 | # Start MCP & A2A services 85 | npm run server 86 | 87 | # Server will be available at http://localhost:3000 88 | ``` 89 | 90 | ## 📚 Usage Examples 91 | 92 | ### Basic Document RAG 93 | 94 | ```typescript 95 | import { DocumentRAGEngine } from 'sentient-framework'; 96 | import { Document } from '@langchain/core/documents'; 97 | 98 | const engine = new DocumentRAGEngine(); 99 | 100 | // Ingest documents 101 | await engine.ingestDocuments([ 102 | new Document({ 103 | pageContent: "Your content here...", 104 | metadata: { source: "doc1.txt" } 105 | }) 106 | ]); 107 | 108 | // Retrieve 109 | const results = await engine.retrieve("your query", 4); 110 | ``` 111 | 112 | ### Hybrid Retrieval with RRF 113 | 114 | ```typescript 115 | import { HybridRetriever, DocumentRAGEngine, KeywordRAGEngine } from 'sentient-framework'; 116 | 117 | const docEngine = new DocumentRAGEngine(); 118 | const keywordEngine = new KeywordRAGEngine(); 119 | 120 | // ... ingest data ... 121 | 122 | const hybrid = new HybridRetriever([docEngine, keywordEngine]); 123 | const results = await hybrid.retrieve("query", 5); 124 | ``` 125 | 126 | ### Context Compression 127 | 128 | ```typescript 129 | import { ContextCompressor } from 'sentient-framework'; 130 | 131 | const compressor = new ContextCompressor(); 132 | 133 | const compressed = await compressor.compress( 134 | "user query", 135 | retrievedDocuments 136 | ); 137 | ``` 138 | 139 | ### Agent Orchestration 140 | 141 | ```typescript 142 | import { OrchestratorAgent, MessageBus, ToolManager } from 'sentient-framework'; 143 | 144 | const messageBus = new MessageBus(); 145 | const toolManager = new ToolManager(); 146 | const orchestrator = new OrchestratorAgent(messageBus, toolManager); 147 | 148 | const payload: SentientPayload = { /* ... */ }; 149 | const result = await orchestrator.process(payload); 150 | ``` 151 | 152 | ## 🔧 API Endpoints 153 | 154 | ### MCP Service (Model Context Protocol) 155 | 156 | ```bash 157 | # List all tools 158 | GET /mcp/tools 159 | 160 | # Execute a tool 161 | POST /mcp/tools/:toolName/execute 162 | Content-Type: application/json 163 | { 164 | "arg1": "value1", 165 | "arg2": "value2" 166 | } 167 | 168 | # Get tool statistics 169 | GET /mcp/tools/:toolName/stats 170 | ``` 171 | 172 | ### A2A Service (Agent-to-Agent Protocol) 173 | 174 | ```bash 175 | # Register an agent 176 | POST /a2a/agents/register 177 | Content-Type: application/json 178 | { 179 | "id": "agent_id", 180 | "name": "Agent Name", 181 | "capabilities": ["capability1", "capability2"], 182 | "endpointUrl": "http://..." 183 | } 184 | 185 | # List agents 186 | GET /a2a/agents 187 | 188 | # Submit a task 189 | POST /a2a/tasks 190 | Content-Type: application/json 191 | { 192 | "targetAgentId": "agent_id", 193 | "description": "Task description", 194 | "payload": { /* ... */ } 195 | } 196 | 197 | # Send a message 198 | POST /a2a/messages 199 | ``` 200 | 201 | ## 🧪 Testing 202 | 203 | ```bash 204 | # Run tests 205 | npm test 206 | 207 | # Run with coverage 208 | npm test -- --coverage 209 | ``` 210 | 211 | ## 📖 Documentation 212 | 213 | - **Architecture Overview**: See [README.md](./README.md) for comprehensive architectural documentation 214 | - **Architecture Diagrams**: See [diagrams/](./diagrams/) for interactive C4 diagrams 215 | - **Quick Start Guide**: See [diagrams/QUICKSTART.md](./diagrams/QUICKSTART.md) 216 | 217 | ## 🔑 Environment Variables 218 | 219 | Required environment variables in `.env`: 220 | 221 | ```bash 222 | # OpenAI (for embeddings and optional LLM) 223 | OPENAI_API_KEY=sk-... 224 | 225 | # Google Gemini (for function calling and compression) 226 | GOOGLE_API_KEY=... 227 | 228 | # Server Configuration 229 | PORT=3000 230 | 231 | # Optional: Graph RAG Service URL 232 | GRAPH_RAG_SERVICE_URL=http://localhost:3001 233 | ``` 234 | 235 | ## 🏆 Key Features 236 | 237 | ### Multi-Modal RAG 238 | - **Document RAG**: Vector embeddings with LangChain.js 239 | - **Graph RAG**: Code structure analysis with tree-sitter 240 | - **Keyword RAG**: BM25 lexical search 241 | - **Hybrid Fusion**: Reciprocal Rank Fusion (RRF) 242 | 243 | ### Intelligent Compression 244 | - EXIT-inspired context-aware sentence classification 245 | - 70-90% compression while retaining relevance 246 | - Parallelized for performance 247 | 248 | ### Agentic Orchestration 249 | - Task decomposition and delegation 250 | - Tool management with MCP protocol 251 | - Inter-agent communication with A2A protocol 252 | 253 | ### Production Ready 254 | - Type-safe with TypeScript 255 | - Standardized interfaces 256 | - Comprehensive error handling 257 | - Extensible architecture 258 | 259 | ## 🔌 Extending the Framework 260 | 261 | ### Adding a Custom Tool 262 | 263 | ```typescript 264 | import { ITool } from 'sentient-framework'; 265 | import { z } from 'zod'; 266 | 267 | export class MyCustomTool implements ITool<{ 268 | param1: z.ZodString; 269 | }> { 270 | readonly name = "my_custom_tool"; 271 | readonly description = "Does something useful"; 272 | readonly schema = z.object({ 273 | param1: z.string().describe("Parameter description") 274 | }); 275 | 276 | async execute(args: z.infer) { 277 | // Your implementation 278 | return { result: "success" }; 279 | } 280 | } 281 | 282 | // Register with ToolManager 283 | toolManager.registerTool(new MyCustomTool()); 284 | ``` 285 | 286 | ### Creating a Custom RAG Engine 287 | 288 | ```typescript 289 | import { IContextEngine } from 'sentient-framework'; 290 | import { Document } from '@langchain/core/documents'; 291 | 292 | export class MyCustomRAGEngine implements IContextEngine { 293 | async retrieve(query: string): Promise { 294 | // Your retrieval logic 295 | return []; 296 | } 297 | } 298 | ``` 299 | 300 | ### Adding a Specialized Agent 301 | 302 | ```typescript 303 | import { IAgent, SentientPayload } from 'sentient-framework'; 304 | 305 | export class MySpecializedAgent implements IAgent { 306 | readonly id = 'my_agent'; 307 | readonly name = 'My Specialized Agent'; 308 | 309 | async process(payload: SentientPayload): Promise { 310 | // Your processing logic 311 | return payload; 312 | } 313 | } 314 | 315 | // Register with orchestrator 316 | orchestrator.registerAgent('my_agent', 'Handles specific tasks'); 317 | ``` 318 | 319 | ## 📊 Performance Considerations 320 | 321 | - **Caching**: Graph RAG results are cached by the RepoMapper service 322 | - **Parallelization**: Compression uses parallel sentence classification 323 | - **Streaming**: Consider streaming for long-running operations 324 | - **Batching**: Hybrid retrieval runs engines in parallel 325 | 326 | ## 🤝 Contributing 327 | 328 | Contributions are welcome! Please: 329 | 1. Read the architecture documentation in `README.md` 330 | 2. Follow the existing code structure 331 | 3. Add tests for new features 332 | 4. Update documentation 333 | 334 | ## 📄 License 335 | 336 | MIT 337 | 338 | ## 🙏 Acknowledgments 339 | 340 | This implementation is based on: 341 | - **LangChain.js** for RAG infrastructure 342 | - **Google Gemini** for function calling and compression 343 | - **Aider's RepoMapper** for Graph RAG inspiration 344 | - **EXIT paper** for context compression methodology 345 | - **MCP & A2A protocols** for standardized agent communication 346 | 347 | ## 🔗 Related Projects 348 | 349 | - [RepoMapper](https://github.com/pdavis68/RepoMapper) - Repository structure analysis 350 | - [LangChain.js](https://github.com/langchain-ai/langchainjs) - RAG framework 351 | - [Model Context Protocol](https://modelcontextprotocol.io/) - Tool standardization 352 | - [Agent-to-Agent Protocol](https://www.ibm.com/think/topics/agent2agent-protocol) - Multi-agent communication 353 | 354 | --- 355 | 356 | **Built with ❤️ for the AI engineering community** 357 | 358 | -------------------------------------------------------------------------------- /src/compression/ContextCompressor.ts: -------------------------------------------------------------------------------- 1 | import { Document } from "@langchain/core/documents"; 2 | import { GoogleGenerativeAI } from "@google/generative-ai"; 3 | 4 | /** 5 | * Sentence with context for relevance classification 6 | */ 7 | interface SentenceWithContext { 8 | sentence: string; 9 | documentContext: string; 10 | sourceDocument: Document; 11 | index: number; 12 | } 13 | 14 | /** 15 | * Context Compressor using EXIT-inspired extractive compression 16 | * 17 | * This module implements a context-aware sentence classification approach 18 | * to intelligently compress retrieved documents while preserving relevance. 19 | */ 20 | export class ContextCompressor { 21 | private geminiClient: GoogleGenerativeAI; 22 | private model: string; 23 | private relevanceThreshold: number; 24 | private maxParallelRequests: number; 25 | 26 | constructor( 27 | apiKey?: string, 28 | model: string = "gemini-2.0-flash-exp", 29 | relevanceThreshold: number = 0.7, 30 | maxParallelRequests: number = 10 31 | ) { 32 | const key = apiKey || process.env.GOOGLE_API_KEY; 33 | if (!key) { 34 | throw new Error("Google API key is required for ContextCompressor"); 35 | } 36 | 37 | this.geminiClient = new GoogleGenerativeAI(key); 38 | this.model = model; 39 | this.relevanceThreshold = relevanceThreshold; 40 | this.maxParallelRequests = maxParallelRequests; 41 | } 42 | 43 | /** 44 | * Compresses a list of documents by extracting only relevant sentences 45 | * 46 | * @param query The original user query 47 | * @param documents Retrieved documents to compress 48 | * @returns Compressed context string 49 | */ 50 | public async compress(query: string, documents: Document[]): Promise { 51 | console.log(`[ContextCompressor] Compressing ${documents.length} documents`); 52 | const startTime = Date.now(); 53 | 54 | // Step 1: Decompose documents into sentences with context 55 | const sentencesWithContext = this.decomposeSentences(documents); 56 | console.log(`[ContextCompressor] Extracted ${sentencesWithContext.length} sentences`); 57 | 58 | // Step 2: Classify each sentence for relevance (parallelized) 59 | const relevantSentences = await this.classifySentences(query, sentencesWithContext); 60 | console.log(`[ContextCompressor] ${relevantSentences.length} relevant sentences found`); 61 | 62 | // Step 3: Reassemble in original order 63 | const compressedContext = this.reassemble(relevantSentences); 64 | 65 | const compressionRatio = compressedContext.length / 66 | documents.reduce((sum, doc) => sum + doc.pageContent.length, 0); 67 | const elapsedTime = Date.now() - startTime; 68 | 69 | console.log(`[ContextCompressor] Compression ratio: ${(compressionRatio * 100).toFixed(1)}% in ${elapsedTime}ms`); 70 | 71 | return compressedContext; 72 | } 73 | 74 | /** 75 | * Step 1: Decompose documents into sentences with their context 76 | */ 77 | private decomposeSentences(documents: Document[]): SentenceWithContext[] { 78 | const sentencesWithContext: SentenceWithContext[] = []; 79 | 80 | documents.forEach(doc => { 81 | const sentences = this.splitIntoSentences(doc.pageContent); 82 | 83 | sentences.forEach((sentence, index) => { 84 | sentencesWithContext.push({ 85 | sentence, 86 | documentContext: doc.pageContent, 87 | sourceDocument: doc, 88 | index 89 | }); 90 | }); 91 | }); 92 | 93 | return sentencesWithContext; 94 | } 95 | 96 | /** 97 | * Step 2: Classify sentences for relevance using LLM 98 | */ 99 | private async classifySentences( 100 | query: string, 101 | sentencesWithContext: SentenceWithContext[] 102 | ): Promise { 103 | const relevantSentences: SentenceWithContext[] = []; 104 | 105 | // Process in batches for parallel execution 106 | const batches = this.createBatches(sentencesWithContext, this.maxParallelRequests); 107 | 108 | for (const batch of batches) { 109 | const batchResults = await Promise.all( 110 | batch.map(sentenceCtx => this.classifySentence(query, sentenceCtx)) 111 | ); 112 | 113 | batchResults.forEach((isRelevant, index) => { 114 | if (isRelevant) { 115 | relevantSentences.push(batch[index]); 116 | } 117 | }); 118 | } 119 | 120 | return relevantSentences; 121 | } 122 | 123 | /** 124 | * Classifies a single sentence for relevance 125 | */ 126 | private async classifySentence( 127 | query: string, 128 | sentenceCtx: SentenceWithContext 129 | ): Promise { 130 | try { 131 | const model = this.geminiClient.getGenerativeModel({ model: this.model }); 132 | 133 | // Create context-aware prompt 134 | const prompt = this.createClassificationPrompt( 135 | query, 136 | sentenceCtx.sentence, 137 | sentenceCtx.documentContext 138 | ); 139 | 140 | const result = await model.generateContent(prompt); 141 | const response = result.response.text().trim().toLowerCase(); 142 | 143 | // Parse response (expecting "yes" or "no" or a score) 144 | if (response.includes('yes') || response.startsWith('1')) { 145 | return true; 146 | } else if (response.includes('no') || response.startsWith('0')) { 147 | return false; 148 | } else { 149 | // Try to extract a numeric score 150 | const scoreMatch = response.match(/(\d+\.?\d*)/); 151 | if (scoreMatch) { 152 | const score = parseFloat(scoreMatch[1]); 153 | return score >= this.relevanceThreshold; 154 | } 155 | } 156 | 157 | return false; 158 | } catch (error) { 159 | console.error('[ContextCompressor] Classification error:', error); 160 | // On error, conservatively include the sentence 161 | return true; 162 | } 163 | } 164 | 165 | /** 166 | * Creates a prompt for sentence classification 167 | */ 168 | private createClassificationPrompt( 169 | query: string, 170 | sentence: string, 171 | documentContext: string 172 | ): string { 173 | // Limit context to reasonable length (first 500 chars) 174 | const contextPreview = documentContext.length > 500 175 | ? documentContext.substring(0, 500) + "..." 176 | : documentContext; 177 | 178 | return `You are a precise information filter. Your task is to determine if a sentence is essential for answering a user's query. 179 | 180 | User Query: "${query}" 181 | 182 | Document Context: 183 | --- 184 | ${contextPreview} 185 | --- 186 | 187 | Sentence to Evaluate: "${sentence}" 188 | 189 | Question: Is this sentence essential for answering the user's query, given the full context of the document it came from? 190 | 191 | Respond with ONLY "yes" or "no". Consider: 192 | 1. Does it directly address the query? 193 | 2. Does it provide crucial supporting information? 194 | 3. Is it necessary for understanding the answer? 195 | 196 | Response:`; 197 | } 198 | 199 | /** 200 | * Step 3: Reassemble relevant sentences in original order 201 | */ 202 | private reassemble(sentencesWithContext: SentenceWithContext[]): string { 203 | // Group by source document to maintain document structure 204 | const byDocument = new Map(); 205 | 206 | sentencesWithContext.forEach(sentenceCtx => { 207 | const docId = sentenceCtx.sourceDocument.metadata?.source || 'unknown'; 208 | if (!byDocument.has(docId)) { 209 | byDocument.set(docId, []); 210 | } 211 | byDocument.get(docId)!.push(sentenceCtx); 212 | }); 213 | 214 | // Reassemble each document's sentences in order 215 | const sections: string[] = []; 216 | 217 | byDocument.forEach((sentences, docId) => { 218 | // Sort by original index 219 | sentences.sort((a, b) => a.index - b.index); 220 | 221 | const reassembled = sentences.map(s => s.sentence).join(' '); 222 | if (reassembled.trim()) { 223 | sections.push(`[Source: ${docId}]\n${reassembled}\n`); 224 | } 225 | }); 226 | 227 | return sections.join('\n---\n\n'); 228 | } 229 | 230 | /** 231 | * Splits text into sentences using simple heuristics 232 | * In production, use a more sophisticated sentence tokenizer 233 | */ 234 | private splitIntoSentences(text: string): string[] { 235 | // Simple sentence splitting on common terminators 236 | const sentences = text 237 | .split(/([.!?]+\s+)/) 238 | .reduce((acc, part, index, array) => { 239 | if (index % 2 === 0) { 240 | const sentence = part + (array[index + 1] || ''); 241 | if (sentence.trim().length > 0) { 242 | acc.push(sentence.trim()); 243 | } 244 | } 245 | return acc; 246 | }, [] as string[]); 247 | 248 | return sentences.filter(s => s.length > 10); // Filter out very short fragments 249 | } 250 | 251 | /** 252 | * Creates batches for parallel processing 253 | */ 254 | private createBatches(items: T[], batchSize: number): T[][] { 255 | const batches: T[][] = []; 256 | for (let i = 0; i < items.length; i += batchSize) { 257 | batches.push(items.slice(i, i + batchSize)); 258 | } 259 | return batches; 260 | } 261 | 262 | /** 263 | * Compresses with detailed metrics 264 | */ 265 | public async compressWithMetrics( 266 | query: string, 267 | documents: Document[] 268 | ): Promise<{ 269 | compressedContext: string; 270 | metrics: { 271 | originalLength: number; 272 | compressedLength: number; 273 | compressionRatio: number; 274 | sentencesTotal: number; 275 | sentencesRetained: number; 276 | processingTime: number; 277 | }; 278 | }> { 279 | const startTime = Date.now(); 280 | const originalLength = documents.reduce((sum, doc) => sum + doc.pageContent.length, 0); 281 | 282 | const sentencesWithContext = this.decomposeSentences(documents); 283 | const relevantSentences = await this.classifySentences(query, sentencesWithContext); 284 | const compressedContext = this.reassemble(relevantSentences); 285 | 286 | return { 287 | compressedContext, 288 | metrics: { 289 | originalLength, 290 | compressedLength: compressedContext.length, 291 | compressionRatio: compressedContext.length / originalLength, 292 | sentencesTotal: sentencesWithContext.length, 293 | sentencesRetained: relevantSentences.length, 294 | processingTime: Date.now() - startTime 295 | } 296 | }; 297 | } 298 | } 299 | 300 | -------------------------------------------------------------------------------- /src/agents/OrchestratorAgent.ts: -------------------------------------------------------------------------------- 1 | import { IAgent } from "../core/interfaces/IAgent.js"; 2 | import { SentientPayload, Task, A2AMessage } from "../core/types.js"; 3 | import { MessageBus } from "./MessageBus.js"; 4 | import { ToolManager } from "./ToolManager.js"; 5 | import { GoogleGenerativeAI } from "@google/generative-ai"; 6 | import { v4 as uuidv4 } from 'uuid'; 7 | 8 | /** 9 | * Orchestrator Agent - The central coordinator of the Sentient Framework 10 | * 11 | * Responsibilities: 12 | * - Receives user requests 13 | * - Decomposes complex queries into subtasks 14 | * - Delegates tasks to specialized agents 15 | * - Synthesizes results into final response 16 | */ 17 | export class OrchestratorAgent implements IAgent { 18 | readonly id: string = 'orchestrator'; 19 | readonly name: string = 'Orchestrator Agent'; 20 | 21 | private messageBus: MessageBus; 22 | private toolManager: ToolManager; 23 | private geminiClient: GoogleGenerativeAI; 24 | private model: string; 25 | private registeredAgents: Map; // agentId -> capabilities description 26 | 27 | constructor( 28 | messageBus: MessageBus, 29 | toolManager: ToolManager, 30 | apiKey?: string, 31 | model: string = "gemini-2.0-flash-exp" 32 | ) { 33 | this.messageBus = messageBus; 34 | this.toolManager = toolManager; 35 | 36 | const key = apiKey || process.env.GOOGLE_API_KEY; 37 | if (!key) { 38 | throw new Error("Google API key is required for OrchestratorAgent"); 39 | } 40 | 41 | this.geminiClient = new GoogleGenerativeAI(key); 42 | this.model = model; 43 | this.registeredAgents = new Map(); 44 | 45 | console.log(`[${this.name}] Initialized`); 46 | } 47 | 48 | /** 49 | * Registers a specialized agent with its capabilities 50 | */ 51 | registerAgent(agentId: string, capabilities: string): void { 52 | this.registeredAgents.set(agentId, capabilities); 53 | console.log(`[${this.name}] Registered agent: ${agentId}`); 54 | } 55 | 56 | /** 57 | * Processes the SentientPayload through the orchestration workflow 58 | */ 59 | async process(payload: SentientPayload): Promise { 60 | console.log(`[${this.name}] Processing query: "${payload.query.substring(0, 50)}..."`); 61 | 62 | try { 63 | // Step 1: Task Decomposition 64 | const tasks = await this.decomposeQuery(payload.query); 65 | payload.state.tasks = tasks; 66 | 67 | // Step 2: Execute tasks 68 | await this.executeTasks(payload); 69 | 70 | // Step 3: Synthesize final response 71 | payload.finalOutput = await this.synthesizeResponse(payload); 72 | 73 | payload.metadata.endTime = new Date(); 74 | console.log(`[${this.name}] Processing complete`); 75 | 76 | return payload; 77 | 78 | } catch (error) { 79 | console.error(`[${this.name}] Error during processing:`, error); 80 | payload.finalOutput = `Error: ${error instanceof Error ? error.message : String(error)}`; 81 | payload.metadata.endTime = new Date(); 82 | return payload; 83 | } 84 | } 85 | 86 | /** 87 | * Decomposes a complex query into executable tasks 88 | */ 89 | private async decomposeQuery(query: string): Promise { 90 | console.log(`[${this.name}] Decomposing query into tasks`); 91 | 92 | const model = this.geminiClient.getGenerativeModel({ model: this.model }); 93 | 94 | const prompt = `You are a task planning expert. Break down the following user query into a sequence of concrete, executable tasks. 95 | 96 | User Query: "${query}" 97 | 98 | Available Agents: 99 | ${Array.from(this.registeredAgents.entries()).map(([id, caps]) => `- ${id}: ${caps}`).join('\n')} 100 | 101 | Available Tools: 102 | ${this.toolManager.listTools().join(', ')} 103 | 104 | Generate a JSON array of tasks with the following structure: 105 | [ 106 | { 107 | "id": "task_1", 108 | "description": "Task description", 109 | "assignedAgent": "agent_id or 'self' for orchestrator", 110 | "dependencies": ["task_id_that_must_complete_first"], 111 | "estimatedComplexity": "low|medium|high" 112 | } 113 | ] 114 | 115 | Only return the JSON array, no additional text.`; 116 | 117 | const result = await model.generateContent(prompt); 118 | const responseText = result.response.text(); 119 | 120 | try { 121 | // Extract JSON from response 122 | const jsonMatch = responseText.match(/\[[\s\S]*\]/); 123 | if (!jsonMatch) { 124 | throw new Error("No JSON array found in response"); 125 | } 126 | 127 | const tasksData = JSON.parse(jsonMatch[0]); 128 | 129 | return tasksData.map((t: any) => ({ 130 | id: t.id || uuidv4(), 131 | description: t.description, 132 | status: 'pending' as const, 133 | assignedAgent: t.assignedAgent || 'self', 134 | dependencies: t.dependencies || [], 135 | result: null 136 | })); 137 | 138 | } catch (error) { 139 | console.error(`[${this.name}] Error parsing task decomposition:`, error); 140 | 141 | // Fallback: create a single task 142 | return [{ 143 | id: uuidv4(), 144 | description: query, 145 | status: 'pending', 146 | assignedAgent: 'self', 147 | dependencies: [] 148 | }]; 149 | } 150 | } 151 | 152 | /** 153 | * Executes tasks respecting dependencies 154 | */ 155 | private async executeTasks(payload: SentientPayload): Promise { 156 | const tasks = payload.state.tasks; 157 | const completed = new Set(); 158 | 159 | while (completed.size < tasks.length) { 160 | // Find tasks ready to execute (dependencies met) 161 | const readyTasks = tasks.filter(task => 162 | task.status === 'pending' && 163 | (task.dependencies || []).every(dep => completed.has(dep)) 164 | ); 165 | 166 | if (readyTasks.length === 0 && completed.size < tasks.length) { 167 | console.error(`[${this.name}] Deadlock detected in task dependencies`); 168 | break; 169 | } 170 | 171 | // Execute ready tasks in parallel 172 | await Promise.all( 173 | readyTasks.map(task => this.executeTask(task, payload)) 174 | ); 175 | 176 | readyTasks.forEach(task => { 177 | if (task.status === 'completed') { 178 | completed.add(task.id); 179 | } 180 | }); 181 | } 182 | } 183 | 184 | /** 185 | * Executes a single task 186 | */ 187 | private async executeTask(task: Task, payload: SentientPayload): Promise { 188 | task.status = 'in_progress'; 189 | payload.state.currentTask = task.id; 190 | 191 | console.log(`[${this.name}] Executing task: ${task.description}`); 192 | 193 | try { 194 | if (task.assignedAgent === 'self') { 195 | // Orchestrator handles the task directly (e.g., using tools) 196 | task.result = await this.executeSelfTask(task, payload); 197 | } else { 198 | // Delegate to another agent via message bus 199 | task.result = await this.delegateTask(task, payload); 200 | } 201 | 202 | task.status = 'completed'; 203 | payload.state.intermediateResults.set(task.id, task.result); 204 | 205 | } catch (error) { 206 | console.error(`[${this.name}] Task execution failed:`, error); 207 | task.status = 'failed'; 208 | task.result = { error: String(error) }; 209 | } 210 | } 211 | 212 | /** 213 | * Executes a task directly using available tools 214 | */ 215 | private async executeSelfTask(task: Task, payload: SentientPayload): Promise { 216 | // Simple implementation: use LLM to decide which tool to use 217 | const availableTools = this.toolManager.listTools(); 218 | 219 | if (availableTools.length === 0) { 220 | return { message: "No tools available for execution" }; 221 | } 222 | 223 | // For now, return a simulated result 224 | return { 225 | message: `Task "${task.description}" executed by Orchestrator`, 226 | timestamp: new Date().toISOString() 227 | }; 228 | } 229 | 230 | /** 231 | * Delegates a task to a specialized agent 232 | */ 233 | private async delegateTask(task: Task, payload: SentientPayload): Promise { 234 | return new Promise((resolve, reject) => { 235 | const messageId = uuidv4(); 236 | const timeoutMs = 30000; // 30 second timeout 237 | 238 | // Set up response handler 239 | const responseHandler = async (message: A2AMessage) => { 240 | if (message.type === 'result' && message.taskId === task.id) { 241 | clearTimeout(timeout); 242 | this.messageBus.unsubscribe(`${this.id}_response_${messageId}`); 243 | resolve(message.payload); 244 | } 245 | }; 246 | 247 | // Subscribe to responses 248 | this.messageBus.subscribe(`${this.id}_response_${messageId}`, responseHandler); 249 | 250 | // Send task message 251 | const taskMessage: A2AMessage = { 252 | id: messageId, 253 | taskId: task.id, 254 | targetAgentId: task.assignedAgent!, 255 | sourceAgentId: this.id, 256 | type: 'task', 257 | payload: { 258 | description: task.description, 259 | context: payload 260 | }, 261 | timestamp: new Date() 262 | }; 263 | 264 | this.messageBus.send(taskMessage); 265 | 266 | // Set timeout 267 | const timeout = setTimeout(() => { 268 | this.messageBus.unsubscribe(`${this.id}_response_${messageId}`); 269 | reject(new Error(`Task delegation timeout: ${task.description}`)); 270 | }, timeoutMs); 271 | }); 272 | } 273 | 274 | /** 275 | * Synthesizes final response from all task results 276 | */ 277 | private async synthesizeResponse(payload: SentientPayload): Promise { 278 | console.log(`[${this.name}] Synthesizing final response`); 279 | 280 | const model = this.geminiClient.getGenerativeModel({ model: this.model }); 281 | 282 | // Gather all task results 283 | const taskResults = payload.state.tasks 284 | .map(task => `Task: ${task.description}\nResult: ${JSON.stringify(task.result, null, 2)}`) 285 | .join('\n\n'); 286 | 287 | // Include compressed context if available 288 | const context = payload.compressedContext || 289 | payload.contextFragments.map(f => f.content).join('\n\n'); 290 | 291 | const prompt = `You are synthesizing a final response for the user based on task execution results. 292 | 293 | Original Query: "${payload.query}" 294 | 295 | Task Results: 296 | ${taskResults} 297 | 298 | Context: 299 | ${context.substring(0, 5000)} 300 | 301 | Generate a comprehensive, well-structured response that directly answers the user's query.`; 302 | 303 | const result = await model.generateContent(prompt); 304 | return result.response.text(); 305 | } 306 | } 307 | 308 | -------------------------------------------------------------------------------- /src/server.ts: -------------------------------------------------------------------------------- 1 | import express, { Request, Response } from 'express'; 2 | import cors from 'cors'; 3 | import { ToolManager } from './agents/ToolManager.js'; 4 | import { MessageBus } from './agents/MessageBus.js'; 5 | import { A2ATask, A2AMessage, AgentCard } from './core/types.js'; 6 | import { v4 as uuidv4 } from 'uuid'; 7 | 8 | /** 9 | * Sentient Framework Server 10 | * Provides MCP (Model Context Protocol) and A2A (Agent-to-Agent) service endpoints 11 | */ 12 | 13 | const app = express(); 14 | app.use(cors()); 15 | app.use(express.json({ limit: '50mb' })); 16 | 17 | const PORT = process.env.PORT || 3000; 18 | 19 | // Initialize core services 20 | const toolManager = new ToolManager(); 21 | const messageBus = new MessageBus(); 22 | const registeredAgents = new Map(); 23 | 24 | // ============================================================================ 25 | // MCP Service Layer - Model Context Protocol 26 | // ============================================================================ 27 | 28 | /** 29 | * GET /mcp/tools 30 | * Returns the manifest of all available tools in MCP format 31 | */ 32 | app.get('/mcp/tools', (req: Request, res: Response) => { 33 | try { 34 | const toolManifest = toolManager.getToolDefinitions(); 35 | 36 | res.json({ 37 | protocol: 'MCP', 38 | version: '1.0', 39 | tools: toolManifest, 40 | timestamp: new Date().toISOString() 41 | }); 42 | } catch (error) { 43 | res.status(500).json({ 44 | error: 'Failed to retrieve tool manifest', 45 | details: error instanceof Error ? error.message : String(error) 46 | }); 47 | } 48 | }); 49 | 50 | /** 51 | * POST /mcp/tools/:toolName/execute 52 | * Executes a specific tool with provided arguments 53 | */ 54 | app.post('/mcp/tools/:toolName/execute', async (req: Request, res: Response) => { 55 | try { 56 | const { toolName } = req.params; 57 | const args = req.body; 58 | 59 | console.log(`[MCP Service] Executing tool: ${toolName}`); 60 | 61 | const result = await toolManager.executeTool(toolName, args); 62 | 63 | if (result.success) { 64 | res.json({ 65 | success: true, 66 | data: result.data, 67 | executionTime: result.executionTime, 68 | timestamp: new Date().toISOString() 69 | }); 70 | } else { 71 | res.status(400).json({ 72 | success: false, 73 | error: result.error, 74 | executionTime: result.executionTime 75 | }); 76 | } 77 | } catch (error) { 78 | res.status(500).json({ 79 | success: false, 80 | error: error instanceof Error ? error.message : String(error) 81 | }); 82 | } 83 | }); 84 | 85 | /** 86 | * GET /mcp/tools/:toolName/stats 87 | * Returns execution statistics for a specific tool 88 | */ 89 | app.get('/mcp/tools/:toolName/stats', (req: Request, res: Response) => { 90 | try { 91 | const { toolName } = req.params; 92 | const stats = toolManager.getToolStats(toolName); 93 | 94 | if (stats) { 95 | res.json(stats); 96 | } else { 97 | res.status(404).json({ 98 | error: `No statistics available for tool: ${toolName}` 99 | }); 100 | } 101 | } catch (error) { 102 | res.status(500).json({ 103 | error: error instanceof Error ? error.message : String(error) 104 | }); 105 | } 106 | }); 107 | 108 | // ============================================================================ 109 | // A2A Service Layer - Agent-to-Agent Protocol 110 | // ============================================================================ 111 | 112 | /** 113 | * POST /a2a/agents/register 114 | * Registers a new agent with the system 115 | */ 116 | app.post('/a2a/agents/register', (req: Request, res: Response) => { 117 | try { 118 | const agentCard: AgentCard = req.body; 119 | 120 | if (!agentCard.id || !agentCard.name || !agentCard.capabilities) { 121 | return res.status(400).json({ 122 | error: 'Invalid agent card: id, name, and capabilities are required' 123 | }); 124 | } 125 | 126 | registeredAgents.set(agentCard.id, agentCard); 127 | console.log(`[A2A Service] Registered agent: ${agentCard.id}`); 128 | 129 | res.status(201).json({ 130 | success: true, 131 | message: `Agent ${agentCard.id} registered successfully`, 132 | agentId: agentCard.id 133 | }); 134 | } catch (error) { 135 | res.status(500).json({ 136 | error: error instanceof Error ? error.message : String(error) 137 | }); 138 | } 139 | }); 140 | 141 | /** 142 | * GET /a2a/agents 143 | * Lists all registered agents 144 | */ 145 | app.get('/a2a/agents', (req: Request, res: Response) => { 146 | try { 147 | const agents = Array.from(registeredAgents.values()); 148 | 149 | res.json({ 150 | protocol: 'A2A', 151 | version: '1.0', 152 | agents, 153 | count: agents.length, 154 | timestamp: new Date().toISOString() 155 | }); 156 | } catch (error) { 157 | res.status(500).json({ 158 | error: error instanceof Error ? error.message : String(error) 159 | }); 160 | } 161 | }); 162 | 163 | /** 164 | * GET /a2a/agents/:agentId 165 | * Gets details of a specific agent 166 | */ 167 | app.get('/a2a/agents/:agentId', (req: Request, res: Response) => { 168 | try { 169 | const { agentId } = req.params; 170 | const agent = registeredAgents.get(agentId); 171 | 172 | if (agent) { 173 | res.json(agent); 174 | } else { 175 | res.status(404).json({ 176 | error: `Agent not found: ${agentId}` 177 | }); 178 | } 179 | } catch (error) { 180 | res.status(500).json({ 181 | error: error instanceof Error ? error.message : String(error) 182 | }); 183 | } 184 | }); 185 | 186 | /** 187 | * POST /a2a/tasks 188 | * Submits a new task to an agent 189 | */ 190 | app.post('/a2a/tasks', async (req: Request, res: Response) => { 191 | try { 192 | const taskData = req.body; 193 | 194 | const task: A2ATask = { 195 | id: taskData.id || uuidv4(), 196 | type: taskData.type || 'general', 197 | description: taskData.description, 198 | status: 'submitted', 199 | targetAgentId: taskData.targetAgentId, 200 | payload: taskData.payload, 201 | artifacts: [] 202 | }; 203 | 204 | // Verify target agent exists 205 | if (!registeredAgents.has(task.targetAgentId)) { 206 | return res.status(404).json({ 207 | error: `Target agent not found: ${task.targetAgentId}` 208 | }); 209 | } 210 | 211 | console.log(`[A2A Service] Task ${task.id} submitted to agent ${task.targetAgentId}`); 212 | 213 | // Create and send A2A message 214 | const message: A2AMessage = { 215 | id: uuidv4(), 216 | taskId: task.id, 217 | targetAgentId: task.targetAgentId, 218 | sourceAgentId: taskData.sourceAgentId || 'api', 219 | type: 'task', 220 | payload: task, 221 | timestamp: new Date() 222 | }; 223 | 224 | await messageBus.send(message); 225 | 226 | res.status(202).json({ 227 | success: true, 228 | taskId: task.id, 229 | status: task.status, 230 | message: 'Task submitted successfully' 231 | }); 232 | 233 | } catch (error) { 234 | res.status(500).json({ 235 | error: error instanceof Error ? error.message : String(error) 236 | }); 237 | } 238 | }); 239 | 240 | /** 241 | * POST /a2a/messages 242 | * Sends a message between agents 243 | */ 244 | app.post('/a2a/messages', async (req: Request, res: Response) => { 245 | try { 246 | const messageData = req.body; 247 | 248 | const message: A2AMessage = { 249 | id: messageData.id || uuidv4(), 250 | taskId: messageData.taskId, 251 | targetAgentId: messageData.targetAgentId, 252 | sourceAgentId: messageData.sourceAgentId, 253 | type: messageData.type || 'task', 254 | payload: messageData.payload, 255 | timestamp: new Date() 256 | }; 257 | 258 | await messageBus.send(message); 259 | 260 | res.json({ 261 | success: true, 262 | messageId: message.id, 263 | message: 'Message sent successfully' 264 | }); 265 | 266 | } catch (error) { 267 | res.status(500).json({ 268 | error: error instanceof Error ? error.message : String(error) 269 | }); 270 | } 271 | }); 272 | 273 | /** 274 | * GET /a2a/bus/stats 275 | * Gets statistics about the message bus 276 | */ 277 | app.get('/a2a/bus/stats', (req: Request, res: Response) => { 278 | try { 279 | const stats = messageBus.getStats(); 280 | res.json(stats); 281 | } catch (error) { 282 | res.status(500).json({ 283 | error: error instanceof Error ? error.message : String(error) 284 | }); 285 | } 286 | }); 287 | 288 | // ============================================================================ 289 | // Health and Info Endpoints 290 | // ============================================================================ 291 | 292 | /** 293 | * GET /health 294 | * Health check endpoint 295 | */ 296 | app.get('/health', (req: Request, res: Response) => { 297 | res.json({ 298 | status: 'healthy', 299 | uptime: process.uptime(), 300 | timestamp: new Date().toISOString(), 301 | services: { 302 | mcp: 'operational', 303 | a2a: 'operational' 304 | } 305 | }); 306 | }); 307 | 308 | /** 309 | * GET / 310 | * API information endpoint 311 | */ 312 | app.get('/', (req: Request, res: Response) => { 313 | res.json({ 314 | name: 'Sentient Framework API', 315 | version: '0.1.0', 316 | description: 'Advanced Context Engineering with Multi-Agent Orchestration', 317 | endpoints: { 318 | mcp: { 319 | tools: 'GET /mcp/tools', 320 | execute: 'POST /mcp/tools/:toolName/execute', 321 | stats: 'GET /mcp/tools/:toolName/stats' 322 | }, 323 | a2a: { 324 | register: 'POST /a2a/agents/register', 325 | agents: 'GET /a2a/agents', 326 | agent: 'GET /a2a/agents/:agentId', 327 | tasks: 'POST /a2a/tasks', 328 | messages: 'POST /a2a/messages', 329 | busStats: 'GET /a2a/bus/stats' 330 | }, 331 | health: 'GET /health' 332 | }, 333 | documentation: 'https://github.com/phodal/codelotus' 334 | }); 335 | }); 336 | 337 | // ============================================================================ 338 | // Server Startup 339 | // ============================================================================ 340 | 341 | app.listen(PORT, () => { 342 | console.log(` 343 | ╔══════════════════════════════════════════════════════════════╗ 344 | ║ ║ 345 | ║ 🚀 Sentient Framework Server ║ 346 | ║ ║ 347 | ║ Advanced Context Engineering System ║ 348 | ║ ║ 349 | ╚══════════════════════════════════════════════════════════════╝ 350 | 351 | Server running on: http://localhost:${PORT} 352 | 353 | Service Endpoints: 354 | 📋 MCP Service: http://localhost:${PORT}/mcp/tools 355 | 🤖 A2A Service: http://localhost:${PORT}/a2a/agents 356 | ❤️ Health: http://localhost:${PORT}/health 357 | 358 | Documentation: https://github.com/phodal/codelotus 359 | `); 360 | }); 361 | 362 | // Export for programmatic usage 363 | export { app, toolManager, messageBus, registeredAgents }; 364 | 365 | -------------------------------------------------------------------------------- /diagrams/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Sentient Framework - Architecture Diagrams 7 | 162 | 163 | 164 | 176 | 177 |
178 |
179 |

Sentient Framework

180 |

Architecture Diagrams

181 |
182 | 183 |
184 |

System Context

185 |
186 | Shows how the Sentient Framework fits into the overall system landscape, including external users, LLM services, vector databases, and code repositories. 187 |
188 | System Context Diagram 189 |
190 | SVG 191 | PNG 192 | PUML 193 |
194 |
195 | 196 |
197 |

Container Diagram

198 |
199 | Major containers and services: Service Layer (MCP Service, A2A Service), Agentic Core (Orchestrator, Tool Manager, Message Bus, Specialized Agents), and Context Engine Layer (Document/Graph/Keyword RAG, Hybrid Retriever, Context Compressor). 200 |
201 | Container Diagram 202 |
203 | SVG 204 | PNG 205 | PUML 206 |
207 |
208 | 209 |
210 |

Service Layer Components

211 |
212 | Detailed view of Service Layer: MCP endpoint (tool discovery, execution), A2A endpoint (task routing, agent discovery), tool registry, and agent card management. 213 |
214 | Service Layer Components 215 |
216 | SVG 217 | PNG 218 | PUML 219 |
220 |
221 | 222 |
223 |

Agentic Core Components

224 |
225 | The "brain" of the system: Orchestrator Agent (task decomposition, scheduling, synthesis), Message Bus, Tool Manager (validation, execution), Specialized Agents, and SentientPayload lifecycle. 226 |
227 | Agentic Core Components 228 |
229 | SVG 230 | PNG 231 | PUML 232 |
233 |
234 | 235 |
236 |

Context Engine Layer Components

237 |
238 | Retrieval and optimization foundation: Document RAG (loader, splitter, embedder, vector store), Graph RAG (RepoMapper), Keyword RAG (BM25), Hybrid Retriever (RRF), and Context Compressor (EXIT-inspired). 239 |
240 | Context Engine Components 241 |
242 | SVG 243 | PNG 244 | PUML 245 |
246 |
247 | 248 |
249 |

Query Flow Sequence

250 |
251 | Complete lifecycle of a complex query: ingestion, task decomposition, parallel retrieval, tool use, context compression, and response synthesis. 252 |
253 | Query Flow Sequence 254 |
255 | SVG 256 | PNG 257 | PUML 258 |
259 |
260 | 261 |
262 |

Deployment Architecture

263 |
264 | Production environment: Kubernetes cluster, Pod types (Service, Agent, Engine), managed services (Vector DB, Redis, PostgreSQL), external services (OpenAI, Gemini), and load balancing. 265 |
266 | Deployment Diagram 267 |
268 | SVG 269 | PNG 270 | PUML 271 |
272 |
273 | 274 |
275 |

Data Flow

276 |
277 | SentientPayload lifecycle: initial creation, state updates during execution, context fragment accumulation, compression, final output generation, and metadata enrichment. 278 |
279 | Data Flow Diagram 280 |
281 | SVG 282 | PNG 283 | PUML 284 |
285 |
286 | 287 |
288 |

Sentient Framework · C4 Model + PlantUML · GitHub

289 |
290 |
291 | 292 | 309 | 310 | 311 | --------------------------------------------------------------------------------