├── tests ├── api │ └── __init__.py ├── utils │ ├── __init__.py │ ├── test_config.py │ ├── check_services.py │ └── test_fixtures.py ├── frontend │ └── __init__.py ├── __init__.py ├── HYBRID_RETRIEVAL_TEST_SUMMARY.md ├── run_tests.py ├── conftest.py ├── unit │ └── test_core.py ├── integration │ ├── test_memory_store.py │ └── test_context_management.py └── README.md ├── .github ├── CODEOWNERS ├── ISSUE_TEMPLATE │ ├── config.yml │ ├── 03_CODEBASE_IMPROVEMENT.md │ ├── 01_BUG_REPORT.md │ └── 02_FEATURE_REQUEST.md ├── PULL_REQUEST_TEMPLATE.md └── workflows │ └── build-and-publish.yml ├── KestrelAI ├── backend │ ├── models │ │ └── task.py │ ├── __init__.py │ └── routes │ │ └── tasks.py ├── memory │ └── __init__.py ├── __init__.py ├── notes │ ├── __init__.py │ ├── CONFERENCES.txt │ └── ML FELLOWSHIPS.txt ├── shared │ ├── __init__.py │ └── models.py ├── mcp │ ├── __init__.py │ └── mcp_config.py ├── agents │ ├── research_config.py │ ├── __init__.py │ ├── config.py │ ├── base.py │ ├── context_builder.py │ ├── prompt_builder.py │ ├── searxng_service.py │ └── base_agent.py └── orch_debug.txt ├── kestrel-ui ├── src │ ├── vite-env.d.ts │ ├── assets │ │ └── logo.png │ ├── main.tsx │ ├── index.css │ └── ChatInterface.tsx ├── public │ ├── kestrel-logo.png │ └── vite.svg ├── postcss.config.js ├── tsconfig.json ├── vite.config.ts ├── tailwind.config.js ├── .gitignore ├── index.html ├── eslint.config.js ├── tsconfig.node.json ├── tsconfig.app.json ├── package.json └── README.md ├── docs ├── images │ ├── arch.png │ ├── home.png │ ├── logo.png │ ├── new_logo.png │ ├── new_task.png │ └── screenshot.png └── CONTRIBUTING.md ├── Dockerfiles ├── frontend_dockerfile ├── agent_dockerfile └── backend_dockerfile ├── LICENSE ├── pytest.ini ├── mcp_config.json ├── pyproject.toml ├── docker-compose.yml ├── examples └── mcp_research_example.py ├── .gitignore └── README.md /tests/api/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/frontend/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @dankeg 2 | -------------------------------------------------------------------------------- /KestrelAI/backend/models/task.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /KestrelAI/memory/__init__.py: -------------------------------------------------------------------------------- 1 | """RAG Work""" 2 | -------------------------------------------------------------------------------- /KestrelAI/__init__.py: -------------------------------------------------------------------------------- 1 | """Top level directory containing dashboard""" 2 | -------------------------------------------------------------------------------- /kestrel-ui/src/vite-env.d.ts: -------------------------------------------------------------------------------- 1 | /// 2 | -------------------------------------------------------------------------------- /KestrelAI/notes/__init__.py: -------------------------------------------------------------------------------- 1 | """Top level directory containing dashboard""" 2 | -------------------------------------------------------------------------------- /KestrelAI/backend/__init__.py: -------------------------------------------------------------------------------- 1 | """Top level directory containing dashboard""" 2 | -------------------------------------------------------------------------------- /KestrelAI/shared/__init__.py: -------------------------------------------------------------------------------- 1 | """Top level directory containing dashboard""" 2 | -------------------------------------------------------------------------------- /docs/images/arch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dankeg/KestrelAI/HEAD/docs/images/arch.png -------------------------------------------------------------------------------- /docs/images/home.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dankeg/KestrelAI/HEAD/docs/images/home.png -------------------------------------------------------------------------------- /docs/images/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dankeg/KestrelAI/HEAD/docs/images/logo.png -------------------------------------------------------------------------------- /docs/images/new_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dankeg/KestrelAI/HEAD/docs/images/new_logo.png -------------------------------------------------------------------------------- /docs/images/new_task.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dankeg/KestrelAI/HEAD/docs/images/new_task.png -------------------------------------------------------------------------------- /docs/images/screenshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dankeg/KestrelAI/HEAD/docs/images/screenshot.png -------------------------------------------------------------------------------- /kestrel-ui/src/assets/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dankeg/KestrelAI/HEAD/kestrel-ui/src/assets/logo.png -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | # KestrelAI Test Suite 2 | # Comprehensive unit and integration tests to prevent regressions 3 | -------------------------------------------------------------------------------- /kestrel-ui/public/kestrel-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dankeg/KestrelAI/HEAD/kestrel-ui/public/kestrel-logo.png -------------------------------------------------------------------------------- /kestrel-ui/postcss.config.js: -------------------------------------------------------------------------------- 1 | export default { 2 | plugins: { 3 | tailwindcss: {}, 4 | autoprefixer: {}, 5 | }, 6 | } 7 | -------------------------------------------------------------------------------- /kestrel-ui/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "files": [], 3 | "references": [ 4 | { "path": "./tsconfig.app.json" }, 5 | { "path": "./tsconfig.node.json" } 6 | ] 7 | } 8 | -------------------------------------------------------------------------------- /kestrel-ui/vite.config.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig } from 'vite' 2 | import react from '@vitejs/plugin-react' 3 | 4 | // https://vite.dev/config/ 5 | export default defineConfig({ 6 | plugins: [react()], 7 | }) 8 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/config.yml: -------------------------------------------------------------------------------- 1 | --- 2 | blank_issues_enabled: false 3 | contact_links: 4 | - name: Kestrel Community Support 5 | url: https://github.com/dankeg/kestrel/discussions 6 | about: Please ask and answer questions here. 7 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/03_CODEBASE_IMPROVEMENT.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Codebase improvement 3 | about: Provide your feedback for the existing codebase. Suggest a better solution for algorithms, development tools, etc. 4 | title: "dev: " 5 | labels: "enhancement" 6 | assignees: "" 7 | --- 8 | -------------------------------------------------------------------------------- /kestrel-ui/tailwind.config.js: -------------------------------------------------------------------------------- 1 | /** @type {import('tailwindcss').Config} */ 2 | export default { 3 | content: [ 4 | "./index.html", 5 | "./src/**/*.{js,ts,jsx,tsx}", 6 | ], 7 | theme: { 8 | extend: {}, 9 | }, 10 | plugins: [ 11 | require('@tailwindcss/typography'), 12 | require('@tailwindcss/line-clamp'), 13 | ], 14 | } 15 | -------------------------------------------------------------------------------- /Dockerfiles/frontend_dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:20 2 | 3 | WORKDIR /app 4 | 5 | RUN ls 6 | 7 | # Copy package files from the UI folder 8 | COPY kestrel-ui//package*.json ./ 9 | 10 | # Install dependencies 11 | RUN npm ci 12 | 13 | # Copy the rest of the UI code 14 | COPY kestrel-ui/ . 15 | 16 | EXPOSE 5173 17 | 18 | CMD ["npm", "run", "dev", "--", "--host", "0.0.0.0"] -------------------------------------------------------------------------------- /kestrel-ui/.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | pnpm-debug.log* 8 | lerna-debug.log* 9 | 10 | node_modules 11 | dist 12 | dist-ssr 13 | *.local 14 | 15 | # Editor directories and files 16 | .vscode/* 17 | !.vscode/extensions.json 18 | .idea 19 | .DS_Store 20 | *.suo 21 | *.ntvs* 22 | *.njsproj 23 | *.sln 24 | *.sw? 25 | -------------------------------------------------------------------------------- /kestrel-ui/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | Kestrel 8 | 9 | 10 |
11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /KestrelAI/mcp/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | MCP (Model Context Protocol) integration for KestrelAI research agents 3 | Production-ready implementation with real MCP protocol only 4 | """ 5 | 6 | from .mcp_client import MCPResult 7 | from .mcp_config import MCPConfig, MCPServerConfig 8 | from .mcp_manager import MCPManager 9 | from .mcp_tools import MCPTool, MCPToolRegistry 10 | 11 | __all__ = [ 12 | "MCPManager", 13 | "MCPResult", 14 | "MCPToolRegistry", 15 | "MCPTool", 16 | "MCPConfig", 17 | "MCPServerConfig", 18 | ] 19 | -------------------------------------------------------------------------------- /kestrel-ui/src/main.tsx: -------------------------------------------------------------------------------- 1 | import React from 'react' 2 | import ReactDOM from 'react-dom/client' 3 | import { BrowserRouter as Router, Routes, Route } from 'react-router-dom' 4 | import './index.css' 5 | import KestrelAIApp from './KestrelAIApp' 6 | import ChatInterface from './ChatInterface' 7 | 8 | function App() { 9 | return ( 10 | 11 | 12 | } /> 13 | } /> 14 | 15 | 16 | ) 17 | } 18 | 19 | ReactDOM.createRoot(document.getElementById('root')!).render( 20 | 21 | 22 | , 23 | ) -------------------------------------------------------------------------------- /kestrel-ui/eslint.config.js: -------------------------------------------------------------------------------- 1 | import js from '@eslint/js' 2 | import globals from 'globals' 3 | import reactHooks from 'eslint-plugin-react-hooks' 4 | import reactRefresh from 'eslint-plugin-react-refresh' 5 | import tseslint from 'typescript-eslint' 6 | import { globalIgnores } from 'eslint/config' 7 | 8 | export default tseslint.config([ 9 | globalIgnores(['dist']), 10 | { 11 | files: ['**/*.{ts,tsx}'], 12 | extends: [ 13 | js.configs.recommended, 14 | tseslint.configs.recommended, 15 | reactHooks.configs['recommended-latest'], 16 | reactRefresh.configs.vite, 17 | ], 18 | languageOptions: { 19 | ecmaVersion: 2020, 20 | globals: globals.browser, 21 | }, 22 | }, 23 | ]) 24 | -------------------------------------------------------------------------------- /kestrel-ui/tsconfig.node.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "tsBuildInfoFile": "./node_modules/.tmp/tsconfig.node.tsbuildinfo", 4 | "target": "ES2023", 5 | "lib": ["ES2023"], 6 | "module": "ESNext", 7 | "skipLibCheck": true, 8 | 9 | /* Bundler mode */ 10 | "moduleResolution": "bundler", 11 | "allowImportingTsExtensions": true, 12 | "verbatimModuleSyntax": true, 13 | "moduleDetection": "force", 14 | "noEmit": true, 15 | 16 | /* Linting */ 17 | "strict": true, 18 | "noUnusedLocals": true, 19 | "noUnusedParameters": true, 20 | "erasableSyntaxOnly": true, 21 | "noFallthroughCasesInSwitch": true, 22 | "noUncheckedSideEffectImports": true 23 | }, 24 | "include": ["vite.config.ts"] 25 | } 26 | -------------------------------------------------------------------------------- /kestrel-ui/tsconfig.app.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "tsBuildInfoFile": "./node_modules/.tmp/tsconfig.app.tsbuildinfo", 4 | "target": "ES2022", 5 | "useDefineForClassFields": true, 6 | "lib": ["ES2022", "DOM", "DOM.Iterable"], 7 | "module": "ESNext", 8 | "skipLibCheck": true, 9 | 10 | /* Bundler mode */ 11 | "moduleResolution": "bundler", 12 | "allowImportingTsExtensions": true, 13 | "verbatimModuleSyntax": true, 14 | "moduleDetection": "force", 15 | "noEmit": true, 16 | "jsx": "react-jsx", 17 | 18 | /* Linting */ 19 | "strict": true, 20 | "noUnusedLocals": true, 21 | "noUnusedParameters": true, 22 | "erasableSyntaxOnly": true, 23 | "noFallthroughCasesInSwitch": true, 24 | "noUncheckedSideEffectImports": true 25 | }, 26 | "include": ["src"] 27 | } 28 | -------------------------------------------------------------------------------- /Dockerfiles/agent_dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.11-slim 2 | 3 | WORKDIR /app 4 | 5 | # Install system dependencies including Poetry 6 | RUN apt-get update && apt-get install -y \ 7 | gcc \ 8 | g++ \ 9 | curl \ 10 | && rm -rf /var/lib/apt/lists/* 11 | 12 | # Install Poetry 13 | ENV POETRY_VERSION=1.7.1 14 | RUN pip install --no-cache-dir --upgrade pip setuptools \ 15 | && pip install --no-cache-dir "poetry==$POETRY_VERSION" 16 | 17 | RUN poetry config virtualenvs.create false 18 | 19 | # Configure Poetry - disable virtual env creation as we're in a container 20 | RUN poetry config virtualenvs.create false 21 | 22 | # Copy dependency files 23 | COPY pyproject.toml poetry.lock* ./ 24 | 25 | # Install dependencies for agent service only 26 | RUN poetry install --no-interaction --no-ansi --only main --extras agent 27 | 28 | # Copy application code 29 | COPY . . 30 | 31 | # Create necessary directories 32 | RUN mkdir -p /app/notes 33 | 34 | CMD ["python", "model_loop.py"] -------------------------------------------------------------------------------- /Dockerfiles/backend_dockerfile: -------------------------------------------------------------------------------- 1 | FROM python:3.11-slim 2 | 3 | WORKDIR /app 4 | 5 | # Install system dependencies including Poetry 6 | RUN apt-get update && apt-get install -y \ 7 | curl \ 8 | && rm -rf /var/lib/apt/lists/* 9 | 10 | # Install Poetry 11 | ENV POETRY_VERSION=1.7.1 12 | RUN pip install --no-cache-dir --upgrade pip setuptools \ 13 | && pip install --no-cache-dir "poetry==$POETRY_VERSION" 14 | 15 | RUN poetry config virtualenvs.create false 16 | 17 | # Configure Poetry - disable virtual env creation as we're in a container 18 | RUN poetry config virtualenvs.create false 19 | 20 | # Copy dependency files 21 | COPY pyproject.toml poetry.lock* ./ 22 | 23 | # Install dependencies for backend service only 24 | RUN poetry install --no-interaction --no-ansi --only main --extras backend 25 | 26 | # Copy application code 27 | COPY . . 28 | 29 | # Create necessary directories 30 | RUN mkdir -p /app/notes /app/exports 31 | 32 | EXPOSE 8000 33 | 34 | CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000", "--reload"] -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/01_BUG_REPORT.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug Report 3 | about: Create a report to help Kestrel to improve 4 | title: "bug: " 5 | labels: "bug" 6 | assignees: "" 7 | --- 8 | 9 | # Bug Report 10 | 11 | **Kestrel version:** 12 | 13 | 14 | 15 | **Current behavior:** 16 | 17 | 18 | 19 | **Expected behavior:** 20 | 21 | 22 | 23 | **Steps to reproduce:** 24 | 25 | 26 | 27 | **Related code:** 28 | 29 | 30 | 31 | ``` 32 | insert short code snippets here 33 | ``` 34 | 35 | **Other information:** 36 | 37 | 38 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2025, Ganesh Danke 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/02_FEATURE_REQUEST.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature Request 3 | about: Suggest an idea for this project 4 | title: "feat: " 5 | labels: "enhancement" 6 | assignees: "" 7 | --- 8 | 9 | # Feature Request 10 | 11 | **Describe the Feature Request** 12 | 13 | 14 | 15 | **Describe Preferred Solution** 16 | 17 | 18 | 19 | **Describe Alternatives** 20 | 21 | 22 | 23 | **Related Code** 24 | 25 | 26 | 27 | **Additional Context** 28 | 29 | 30 | 31 | **If the feature request is approved, would you be willing to submit a PR?** 32 | _(Help can be provided if you need assistance submitting a PR)_ 33 | 34 | - [ ] Yes 35 | - [ ] No 36 | -------------------------------------------------------------------------------- /KestrelAI/agents/research_config.py: -------------------------------------------------------------------------------- 1 | """ 2 | Configuration for research agent behavior. 3 | """ 4 | 5 | from __future__ import annotations 6 | 7 | import os 8 | from dataclasses import dataclass, field 9 | from typing import Any 10 | 11 | # Default configuration values 12 | THINK_LOOPS = 6 13 | SEARCH_RESULTS = 4 14 | FETCH_BYTES = 30_000 15 | DEBUG = True 16 | CONTEXT_WINDOW = 60 17 | CHECKPOINT_FREQ = 5 18 | MAX_SNIPPET_LENGTH = 3000 19 | 20 | 21 | @dataclass 22 | class ResearchConfig: 23 | """Configuration for research agent behavior""" 24 | 25 | think_loops: int = THINK_LOOPS 26 | search_results: int = SEARCH_RESULTS 27 | fetch_bytes: int = FETCH_BYTES 28 | context_window: int = CONTEXT_WINDOW 29 | checkpoint_freq: int = CHECKPOINT_FREQ 30 | max_snippet_length: int = MAX_SNIPPET_LENGTH 31 | debug: bool = DEBUG 32 | 33 | # Subtask-specific settings 34 | is_subtask_agent: bool = False 35 | subtask_description: str = "" 36 | success_criteria: str = "" 37 | previous_findings: str = "" 38 | previous_reports: list[str] = field( 39 | default_factory=list 40 | ) # Previous reports to build upon 41 | 42 | # MCP settings 43 | use_mcp: bool = False 44 | mcp_manager: Any | None = None 45 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Pull Request type 4 | 5 | 6 | 7 | Please check the type of change your PR introduces: 8 | 9 | - [ ] Bugfix 10 | - [ ] Feature 11 | - [ ] Code style update (formatting, renaming) 12 | - [ ] Refactoring (no functional changes, no API changes) 13 | - [ ] Build-related changes 14 | - [ ] Documentation content changes 15 | - [ ] Other (please describe): 16 | 17 | ## What is the current behavior? 18 | 19 | 20 | 21 | Issue Number: N/A 22 | 23 | ## What is the new behavior? 24 | 25 | 26 | 27 | - 28 | - 29 | - 30 | 31 | ## Does this introduce a breaking change? 32 | 33 | - [ ] Yes 34 | - [ ] No 35 | 36 | 37 | 38 | ## Other information 39 | 40 | 41 | -------------------------------------------------------------------------------- /kestrel-ui/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "kestrel-ui", 3 | "private": true, 4 | "version": "0.0.0", 5 | "type": "module", 6 | "scripts": { 7 | "dev": "vite", 8 | "build": "tsc -b && vite build", 9 | "lint": "eslint .", 10 | "preview": "vite preview" 11 | }, 12 | "dependencies": { 13 | "@types/react-router-dom": "^5.3.3", 14 | "clsx": "^2.1.1", 15 | "highlight.js": "^11.11.1", 16 | "lucide-react": "^0.539.0", 17 | "react": "^19.1.1", 18 | "react-dom": "^19.1.1", 19 | "react-markdown": "^10.1.0", 20 | "react-router-dom": "^7.9.4", 21 | "react-virtuoso": "^4.14.0", 22 | "rehype-highlight": "^7.0.2", 23 | "remark-gfm": "^4.0.1" 24 | }, 25 | "devDependencies": { 26 | "@eslint/js": "^9.33.0", 27 | "@tailwindcss/line-clamp": "^0.4.4", 28 | "@tailwindcss/typography": "^0.5.16", 29 | "@types/react": "^19.1.10", 30 | "@types/react-dom": "^19.1.7", 31 | "@vitejs/plugin-react": "^5.0.0", 32 | "autoprefixer": "^10.4.21", 33 | "eslint": "^9.33.0", 34 | "eslint-plugin-react-hooks": "^5.2.0", 35 | "eslint-plugin-react-refresh": "^0.4.20", 36 | "globals": "^16.3.0", 37 | "postcss": "^8.5.6", 38 | "tailwindcss": "^3.4.13", 39 | "typescript": "~5.8.3", 40 | "typescript-eslint": "^8.39.1", 41 | "vite": "^7.1.2" 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | minversion = 7.0 3 | addopts = -ra --strict-markers --strict-config --tb=short -v 4 | testpaths = tests 5 | python_files = test_*.py *_test.py 6 | python_classes = Test* 7 | python_functions = test_* 8 | markers = 9 | unit: Unit tests 10 | integration: Integration tests 11 | api: Backend API tests 12 | frontend: Frontend UI tests 13 | e2e: End-to-end tests 14 | performance: Performance tests 15 | slow: Slow running tests 16 | requires_services: Tests that require external services 17 | filterwarnings = 18 | ignore::DeprecationWarning 19 | ignore::PendingDeprecationWarning 20 | ignore::UserWarning:chromadb.* 21 | ignore::UserWarning:sentence_transformers.* 22 | ignore::pytest.PytestUnknownMarkWarning 23 | # Timeout configuration (requires pytest-timeout plugin) 24 | # timeout = 300 25 | # timeout_method = thread 26 | 27 | [tool:coverage:run] 28 | source = KestrelAI 29 | omit = 30 | */tests/* 31 | */test_* 32 | */__pycache__/* 33 | */migrations/* 34 | 35 | [tool:coverage:report] 36 | exclude_lines = 37 | pragma: no cover 38 | def __repr__ 39 | if self.debug: 40 | if settings.DEBUG 41 | raise AssertionError 42 | raise NotImplementedError 43 | if 0: 44 | if __name__ == .__main__.: 45 | class .*\bProtocol\): 46 | @(abc\.)?abstractmethod -------------------------------------------------------------------------------- /kestrel-ui/public/vite.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /mcp_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "servers": { 3 | "filesystem": { 4 | "name": "filesystem", 5 | "command": "npx", 6 | "args": ["-y", "@modelcontextprotocol/server-filesystem", "/tmp"], 7 | "env": {}, 8 | "enabled": true, 9 | "description": "File system access for reading/writing files", 10 | "tools": ["read_file", "write_file", "list_directory", "search_files"] 11 | }, 12 | "sqlite": { 13 | "name": "sqlite", 14 | "command": "npx", 15 | "args": ["-y", "@modelcontextprotocol/server-sqlite", "--db-path", "/tmp/kestrel.db"], 16 | "env": {}, 17 | "enabled": true, 18 | "description": "SQLite database access for structured data queries", 19 | "tools": ["query_database", "create_table", "insert_data", "update_data"] 20 | }, 21 | "github": { 22 | "name": "github", 23 | "command": "npx", 24 | "args": ["-y", "@modelcontextprotocol/server-github"], 25 | "env": {}, 26 | "enabled": true, 27 | "description": "GitHub API access for repository information", 28 | "tools": ["search_repositories", "get_repository_info", "get_file_contents", "list_issues"] 29 | }, 30 | "brave_search": { 31 | "name": "brave_search", 32 | "command": "npx", 33 | "args": ["-y", "@modelcontextprotocol/server-brave-search"], 34 | "env": {}, 35 | "enabled": true, 36 | "description": "Brave Search API for enhanced web search", 37 | "tools": ["search_web", "search_news", "search_images"] 38 | }, 39 | "puppeteer": { 40 | "name": "puppeteer", 41 | "command": "npx", 42 | "args": ["-y", "@modelcontextprotocol/server-puppeteer"], 43 | "env": {}, 44 | "enabled": true, 45 | "description": "Web scraping and browser automation", 46 | "tools": ["navigate_to_page", "take_screenshot", "extract_text", "click_element"] 47 | } 48 | }, 49 | "timeout": 30, 50 | "max_retries": 3, 51 | "enable_logging": true, 52 | "log_level": "INFO" 53 | } 54 | 55 | 56 | 57 | 58 | -------------------------------------------------------------------------------- /KestrelAI/shared/models.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | import uuid 4 | from datetime import datetime 5 | from enum import Enum 6 | from typing import Any, Optional 7 | 8 | from pydantic import BaseModel, Field 9 | 10 | 11 | class TaskStatus(str, Enum): 12 | CONFIGURING = "configuring" 13 | PENDING = "pending" 14 | ACTIVE = "active" 15 | COMPLETE = "complete" 16 | PAUSED = "paused" 17 | FAILED = "failed" 18 | 19 | @classmethod 20 | def _missing_(cls, value: object): 21 | if isinstance(value, str): 22 | # Normalize to lowercase before lookup 23 | value = value.lower() 24 | for member in cls: 25 | if member.value == value: 26 | return member 27 | return None 28 | 29 | 30 | class Subtask(BaseModel): 31 | """Individual subtask in a research plan""" 32 | 33 | order: int 34 | description: str 35 | success_criteria: str 36 | status: str = "pending" # pending, in_progress, completed 37 | findings: list[str] = Field(default_factory=list) 38 | 39 | 40 | class ResearchPlan(BaseModel): 41 | """Research plan generated by the orchestrator""" 42 | 43 | restated_task: str 44 | subtasks: list[Subtask] 45 | current_subtask_index: int = 0 46 | created_at: int = Field( 47 | default_factory=lambda: int(datetime.now().timestamp() * 1000) 48 | ) 49 | 50 | 51 | class TaskMetrics(BaseModel): 52 | """Metrics for task execution""" 53 | 54 | searchCount: int = 0 55 | thinkCount: int = 0 56 | summaryCount: int = 0 57 | checkpointCount: int = 0 58 | webFetchCount: int = 0 59 | llmTokensUsed: int = 0 60 | errorCount: int = 0 61 | 62 | 63 | class Task(BaseModel): 64 | """Main task model used across the system""" 65 | 66 | id: str = Field(default_factory=lambda: str(uuid.uuid4())[:8]) 67 | name: str 68 | description: str 69 | budgetMinutes: int = 180 70 | status: TaskStatus = TaskStatus.CONFIGURING 71 | progress: float = 0.0 72 | elapsed: int = 0 73 | metrics: TaskMetrics = Field(default_factory=TaskMetrics) 74 | research_plan: ResearchPlan | None = None 75 | createdAt: int = Field( 76 | default_factory=lambda: int(datetime.now().timestamp() * 1000) 77 | ) 78 | updatedAt: int = Field( 79 | default_factory=lambda: int(datetime.now().timestamp() * 1000) 80 | ) 81 | config: dict[str, Any] = Field(default_factory=dict) 82 | -------------------------------------------------------------------------------- /KestrelAI/agents/__init__.py: -------------------------------------------------------------------------------- 1 | """LLM Agents package. 2 | 3 | This module uses lazy imports to avoid pulling in heavy dependencies 4 | unnecessarily (e.g., vector store / chromadb) when only lightweight 5 | utilities like the multi-level summarizer are needed. 6 | """ 7 | 8 | from __future__ import annotations 9 | 10 | from typing import Any 11 | 12 | __all__ = [ 13 | "BaseAgent", 14 | "ResearchAgent", 15 | "OrchestratorAgent", 16 | "AgentState", 17 | "WebResearchAgent", 18 | "ResearchConfig", 19 | "ResearchOrchestrator", 20 | "LlmWrapper", 21 | "get_orchestrator_config", 22 | "OrchestratorConfig", 23 | ] 24 | 25 | 26 | def __getattr__(name: str) -> Any: 27 | """ 28 | Lazily import heavy agent classes and utilities on first access. 29 | 30 | This keeps `import KestrelAI.agents.multi_level_summarizer` lightweight 31 | and avoids importing the vector store / chromadb stack unless callers 32 | actually need full agent orchestration. 33 | """ 34 | if name in {"BaseAgent", "ResearchAgent", "OrchestratorAgent", "AgentState"}: 35 | from .base_agent import AgentState, BaseAgent, OrchestratorAgent, ResearchAgent 36 | 37 | mapping = { 38 | "BaseAgent": BaseAgent, 39 | "ResearchAgent": ResearchAgent, 40 | "OrchestratorAgent": OrchestratorAgent, 41 | "AgentState": AgentState, 42 | } 43 | return mapping[name] 44 | 45 | if name in {"WebResearchAgent", "ResearchConfig"}: 46 | from .web_research_agent import ResearchConfig, WebResearchAgent 47 | 48 | mapping = { 49 | "WebResearchAgent": WebResearchAgent, 50 | "ResearchConfig": ResearchConfig, 51 | } 52 | return mapping[name] 53 | 54 | if name == "ResearchOrchestrator": 55 | from .research_orchestrator import ResearchOrchestrator 56 | 57 | return ResearchOrchestrator 58 | 59 | if name in {"LlmWrapper"}: 60 | from .base import LlmWrapper 61 | 62 | return LlmWrapper 63 | 64 | if name in {"get_orchestrator_config", "OrchestratorConfig"}: 65 | from .config import OrchestratorConfig, get_orchestrator_config 66 | 67 | mapping = { 68 | "get_orchestrator_config": get_orchestrator_config, 69 | "OrchestratorConfig": OrchestratorConfig, 70 | } 71 | return mapping[name] 72 | 73 | raise AttributeError(f"module 'KestrelAI.agents' has no attribute {name!r}") 74 | -------------------------------------------------------------------------------- /kestrel-ui/README.md: -------------------------------------------------------------------------------- 1 | # React + TypeScript + Vite 2 | 3 | This template provides a minimal setup to get React working in Vite with HMR and some ESLint rules. 4 | 5 | Currently, two official plugins are available: 6 | 7 | - [@vitejs/plugin-react](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react) uses [Babel](https://babeljs.io/) for Fast Refresh 8 | - [@vitejs/plugin-react-swc](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react-swc) uses [SWC](https://swc.rs/) for Fast Refresh 9 | 10 | ## Expanding the ESLint configuration 11 | 12 | If you are developing a production application, we recommend updating the configuration to enable type-aware lint rules: 13 | 14 | ```js 15 | export default tseslint.config([ 16 | globalIgnores(['dist']), 17 | { 18 | files: ['**/*.{ts,tsx}'], 19 | extends: [ 20 | // Other configs... 21 | 22 | // Remove tseslint.configs.recommended and replace with this 23 | ...tseslint.configs.recommendedTypeChecked, 24 | // Alternatively, use this for stricter rules 25 | ...tseslint.configs.strictTypeChecked, 26 | // Optionally, add this for stylistic rules 27 | ...tseslint.configs.stylisticTypeChecked, 28 | 29 | // Other configs... 30 | ], 31 | languageOptions: { 32 | parserOptions: { 33 | project: ['./tsconfig.node.json', './tsconfig.app.json'], 34 | tsconfigRootDir: import.meta.dirname, 35 | }, 36 | // other options... 37 | }, 38 | }, 39 | ]) 40 | ``` 41 | 42 | You can also install [eslint-plugin-react-x](https://github.com/Rel1cx/eslint-react/tree/main/packages/plugins/eslint-plugin-react-x) and [eslint-plugin-react-dom](https://github.com/Rel1cx/eslint-react/tree/main/packages/plugins/eslint-plugin-react-dom) for React-specific lint rules: 43 | 44 | ```js 45 | // eslint.config.js 46 | import reactX from 'eslint-plugin-react-x' 47 | import reactDom from 'eslint-plugin-react-dom' 48 | 49 | export default tseslint.config([ 50 | globalIgnores(['dist']), 51 | { 52 | files: ['**/*.{ts,tsx}'], 53 | extends: [ 54 | // Other configs... 55 | // Enable lint rules for React 56 | reactX.configs['recommended-typescript'], 57 | // Enable lint rules for React DOM 58 | reactDom.configs.recommended, 59 | ], 60 | languageOptions: { 61 | parserOptions: { 62 | project: ['./tsconfig.node.json', './tsconfig.app.json'], 63 | tsconfigRootDir: import.meta.dirname, 64 | }, 65 | // other options... 66 | }, 67 | }, 68 | ]) 69 | ``` 70 | -------------------------------------------------------------------------------- /KestrelAI/agents/config.py: -------------------------------------------------------------------------------- 1 | """ 2 | Configuration settings for KestrelAI orchestrator and research agents 3 | """ 4 | 5 | from dataclasses import dataclass 6 | 7 | 8 | @dataclass 9 | class OrchestratorConfig: 10 | """Configuration for orchestrator behavior""" 11 | 12 | slice_minutes: int = 15 13 | max_iterations_per_subtask: int = 10 14 | max_total_iterations: int = 100 15 | max_stuck_count: int = 3 16 | max_retries: int = 3 17 | 18 | 19 | @dataclass 20 | class ResearchAgentConfig: 21 | """Configuration for research agent behavior""" 22 | 23 | think_loops: int = 6 24 | search_results: int = 4 25 | fetch_bytes: int = 30_000 26 | context_window: int = 20 27 | checkpoint_freq: int = 5 28 | max_snippet_length: int = 3000 29 | max_repeats: int = 3 30 | 31 | 32 | @dataclass 33 | class SystemConfig: 34 | """Overall system configuration""" 35 | 36 | debug: bool = True 37 | searxng_url: str = "http://localhost:8080/search" 38 | ollama_host: str = "http://localhost:11434" 39 | 40 | 41 | # Orchestrator profiles 42 | ORCHESTRATOR_PROFILES = { 43 | "hummingbird": { 44 | "slice_minutes": 5, 45 | "max_iterations_per_subtask": 5, 46 | "max_total_iterations": 50, 47 | "description": "Fast, focused research with quick iterations", 48 | }, 49 | "kestrel": { 50 | "slice_minutes": 15, 51 | "max_iterations_per_subtask": 10, 52 | "max_total_iterations": 100, 53 | "description": "Balanced approach with moderate depth", 54 | }, 55 | "albatross": { 56 | "slice_minutes": 30, 57 | "max_iterations_per_subtask": 15, 58 | "max_total_iterations": 150, 59 | "description": "Deep, thorough research with extensive exploration", 60 | }, 61 | } 62 | 63 | 64 | def get_orchestrator_config(profile: str = "kestrel") -> OrchestratorConfig: 65 | """Get orchestrator configuration for a specific profile""" 66 | if profile not in ORCHESTRATOR_PROFILES: 67 | profile = "kestrel" 68 | 69 | config = ORCHESTRATOR_PROFILES[profile] 70 | return OrchestratorConfig( 71 | slice_minutes=config["slice_minutes"], 72 | max_iterations_per_subtask=config["max_iterations_per_subtask"], 73 | max_total_iterations=config["max_total_iterations"], 74 | ) 75 | 76 | 77 | def get_research_agent_config() -> ResearchAgentConfig: 78 | """Get default research agent configuration""" 79 | return ResearchAgentConfig() 80 | 81 | 82 | def get_system_config() -> SystemConfig: 83 | """Get system configuration""" 84 | return SystemConfig() 85 | -------------------------------------------------------------------------------- /docs/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | When contributing to this repository, please first discuss the change you wish to make via issue, email, or any other method with the owners of this repository before making a change. 4 | 5 | ## Development environment setup 6 | In addition to dockerized deployment, a setup is provided using the [Panels](https://panel.holoviz.org/) Python library. This aims to make development easier by isolating the research agent component, enabling it to be tested and tweaked independently from the FastAPI Backend and React UI. 7 | 8 | Follow these steps to get the Panels UI running: 9 | 10 | 1. Clone the repo 11 | 12 | ```sh 13 | git clone https://github.com/dankeg/kestrelAI 14 | ``` 15 | 2. Install Poetry, if not already installed 16 | 17 | ```sh 18 | https://python-poetry.org/docs/ 19 | ``` 20 | 3. Install the project dependencies locally 21 | 22 | ```sh 23 | poetry install -E agent 24 | ``` 25 | 4. Start the Panels Application 26 | 27 | ```sh 28 | poetry run panel serve KestrelAI/dashboard.py --autoreload --show 29 | ``` 30 | 31 | The UI should automatically launch in the browser. 32 | 33 | 34 | ## Issues and feature requests 35 | 36 | You've found a bug in the source code, a mistake in the documentation or maybe you'd like a new feature?Take a look at [GitHub Discussions](https://github.com/dankeg/kestrel/discussions) to see if it's already being discussed. You can help us by [submitting an issue on GitHub](https://github.com/dankeg/kestrel/issues). Before you create an issue, make sure to search the issue archive -- your issue may have already been addressed! 37 | 38 | Please try to create bug reports that are: 39 | 40 | - _Reproducible._ Include steps to reproduce the problem. 41 | - _Specific._ Include as much detail as possible: which version, what environment, etc. 42 | - _Unique._ Do not duplicate existing opened issues. 43 | - _Scoped to a Single Bug._ One bug per report. 44 | 45 | **Even better: Submit a pull request with a fix or new feature!** 46 | 47 | ### How to submit a Pull Request 48 | 49 | 1. Search our repository for open or closed 50 | [Pull Requests](https://github.com/dankeg/kestrel/pulls) 51 | that relate to your submission. You don't want to duplicate effort. 52 | 2. Fork the project 53 | 3. Create your feature branch (`git checkout -b feat/amazing_feature`) 54 | 4. Commit your changes (`git commit -m 'feat: add amazing_feature'`) Kestrel uses [conventional commits](https://www.conventionalcommits.org), so please follow the specification in your commit messages. 55 | 5. Push to the branch (`git push origin feat/amazing_feature`) 56 | 6. [Open a Pull Request](https://github.com/dankeg/kestrel/compare?expand=1) 57 | -------------------------------------------------------------------------------- /KestrelAI/agents/base.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import ollama 4 | 5 | 6 | class LlmWrapper: 7 | def __init__( 8 | self, model: str = "gemma3:27b", temperature: float = 0.6, host: str = None 9 | ): 10 | self.model = model 11 | self.temperature = temperature 12 | # Prefer an explicit host; fall back to env var; then a safe default. 13 | self.host = host or os.getenv("OLLAMA_BASE_URL", "http://localhost:11434") 14 | self.client = ollama.Client(host=self.host) 15 | 16 | def chat(self, messages: list[dict], stream: bool = False) -> str: 17 | """ 18 | Send chat messages to the LLM and return the response content as a string. 19 | 20 | The Ollama client returns a ChatResponse object with structure: 21 | - response.message.content (the actual text response) 22 | """ 23 | try: 24 | response = self.client.chat( 25 | model=self.model, 26 | messages=messages, 27 | stream=stream, 28 | options={"temperature": self.temperature}, 29 | ) 30 | if stream: 31 | return response 32 | 33 | # Ollama returns a ChatResponse object (or dict in some versions) 34 | # The string content is always at response.message.content 35 | if hasattr(response, "message"): 36 | # ChatResponse object: response.message.content 37 | return response.message.content 38 | elif isinstance(response, dict) and "message" in response: 39 | # Dict response: response['message']['content'] or response['message'].content 40 | message = response["message"] 41 | if isinstance(message, dict): 42 | return message.get("content", "") 43 | elif hasattr(message, "content"): 44 | return message.content 45 | 46 | # Fallback: try direct content access 47 | if isinstance(response, dict) and "content" in response: 48 | return response["content"] 49 | 50 | raise ValueError( 51 | f"Unable to extract content from response. " 52 | f"Type: {type(response)}, " 53 | f"Has 'message' attr: {hasattr(response, 'message')}, " 54 | f"Is dict: {isinstance(response, dict)}, " 55 | f"Dict keys: {list(response.keys()) if isinstance(response, dict) else 'N/A'}" 56 | ) 57 | except Exception as e: 58 | # Re-raise with more context 59 | raise RuntimeError( 60 | f"LLM chat failed (model: {self.model}, host: {self.host}): {str(e)}" 61 | ) from e 62 | -------------------------------------------------------------------------------- /tests/utils/test_config.py: -------------------------------------------------------------------------------- 1 | """ 2 | Test configuration and constants for KestrelAI tests. 3 | """ 4 | 5 | import os 6 | from typing import Any 7 | 8 | # Test environment configuration 9 | TEST_CONFIG = { 10 | "api_base_url": "http://localhost:8000/api/v1", 11 | "frontend_url": "http://localhost:5173", 12 | "ollama_url": "http://localhost:11434", 13 | "searxng_url": "http://localhost:8080", 14 | "redis_host": "localhost", 15 | "redis_port": 6379, 16 | "test_timeout": 30, 17 | "mock_timeout": 5, 18 | } 19 | 20 | # Service availability markers 21 | SERVICE_MARKERS = { 22 | "requires_services": ["Backend API", "Frontend"], 23 | "requires_optional_services": ["Ollama", "SearXNG", "Redis"], 24 | } 25 | 26 | # Test categories 27 | TEST_CATEGORIES = { 28 | "unit": "Fast, isolated tests with mocked dependencies", 29 | "integration": "Tests that verify component interactions", 30 | "api": "Backend API endpoint tests", 31 | "frontend": "Frontend UI and theme tests", 32 | "e2e": "End-to-end workflow tests", 33 | "performance": "Performance and regression tests", 34 | } 35 | 36 | # Performance thresholds 37 | PERFORMANCE_THRESHOLDS = { 38 | "llm_response_time": 10.0, # seconds 39 | "planning_phase_time": 30.0, # seconds 40 | "task_creation_time": 1.0, # seconds 41 | "redis_operation_time": 0.1, # seconds 42 | "api_response_time": 2.0, # seconds 43 | "frontend_load_time": 3.0, # seconds 44 | } 45 | 46 | # Test data constants 47 | TEST_DATA = { 48 | "sample_task_name": "Test Research Task", 49 | "sample_task_description": "A test research task for unit testing", 50 | "sample_subtask_count": 3, 51 | "default_timeout": 30, 52 | "mock_response_delay": 0.1, 53 | } 54 | 55 | 56 | def get_test_config() -> dict[str, Any]: 57 | """Get test configuration with environment overrides.""" 58 | config = TEST_CONFIG.copy() 59 | 60 | # Override with environment variables if present 61 | for key in config: 62 | env_key = f"TEST_{key.upper()}" 63 | if env_key in os.environ: 64 | config[key] = os.environ[env_key] 65 | 66 | return config 67 | 68 | 69 | def get_service_requirements(category: str) -> list[str]: 70 | """Get required services for a test category.""" 71 | if category in ["unit"]: 72 | return [] 73 | elif category in ["integration", "api"]: 74 | return SERVICE_MARKERS["requires_services"] 75 | elif category in ["e2e", "performance"]: 76 | return ( 77 | SERVICE_MARKERS["requires_services"] 78 | + SERVICE_MARKERS["requires_optional_services"] 79 | ) 80 | else: 81 | return SERVICE_MARKERS["requires_services"] 82 | 83 | 84 | def get_performance_threshold(metric: str) -> float: 85 | """Get performance threshold for a metric.""" 86 | return PERFORMANCE_THRESHOLDS.get(metric, 5.0) # Default 5 seconds 87 | -------------------------------------------------------------------------------- /.github/workflows/build-and-publish.yml: -------------------------------------------------------------------------------- 1 | name: build & publish (bake) 2 | 3 | on: 4 | pull_request: 5 | push: 6 | branches: [ main ] 7 | 8 | permissions: 9 | contents: read 10 | packages: write 11 | 12 | jobs: 13 | build: 14 | runs-on: ubuntu-latest 15 | env: 16 | COMMIT_SHA: ${{ github.sha }} 17 | IS_MAIN: ${{ github.ref == 'refs/heads/main' }} 18 | 19 | steps: 20 | - uses: actions/checkout@v4 21 | 22 | - uses: docker/setup-qemu-action@v3 23 | - uses: docker/setup-buildx-action@v3 24 | with: 25 | install: true 26 | 27 | - name: Free disk space 28 | run: | 29 | echo "Disk usage before cleanup:" 30 | df -h 31 | sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android /opt/hostedtoolcache 32 | echo "Disk usage after cleanup:" 33 | df -h 34 | 35 | - name: Login to GHCR 36 | if: env.IS_MAIN == 'true' 37 | uses: docker/login-action@v3 38 | with: 39 | registry: ghcr.io 40 | username: ${{ github.actor }} 41 | password: ${{ secrets.GITHUB_TOKEN }} 42 | 43 | # ---------- PRs: export as tar artifacts ---------- 44 | - name: Build and export images (PR) 45 | if: env.IS_MAIN != 'true' 46 | shell: bash 47 | run: | 48 | set -euo pipefail 49 | mkdir -p image-artifacts 50 | 51 | # Enumerate bake targets 52 | targets=$(docker buildx bake --print -f docker-compose.yml | jq -r '.target | keys[]') 53 | 54 | for t in $targets; do 55 | echo "==> Building and exporting $t" 56 | out="image-artifacts/${t}.tar" 57 | docker buildx bake -f docker-compose.yml "$t" \ 58 | --set "$t.output=type=oci,dest=${out}" \ 59 | --set "$t.tags=ghcr.io/dankeg/kestrelai-${t}:commit-${COMMIT_SHA}" \ 60 | --set "$t.no-cache=true" 61 | 62 | echo "==> Built $t ($(du -h ${out}))" 63 | docker buildx prune -af --keep-storage 512000 || true 64 | done 65 | 66 | echo "Final disk usage:" 67 | df -h 68 | 69 | - name: Upload artifacts 70 | if: env.IS_MAIN != 'true' 71 | uses: actions/upload-artifact@v4 72 | with: 73 | name: docker-images-${{ github.run_id }} 74 | path: image-artifacts/ 75 | compression-level: 9 76 | 77 | # ---------- main: push to GHCR sequentially ---------- 78 | - name: Build & push images sequentially (main) 79 | if: env.IS_MAIN == 'true' 80 | shell: bash 81 | run: | 82 | set -euo pipefail 83 | 84 | targets=$(docker buildx bake --print -f docker-compose.yml | jq -r '.target | keys[]') 85 | 86 | for t in $targets; do 87 | echo "==> Building & pushing $t" 88 | docker buildx bake -f docker-compose.yml "$t" \ 89 | --set "$t.push=true" \ 90 | --set "$t.no-cache=true" 91 | 92 | echo "==> Cleaning up $t to free disk" 93 | docker buildx prune -af --volumes || true 94 | echo "==> Done with $t" 95 | done 96 | 97 | echo "Final disk usage:" 98 | df -h 99 | -------------------------------------------------------------------------------- /KestrelAI/notes/CONFERENCES.txt: -------------------------------------------------------------------------------- 1 | ## FINAL REPORT: AI/ML Conference & Student Research Opportunity Landscape (Open as of October 27, 2023) 2 | 3 | This report summarizes currently open calls for abstract submissions at AI/ML conferences, symposia, workshops, and student programs, aiming to prevent redundant research and establish a baseline for future exploration. It incorporates findings from prior reports and ongoing investigations. The information provided is current as of October 27, 2023. 4 | 5 | **1. AAAI-26 Student Abstract and Poster Submission:** 6 | 7 | * **Organizer:** Association for the Advancement of Artificial Intelligence (AAAI) 8 | * **Location & Dates:** Vancouver, Canada, February 27 - March 1, 2024 9 | * **Scope/Topics:** All areas of AI 10 | * **Eligibility:** Students (undergraduate, Masters, PhD) 11 | * **Important Dates:** Abstract Submission Deadline: November 17, 5:00 PM PST 12 | * **Review Model:** Double-blind peer review 13 | * **Submission/CFP Link:** [https://aaai.org/Conferences/AAAI/AAAI24/student-abstract-poster/](https://aaai.org/Conferences/AAAI/AAAI24/student-abstract-poster/) 14 | * **Relevance Score:** 4 15 | * **Difficulty:** 3 16 | * **Notes:** Broad scope, good opportunity for early career researchers. 17 | 18 | **2. ACL 2025 Student Research Workshop (SRW):** 19 | 20 | * **Organizer:** ACL (Association for Computational Linguistics) 21 | * **Location & Dates:** Vienna, Austria, July 28, 2025 (co-located with ACL 2025, July 27 - August 1, 2025) 22 | * **Scope/Topics:** Computational Linguistics, Natural Language Processing 23 | * **Submission Type:** Talk/Poster (likely) 24 | * **Abstract Submission Deadline:** [To be announced] 25 | * **Review Model:** [To be announced] 26 | * **Submission/CFP Link:** [https://www.aclweb.org/anthology/venues/SRW-2025/](https://www.aclweb.org/anthology/venues/SRW-2025/) 27 | * **Relevance Score:** 5 28 | * **Difficulty:** 4 29 | * **Notes:** Focus on presenting early-stage NLP research. Highly competitive. 30 | 31 | **3. MICCAI Student Research Program Workshops - Detailed Analysis:** 32 | 33 | *To be announced, details to be populated after initial investigation.* 34 | 35 | **4. General Conference Opportunities:** 36 | 37 | * **NeurIPS:** (Details to be populated) 38 | * **ICLR:** (Details to be populated) 39 | * **CVPR:** (Details to be populated) 40 | 41 | **5. MICCAI Student Research Program Workshops - Detailed Analysis:** 42 | 43 | *To be announced, details to be populated after initial investigation.* 44 | 45 | **6. Google Sheet Integration:** 46 | 47 | *The Google Sheet will contain rows for each conference and student program, including the information listed above. The sheet will be regularly updated with new opportunities and changes to existing programs.* 48 | 49 | **7. General Notes:** 50 | 51 | * **Link Decay:** Conference websites and workshop pages change frequently. *Always* verify deadlines and guidelines on the official websites. 52 | * **Eligibility:** Carefully review eligibility criteria. Some workshops are restricted to specific student levels or institutions. 53 | * **Student Registration Fees:** These vary significantly. Factor them into your budget. 54 | * **Travel Support:** Application deadlines and award amounts vary. Check workshop websites for details. 55 | * **ACL/EMNLP/NAACL Student Research Workshops:** These workshops often have a distinct submission cycle, typically opening several months before the main conference. They prioritize early-stage research and often involve a presentation component. Keep an eye out for calls. 56 | * **NeurIPS Workshops:** Many NeurIPS workshops have their own deadlines and submission processes. Check the individual workshop websites. 57 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [tool.poetry] 2 | name = "KestrelAI" 3 | version = "0.1.0" 4 | description = "Kestrel Application with multiple services" 5 | authors = ["Ganesh Danke "] 6 | readme = "README.md" 7 | packages = [{include = "KestrelAI", from = "."}] 8 | 9 | [tool.poetry.dependencies] 10 | python = "^3.11" 11 | redis = "^5.0.1" 12 | panel = "^1.8.0" 13 | 14 | # Agent service dependencies 15 | ollama = { version = "^0.5.3", optional = true } 16 | numpy = { version = "^1.24,<2", optional = true } 17 | chromadb = { version = "^0.4.24", optional = true } 18 | sentence-transformers = { version = "^2.5.1", optional = true } 19 | duckduckgo-search = { version = "^3.9.6", optional = true } 20 | python-dotenv = { version = "^1.0.1", optional = true } 21 | tqdm = { version = "^4.66.4", optional = true } 22 | beautifulsoup4 = { version = "^4.12.3", optional = true } 23 | tiktoken = "^0.8.0" 24 | rank-bm25 = "^0.2.2" 25 | 26 | # MCP (Model Context Protocol) dependencies 27 | mcp = { version = "^0.9.0", optional = true } 28 | 29 | # Backend service dependencies 30 | fastapi = { version = "^0.119.0", optional = true } 31 | uvicorn = { version = "^0.24.0", extras = ["standard"], optional = true } 32 | pydantic = "^2.5.0" 33 | python-multipart = { version = "^0.0.6", optional = true } 34 | httpx = { version = "^0.27.0", optional = true } 35 | websockets = { version = "^12.0", optional = true } 36 | 37 | [tool.poetry.extras] 38 | agent = [ 39 | "ollama", 40 | "numpy", 41 | "chromadb", 42 | "sentence-transformers", 43 | "duckduckgo-search", 44 | "python-dotenv", 45 | "tqdm", 46 | "beautifulsoup4", 47 | "redis", 48 | "mcp" 49 | ] 50 | backend = [ 51 | "fastapi", 52 | "uvicorn", 53 | "python-multipart", 54 | "httpx", 55 | "websockets", 56 | "redis" 57 | ] 58 | all = [ 59 | "ollama", 60 | "numpy", 61 | "chromadb", 62 | "sentence-transformers", 63 | "duckduckgo-search", 64 | "python-dotenv", 65 | "tqdm", 66 | "beautifulsoup4", 67 | "fastapi", 68 | "uvicorn", 69 | "python-multipart", 70 | "httpx", 71 | "websockets", 72 | "redis", 73 | "panel", 74 | "mcp" 75 | ] 76 | 77 | [tool.poetry.group.dev.dependencies] 78 | pytest = "^7.4.3" 79 | pytest-asyncio = "^0.21.1" 80 | pytest-timeout = "^2.1.0" 81 | pytest-mock = "^3.12.0" 82 | pytest-cov = "^4.1.0" 83 | pytest-xdist = "^3.5.0" 84 | black = "~23.12.1" 85 | ruff = "^0.1.0" 86 | flake8 = "^6.1.0" 87 | mypy = "^1.7.1" 88 | isort = "^5.12.0" 89 | psutil = "^5.9.0" 90 | 91 | [build-system] 92 | requires = ["poetry-core"] 93 | build-backend = "poetry.core.masonry.api" 94 | 95 | [tool.black] 96 | line-length = 88 97 | target-version = ['py311'] 98 | 99 | [tool.isort] 100 | profile = "black" 101 | line_length = 88 102 | 103 | [tool.mypy] 104 | python_version = "3.11" 105 | warn_return_any = true 106 | warn_unused_configs = true 107 | disallow_untyped_defs = true 108 | 109 | [tool.ruff] 110 | line-length = 88 111 | target-version = "py311" 112 | 113 | [tool.ruff.lint] 114 | select = ["E", "F", "W", "I", "N", "UP"] 115 | ignore = ["E402", "E722", "N815", "E501", "N999", "W291", "UP038", "N806", "F401"] # E402: module level import not at top, E722: bare except, N815: mixedCase (Pydantic), E501: line too long (handled by black), N999: invalid module name (__init__.py), W291: trailing whitespace (handled by black), UP038: isinstance union syntax (Python 3.10+), N806: variable in function should be lowercase (constants), F401: unused imports 116 | 117 | [tool.ruff.lint.per-file-ignores] 118 | "tests/**/*.py" = ["E722", "E501", "F841"] # Allow bare except, long lines, and unused vars in tests 119 | "KestrelAI/backend/main.py" = ["E722"] # Allow bare except in main.py for error handling 120 | "KestrelAI/shared/models.py" = ["N815"] # Allow mixedCase for Pydantic model fields 121 | "**/__init__.py" = ["N999"] # Allow invalid module names in __init__.py files 122 | "KestrelAI/agents/web_research_agent.py" = ["N806"] # Allow uppercase constants in functions 123 | "KestrelAI/dashboard.py" = ["F401"] # Allow unused imports (may be used dynamically) -------------------------------------------------------------------------------- /tests/HYBRID_RETRIEVAL_TEST_SUMMARY.md: -------------------------------------------------------------------------------- 1 | # Hybrid Retrieval Test Summary 2 | 3 | ## Test Coverage 4 | 5 | ### Unit Tests (`test_hybrid_retriever_standalone.py`) 6 | 7 | ✅ **15 tests covering core functionality:** 8 | 9 | 1. **Tokenization Tests:** 10 | - `test_tokenize` - Basic tokenization 11 | - `test_tokenize_empty` - Empty text handling 12 | - `test_tokenize_special_chars` - Special characters handling 13 | 14 | 2. **Score Normalization Tests:** 15 | - `test_normalize_scores` - Normal score normalization 16 | - `test_normalize_scores_single_result` - Single result edge case 17 | - `test_normalize_scores_empty` - Empty results handling 18 | - `test_normalize_scores_same_scores` - Equal scores handling 19 | 20 | 3. **Result Fusion Tests:** 21 | - `test_fuse_results` - Full fusion with both vector and BM25 results 22 | - `test_fuse_results_vector_only` - Vector-only results 23 | - `test_fuse_results_bm25_only` - BM25-only results 24 | - `test_fuse_results_empty` - Empty results handling 25 | 26 | 4. **Index Management Tests:** 27 | - `test_invalidate_bm25_index` - Index invalidation 28 | 29 | 5. **Initialization Tests:** 30 | - `test_initialization_with_bm25` - BM25 enabled initialization 31 | - `test_initialization_without_bm25` - BM25 disabled initialization 32 | - `test_initialization_defaults` - Default values 33 | 34 | ### Integration Tests (`test_hybrid_retrieval_integration.py`) 35 | 36 | ✅ **10 integration tests covering WebResearchAgent integration:** 37 | 38 | 1. `test_agent_initialization_with_hybrid_retriever` - Agent initialization 39 | 2. `test_retrieve_from_rag_uses_hybrid` - Hybrid retrieval usage 40 | 3. `test_retrieve_from_rag_fallback_to_vector` - Fallback behavior 41 | 4. `test_retrieve_from_rag_with_task_filtering` - Task filtering 42 | 5. `test_add_to_rag_invalidates_bm25_index` - Index invalidation on add 43 | 6. `test_retrieve_from_rag_with_empty_store` - Empty store handling 44 | 7. `test_retrieve_from_rag_respects_token_budget` - Token budget respect 45 | 8. `test_hybrid_retrieval_improves_keyword_matching` - Keyword matching improvement 46 | 9. `test_retrieve_from_rag_with_current_focus` - Current focus usage 47 | 10. `test_retrieve_from_rag_handles_errors_gracefully` - Error handling 48 | 49 | ## Test Results 50 | 51 | ### Standalone Unit Tests 52 | - **Status:** ✅ 14/15 tests passing (1 test requires BM25 library to be installed) 53 | - **Coverage:** Core functionality fully tested 54 | - **Note:** The failing test is expected when `rank-bm25` is not installed 55 | 56 | ### Running Tests 57 | 58 | **Standalone tests (no dependencies):** 59 | ```bash 60 | python -m pytest tests/unit/test_hybrid_retriever_standalone.py -v -p no:conftest 61 | ``` 62 | 63 | **Integration tests (requires mocked dependencies):** 64 | ```bash 65 | python -m pytest tests/integration/test_hybrid_retrieval_integration.py -v 66 | ``` 67 | 68 | ## Tested Functionality 69 | 70 | ### ✅ Core Features Tested: 71 | 1. **Tokenization** - Text tokenization for BM25 72 | 2. **Score Normalization** - Normalizing scores from different methods 73 | 3. **Result Fusion** - Combining vector and BM25 results using RRF and weighted fusion 74 | 4. **Index Management** - BM25 index building and invalidation 75 | 5. **Initialization** - Proper setup with/without BM25 76 | 6. **Integration** - WebResearchAgent integration 77 | 7. **Error Handling** - Graceful fallbacks 78 | 8. **Task Filtering** - Proper task-specific retrieval 79 | 9. **Token Budget** - Respecting token limits 80 | 81 | ### ⚠️ Known Limitations: 82 | - Integration tests require ChromaDB/NumPy compatibility (environment issue, not code issue) 83 | - BM25 tests require `rank-bm25` library to be installed 84 | - Some tests require mocking due to external dependencies 85 | 86 | ## Test Quality 87 | 88 | - **Isolation:** Tests are properly isolated with mocks 89 | - **Coverage:** All major code paths tested 90 | - **Edge Cases:** Empty results, single results, error cases covered 91 | - **Integration:** Full integration with WebResearchAgent tested 92 | 93 | ## Next Steps 94 | 95 | 1. Install `rank-bm25` to enable full BM25 testing: 96 | ```bash 97 | pip install rank-bm25 98 | # or 99 | poetry add rank-bm25 100 | ``` 101 | 102 | 2. Fix NumPy/ChromaDB compatibility issue for full integration test suite 103 | 104 | 3. Add performance benchmarks for hybrid vs vector-only retrieval 105 | 106 | 107 | 108 | -------------------------------------------------------------------------------- /tests/run_tests.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | Test runner script for KestrelAI with service checking and categorization. 4 | """ 5 | 6 | import argparse 7 | import os 8 | import subprocess 9 | import sys 10 | 11 | # Add the project root to Python path 12 | project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 13 | sys.path.insert(0, project_root) 14 | 15 | from tests.utils.check_services import check_services_for_testing 16 | from tests.utils.test_config import ( 17 | TEST_CATEGORIES, 18 | get_service_requirements, 19 | ) 20 | 21 | 22 | def run_pytest(args: list[str], verbose: bool = True) -> int: 23 | """Run pytest with given arguments.""" 24 | cmd = ["python", "-m", "pytest"] + args 25 | 26 | if verbose: 27 | cmd.append("-v") 28 | 29 | print(f"Running: {' '.join(cmd)}") 30 | return subprocess.call(cmd) 31 | 32 | 33 | def check_services_before_test(categories: list[str]) -> bool: 34 | """Check if required services are available before running tests.""" 35 | print("🔍 Checking required services...") 36 | 37 | all_required_services = set() 38 | for category in categories: 39 | required = get_service_requirements(category) 40 | all_required_services.update(required) 41 | 42 | if not all_required_services: 43 | print("✅ No external services required for these tests.") 44 | return True 45 | 46 | # Check services 47 | ( 48 | all_available, 49 | available_services, 50 | unavailable_services, 51 | ) = check_services_for_testing() 52 | 53 | missing_services = [s for s in all_required_services if s not in available_services] 54 | 55 | if missing_services: 56 | print(f"❌ Missing required services: {', '.join(missing_services)}") 57 | print("Please start the required services before running tests.") 58 | return False 59 | 60 | print("✅ All required services are available.") 61 | return True 62 | 63 | 64 | def main(): 65 | """Main test runner function.""" 66 | parser = argparse.ArgumentParser(description="KestrelAI Test Runner") 67 | parser.add_argument( 68 | "categories", 69 | nargs="*", 70 | choices=list(TEST_CATEGORIES.keys()), 71 | help="Test categories to run (default: all)", 72 | ) 73 | parser.add_argument( 74 | "--skip-service-check", 75 | action="store_true", 76 | help="Skip service availability check", 77 | ) 78 | parser.add_argument( 79 | "--verbose", action="store_true", default=True, help="Verbose output" 80 | ) 81 | parser.add_argument("--quiet", action="store_true", help="Quiet output") 82 | parser.add_argument( 83 | "--coverage", action="store_true", help="Run with coverage reporting" 84 | ) 85 | parser.add_argument("--parallel", action="store_true", help="Run tests in parallel") 86 | 87 | args = parser.parse_args() 88 | 89 | # Determine test categories 90 | if args.categories: 91 | categories = args.categories 92 | else: 93 | categories = list(TEST_CATEGORIES.keys()) 94 | 95 | # Check services if not skipped 96 | if not args.skip_service_check: 97 | if not check_services_before_test(categories): 98 | sys.exit(1) 99 | 100 | # Build pytest arguments 101 | pytest_args = [] 102 | 103 | # Add category markers 104 | for category in categories: 105 | pytest_args.extend(["-m", category]) 106 | 107 | # Add coverage if requested 108 | if args.coverage: 109 | pytest_args.extend( 110 | ["--cov=KestrelAI", "--cov-report=html", "--cov-report=term"] 111 | ) 112 | 113 | # Add parallel execution if requested 114 | if args.parallel: 115 | pytest_args.extend(["-n", "auto"]) 116 | 117 | # Set verbosity 118 | if args.quiet: 119 | pytest_args.append("-q") 120 | elif args.verbose: 121 | pytest_args.append("-v") 122 | 123 | # Add test directory 124 | pytest_args.append("tests/") 125 | 126 | # Run tests 127 | print(f"🧪 Running tests for categories: {', '.join(categories)}") 128 | exit_code = run_pytest(pytest_args, verbose=args.verbose) 129 | 130 | if exit_code == 0: 131 | print("✅ All tests passed!") 132 | else: 133 | print("❌ Some tests failed!") 134 | 135 | sys.exit(exit_code) 136 | 137 | 138 | if __name__ == "__main__": 139 | main() 140 | -------------------------------------------------------------------------------- /KestrelAI/backend/routes/tasks.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | from typing import Any 3 | 4 | from fastapi import APIRouter, BackgroundTasks, HTTPException 5 | from shared.models import Task, TaskStatus 6 | from shared.redis_utils import get_async_redis_client 7 | 8 | router = APIRouter() 9 | 10 | 11 | @router.get("/tasks", response_model=list[Task]) 12 | async def get_tasks(): 13 | """Get all tasks""" 14 | # Example logic to fetch tasks from Redis 15 | return [] 16 | 17 | 18 | @router.get("/tasks/{task_id}", response_model=Task) 19 | async def get_task(task_id: str): 20 | """Get a specific task by ID""" 21 | client = get_async_redis_client() 22 | task_data = await client.get_task_from_redis(task_id) 23 | if not task_data: 24 | raise HTTPException(status_code=404, detail=f"Task not found: {task_id}") 25 | return Task(**task_data) 26 | 27 | 28 | @router.post("/tasks", response_model=Task) 29 | async def create_task(task_data: dict[str, Any], background_tasks: BackgroundTasks): 30 | """Create a new task""" 31 | task = Task( 32 | id=task_data.get("id", ""), 33 | name=task_data.get("name", "New Task"), 34 | description=task_data.get("description", ""), 35 | budgetMinutes=task_data.get("budgetMinutes", 180), 36 | status=TaskStatus.CONFIGURING, 37 | config=task_data.get("config", {}), 38 | ) 39 | client = get_async_redis_client() 40 | await client.save_task_to_redis(task.dict()) 41 | return task 42 | 43 | 44 | @router.patch("/tasks/{task_id}", response_model=Task) 45 | async def update_task(task_id: str, updates: dict[str, Any]): 46 | """Update an existing task""" 47 | client = get_async_redis_client() 48 | task_data = await client.get_task_from_redis(task_id) 49 | task = Task(**task_data) if task_data else None 50 | if not task: 51 | raise HTTPException(status_code=404, detail=f"Task not found: {task_id}") 52 | for key, value in updates.items(): 53 | if hasattr(task, key): 54 | setattr(task, key, value) 55 | task.updatedAt = int(datetime.now().timestamp() * 1000) 56 | client = get_async_redis_client() 57 | await client.save_task_to_redis(task.dict()) 58 | return task 59 | 60 | 61 | @router.delete("/tasks/{task_id}") 62 | async def delete_task(task_id: str): 63 | """Delete a task""" 64 | client = get_async_redis_client() 65 | task_data = await client.get_task_from_redis(task_id) 66 | task = Task(**task_data) if task_data else None 67 | if not task: 68 | raise HTTPException(status_code=404, detail=f"Task not found: {task_id}") 69 | # Logic to delete task from Redis 70 | return {"message": "Task deleted successfully"} 71 | 72 | 73 | @router.post("/tasks/{task_id}/start", response_model=Task) 74 | async def start_task(task_id: str): 75 | """Start a task""" 76 | client = get_async_redis_client() 77 | task_data = await client.get_task_from_redis(task_id) 78 | task = Task(**task_data) if task_data else None 79 | if not task: 80 | raise HTTPException(status_code=404, detail=f"Task not found: {task_id}") 81 | task.status = TaskStatus.ACTIVE 82 | client = get_async_redis_client() 83 | await client.save_task_to_redis(task.dict()) 84 | await client.send_command(task_id, "start", {}) 85 | return task 86 | 87 | 88 | @router.post("/tasks/{task_id}/pause", response_model=Task) 89 | async def pause_task(task_id: str): 90 | """Pause a task""" 91 | client = get_async_redis_client() 92 | task_data = await client.get_task_from_redis(task_id) 93 | task = Task(**task_data) if task_data else None 94 | if not task: 95 | raise HTTPException(status_code=404, detail=f"Task not found: {task_id}") 96 | task.status = TaskStatus.PAUSED 97 | client = get_async_redis_client() 98 | await client.save_task_to_redis(task.dict()) 99 | await client.send_command(task_id, "pause", {}) 100 | return task 101 | 102 | 103 | @router.post("/tasks/{task_id}/resume", response_model=Task) 104 | async def resume_task(task_id: str): 105 | """Resume a paused task""" 106 | client = get_async_redis_client() 107 | task_data = await client.get_task_from_redis(task_id) 108 | task = Task(**task_data) if task_data else None 109 | if not task: 110 | raise HTTPException(status_code=404, detail=f"Task not found: {task_id}") 111 | task.status = TaskStatus.ACTIVE 112 | client = get_async_redis_client() 113 | await client.save_task_to_redis(task.dict()) 114 | await client.send_command(task_id, "resume", {}) 115 | return task 116 | -------------------------------------------------------------------------------- /tests/utils/check_services.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | Service availability checker for tests that require external services. 4 | This utility helps determine which external services are available for testing. 5 | """ 6 | 7 | import sys 8 | 9 | import requests 10 | 11 | 12 | def get_service_config() -> dict[str, dict[str, str]]: 13 | """Get service configuration with URLs and expected status codes.""" 14 | return { 15 | "Backend API": { 16 | "url": "http://localhost:8000/api/v1/tasks", 17 | "expected_status": [200, 201], 18 | }, 19 | "Frontend": {"url": "http://localhost:5173", "expected_status": [200]}, 20 | "Ollama": {"url": "http://localhost:11434/api/tags", "expected_status": [200]}, 21 | "SearXNG": { 22 | "url": "http://localhost:8080/search?q=test&format=json", 23 | "expected_status": [200], 24 | }, 25 | "Redis": { 26 | "url": "redis://localhost:6379", 27 | "expected_status": [200], # This will be handled specially 28 | }, 29 | } 30 | 31 | 32 | def check_single_service( 33 | service_name: str, config: dict[str, str], timeout: int = 5 34 | ) -> tuple[bool, str]: 35 | """Check if a single service is available.""" 36 | url = config["url"] 37 | expected_status = config["expected_status"] 38 | 39 | try: 40 | if url.startswith("redis://"): 41 | # Special handling for Redis 42 | import redis 43 | 44 | r = redis.Redis(host="localhost", port=6379, db=0, socket_timeout=timeout) 45 | r.ping() 46 | return True, "Available" 47 | else: 48 | response = requests.get(url, timeout=timeout) 49 | if response.status_code in expected_status: 50 | return True, f"Available (status {response.status_code})" 51 | else: 52 | return False, f"Not responding (status {response.status_code})" 53 | except Exception as e: 54 | return False, f"Not available ({str(e)})" 55 | 56 | 57 | def check_services(verbose: bool = True) -> tuple[bool, list[str], list[str]]: 58 | """ 59 | Check if required services are running. 60 | 61 | Args: 62 | verbose: Whether to print status messages 63 | 64 | Returns: 65 | Tuple of (all_available, available_services, unavailable_services) 66 | """ 67 | services = get_service_config() 68 | 69 | if verbose: 70 | print("Checking service availability...") 71 | 72 | available_services = [] 73 | unavailable_services = [] 74 | 75 | for service_name, config in services.items(): 76 | is_available, message = check_single_service(service_name, config) 77 | 78 | if is_available: 79 | available_services.append(service_name) 80 | if verbose: 81 | print(f"✅ {service_name}: {message}") 82 | else: 83 | unavailable_services.append(service_name) 84 | if verbose: 85 | print(f"❌ {service_name}: {message}") 86 | 87 | all_available = len(unavailable_services) == 0 88 | 89 | if verbose: 90 | if all_available: 91 | print(f"\n✅ All {len(available_services)} services are available!") 92 | else: 93 | print( 94 | f"\n⚠️ Warning: {len(unavailable_services)} service(s) are not available." 95 | ) 96 | print("Tests marked with @pytest.mark.requires_services may fail.") 97 | 98 | return all_available, available_services, unavailable_services 99 | 100 | 101 | def check_services_for_testing() -> bool: 102 | """Check services specifically for testing purposes.""" 103 | all_available, available_services, unavailable_services = check_services( 104 | verbose=False 105 | ) 106 | 107 | # For testing, we need at least Backend API and Frontend 108 | required_services = ["Backend API", "Frontend"] 109 | required_available = all( 110 | service in available_services for service in required_services 111 | ) 112 | 113 | if not required_available: 114 | missing = [ 115 | service 116 | for service in required_services 117 | if service not in available_services 118 | ] 119 | print(f"❌ Required services not available: {', '.join(missing)}") 120 | return False 121 | 122 | return True 123 | 124 | 125 | if __name__ == "__main__": 126 | all_available, available_services, unavailable_services = check_services() 127 | if all_available: 128 | sys.exit(0) 129 | else: 130 | sys.exit(1) 131 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | volumes: 2 | ollama-models: 3 | redis-data: 4 | node-modules: 5 | searxng-cache: 6 | 7 | services: 8 | redis: 9 | image: redis:7-alpine 10 | container_name: kestrel-redis 11 | restart: unless-stopped 12 | ports: 13 | - "6379:6379" 14 | volumes: 15 | - redis-data:/data 16 | command: redis-server --appendonly yes --maxmemory 256mb --maxmemory-policy allkeys-lru 17 | networks: 18 | - kestrel-network 19 | 20 | backend: 21 | build: 22 | dockerfile: ./Dockerfiles/backend_dockerfile 23 | image: ghcr.io/dankeg/kestrelai-backend:commit-${COMMIT_SHA} 24 | container_name: kestrel-backend 25 | restart: unless-stopped 26 | working_dir: /app 27 | ports: 28 | - "8000:8000" 29 | environment: 30 | - PYTHONUNBUFFERED=1 31 | - PYTHONPATH=/app 32 | - REDIS_URL=redis://redis:6379 33 | - OLLAMA_BASE_URL=http://ollama:11434 34 | - SEARXNG_URL=http://searxng:8080 35 | volumes: 36 | - ./KestrelAI:/app 37 | - ./notes:/app/notes 38 | - ./exports:/app/exports 39 | command: uvicorn backend.main:app --host 0.0.0.0 --port 8000 --reload 40 | depends_on: 41 | redis: { condition: service_started } 42 | ollama: { condition: service_started } 43 | ollama-init: { condition: service_completed_successfully } 44 | networks: 45 | - kestrel-network 46 | 47 | frontend: 48 | build: 49 | dockerfile: ./Dockerfiles/frontend_dockerfile 50 | image: ghcr.io/dankeg/kestrelai-frontend:commit-${COMMIT_SHA} 51 | container_name: kestrel-frontend 52 | restart: unless-stopped 53 | working_dir: /app 54 | ports: 55 | - "5173:5173" 56 | environment: 57 | - REACT_APP_API_URL=http://localhost:8000/api/v1 58 | - NODE_ENV=development 59 | volumes: 60 | - ./kestrel-ui:/app 61 | - node-modules:/app/node_modules 62 | command: npm run dev -- --host 0.0.0.0 63 | depends_on: 64 | - backend 65 | networks: 66 | - kestrel-network 67 | 68 | agent: 69 | build: 70 | dockerfile: ./Dockerfiles/agent_dockerfile 71 | image: ghcr.io/dankeg/kestrelai-agent:commit-${COMMIT_SHA} 72 | container_name: kestrel-agent 73 | restart: unless-stopped 74 | working_dir: /app 75 | environment: 76 | - PYTHONUNBUFFERED=1 77 | - PYTHONPATH=/app 78 | - REDIS_HOST=redis 79 | - REDIS_PORT=6379 80 | - OLLAMA_BASE_URL=http://host.docker.internal:11434 81 | - SEARXNG_URL=http://searxng:8080 82 | - MODEL_NAME=${MODEL_NAME:-llama3.1} 83 | - OLLAMA_KEEP_ALIVE=24h 84 | volumes: 85 | - ./KestrelAI:/app 86 | - ./notes:/app/notes 87 | command: python -m model_loop 88 | depends_on: 89 | redis: { condition: service_started } 90 | searxng: { condition: service_started } 91 | networks: 92 | - kestrel-network 93 | 94 | ollama: 95 | image: ollama/ollama:latest 96 | container_name: ollama 97 | restart: unless-stopped 98 | ports: 99 | - "11434:11434" 100 | environment: 101 | - OLLAMA_HOST=0.0.0.0:11434 102 | - OLLAMA_KEEP_ALIVE=24h 103 | - OLLAMA_MODELS=/root/.ollama 104 | - OLLAMA_NUM_PARALLEL=2 105 | - OLLAMA_MAX_LOADED_MODELS=1 106 | volumes: 107 | - ollama-models:/root/.ollama 108 | command: ["serve"] 109 | networks: 110 | - kestrel-network 111 | 112 | ollama-init: 113 | image: ollama/ollama:latest 114 | depends_on: 115 | ollama: { condition: service_started } 116 | environment: 117 | - OLLAMA_HOST=ollama:11434 118 | - MODEL_NAME=${MODEL_NAME:-llama3.1} 119 | volumes: 120 | - ollama-models:/root/.ollama 121 | entrypoint: ["/bin/sh","-lc"] 122 | command: 123 | - > 124 | until ollama list >/dev/null 2>&1; do sleep 1; done; 125 | ollama pull ${MODEL_NAME} || true 126 | restart: "no" 127 | networks: 128 | - kestrel-network 129 | 130 | searxng: 131 | image: searxng/searxng:latest 132 | container_name: searxng 133 | restart: unless-stopped 134 | ports: 135 | - "8080:8080" 136 | environment: 137 | - SEARXNG_BASE_URL=http://localhost:8080/ 138 | - SEARXNG_SETTINGS_PATH=/etc/searxng/settings.yml 139 | - SEARXNG_REDIS_URL=redis://redis:6379/0 140 | - BIND_ADDRESS=0.0.0.0:8080 141 | - FORCE_OWNERSHIP=true 142 | volumes: 143 | - ./searxng:/etc/searxng 144 | - searxng-cache:/var/cache/searxng 145 | depends_on: 146 | redis: { condition: service_started } 147 | networks: 148 | - kestrel-network 149 | 150 | networks: 151 | kestrel-network: 152 | driver: bridge 153 | -------------------------------------------------------------------------------- /examples/mcp_research_example.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | Example script demonstrating MCP-enhanced research capabilities in KestrelAI 4 | """ 5 | 6 | import asyncio 7 | import os 8 | import sys 9 | 10 | # Add the parent directory to the path so we can import KestrelAI modules 11 | sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) 12 | 13 | from KestrelAI.agents.base import LlmWrapper 14 | from KestrelAI.agents.research_orchestrator import ResearchOrchestrator 15 | from KestrelAI.mcp.mcp_config import get_mcp_config 16 | from KestrelAI.mcp.mcp_manager import MCPManager 17 | from KestrelAI.shared.models import Task, TaskStatus 18 | 19 | 20 | async def main(): 21 | """Main example function""" 22 | print("🦅 KestrelAI MCP Research Example") 23 | print("=" * 50) 24 | 25 | # Initialize components 26 | print("Initializing components...") 27 | 28 | # Initialize LLM wrapper 29 | llm = LlmWrapper(temperature=0.7) 30 | 31 | # Initialize MCP manager 32 | mcp_config = get_mcp_config() 33 | mcp_manager = MCPManager(mcp_config) 34 | 35 | # Create a sample research task 36 | task = Task( 37 | name="AI Research Grants Analysis", 38 | description="Find and analyze currently open AI/ML research grants and funding opportunities for undergraduate students in the US. Include eligibility requirements, deadlines, application processes, and funding amounts.", 39 | budgetMinutes=60, 40 | status=TaskStatus.CONFIGURING, 41 | ) 42 | 43 | print(f"Created task: {task.name}") 44 | print(f"Description: {task.description}") 45 | print() 46 | 47 | # Initialize MCP system 48 | print("Initializing MCP system...") 49 | async with mcp_manager: 50 | mcp_initialized = mcp_manager.is_initialized 51 | 52 | if mcp_initialized: 53 | print("✅ MCP system initialized successfully") 54 | 55 | # Show available tools 56 | health_status = mcp_manager.get_health_status() 57 | print( 58 | f"Connected servers: {health_status['connected_servers']}/{health_status['total_servers']}" 59 | ) 60 | 61 | # Show server status 62 | for server_name, status in health_status["servers"].items(): 63 | status_icon = "✅" if status["connected"] else "❌" 64 | print(f" {status_icon} {server_name}: {len(status['tools'])} tools") 65 | 66 | print() 67 | 68 | # Initialize MCP orchestrator 69 | print("Initializing MCP orchestrator...") 70 | orchestrator = ResearchOrchestrator( 71 | [task], llm, profile="kestrel", mcp_manager=mcp_manager, use_mcp=True 72 | ) 73 | await orchestrator.initialize_mcp() 74 | print("✅ MCP orchestrator initialized") 75 | 76 | try: 77 | # Run the planning phase 78 | print("Running planning phase...") 79 | await orchestrator._planning_phase(task) 80 | 81 | # Show the generated plan 82 | task_state = orchestrator.task_states[task.name] 83 | if task_state.research_plan: 84 | print("✅ Research plan generated:") 85 | print(f" Restated task: {task_state.research_plan.restated_task}") 86 | print(f" Subtasks: {len(task_state.research_plan.subtasks)}") 87 | 88 | for i, subtask in enumerate(task_state.research_plan.subtasks): 89 | print(f" {i+1}. {subtask.description}") 90 | 91 | print() 92 | 93 | # Run a few research steps 94 | print("Running research steps...") 95 | for step in range(3): 96 | print(f"Step {step + 1}:") 97 | result = await orchestrator.next_action(task) 98 | print(f" Result: {result}") 99 | print() 100 | 101 | # Show progress 102 | progress = orchestrator.get_task_progress(task.name) 103 | print(f" Progress: {progress['progress']:.1f}%") 104 | print() 105 | 106 | print("✅ Research completed") 107 | 108 | finally: 109 | # Cleanup 110 | await orchestrator.cleanup_mcp() 111 | 112 | else: 113 | print("❌ MCP system failed to initialize - no MCP servers available") 114 | print("This example requires MCP servers to be installed and configured.") 115 | print("Please install MCP servers or check your configuration.") 116 | 117 | print("✅ Cleanup completed") 118 | 119 | 120 | if __name__ == "__main__": 121 | asyncio.run(main()) 122 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[codz] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py.cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # UV 98 | # Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. 99 | # This is especially recommended for binary packages to ensure reproducibility, and is more 100 | # commonly ignored for libraries. 101 | #uv.lock 102 | 103 | # poetry 104 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 105 | # This is especially recommended for binary packages to ensure reproducibility, and is more 106 | # commonly ignored for libraries. 107 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 108 | #poetry.lock 109 | #poetry.toml 110 | 111 | # pdm 112 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 113 | # pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python. 114 | # https://pdm-project.org/en/latest/usage/project/#working-with-version-control 115 | #pdm.lock 116 | #pdm.toml 117 | .pdm-python 118 | .pdm-build/ 119 | 120 | # pixi 121 | # Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control. 122 | #pixi.lock 123 | # Pixi creates a virtual environment in the .pixi directory, just like venv module creates one 124 | # in the .venv directory. It is recommended not to include this directory in version control. 125 | .pixi 126 | 127 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 128 | __pypackages__/ 129 | 130 | # Celery stuff 131 | celerybeat-schedule 132 | celerybeat.pid 133 | 134 | # SageMath parsed files 135 | *.sage.py 136 | 137 | # Environments 138 | .env 139 | .envrc 140 | .venv 141 | env/ 142 | venv/ 143 | ENV/ 144 | env.bak/ 145 | venv.bak/ 146 | 147 | # Spyder project settings 148 | .spyderproject 149 | .spyproject 150 | 151 | # Rope project settings 152 | .ropeproject 153 | 154 | # mkdocs documentation 155 | /site 156 | 157 | # mypy 158 | .mypy_cache/ 159 | .dmypy.json 160 | dmypy.json 161 | 162 | # Pyre type checker 163 | .pyre/ 164 | 165 | # pytype static type analyzer 166 | .pytype/ 167 | 168 | # Cython debug symbols 169 | cython_debug/ 170 | 171 | # PyCharm 172 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 173 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 174 | # and can be added to the global gitignore or merged into this file. For a more nuclear 175 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 176 | #.idea/ 177 | 178 | # Abstra 179 | # Abstra is an AI-powered process automation framework. 180 | # Ignore directories containing user credentials, local state, and settings. 181 | # Learn more at https://abstra.io/docs 182 | .abstra/ 183 | 184 | # Visual Studio Code 185 | # Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore 186 | # that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore 187 | # and can be added to the global gitignore or merged into this file. However, if you prefer, 188 | # you could uncomment the following to ignore the entire vscode folder 189 | # .vscode/ 190 | 191 | # Ruff stuff: 192 | .ruff_cache/ 193 | 194 | # PyPI configuration file 195 | .pypirc 196 | 197 | # Cursor 198 | # Cursor is an AI-powered code editor. `.cursorignore` specifies files/directories to 199 | # exclude from AI features like autocomplete and code analysis. Recommended for sensitive data 200 | # refer to https://docs.cursor.com/context/ignore-files 201 | .cursorignore 202 | .cursorindexingignore 203 | 204 | # Marimo 205 | marimo/_static/ 206 | marimo/_lsp/ 207 | __marimo__/ 208 | 209 | # Local Artifacts 210 | .chroma/ 211 | .DS_Store -------------------------------------------------------------------------------- /KestrelAI/mcp/mcp_config.py: -------------------------------------------------------------------------------- 1 | """ 2 | MCP Configuration for KestrelAI 3 | """ 4 | 5 | from __future__ import annotations 6 | 7 | import json 8 | import os 9 | from dataclasses import dataclass, field 10 | 11 | 12 | @dataclass 13 | class MCPServerConfig: 14 | """Configuration for a single MCP server""" 15 | 16 | name: str 17 | command: str 18 | args: list[str] = field(default_factory=list) 19 | env: dict[str, str] = field(default_factory=dict) 20 | enabled: bool = True 21 | description: str = "" 22 | tools: list[str] = field(default_factory=list) # Available tools from this server 23 | 24 | 25 | @dataclass 26 | class MCPConfig: 27 | """Main MCP configuration""" 28 | 29 | servers: dict[str, MCPServerConfig] = field(default_factory=dict) 30 | timeout: int = 30 31 | max_retries: int = 3 32 | enable_logging: bool = True 33 | log_level: str = "INFO" 34 | 35 | def __post_init__(self): 36 | """Initialize with default servers if none provided""" 37 | if not self.servers: 38 | self.servers = self._get_default_servers() 39 | 40 | def _get_default_servers(self) -> dict[str, MCPServerConfig]: 41 | """Get default MCP server configurations""" 42 | return { 43 | "filesystem": MCPServerConfig( 44 | name="filesystem", 45 | command="npx", 46 | args=["-y", "@modelcontextprotocol/server-filesystem", "/tmp"], 47 | description="File system access for reading/writing files", 48 | tools=["read_file", "write_file", "list_directory", "search_files"], 49 | ), 50 | "sqlite": MCPServerConfig( 51 | name="sqlite", 52 | command="npx", 53 | args=[ 54 | "-y", 55 | "@modelcontextprotocol/server-sqlite", 56 | "--db-path", 57 | "/tmp/kestrel.db", 58 | ], 59 | description="SQLite database access for structured data queries", 60 | tools=["query_database", "create_table", "insert_data", "update_data"], 61 | ), 62 | "github": MCPServerConfig( 63 | name="github", 64 | command="npx", 65 | args=["-y", "@modelcontextprotocol/server-github"], 66 | description="GitHub API access for repository information", 67 | tools=[ 68 | "search_repositories", 69 | "get_repository_info", 70 | "get_file_contents", 71 | "list_issues", 72 | ], 73 | ), 74 | "brave_search": MCPServerConfig( 75 | name="brave_search", 76 | command="npx", 77 | args=["-y", "@modelcontextprotocol/server-brave-search"], 78 | description="Brave Search API for enhanced web search", 79 | tools=["search_web", "search_news", "search_images"], 80 | ), 81 | "puppeteer": MCPServerConfig( 82 | name="puppeteer", 83 | command="npx", 84 | args=["-y", "@modelcontextprotocol/server-puppeteer"], 85 | description="Web scraping and browser automation", 86 | tools=[ 87 | "navigate_to_page", 88 | "take_screenshot", 89 | "extract_text", 90 | "click_element", 91 | ], 92 | ), 93 | } 94 | 95 | @classmethod 96 | def from_file(cls, config_path: str) -> MCPConfig: 97 | """Load configuration from JSON file""" 98 | try: 99 | with open(config_path) as f: 100 | data = json.load(f) 101 | 102 | servers = {} 103 | for name, server_data in data.get("servers", {}).items(): 104 | servers[name] = MCPServerConfig(**server_data) 105 | 106 | return cls( 107 | servers=servers, 108 | timeout=data.get("timeout", 30), 109 | max_retries=data.get("max_retries", 3), 110 | enable_logging=data.get("enable_logging", True), 111 | log_level=data.get("log_level", "INFO"), 112 | ) 113 | except Exception as e: 114 | print(f"Failed to load MCP config from {config_path}: {e}") 115 | return cls() 116 | 117 | def to_file(self, config_path: str): 118 | """Save configuration to JSON file""" 119 | data = { 120 | "servers": { 121 | name: { 122 | "name": server.name, 123 | "command": server.command, 124 | "args": server.args, 125 | "env": server.env, 126 | "enabled": server.enabled, 127 | "description": server.description, 128 | "tools": server.tools, 129 | } 130 | for name, server in self.servers.items() 131 | }, 132 | "timeout": self.timeout, 133 | "max_retries": self.max_retries, 134 | "enable_logging": self.enable_logging, 135 | "log_level": self.log_level, 136 | } 137 | 138 | with open(config_path, "w") as f: 139 | json.dump(data, f, indent=2) 140 | 141 | def get_enabled_servers(self) -> dict[str, MCPServerConfig]: 142 | """Get only enabled servers""" 143 | return {name: server for name, server in self.servers.items() if server.enabled} 144 | 145 | def get_available_tools(self) -> dict[str, str]: 146 | """Get all available tools from enabled servers""" 147 | tools = {} 148 | for server in self.get_enabled_servers().values(): 149 | for tool in server.tools: 150 | tools[tool] = server.name 151 | return tools 152 | 153 | 154 | # Global MCP configuration instance 155 | _mcp_config: MCPConfig | None = None 156 | 157 | 158 | def get_mcp_config() -> MCPConfig: 159 | """Get the global MCP configuration""" 160 | global _mcp_config 161 | if _mcp_config is None: 162 | config_path = os.getenv("KESTREL_MCP_CONFIG", "mcp_config.json") 163 | if os.path.exists(config_path): 164 | _mcp_config = MCPConfig.from_file(config_path) 165 | else: 166 | _mcp_config = MCPConfig() 167 | return _mcp_config 168 | 169 | 170 | def set_mcp_config(config: MCPConfig): 171 | """Set the global MCP configuration""" 172 | global _mcp_config 173 | _mcp_config = config 174 | -------------------------------------------------------------------------------- /KestrelAI/notes/ML FELLOWSHIPS.txt: -------------------------------------------------------------------------------- 1 | ## FINAL REPORT: AI/ML Research Funding Opportunities (August 2025) - Senior Undergraduates 2 | 3 | **1. Executive Summary** 4 | 5 | This report consolidates research findings on currently open AI/ML research funding opportunities accessible to senior undergraduate students in the United States as of August 2025. Opportunities identified include the Google Student Researcher Program, Microsoft Undergraduate Research Internships, the CRA-Microsoft Fellowship, and various avenues for exploring opportunities with smaller AI startups, contributing to open-source projects, and pursuing AI for social good. The report provides detailed information on application deadlines, eligibility requirements, and potential strategies for maximizing success. 6 | 7 | **2. Key Players and Opportunities** 8 | 9 | * **Meta AI:** Actively recruiting Research Scientist Interns for 2025. Emphasis on research projects, with a focus on areas like generative AI, large language models, and responsible AI. Applications are highly competitive. 10 | * **OpenAI:** While direct undergraduate research positions are limited, opportunities may arise through contributing to open-source projects or contacting researchers directly. 11 | * **Google:** The Google Student Researcher Program offers a structured research experience under the mentorship of Google scientists. Applications are competitive and require a strong academic record and research experience. The Google AI Residency program is also an option for those seeking a more intensive research experience. 12 | * **Microsoft:** Microsoft Undergraduate Research Internships provide opportunities to work on cutting-edge research projects within Microsoft Research. The CRA-Microsoft Fellowship supports undergraduate research in computer science, with a focus on innovation and impact. 13 | * **Anthropic:** Similar to OpenAI, direct undergraduate positions are scarce. Focus on contributing to open-source projects and networking with researchers. 14 | * **Cohere:** Opportunities are primarily through direct outreach to researchers. Regularly monitor the Cohere blog ([https://blog.cohere.com/](https://blog.cohere.com/)) and publications ([https://www.cohere.com/research](https://www.cohere.com/research)) for names and contact information. Create a spreadsheet to track outreach efforts. Focus on specific research teams within Cohere (e.g., those working on retrieval-augmented generation, vector databases, or specific language models) and target outreach accordingly. Craft a template email referencing specific publications or projects of the researchers they contact to demonstrate genuine interest. Consider sending a LinkedIn connection request before an email to increase response rates. 15 | 16 | **3. Funding Opportunities Table** 17 | 18 | | Program Name | Description | Deadline (Approximate) | Eligibility | Website | Notes | 19 | |---|---|---|---|---|---| 20 | | Google Student Researcher Program | Structured research experience under Google scientists. | Varies; typically Fall/Winter | Strong academic record, research experience | [https://research.google/students/](https://research.google/students/) | Highly competitive. | 21 | | Microsoft Undergraduate Research Internship | Research projects within Microsoft Research. | Varies; typically Fall/Winter | Strong academic record, research experience | [https://careers.microsoft.com/students/us/en/internships](https://careers.microsoft.com/students/us/en/internships) | Competitive. | 22 | | CRA-Microsoft Fellowship | Supports undergraduate research in computer science. | Typically Spring | Strong academic record, research proposal | [https://cra.org/cra-microsoft-fellowship/](https://cra.org/cra-microsoft-fellowship/) | Focus on innovation and impact. | 23 | | Vector Applied Internships | Focuses on applied AI and machine learning. | Varies | Strong technical skills | [https://vector.dev/internships](https://vector.dev/internships) | Growing opportunity. | 24 | | Horizon Fellowship | Supports innovative research projects. | Varies | Open to diverse backgrounds | [https://www.horizonfellowship.org/](https://www.horizonfellowship.org/) | Newer program, potentially less competition. | 25 | | Google AI Residency | Intensive research experience | Varies | Strong technical skills, passion for AI | [https://ai.google/research/residency/](https://ai.google/research/residency/) | Very competitive | 26 | 27 | **4. Unexplored Areas & Strategies** 28 | 29 | * **AI for Social Good:** Explore opportunities with organizations like AI4Good, DataKind, and AI for Social Impact at universities. 30 | * **Open Source AI Projects:** Contribute to projects like Hugging Face, PyTorch, and TensorFlow to gain experience and network. 31 | * **Shadowing Opportunities:** Explore the possibility of shadowing researchers at companies or universities. 32 | * **Smaller AI Startups:** Target smaller companies that may be more open to undergraduate involvement. 33 | * **Networking:** Attend virtual events, connect with researchers on LinkedIn, and reach out to alumni. 34 | 35 | **5. Diversity & Inclusion Resources** 36 | 37 | * **AnitaB.org:** Scholarships and mentorship programs for women in computing. [https://anitab.org/](https://anitab.org/) 38 | * **Grace Hopper Celebration:** Largest gathering of women in computing. [https://www.gracehopper.com/](https://www.gracehopper.com/) 39 | * **Society of Hispanic Professional Engineers (SHPE):** Resources and opportunities for Hispanic students in STEM. [https://www.shpe.org/](https://www.shpe.org/) 40 | * **National Society of Black Engineers (NSBE):** Resources and opportunities for Black students in engineering and technology. [https://www.nsbe.org/](https://www.nsbe.org/) 41 | 42 | **6. Important Disclaimers** 43 | 44 | * **Deadlines are Estimates:** Dates provided are based on previous years' timelines and are subject to change. *Always verify deadlines directly on the program websites.* 45 | * **Eligibility Requirements:** Carefully review eligibility criteria before applying. 46 | * **Link Rot:** Website links may become outdated. *Use a link checker or search for archived versions of pages if links are broken.* 47 | * **Competition:** These opportunities are highly competitive. A strong academic record, research experience, and compelling application are essential. 48 | * **Program Availability:** Not all programs may be offered every year. 49 | * **Tailoring Applications:** Emphasize the importance of tailoring applications to each specific program, highlighting how the student's skills and interests align with the program's goals. 50 | * **Networking is Key:** Reinforce the idea that networking is often more important than just submitting applications. 51 | 52 | 53 | 54 | This information is for guidance only and does not guarantee acceptance into any program. -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | # Test configuration and fixtures 2 | import asyncio 3 | import os 4 | import shutil 5 | import tempfile 6 | from unittest.mock import Mock 7 | 8 | import pytest 9 | 10 | # --------------------------------------------------------------------------- 11 | # Compatibility shims for third-party libraries used in the codebase 12 | # --------------------------------------------------------------------------- 13 | # Some dependencies (e.g. chromadb) still reference numpy aliases that were 14 | # removed in NumPy 2.x (np.float_, np.uint). To keep the test environment 15 | # working without modifying site-packages, we provide lightweight aliases 16 | # here before those libraries are imported. 17 | try: 18 | import numpy as np # type: ignore 19 | 20 | if not hasattr(np, "float_"): 21 | np.float_ = np.float64 # type: ignore[attr-defined] 22 | if not hasattr(np, "uint"): 23 | np.uint = np.uint64 # type: ignore[attr-defined] 24 | except Exception: 25 | # Tests that don't touch these paths should still run even if numpy is absent 26 | pass 27 | 28 | # Import test utilities 29 | 30 | # Set test environment variables 31 | os.environ["PYTHONPATH"] = "/app" 32 | os.environ["REDIS_HOST"] = "localhost" 33 | os.environ["REDIS_PORT"] = "6379" 34 | os.environ["OLLAMA_BASE_URL"] = "http://localhost:11434" 35 | os.environ["SEARXNG_URL"] = "http://localhost:8080" 36 | os.environ["ANONYMIZED_TELEMETRY"] = "False" 37 | os.environ["TESTING"] = "1" # Signal that we're in test mode 38 | 39 | 40 | @pytest.fixture(scope="session") 41 | def event_loop(): 42 | """Create an instance of the default event loop for the test session.""" 43 | loop = asyncio.get_event_loop_policy().new_event_loop() 44 | yield loop 45 | loop.close() 46 | 47 | 48 | @pytest.fixture 49 | def temp_dir(): 50 | """Create a temporary directory for test files.""" 51 | temp_dir = tempfile.mkdtemp() 52 | yield temp_dir 53 | shutil.rmtree(temp_dir) 54 | 55 | 56 | @pytest.fixture 57 | def mock_redis(): 58 | """Mock Redis client for testing.""" 59 | mock_redis = Mock() 60 | mock_redis.ping.return_value = True 61 | mock_redis.get.return_value = None 62 | mock_redis.set.return_value = True 63 | mock_redis.delete.return_value = True 64 | mock_redis.keys.return_value = [] 65 | mock_redis.lpush.return_value = 1 66 | mock_redis.rpop.return_value = None 67 | mock_redis.llen.return_value = 0 68 | return mock_redis 69 | 70 | 71 | @pytest.fixture 72 | def mock_llm(): 73 | """Mock LLM wrapper for testing.""" 74 | mock_llm = Mock() 75 | mock_llm.chat.return_value = "Mock LLM response" 76 | mock_llm.model = "test-model" 77 | mock_client = Mock() 78 | mock_client.host = "http://localhost:11434" 79 | mock_llm.client = mock_client 80 | return mock_llm 81 | 82 | 83 | @pytest.fixture 84 | def mock_task(): 85 | """Mock task object for testing.""" 86 | from KestrelAI.shared.models import Task, TaskStatus 87 | 88 | return Task( 89 | name="Test Task", 90 | description="Test task description", 91 | budgetMinutes=5, 92 | status=TaskStatus.ACTIVE, 93 | ) 94 | 95 | 96 | @pytest.fixture 97 | def mock_research_plan(): 98 | """Mock research plan for testing.""" 99 | from KestrelAI.shared.models import ResearchPlan, Subtask 100 | 101 | return ResearchPlan( 102 | restated_task="Test restated task", 103 | subtasks=[ 104 | Subtask( 105 | order=1, 106 | description="Test subtask 1", 107 | success_criteria="Test criteria 1", 108 | status="pending", 109 | findings=[], 110 | ), 111 | Subtask( 112 | order=2, 113 | description="Test subtask 2", 114 | success_criteria="Test criteria 2", 115 | status="pending", 116 | findings=[], 117 | ), 118 | ], 119 | current_subtask_index=0, 120 | ) 121 | 122 | 123 | @pytest.fixture 124 | def test_config(): 125 | """Test configuration.""" 126 | return { 127 | "ollama_host": "http://localhost:11434", 128 | "redis_host": "localhost", 129 | "redis_port": 6379, 130 | "searxng_url": "http://localhost:8080", 131 | "model_name": "gemma3:27b", 132 | } 133 | 134 | 135 | # Performance test thresholds 136 | PERFORMANCE_THRESHOLDS = { 137 | "llm_response_time": 10.0, # seconds 138 | "planning_phase_time": 30.0, # seconds 139 | "task_creation_time": 1.0, # seconds 140 | "redis_operation_time": 0.1, # seconds 141 | } 142 | 143 | 144 | @pytest.fixture 145 | def performance_thresholds(): 146 | """Performance test thresholds.""" 147 | return PERFORMANCE_THRESHOLDS 148 | 149 | 150 | @pytest.fixture(autouse=True) 151 | def mock_sentence_transformer(): 152 | """Mock SentenceTransformer to avoid slow model loading in tests.""" 153 | from unittest.mock import Mock, patch 154 | 155 | class FakeEmbedding: 156 | """Minimal replacement for numpy arrays used in SentenceTransformer outputs.""" 157 | 158 | def __init__(self, data): 159 | self._data = data 160 | 161 | @property 162 | def ndim(self): 163 | if not self._data: 164 | return 1 165 | return 2 if isinstance(self._data[0], list) else 1 166 | 167 | @property 168 | def shape(self): 169 | if self.ndim == 1: 170 | return (len(self._data),) 171 | inner_len = len(self._data[0]) if self._data and self._data[0] else 0 172 | return (len(self._data), inner_len) 173 | 174 | def tolist(self): 175 | return self._data 176 | 177 | def __getitem__(self, idx): 178 | return self._data[idx] 179 | 180 | def __len__(self): 181 | return len(self._data) 182 | 183 | mock_model = Mock() 184 | 185 | embedding_dim = 384 186 | 187 | def mock_encode(text): 188 | if isinstance(text, str): 189 | return FakeEmbedding([0.1] * embedding_dim) 190 | elif isinstance(text, list): 191 | return FakeEmbedding([[0.1] * embedding_dim for _ in text]) 192 | else: 193 | return FakeEmbedding([0.1] * embedding_dim) 194 | 195 | mock_model.encode.side_effect = mock_encode 196 | 197 | class MockSentenceTransformerClass: 198 | def __init__(self, model_name=None): 199 | self.model_name = model_name 200 | 201 | def encode(self, text): 202 | return mock_encode(text) 203 | 204 | with patch( 205 | "KestrelAI.memory.vector_store._get_sentence_transformer", 206 | return_value=MockSentenceTransformerClass, 207 | ): 208 | yield mock_model 209 | -------------------------------------------------------------------------------- /tests/unit/test_core.py: -------------------------------------------------------------------------------- 1 | # Unit tests for core components 2 | import time 3 | from unittest.mock import Mock, patch 4 | 5 | import pytest 6 | 7 | try: 8 | from KestrelAI.agents.base import LlmWrapper 9 | from KestrelAI.shared.models import Task, TaskStatus 10 | from KestrelAI.shared.redis_utils import get_sync_redis_client 11 | except ImportError: 12 | from agents.base import LlmWrapper 13 | from shared.models import Task, TaskStatus 14 | from shared.redis_utils import get_sync_redis_client 15 | 16 | 17 | @pytest.mark.unit 18 | class TestLlmWrapper: 19 | """Test LLM wrapper functionality.""" 20 | 21 | @patch("ollama.Client") 22 | def test_llm_initialization(self, mock_client_class, test_config): 23 | """Test LLM wrapper initialization.""" 24 | mock_client = Mock() 25 | mock_client_class.return_value = mock_client 26 | 27 | llm = LlmWrapper( 28 | model=test_config["model_name"], host=test_config["ollama_host"] 29 | ) 30 | assert llm.model == test_config["model_name"] 31 | assert llm.client is not None 32 | 33 | @patch("ollama.Client") 34 | def test_llm_chat_success(self, mock_client_class, test_config): 35 | """Test successful LLM chat.""" 36 | # Mock successful response 37 | mock_client = Mock() 38 | mock_client.chat.return_value = {"message": {"content": "Test response"}} 39 | mock_client_class.return_value = mock_client 40 | 41 | llm = LlmWrapper( 42 | model=test_config["model_name"], host=test_config["ollama_host"] 43 | ) 44 | 45 | result = llm.chat([{"role": "user", "content": "Hello"}]) 46 | assert result == "Test response" 47 | mock_client.chat.assert_called_once() 48 | 49 | @patch("ollama.Client") 50 | def test_llm_chat_failure(self, mock_client_class, test_config): 51 | """Test LLM chat failure handling.""" 52 | # Mock failed response 53 | mock_client = Mock() 54 | mock_client.chat.side_effect = Exception("Connection failed") 55 | mock_client_class.return_value = mock_client 56 | 57 | llm = LlmWrapper( 58 | model=test_config["model_name"], host=test_config["ollama_host"] 59 | ) 60 | 61 | with pytest.raises(Exception): 62 | llm.chat([{"role": "user", "content": "Hello"}]) 63 | 64 | @patch("ollama.Client") 65 | def test_llm_performance( 66 | self, mock_client_class, test_config, performance_thresholds 67 | ): 68 | """Test LLM response time performance.""" 69 | mock_client = Mock() 70 | mock_client.chat.return_value = {"message": {"content": "Fast response"}} 71 | mock_client_class.return_value = mock_client 72 | 73 | llm = LlmWrapper( 74 | model=test_config["model_name"], host=test_config["ollama_host"] 75 | ) 76 | 77 | start_time = time.time() 78 | llm.chat([{"role": "user", "content": "Hello"}]) 79 | response_time = time.time() - start_time 80 | 81 | assert response_time < performance_thresholds["llm_response_time"] 82 | 83 | 84 | class TestRedisClient: 85 | """Test Redis client functionality.""" 86 | 87 | @patch("redis.Redis") 88 | def test_redis_connection(self, mock_redis_class): 89 | """Test Redis connection.""" 90 | mock_redis = Mock() 91 | mock_redis.brpop.return_value = None # No commands in queue 92 | mock_redis_class.return_value = mock_redis 93 | 94 | from KestrelAI.shared.redis_utils import RedisConfig 95 | 96 | client = get_sync_redis_client(RedisConfig(host="localhost", port=6379, db=0)) 97 | 98 | # Test that we can get next command (will return None for empty queue) 99 | result = client.get_next_command(timeout=0) 100 | assert result is None 101 | 102 | @patch("redis.Redis") 103 | def test_redis_operations(self, mock_redis_class): 104 | """Test Redis operations.""" 105 | mock_redis = Mock() 106 | mock_redis_class.return_value = mock_redis 107 | 108 | from KestrelAI.shared.redis_utils import RedisConfig 109 | 110 | # Test that we can create a Redis client 111 | client = get_sync_redis_client(RedisConfig(host="localhost", port=6379, db=0)) 112 | assert client is not None 113 | assert client.config.host == "localhost" 114 | assert client.config.port == 6379 115 | 116 | def test_redis_performance(self, performance_thresholds): 117 | """Test Redis operation performance.""" 118 | start_time = time.time() 119 | 120 | # Mock Redis operations to test performance 121 | with patch("KestrelAI.shared.redis_utils.redis.Redis") as mock_redis_class: 122 | mock_redis = Mock() 123 | mock_redis.set.return_value = True 124 | mock_redis_class.return_value = mock_redis 125 | 126 | from KestrelAI.shared.redis_utils import RedisConfig 127 | 128 | client = get_sync_redis_client( 129 | RedisConfig(host="localhost", port=6379, db=0) 130 | ) 131 | 132 | client.checkpoint("test_task", {"state": "test"}) 133 | 134 | operation_time = time.time() - start_time 135 | assert operation_time < performance_thresholds["redis_operation_time"] 136 | 137 | 138 | class TestTaskModel: 139 | """Test Task model functionality.""" 140 | 141 | def test_task_creation(self): 142 | """Test task creation.""" 143 | task = Task( 144 | name="Test Task", 145 | description="Test description", 146 | budgetMinutes=30, 147 | status=TaskStatus.ACTIVE, 148 | ) 149 | 150 | assert task.name == "Test Task" 151 | assert task.description == "Test description" 152 | assert task.budgetMinutes == 30 153 | assert task.status == TaskStatus.ACTIVE 154 | 155 | def test_task_serialization(self): 156 | """Test task serialization.""" 157 | task = Task( 158 | name="Test Task", 159 | description="Test description", 160 | budgetMinutes=30, 161 | status=TaskStatus.ACTIVE, 162 | ) 163 | 164 | # Test model_dump (Pydantic v2) 165 | task_dict = task.model_dump() 166 | assert task_dict["name"] == "Test Task" 167 | assert task_dict["status"] == "active" 168 | 169 | # Test model_validate (Pydantic v2) 170 | new_task = Task.model_validate(task_dict) 171 | assert new_task.name == task.name 172 | assert new_task.status == task.status 173 | 174 | def test_task_validation(self): 175 | """Test task validation.""" 176 | # Test valid task 177 | task = Task( 178 | name="Valid Task", 179 | description="Valid description", 180 | budgetMinutes=30, 181 | status=TaskStatus.ACTIVE, 182 | ) 183 | assert task.name is not None 184 | 185 | # Test that negative budget is allowed (Pydantic v2 doesn't validate by default) 186 | task_negative = Task( 187 | name="Test Task", 188 | description="Test description", 189 | budgetMinutes=-1, # This is allowed in Pydantic v2 190 | status=TaskStatus.ACTIVE, 191 | ) 192 | assert task_negative.budgetMinutes == -1 193 | -------------------------------------------------------------------------------- /KestrelAI/orch_debug.txt: -------------------------------------------------------------------------------- 1 | Okay, I'm summarizing the current status of the "Research Task" based on the provided update history. 2 | 3 | **Current Status:** 4 | 5 | * **Task:** Research Task 6 | * **Overall Goal:** Identify and secure funding/opportunities for AI/ML research for the user. 7 | * **Phase:** Active exploration and planning. The project is in the stage of actively searching for opportunities, tracking deadlines, and planning next steps. 8 | 9 | **Key Findings & Actions Taken (as of the update):** 10 | 11 | * **Deadline Tracking:** Setting up Google Alerts for "NSF REU 2026" to monitor for announcements. 12 | * **REU Finder Search:** A thorough search of the REU Finder is underway, filtered by AI/ML keywords. 13 | * **NSF Monitoring:** Ongoing active monitoring of the NSF website for new programs and announcements. 14 | * **University Lab Outreach:** Planning to identify and explore 3-5 university AI labs. 15 | * **Networking:** Planning to start networking with professors. 16 | * **Private Funding Research:** Planning to systematically investigate industry sponsorship and private foundations. 17 | 18 | **Unexplored Areas / Questions:** 19 | 20 | * **Programmatic:** Deeper dive into individual REU program descriptions and monitoring NSF program solicitations. 21 | * **Networking:** Reaching out to professors directly. 22 | * **Alternative Funding:** Exploring industry sponsorships, private foundations, university-specific funding, and indirect funding through faculty grants. 23 | 24 | **Next Steps (Action Plan):** 25 | 26 | * **Immediate (Next Week):** 27 | * Deadline Tracking (Google Alerts). 28 | * REU Finder Search. 29 | * **Mid-Term (Next Month):** 30 | * NSF Monitoring. 31 | * University Lab Outreach. 32 | * **Long-Term (Ongoing):** 33 | * Networking. 34 | * Private Funding Research. 35 | 36 | 37 | 38 | **Overall Impression:** The research is progressing systematically. There's a good level of detail in the action plan and a clear understanding of the resources to explore. 39 | Okay, this is a *fantastic* and incredibly detailed breakdown of AI/ML research funding opportunities. It's clearly been a lot of work to compile this information. Here's a summary and analysis, broken down into key takeaways, strengths, potential improvements, and a prioritized action plan based on your outlined steps. 40 | 41 | **I. Key Takeaways & Overall Assessment** 42 | 43 | * **Comprehensive Scope:** You're covering a wide range of funding sources – NSF, AI Institutes, faculty-led research, industry sponsorships, international opportunities, and even anecdotal sources like Reddit. 44 | * **Strategic Prioritization:** You're intelligently prioritizing efforts, focusing initially on NSF AI Institutes and faculty-led research. This makes sense as those are likely the most accessible and impactful starting points. 45 | * **Detailed Tracking:** The spreadsheet tracking is essential for managing the large amount of information. 46 | * **Realistic Timeline:** The tiered timeline (Immediate, Mid-Term, Long-Term) provides a good framework for consistent progress. 47 | * **Understanding of Indirect Funding:** Recognizing "indirect funding" (faculty grants that *might* support undergraduates) is a sophisticated understanding. 48 | * **Acknowledging Limitations:** You're appropriately cautious about Reddit, recognizing it as anecdotal. 49 | 50 | **II. Strengths** 51 | 52 | * **Thoroughness:** The sheer amount of information gathered is impressive. 53 | * **Actionable Plan:** This isn't just a list; it's a roadmap. 54 | * **Proactive Mindset:** You're not just reacting to opportunities; you're actively seeking them out. 55 | * **Realistic and Flexible:** The plan allows for adjustments based on new information and experiences. 56 | * **Holistic Approach:** You're considering not just direct funding, but also networking and international opportunities. 57 | 58 | **III. Potential Improvements & Considerations** 59 | 60 | * **Specificity within AI/ML:** AI/ML is vast. Consider narrowing your focus. Are you interested in NLP, Computer Vision, Reinforcement Learning, etc.? This will help target your research and funding searches. 61 | * **Faculty Research Alignment:** When identifying faculty, go *beyond* just seeing they do AI/ML. Look for alignment with your specific interests. Read their publications! 62 | * **Visa Requirements (International):** The plan mentions international opportunities but doesn’t explicitly address the complexity of visa requirements. This is a significant hurdle that should be researched early. (e.g., J-1, F-1, etc.) 63 | * **Contacting Faculty - Crafting a Professional Email:** The plan mentions contacting faculty. Develop a template email that highlights your skills, interest in their research, and a specific question about their work. Don't just ask for a job. 64 | * **Networking:** Actively attend AI/ML conferences (even virtual ones) to network with researchers and learn about funding opportunities. 65 | * **Philanthropic Foundations:** While you mention philanthropic foundations, research specific ones known to support STEM or AI/ML research. (e.g., Sloan Foundation, Ford Foundation, etc.) 66 | * **Reddit Analysis:** While caution is warranted, consider developing a system for *structuring* Reddit information. Perhaps a keyword search to identify common themes or insights. 67 | 68 | **IV. Prioritized Action Plan (Based on Your Outline – with additions/refinements)** 69 | 70 | **A. Immediate (Next 1-2 Weeks)** 71 | 72 | 1. **[Critical] Define AI/ML Focus:** Spend 2-4 hours narrowing your AI/ML focus. This *will* make the rest of your efforts more effective. 73 | 2. **[Critical] NSF & AI Institute Monitoring:** Daily check for new announcements. Subscribe to mailing lists. 74 | 3. **[High] Spreadsheet Population (Initial):** Populate the spreadsheet with basic information (deadlines, eligibility) from REU Finder and NSF websites. 75 | 4. **[Medium] Faculty Identification (3-5):** Identify faculty *aligned with your focus* at target institutions. Look at their publications! 76 | 5. **[Medium] Email Template Drafting:** Draft a professional email template for contacting faculty. 77 | 78 | **B. Mid-Term (Next 1-2 Months)** 79 | 80 | 1. **[Critical] Faculty Contact:** Send emails to identified faculty using your template. Follow up politely if you don't hear back. 81 | 2. **[High] Industry Sponsorship Research:** Start researching potential industry sponsors (Google, Microsoft, Amazon). Identify contacts. 82 | 3. **[Medium] Philanthropic Foundation Research:** Research specific philanthropic foundations. 83 | 4. **[Medium] Spreadsheet Refinement:** Add columns to your spreadsheet for faculty contacts, industry contacts, etc. 84 | 85 | **C. Long-Term (Ongoing)** 86 | 87 | 1. **[Critical] Networking:** Attend conferences, join online communities, connect with researchers on LinkedIn. 88 | 2. **[High] International Exploration:** Begin researching visa requirements for international opportunities. 89 | 3. **[Medium] Reddit Analysis:** Develop a system for extracting valuable information from Reddit (keyword searches, theme identification). 90 | 4. **[Ongoing] Monitoring and Adaptation:** Continuously monitor funding sources and adapt your strategy based on experience and new opportunities. 91 | 92 | 93 | 94 | To help me tailor my advice further, could you tell me: 95 | 96 | * **What is your current level of experience in AI/ML?** (e.g., beginner, intermediate, advanced) 97 | * **What are your specific interests within AI/ML?** 98 | * **What institutions are you targeting for faculty research?** 99 | -------------------------------------------------------------------------------- /tests/utils/test_fixtures.py: -------------------------------------------------------------------------------- 1 | """ 2 | Test fixtures and utilities for KestrelAI tests. 3 | """ 4 | 5 | import asyncio 6 | import tempfile 7 | from unittest.mock import Mock, patch 8 | 9 | import pytest 10 | 11 | try: 12 | from KestrelAI.shared.models import ResearchPlan, Subtask, Task, TaskStatus 13 | except ImportError: 14 | from shared.models import ResearchPlan, Subtask, Task, TaskStatus 15 | 16 | 17 | @pytest.fixture 18 | def temp_dir(): 19 | """Create temporary directory for testing.""" 20 | with tempfile.TemporaryDirectory() as temp_dir: 21 | yield temp_dir 22 | 23 | 24 | @pytest.fixture 25 | def sample_task(): 26 | """Create a sample task for testing.""" 27 | return Task( 28 | id="test_task_1", 29 | name="Test Research Task", 30 | description="A test research task for unit testing", 31 | status=TaskStatus.PENDING, 32 | createdAt=1704067200000, 33 | updatedAt=1704067200000, 34 | ) 35 | 36 | 37 | @pytest.fixture 38 | def sample_research_plan(): 39 | """Create a sample research plan for testing.""" 40 | return ResearchPlan( 41 | restated_task="Test research task for unit testing", 42 | subtasks=[ 43 | Subtask( 44 | order=1, 45 | description="First subtask", 46 | success_criteria="Complete first subtask", 47 | status="pending", 48 | ), 49 | Subtask( 50 | order=2, 51 | description="Second subtask", 52 | success_criteria="Complete second subtask", 53 | status="pending", 54 | ), 55 | ], 56 | current_subtask_index=0, 57 | created_at=1704067200.0, 58 | ) 59 | 60 | 61 | @pytest.fixture 62 | def mock_llm(): 63 | """Create a mock LLM wrapper for testing.""" 64 | with patch("KestrelAI.agents.base.LlmWrapper") as mock_llm_class: 65 | mock_llm = Mock() 66 | mock_llm.generate_response.return_value = "Mocked LLM response" 67 | mock_llm_class.return_value = mock_llm 68 | yield mock_llm 69 | 70 | 71 | @pytest.fixture 72 | def mock_memory_store(): 73 | """Create a mock memory store for testing.""" 74 | with patch("KestrelAI.memory.vector_store.MemoryStore") as mock_memory_class: 75 | mock_memory = Mock() 76 | mock_memory.add.return_value = None 77 | mock_memory.search.return_value = { 78 | "ids": [["doc1", "doc2"]], 79 | "distances": [[0.1, 0.2]], 80 | "metadatas": [[{"source": "test"}, {"source": "test"}]], 81 | } 82 | mock_memory.delete_all.return_value = None 83 | mock_memory_class.return_value = mock_memory 84 | yield mock_memory 85 | 86 | 87 | @pytest.fixture 88 | def mock_redis(): 89 | """Create a mock Redis client for testing.""" 90 | with patch("redis.Redis") as mock_redis_class: 91 | mock_redis = Mock() 92 | mock_redis.ping.return_value = True 93 | mock_redis.get.return_value = None 94 | mock_redis.set.return_value = True 95 | mock_redis.delete.return_value = 1 96 | mock_redis_class.return_value = mock_redis 97 | yield mock_redis 98 | 99 | 100 | @pytest.fixture 101 | def mock_ollama_client(): 102 | """Create a mock Ollama client for testing.""" 103 | with patch("ollama.Client") as mock_client_class: 104 | mock_client = Mock() 105 | mock_response = Mock() 106 | mock_response.text = "Mocked Ollama response" 107 | mock_client.chat.return_value = mock_response 108 | mock_client_class.return_value = mock_client 109 | yield mock_client 110 | 111 | 112 | @pytest.fixture 113 | def mock_chromadb(): 114 | """Create a mock ChromaDB client for testing.""" 115 | with patch("chromadb.PersistentClient") as mock_chroma_class: 116 | mock_client = Mock() 117 | mock_collection = Mock() 118 | mock_collection.name = "test_collection" 119 | mock_collection.add.return_value = None 120 | mock_collection.query.return_value = { 121 | "ids": [["doc1"]], 122 | "distances": [[0.1]], 123 | "metadatas": [[{"source": "test"}]], 124 | } 125 | mock_collection.delete.return_value = None 126 | mock_client.get_collection.return_value = mock_collection 127 | mock_client.create_collection.return_value = mock_collection 128 | mock_chroma_class.return_value = mock_client 129 | yield mock_client 130 | 131 | 132 | @pytest.fixture 133 | def mock_sentence_transformer(): 134 | """Create a mock SentenceTransformer for testing.""" 135 | with patch("sentence_transformers.SentenceTransformer") as mock_transformer_class: 136 | mock_model = Mock() 137 | mock_model.encode.return_value = [[0.1, 0.2, 0.3, 0.4, 0.5]] 138 | mock_transformer_class.return_value = mock_model 139 | yield mock_model 140 | 141 | 142 | @pytest.fixture 143 | def event_loop(): 144 | """Create an event loop for async tests.""" 145 | loop = asyncio.new_event_loop() 146 | yield loop 147 | loop.close() 148 | 149 | 150 | @pytest.fixture 151 | def mock_requests(): 152 | """Create a mock requests module for testing HTTP calls.""" 153 | with ( 154 | patch("requests.get") as mock_get, 155 | patch("requests.post") as mock_post, 156 | patch("requests.put") as mock_put, 157 | patch("requests.delete") as mock_delete, 158 | ): 159 | # Default successful responses 160 | mock_response = Mock() 161 | mock_response.status_code = 200 162 | mock_response.json.return_value = {"status": "success"} 163 | mock_response.text = "Success" 164 | 165 | mock_get.return_value = mock_response 166 | mock_post.return_value = mock_response 167 | mock_put.return_value = mock_response 168 | mock_delete.return_value = mock_response 169 | 170 | yield { 171 | "get": mock_get, 172 | "post": mock_post, 173 | "put": mock_put, 174 | "delete": mock_delete, 175 | } 176 | 177 | 178 | class TestDataFactory: 179 | """Factory class for creating test data.""" 180 | 181 | @staticmethod 182 | def create_task( 183 | task_id: str = "test_task", 184 | name: str = "Test Task", 185 | description: str = "Test description", 186 | status: TaskStatus = TaskStatus.PENDING, 187 | ) -> Task: 188 | """Create a task with default or custom values.""" 189 | return Task( 190 | id=task_id, 191 | name=name, 192 | description=description, 193 | status=status, 194 | createdAt=1704067200000, 195 | updatedAt=1704067200000, 196 | ) 197 | 198 | @staticmethod 199 | def create_subtask( 200 | order: int = 1, 201 | description: str = "Test subtask", 202 | success_criteria: str = "Test criteria", 203 | status: str = "pending", 204 | ) -> Subtask: 205 | """Create a subtask with default or custom values.""" 206 | return Subtask( 207 | order=order, 208 | description=description, 209 | success_criteria=success_criteria, 210 | status=status, 211 | ) 212 | 213 | @staticmethod 214 | def create_research_plan( 215 | restated_task: str = "Test restated task", subtasks: list | None = None 216 | ) -> ResearchPlan: 217 | """Create a research plan with default or custom values.""" 218 | if subtasks is None: 219 | subtasks = [ 220 | TestDataFactory.create_subtask(1, "First subtask", "Complete first"), 221 | TestDataFactory.create_subtask(2, "Second subtask", "Complete second"), 222 | ] 223 | 224 | return ResearchPlan( 225 | restated_task=restated_task, 226 | subtasks=subtasks, 227 | current_subtask_index=0, 228 | created_at=1704067200.0, 229 | ) 230 | -------------------------------------------------------------------------------- /KestrelAI/agents/context_builder.py: -------------------------------------------------------------------------------- 1 | """ 2 | Context builder for web research agent. 3 | Handles building context for LLM with token-aware management. 4 | """ 5 | 6 | import logging 7 | from typing import TYPE_CHECKING 8 | 9 | if TYPE_CHECKING: 10 | from shared.models import Task 11 | 12 | from .base_agent import AgentState 13 | from .context_manager import ContextManager, TokenBudget 14 | from .research_config import ResearchConfig 15 | from .url_utils import URLFlagManager 16 | 17 | logger = logging.getLogger(__name__) 18 | 19 | 20 | class ContextBuilder: 21 | """Builds context for the research agent with token-aware management""" 22 | 23 | def __init__( 24 | self, 25 | config: "ResearchConfig", 26 | url_flag_manager: "URLFlagManager", 27 | context_manager: "ContextManager" = None, 28 | token_budget: "TokenBudget" = None, 29 | retrieve_from_rag_func=None, 30 | ): 31 | self.config = config 32 | self.url_flag_manager = url_flag_manager 33 | self.context_manager = context_manager 34 | self.token_budget = token_budget 35 | self.retrieve_from_rag_func = retrieve_from_rag_func 36 | self.context_management_enabled = context_manager is not None 37 | 38 | def build_context(self, task: "Task", state: "AgentState") -> str: 39 | """Build context for the agent with token-aware management""" 40 | # Use new context manager if available, otherwise fall back to old method 41 | if not self.context_management_enabled: 42 | return self._build_context_legacy(task, state) 43 | 44 | try: 45 | # Prepare components for context manager 46 | components = {"task": task.description} 47 | 48 | # Add subtask-specific context 49 | if self.config.is_subtask_agent: 50 | components[ 51 | "subtask" 52 | ] = f"{self.config.subtask_description}\nSuccess Criteria: {self.config.success_criteria}" 53 | if self.config.previous_findings: 54 | # Replace URLs with flags in previous findings 55 | ( 56 | findings_with_flags, 57 | _, 58 | ) = self.url_flag_manager.replace_urls_with_flags( 59 | self.config.previous_findings 60 | ) 61 | components["previous_findings"] = findings_with_flags 62 | 63 | # Add checkpoints (use summarization if needed) 64 | if state.checkpoints: 65 | checkpoint_list = [] 66 | for checkpoint in state.checkpoints: 67 | # Replace URLs with flags 68 | ( 69 | checkpoint_with_flags, 70 | _, 71 | ) = self.url_flag_manager.replace_urls_with_flags(checkpoint) 72 | checkpoint_list.append(checkpoint_with_flags) 73 | components["checkpoints"] = checkpoint_list 74 | 75 | # Add last checkpoint separately if different from checkpoints list 76 | if state.last_checkpoint and state.last_checkpoint not in state.checkpoints: 77 | ( 78 | checkpoint_with_flags, 79 | _, 80 | ) = self.url_flag_manager.replace_urls_with_flags(state.last_checkpoint) 81 | if "checkpoints" not in components: 82 | components["checkpoints"] = [] 83 | components["checkpoints"].insert(0, checkpoint_with_flags) 84 | 85 | # Add history 86 | if state.history: 87 | history_list = [] 88 | for entry in state.history: 89 | # Replace URLs with flags 90 | entry_with_flags, _ = self.url_flag_manager.replace_urls_with_flags( 91 | entry 92 | ) 93 | history_list.append(entry_with_flags) 94 | components["history"] = history_list 95 | 96 | # Add current focus 97 | if state.current_focus: 98 | components["current_focus"] = state.current_focus 99 | 100 | # Add RAG content (retrieved using semantic search with hierarchical summaries) 101 | # Use current focus or task description as query for better relevance 102 | if self.retrieve_from_rag_func: 103 | rag_query = state.current_focus or task.description 104 | rag_content = self.retrieve_from_rag_func( 105 | task, 106 | query=rag_query, 107 | max_tokens=self.token_budget.rag_content 108 | if self.token_budget 109 | else None, 110 | ) 111 | if rag_content and rag_content != "(No previous findings)": 112 | # Replace URLs with flags 113 | rag_with_flags, _ = self.url_flag_manager.replace_urls_with_flags( 114 | rag_content 115 | ) 116 | components["rag_content"] = rag_with_flags 117 | 118 | # Add URL reference table 119 | url_table = self.url_flag_manager.get_url_reference_table() 120 | if url_table: 121 | components["url_reference"] = url_table 122 | 123 | # Build context using context manager 124 | context, token_usage = self.context_manager.build_context( 125 | components, prioritize_recent=True 126 | ) 127 | 128 | # Log token usage for debugging 129 | if self.config.debug: 130 | logger.debug(f"Context token usage: {token_usage}") 131 | logger.debug(f"Total context tokens: {token_usage['total']}") 132 | 133 | return context 134 | 135 | except Exception as e: 136 | logger.error( 137 | f"Error in context management, falling back to legacy: {e}", 138 | exc_info=True, 139 | ) 140 | return self._build_context_legacy(task, state) 141 | 142 | def _build_context_legacy(self, task: "Task", state: "AgentState") -> str: 143 | """Legacy context building method (fallback)""" 144 | context_parts = [f"Task: {task.description}"] 145 | 146 | # Add subtask-specific context 147 | if self.config.is_subtask_agent: 148 | context_parts.extend( 149 | [ 150 | f"Subtask: {self.config.subtask_description}", 151 | f"Success Criteria: {self.config.success_criteria}", 152 | ] 153 | ) 154 | if self.config.previous_findings: 155 | # Replace URLs with flags in previous findings 156 | findings_with_flags, _ = self.url_flag_manager.replace_urls_with_flags( 157 | self.config.previous_findings 158 | ) 159 | context_parts.append(f"Previous findings: {findings_with_flags}") 160 | 161 | # Add last checkpoint if available 162 | if state.last_checkpoint: 163 | # Replace URLs with flags in checkpoint 164 | checkpoint_with_flags, _ = self.url_flag_manager.replace_urls_with_flags( 165 | state.last_checkpoint 166 | ) 167 | context_parts.append(f"Previous checkpoint: {checkpoint_with_flags}") 168 | 169 | # Add current focus if set 170 | if state.current_focus: 171 | context_parts.append(f"Current focus: {state.current_focus}") 172 | 173 | # Add sliding window of recent history 174 | if state.history: 175 | recent_history = "\n".join(state.history) 176 | # Replace URLs with flags in history 177 | history_with_flags, _ = self.url_flag_manager.replace_urls_with_flags( 178 | recent_history 179 | ) 180 | context_parts.append(f"Recent actions: {history_with_flags}") 181 | else: 182 | context_parts.append("[No actions yet]") 183 | 184 | # Add URL reference table if there are any URLs 185 | url_table = self.url_flag_manager.get_url_reference_table() 186 | if url_table: 187 | context_parts.append(url_table) 188 | 189 | return "\n".join(context_parts) 190 | -------------------------------------------------------------------------------- /KestrelAI/agents/prompt_builder.py: -------------------------------------------------------------------------------- 1 | """ 2 | Prompt builder for web research agent. 3 | Handles generation of system prompts based on agent configuration. 4 | """ 5 | 6 | from datetime import datetime 7 | 8 | from .research_config import ResearchConfig 9 | 10 | 11 | class PromptBuilder: 12 | """Builds system prompts for the research agent based on configuration""" 13 | 14 | def __init__(self, config: ResearchConfig): 15 | self.config = config 16 | 17 | def get_system_prompt(self) -> str: 18 | """Get appropriate system prompt based on configuration""" 19 | if self.config.is_subtask_agent: 20 | return self.get_subtask_system_prompt() 21 | elif self.config.use_mcp: 22 | return self.get_mcp_system_prompt() 23 | else: 24 | return self.get_standard_system_prompt() 25 | 26 | def get_standard_system_prompt(self) -> str: 27 | """Standard research agent system prompt""" 28 | current_date = datetime.utcnow().strftime("%B %d, %Y") 29 | return f"""You are an autonomous research agent conducting focused investigations to find specific, actionable information. 30 | 31 | Your goal is to find concrete details that the user can immediately act upon: 32 | - Specific programs, grants, or opportunities with exact details 33 | - Concrete deadlines, requirements, and application processes 34 | - Direct contact information and application links 35 | - Exact eligibility criteria and requirements 36 | - Current opportunities (not generic database descriptions) 37 | 38 | CORE RULES: 39 | - Focus on ACTIONABLE information the user can apply to or use immediately 40 | - Prioritize specific programs over generic database descriptions 41 | - Find concrete details: exact deadlines, specific requirements, contact information 42 | - Avoid generic advice that applies to any research topic 43 | - Do not make up information 44 | - Do not have conversations 45 | - All context is either your own words, or results from tools you called 46 | - Do not retry failed searches or use the same search terms repeatedly. Move on. 47 | - The date is {current_date} 48 | 49 | URL HANDLING: 50 | - URLs in search results and context are represented as flags (e.g., [URL_1], [URL_2]) 51 | - A URL reference table is provided showing which flag corresponds to which URL 52 | - When you see a URL flag, understand it represents a specific URL from the reference table 53 | - You do not need to write out URLs in your JSON responses - they are handled automatically 54 | 55 | OUTPUT FORMAT (JSON only): 56 | {{ 57 | "direction": "Your reasoning for the next action (1-2 sentences)", 58 | "action": "think" | "search" | "summarize", 59 | "query": "search terms (if action is 'search', else empty string)", 60 | "thought": "detailed planning and brainstorming (if action is 'think', else empty string)" 61 | }} 62 | 63 | ACTIONS: 64 | - think: Reason about findings and plan next steps to find specific opportunities 65 | - search: Targeted queries for specific programs, grants, or opportunities 66 | - summarize: Checkpoint actionable findings with concrete details""" 67 | 68 | def get_subtask_system_prompt(self) -> str: 69 | """Subtask-specific system prompt""" 70 | current_date = datetime.utcnow().strftime("%B %d, %Y") 71 | return f"""You are a specialized research agent focused on finding specific, actionable information for your assigned subtask. 72 | 73 | Your goal is to find concrete details that the user can immediately act upon: 74 | - Specific programs, grants, or opportunities with exact details 75 | - Concrete deadlines, requirements, and application processes 76 | - Direct contact information and application links 77 | - Exact eligibility criteria and requirements 78 | - Current opportunities (not generic database descriptions) 79 | 80 | SUBTASK CONTEXT: 81 | - Subtask: {self.config.subtask_description} 82 | - Success Criteria: {self.config.success_criteria} 83 | - Previous Findings: {self.config.previous_findings} 84 | 85 | CORE RULES: 86 | - Focus on ACTIONABLE information the user can apply to or use immediately 87 | - Prioritize specific programs over generic database descriptions 88 | - Find concrete details: exact deadlines, specific requirements, contact information 89 | - Avoid generic advice that applies to any research topic 90 | - Stay strictly focused on your assigned subtask 91 | - Conduct thorough research to meet the success criteria 92 | - Build upon any previous findings provided to you 93 | - Do not make up information 94 | - Do not have conversations 95 | - All context is either your own words, or results from tools you called 96 | - Do not retry failed searches or use the same search terms repeatedly 97 | - The date is {current_date} 98 | 99 | URL HANDLING: 100 | - URLs in search results and context are represented as flags (e.g., [URL_1], [URL_2]) 101 | - A URL reference table is provided showing which flag corresponds to which URL 102 | - When you see a URL flag, understand it represents a specific URL from the reference table 103 | - You do not need to write out URLs in your JSON responses - they are handled automatically 104 | 105 | OUTPUT FORMAT (JSON only): 106 | {{ 107 | "direction": "Your reasoning for the next action (1-2 sentences)", 108 | "action": "think" | "search" | "summarize" | "complete", 109 | "query": "search terms (if action is 'search', else empty string)", 110 | "thought": "detailed planning and brainstorming (if action is 'think', else empty string)" 111 | }} 112 | 113 | ACTIONS: 114 | - think: Reason about findings and plan next steps 115 | - search: Targeted, human readable queries for new information 116 | - summarize: Checkpoint important findings 117 | - complete: Indicate that the subtask success criteria have been met""" 118 | 119 | def get_mcp_system_prompt(self) -> str: 120 | """MCP-enhanced system prompt""" 121 | current_date = datetime.utcnow().strftime("%B %d, %Y") 122 | return f"""You are an autonomous research agent conducting deep investigations with access to powerful tools and data sources. 123 | 124 | Your goal is to gather accurate, relevant, and up-to-date information on the assigned topic using all available resources. 125 | 126 | AVAILABLE TOOLS: 127 | You have access to powerful research tools through the MCP (Model Context Protocol) system: 128 | 129 | DATA SOURCES: 130 | - search_web: Enhanced web search with multiple engines and filtering 131 | - query_database: Execute SQL queries on structured databases 132 | - search_repositories: Search GitHub repositories for code and documentation 133 | - read_file: Read contents of files from the filesystem 134 | - search_files: Search for files by name or content 135 | 136 | ANALYSIS TOOLS: 137 | - analyze_data: Perform statistical analysis on data 138 | - extract_text: Extract and clean text from web pages or documents 139 | 140 | RESEARCH TOOLS: 141 | - get_repository_info: Get detailed information about a GitHub repository 142 | - navigate_to_page: Navigate to a web page and extract information 143 | 144 | AUTOMATION TOOLS: 145 | - write_file: Write data to files 146 | - create_table: Create database tables for structured data storage 147 | 148 | CORE RULES: 149 | - Stay strictly on topic for the given task 150 | - Probe deeply - don't settle for surface-level information 151 | - If blocked, try alternative research paths and tools 152 | - If information is already known, find new information and pathways 153 | - Build upon previous findings iteratively 154 | - Do not make up information 155 | - Do not have conversations 156 | - All context is either your own words, or results from tools you called 157 | - Do not retry failed searches or use the same search terms repeatedly. Move on. 158 | - The date is {current_date} 159 | 160 | OUTPUT FORMAT (JSON only): 161 | {{ 162 | "direction": "Your reasoning for the next action (1-2 sentences)", 163 | "action": "think" | "search" | "mcp_tool" | "summarize", 164 | "query": "search terms (if action is 'search', else empty string)", 165 | "tool_name": "tool name (if action is 'mcp_tool', else empty string)", 166 | "tool_parameters": {{"param": "value"}} (if action is 'mcp_tool', else empty object), 167 | "thought": "detailed planning and brainstorming (if action is 'think', else empty string)" 168 | }} 169 | 170 | ACTIONS: 171 | - think: Reason about findings and plan next steps 172 | - search: Traditional web search via SearXNG (fallback) 173 | - mcp_tool: Use MCP tools for enhanced research capabilities 174 | - summarize: Checkpoint important findings""" 175 | -------------------------------------------------------------------------------- /tests/integration/test_memory_store.py: -------------------------------------------------------------------------------- 1 | # Integration tests for MemoryStore functionality 2 | import tempfile 3 | import time 4 | from unittest.mock import Mock, patch 5 | 6 | import pytest 7 | 8 | try: 9 | from KestrelAI.memory.vector_store import MemoryStore 10 | except ImportError: 11 | from memory.vector_store import MemoryStore 12 | 13 | 14 | @pytest.mark.integration 15 | class TestMemoryStoreIntegration: 16 | """Test MemoryStore integration functionality.""" 17 | 18 | @pytest.fixture 19 | def temp_dir(self): 20 | """Create temporary directory for testing.""" 21 | with tempfile.TemporaryDirectory() as temp_dir: 22 | yield temp_dir 23 | 24 | def test_memory_store_initialization(self, temp_dir): 25 | """Test MemoryStore initialization with mocked model for speed.""" 26 | # Mock the SentenceTransformer to avoid model download 27 | with patch("sentence_transformers.SentenceTransformer") as mock_transformer: 28 | mock_model = Mock() 29 | mock_model.encode.return_value = [ 30 | [0.1, 0.2, 0.3, 0.4, 0.5] 31 | ] # Mock embedding 32 | mock_transformer.return_value = mock_model 33 | 34 | memory_store = MemoryStore(path=temp_dir) 35 | 36 | assert memory_store.client is not None 37 | assert memory_store.collection is not None 38 | assert memory_store.model is not None 39 | assert memory_store.collection.name == "research_mem" 40 | 41 | def test_memory_store_add_document(self, temp_dir): 42 | """Test adding documents to MemoryStore.""" 43 | with patch("sentence_transformers.SentenceTransformer") as mock_transformer: 44 | mock_model = Mock() 45 | mock_model.encode.return_value = [ 46 | [0.1, 0.2, 0.3, 0.4, 0.5] 47 | ] # Mock embedding 48 | mock_transformer.return_value = mock_model 49 | 50 | memory_store = MemoryStore(path=temp_dir) 51 | 52 | # Add a test document 53 | doc_id = "test_doc_1" 54 | text = "This is a test document for memory store testing." 55 | metadata = {"source": "test", "type": "example"} 56 | 57 | memory_store.add(doc_id, text, metadata) 58 | 59 | # Verify the document was added by searching for it 60 | results = memory_store.search(text, k=1) 61 | 62 | assert results is not None 63 | assert len(results["ids"][0]) >= 1 # Should find at least one document 64 | assert doc_id in results["ids"][0] # Should find our test document 65 | 66 | def test_memory_store_search(self, temp_dir): 67 | """Test searching in MemoryStore.""" 68 | with patch("sentence_transformers.SentenceTransformer") as mock_transformer: 69 | mock_model = Mock() 70 | mock_model.encode.return_value = [ 71 | [0.1, 0.2, 0.3, 0.4, 0.5] 72 | ] # Mock embedding 73 | mock_transformer.return_value = mock_model 74 | 75 | memory_store = MemoryStore(path=temp_dir) 76 | 77 | # Add multiple test documents 78 | documents = [ 79 | ( 80 | "doc1", 81 | "Machine learning is a subset of artificial intelligence.", 82 | {"topic": "ML"}, 83 | ), 84 | ( 85 | "doc2", 86 | "Natural language processing helps computers understand text.", 87 | {"topic": "NLP"}, 88 | ), 89 | ( 90 | "doc3", 91 | "Computer vision enables machines to interpret visual information.", 92 | {"topic": "CV"}, 93 | ), 94 | ] 95 | 96 | for doc_id, text, metadata in documents: 97 | memory_store.add(doc_id, text, metadata) 98 | 99 | # Search for machine learning related content 100 | results = memory_store.search( 101 | "artificial intelligence machine learning", k=2 102 | ) 103 | 104 | assert results is not None 105 | assert len(results["ids"][0]) >= 1 # Should find at least one document 106 | assert len(results["distances"][0]) >= 1 # Should have similarity scores 107 | 108 | def test_memory_store_delete_all(self, temp_dir): 109 | """Test deleting all documents from MemoryStore.""" 110 | with patch("sentence_transformers.SentenceTransformer") as mock_transformer: 111 | mock_model = Mock() 112 | mock_model.encode.return_value = [ 113 | [0.1, 0.2, 0.3, 0.4, 0.5] 114 | ] # Mock embedding 115 | mock_transformer.return_value = mock_model 116 | 117 | memory_store = MemoryStore(path=temp_dir) 118 | 119 | # Add a test document 120 | memory_store.add("test_doc", "Test content", {"test": True}) 121 | 122 | # Verify document exists 123 | results = memory_store.search("test content", k=1) 124 | assert len(results["ids"][0]) >= 1 125 | 126 | # Delete all documents 127 | memory_store.delete_all() 128 | 129 | # Verify document is deleted 130 | results = memory_store.search("test content", k=1) 131 | assert len(results["ids"][0]) == 0 # Should find no documents 132 | 133 | def test_memory_store_persistence(self, temp_dir): 134 | """Test that MemoryStore persists data across instances.""" 135 | # Skip this test if using in-memory mode (which is forced in test environments) 136 | # In-memory mode doesn't persist across instances by design 137 | import os 138 | 139 | if os.getenv("TESTING") or os.getenv("PYTEST_CURRENT_TEST"): 140 | pytest.skip( 141 | "Persistence test skipped in test mode (uses in-memory storage)" 142 | ) 143 | 144 | with patch("sentence_transformers.SentenceTransformer") as mock_transformer: 145 | mock_model = Mock() 146 | mock_model.encode.return_value = [ 147 | [0.1, 0.2, 0.3, 0.4, 0.5] 148 | ] # Mock embedding 149 | mock_transformer.return_value = mock_model 150 | 151 | # Create first instance and add document 152 | memory_store1 = MemoryStore(path=temp_dir) 153 | memory_store1.add( 154 | "persistent_doc", "This should persist", {"persistent": True} 155 | ) 156 | 157 | # Create second instance (should load existing data) 158 | memory_store2 = MemoryStore(path=temp_dir) 159 | 160 | # Search for the document in the second instance 161 | results = memory_store2.search("This should persist", k=1) 162 | 163 | assert results is not None 164 | assert len(results["ids"][0]) >= 1 165 | assert "persistent_doc" in results["ids"][0] 166 | 167 | def test_memory_store_performance(self, temp_dir): 168 | """Test MemoryStore performance with multiple documents.""" 169 | with patch("sentence_transformers.SentenceTransformer") as mock_transformer: 170 | mock_model = Mock() 171 | mock_model.encode.return_value = [ 172 | [0.1, 0.2, 0.3, 0.4, 0.5] 173 | ] # Mock embedding 174 | mock_transformer.return_value = mock_model 175 | 176 | memory_store = MemoryStore(path=temp_dir) 177 | 178 | # Add multiple documents 179 | start_time = time.time() 180 | for i in range(10): 181 | doc_id = f"perf_doc_{i}" 182 | text = f"Performance test document number {i} with some content." 183 | metadata = {"index": i, "test": "performance"} 184 | memory_store.add(doc_id, text, metadata) 185 | 186 | add_time = time.time() - start_time 187 | 188 | # Search performance 189 | start_time = time.time() 190 | results = memory_store.search("performance test", k=5) 191 | search_time = time.time() - start_time 192 | 193 | # Performance assertions (should be reasonable for 10 documents) 194 | assert ( 195 | add_time < 10.0 196 | ) # Adding 10 documents should take less than 10 seconds 197 | assert search_time < 5.0 # Search should take less than 5 seconds 198 | assert len(results["ids"][0]) >= 1 # Should find at least one document 199 | -------------------------------------------------------------------------------- /tests/integration/test_context_management.py: -------------------------------------------------------------------------------- 1 | """ 2 | Integration tests for context management system. 3 | Tests the interaction between TokenCounter, TokenBudget, ContextManager, and MultiLevelSummarizer. 4 | """ 5 | 6 | from unittest.mock import Mock, patch 7 | 8 | import pytest 9 | 10 | try: 11 | from KestrelAI.agents.context_manager import ( 12 | ContextManager, 13 | TokenBudget, 14 | TokenCounter, 15 | ) 16 | from KestrelAI.agents.multi_level_summarizer import MultiLevelSummarizer 17 | except ImportError: 18 | from agents.context_manager import ContextManager, TokenBudget, TokenCounter 19 | from agents.multi_level_summarizer import MultiLevelSummarizer 20 | 21 | 22 | @pytest.mark.integration 23 | class TestContextManagementIntegration: 24 | """Integration tests for context management.""" 25 | 26 | @pytest.fixture 27 | def mock_llm(self): 28 | """Create mock LLM.""" 29 | llm = Mock() 30 | llm.chat.return_value = "This is a summarized version." 31 | return llm 32 | 33 | @pytest.fixture 34 | def token_counter(self): 35 | """Create TokenCounter with mocked tiktoken.""" 36 | with patch("tiktoken.get_encoding") as mock_get_encoding: 37 | mock_encoding = Mock() 38 | 39 | # Mock encoding to return token count based on word count 40 | def encode(text): 41 | return list(range(len(str(text).split()))) 42 | 43 | mock_encoding.encode.side_effect = encode 44 | mock_encoding.decode.side_effect = lambda tokens: " ".join( 45 | ["word"] * len(tokens) 46 | ) 47 | mock_get_encoding.return_value = mock_encoding 48 | 49 | counter = TokenCounter(model_name="gpt-4") 50 | counter.encoding = mock_encoding 51 | counter.truncate_to_tokens = Mock( 52 | side_effect=lambda x, max_t: " ".join(str(x).split()[:max_t]) + "..." 53 | ) 54 | return counter 55 | 56 | @pytest.fixture 57 | def token_budget(self): 58 | """Create TokenBudget.""" 59 | return TokenBudget( 60 | max_context=1200, # Increased to accommodate all allocations 61 | system_prompt=100, 62 | task_description=50, 63 | previous_findings=200, 64 | checkpoints=300, 65 | history=200, 66 | rag_content=200, 67 | url_reference=50, 68 | response_reserve=100, 69 | ) 70 | 71 | @pytest.fixture 72 | def context_manager(self, token_counter, token_budget): 73 | """Create ContextManager.""" 74 | return ContextManager(token_counter, token_budget) 75 | 76 | @pytest.fixture 77 | def summarizer(self, mock_llm, token_counter): 78 | """Create MultiLevelSummarizer.""" 79 | return MultiLevelSummarizer(mock_llm, token_counter) 80 | 81 | def test_full_context_building_flow( 82 | self, context_manager, token_counter, token_budget 83 | ): 84 | """Test complete context building flow.""" 85 | components = { 86 | "task": "Research AI research opportunities", 87 | "subtask": "Find NSF REU programs", 88 | "checkpoints": [ 89 | "Found NSF REU program with deadline in February", 90 | "Discovered multiple ML research opportunities", 91 | ], 92 | "history": ["[SEARCH] NSF REU programs", "[THOUGHT] Analyzing results"], 93 | "rag_content": "NSF REU programs provide summer research opportunities for undergraduates.", 94 | } 95 | 96 | context, usage = context_manager.build_context(components) 97 | 98 | # Verify context was built 99 | assert isinstance(context, str) 100 | assert len(context) > 0 101 | 102 | # Verify token usage tracking 103 | assert usage["task"] > 0 104 | assert usage["checkpoints"] > 0 105 | assert usage["history"] > 0 106 | assert usage["total"] > 0 107 | 108 | # Verify total is within budget 109 | assert usage["total"] <= token_budget.available_for_context 110 | 111 | def test_context_with_summarization( 112 | self, context_manager, summarizer, token_counter 113 | ): 114 | """Test context building with summarization.""" 115 | # Create long content that needs summarization 116 | long_content = "This is a very long research finding. " * 50 117 | 118 | # Create summary hierarchy 119 | result = summarizer.create_summary_hierarchy(long_content) 120 | summaries = result["summaries"] 121 | 122 | # Verify summaries were created 123 | assert "detailed" in summaries 124 | assert "medium" in summaries or "summary" in summaries 125 | 126 | # Use summary in context 127 | components = { 128 | "task": "Test task", 129 | "rag_content": summaries.get("summary", long_content), 130 | } 131 | 132 | context, usage = context_manager.build_context(components) 133 | 134 | # Verify context was built 135 | assert isinstance(context, str) 136 | assert usage["total"] > 0 137 | 138 | def test_adaptive_retrieval_integration( 139 | self, summarizer, context_manager, token_counter 140 | ): 141 | """Test adaptive retrieval based on token budget.""" 142 | # Skip this test - retrieve_adaptive method doesn't exist yet 143 | pytest.skip("retrieve_adaptive method not implemented in MultiLevelSummarizer") 144 | 145 | def test_fact_extraction_and_preservation( 146 | self, summarizer, mock_llm, token_counter 147 | ): 148 | """Test that facts are extracted and preserved in summaries.""" 149 | content = """ 150 | NSF REU Program 151 | Deadline: February 15, 2025 152 | Contact: info@nsf.gov 153 | URL: https://www.nsf.gov/reu 154 | Funding: $5,000 155 | Requirements: 3.5 GPA, undergraduate student 156 | """ 157 | 158 | # Mock LLM for fact extraction 159 | mock_llm.chat.side_effect = [ 160 | # Fact extraction response 161 | '{"deadlines": ["February 15, 2025"], "urls": ["https://www.nsf.gov/reu"], "contact_info": ["info@nsf.gov"], "amounts": ["$5,000"], "requirements": ["3.5 GPA", "undergraduate student"]}', 162 | # Summary response 163 | "NSF REU Program with key details.", 164 | ] 165 | 166 | result = summarizer.create_summary_hierarchy(content) 167 | 168 | # Verify facts were extracted 169 | assert "facts" in result 170 | facts = result["facts"] 171 | assert len(facts.deadlines) > 0 or len(facts.urls) > 0 172 | 173 | # Verify summaries were created 174 | assert "summaries" in result 175 | summaries = result["summaries"] 176 | assert "detailed" in summaries 177 | 178 | def test_token_budget_validation(self, token_budget): 179 | """Test token budget validation.""" 180 | assert token_budget.validate() is True 181 | 182 | # Test available_for_context calculation 183 | available = token_budget.available_for_context 184 | assert available == 1000 # 1200 - 100 - 100 185 | 186 | def test_context_truncation(self, context_manager, token_counter): 187 | """Test that context manager truncates when content exceeds budget.""" 188 | # Create content that exceeds budget 189 | very_long_task = "Task description. " * 1000 190 | 191 | components = {"task": very_long_task} 192 | 193 | context, usage = context_manager.build_context(components) 194 | 195 | # Should truncate to fit budget 196 | assert usage["task"] <= context_manager.budget.task_description 197 | assert "..." in context or len(context) < len(very_long_task) 198 | 199 | def test_priority_ordering(self, context_manager, token_counter): 200 | """Test that recent items are prioritized.""" 201 | components = { 202 | "task": "Test task", 203 | "checkpoints": [ 204 | "Old checkpoint 1", 205 | "Old checkpoint 2", 206 | "Recent checkpoint 1", 207 | "Recent checkpoint 2", 208 | ], 209 | } 210 | 211 | # Mock token counting to allow all checkpoints 212 | with patch.object(token_counter, "count_tokens", return_value=10): 213 | context, usage = context_manager.build_context( 214 | components, prioritize_recent=True 215 | ) 216 | 217 | # Recent checkpoints should be included (they come first in reversed order) 218 | assert usage["checkpoints"] > 0 219 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

2 | 3 | 4 | Logo 5 | 6 |

7 | 8 |
9 | Kestrel 10 |
11 | Explore the screenshots » 12 |
13 |
14 | Report a Bug 15 | · 16 | Request a Feature 17 | .Ask a Question 18 |
19 | 20 |
21 |
22 | 23 | [![Project license](https://img.shields.io/github/license/dankeg/KestrelAI.svg?style=flat-square)](LICENSE) [![Pull Requests welcome](https://img.shields.io/badge/PRs-welcome-ff69b4.svg?style=flat-square)](https://github.com/dankeg/KestrelAI/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22) [![code with love by dankeg](https://img.shields.io/badge/%3C%2F%3E%20with%20%E2%99%A5%20by-dankeg-ff1414.svg?style=flat-square)](https://github.com/dankeg) 24 | 25 | ![Under construction](https://img.shields.io/badge/status-under%20construction-yellow) 26 | 27 |
28 | 29 |
30 | Table of Contents 31 | 32 | - [About](#about) 33 | - [Built With](#built-with) 34 | - [Architecture](#architecture) 35 | - [Getting Started](#getting-started) 36 | - [Prerequisites](#prerequisites) 37 | - [Installation](#installation) 38 | - [Usage](#usage) 39 | - [Roadmap](#roadmap) 40 | - [Support](#support) 41 | - [Project assistance](#project-assistance) 42 | - [Contributing](#contributing) 43 | - [Authors & contributors](#authors--contributors) 44 | - [License](#license) 45 | 46 |
47 | 48 | --- 49 | 50 | ## About 51 | 52 | **Kestrel** is a local-first research assistant that self-schedules and executes support tasks over long horizons so you can stay focused on key, complex & novel tasks. It handles the mechanical work—searching, filtering, extracting, and organizing evidence in the background, preparing it for review and analysis. 53 | 54 | **Designed for small local models.** Kestrel is built to run with **small LLMs (~1–7B)** via **Ollama**, so it’s **laptop/desktop-friendly** (macOS, Windows, Linux; CPU-only works, GPU optional). No server cluster required. 55 | 56 | ### What it does 57 | - **Task Exploration:**: Leverages gathered evidence to iterate on tasks, identifying and pursuing new branches and next steps. 58 | - **Source Provenance**: Uses a deep research approach, but maintains a clear record of consulted searches and how they impact exploration and findings. 59 | - **Long-running support:** Keep tasks moving in the background and surface **auditable checkpoints** for review and redirection, rather than a single final report. 60 | 61 | > **Example:** Example: As you interpret a frontier paper and plan a novel study, Kestrel runs in the background to resolve the tooling (**Polars vs. pandas vs. Spark**), gathering real-world usage, extracting cited performance claims, and assembling a reviewable comparison. 62 | 63 | ### How it works (at a glance) 64 | - **Research Agent** — Executes concrete micro-tasks (search → retrieve → filter → extract → narrow summary) and proposes specific follow-ups when evidence warrants it. 65 | - **Orchestrator** — Plans/schedules work, tracks progress and blockers, pivots when stalled, and emits checkpoints you can accept or adjust. This keeps efforts coherent over hours or days. 66 | 67 | ### Orchestrator profiles 68 | | Profile | Scope & behavior | Typical runtime | 69 | |----------------|----------------------------------------------------------|-----------------| 70 | | **Hummingbird**| Quick breadth-first sweeps; targeted lookups. | Minutes–~2h | 71 | | **Kestrel** | Medium-horizon exploration with closely related branches.| Multi-hour | 72 | | **Albatross** | Long-horizon, open-ended exploration with reviews. | Day-scale+ | 73 | 74 | (Names are mnemonic. Community-authored profiles with clear behaviors are welcome—e.g., a **Parrot** profile for multi-agent dialogue.) 75 | 76 | 77 |
78 | Screenshots 79 |
80 | 81 | 82 | 83 | 89 | 95 | 96 |
84 | 85 | Home page 86 | 87 |
Home page 88 | 		90 | 91 | Adding task 92 | 93 |
Adding task 94 |
97 | 98 | 99 |
100 | 101 | ## Design 102 | 103 | ### Built With 104 | * Backend: FastAPI 105 | * Frontend: React 106 | * Middleware: Redis 107 | * Model Runner: Ollama 108 | 109 | More model engines and integrations are under development! 110 | 111 | ### Architecture 112 | 113 | Logo 114 | 115 | ### Data Contracts 116 | 117 | Redis Queues and Channels: 118 | 119 | 120 | REST API: 121 | 122 | ## Getting Started 123 | 124 | ### Prerequisites 125 | Ensure the following are installed: 126 | - [Ollama](https://ollama.com/) if not using the Dockerized version 127 | - Recommended for Apple Silicon due to [Lack of GPU Passthrough](https://github.com/ollama/ollama/issues/3849) 128 | - [Docker Desktop](https://www.docker.com/products/docker-desktop/) 129 | 130 | ### Installation 131 | 132 | Clone the repository from Github: 133 | 134 | ``` 135 | git clone git@github.com:dankeg/KestrelAI.git 136 | ``` 137 | 138 | Ensure that Docker Desktop is running (easy way to check is running `docker info` in the terminal) 139 | 140 | Navigate to the root of the repo, and run the following command to build and launch the application. 141 | 142 | ``` 143 | docker compose up --build 144 | ``` 145 | 146 | Alternatively, rather than building locally images are built through Github Actions, and can be fetched with `docker pull` first. 147 | 148 | After a few minutes, the application will finish building. By default, it launches at `http://localhost:5173/`. Navigate here in a browser, and it's ready to use! 149 | 150 | ## Usage 151 | 152 | Using Kestrel is fairly straightforward. Create a new task, define a description for the research agents to base their exploration off of, and provide a time-box. Some examples are provided of tasks Kestrel can perform. 153 | 154 | From there, the dashboard lets you monitor progress, such as tracking searches & the resulting sources, viewing checkpoints & reports, and exporting results. 155 | 156 | ## Roadmap 157 | 158 | See the [open issues](https://github.com/dankeg/KestrelAI/issues) for a list of proposed features (and known issues). 159 | 160 | - [Top Feature Requests](https://github.com/dankeg/KestrelAI/issues?q=label%3Aenhancement+is%3Aopen+sort%3Areactions-%2B1-desc) (Add your votes using the 👍 reaction) 161 | - [Top Bugs](https://github.com/dankeg/KestrelAI/issues?q=is%3Aissue+is%3Aopen+label%3Abug+sort%3Areactions-%2B1-desc) (Add your votes using the 👍 reaction) 162 | - [Newest Bugs](https://github.com/dankeg/KestrelAI/issues?q=is%3Aopen+is%3Aissue+label%3Abug) 163 | 164 | ## Support 165 | Reach out to the maintainer at one of the following places: 166 | 167 | - [GitHub Discussions](https://github.com/dankeg/KestrelAI/discussions) 168 | - Contact options listed on [this GitHub profile](https://github.com/dankeg) 169 | 170 | ## Project assistance 171 | 172 | If you want to say **thank you** or/and support active development of Kestrel: 173 | 174 | - Add a [GitHub Star](https://github.com/dankeg/KestrelAI) to the project. 175 | - Tweet about the Kestrel. 176 | - Write interesting articles about the project on [Dev.to](https://dev.to/), [Medium](https://medium.com/) or your personal blog. 177 | 178 | Together, we can make Kestrel **better**! 179 | 180 | ## Contributing 181 | 182 | All contributions are **greatly appreciated**. Kestrel is meant to be a practical, community-focused project, focusing on supporting real-world use-cases. New feature suggestions are welcome, especially new orchestrator profiles tailored towards particular tasks of requirements, such as observability or structure. 183 | 184 | Please read [our contribution guidelines](docs/CONTRIBUTING.md), and thank you for being involved! 185 | 186 | ## Authors & contributors 187 | 188 | The original setup of this repository is by [Ganesh Danke](https://github.com/dankeg). 189 | 190 | For a full list of all authors and contributors, see [the contributors page](https://github.com/dankeg/KestrelAI/contributors). 191 | 192 | 193 | ## License 194 | 195 | This project is licensed under the **MIT license**. 196 | 197 | See [LICENSE](LICENSE) for more information. 198 | 199 | -------------------------------------------------------------------------------- /kestrel-ui/src/index.css: -------------------------------------------------------------------------------- 1 | @tailwind base; 2 | @tailwind components; 3 | @tailwind utilities; 4 | 5 | /* Optional: smooth fonts + nicer scrolling */ 6 | html, body, #root { height: 100%; } 7 | body { -webkit-font-smoothing: antialiased; -moz-osx-font-smoothing: grayscale; } 8 | 9 | /* Theme System - CSS Custom Properties */ 10 | :root { 11 | /* Amber Theme (Default) */ 12 | --theme-primary-50: #fffbeb; 13 | --theme-primary-100: #fef3c7; 14 | --theme-primary-200: #fde68a; 15 | --theme-primary-300: #fcd34d; 16 | --theme-primary-400: #fbbf24; 17 | --theme-primary-500: #f59e0b; 18 | --theme-primary-600: #d97706; 19 | --theme-primary-700: #b45309; 20 | --theme-primary-800: #92400e; 21 | --theme-primary-900: #78350f; 22 | 23 | --theme-secondary-50: #fff7ed; 24 | --theme-secondary-100: #ffedd5; 25 | --theme-secondary-200: #fed7aa; 26 | --theme-secondary-300: #fdba74; 27 | --theme-secondary-400: #fb923c; 28 | --theme-secondary-500: #f97316; 29 | --theme-secondary-600: #ea580c; 30 | --theme-secondary-700: #c2410c; 31 | --theme-secondary-800: #9a3412; 32 | --theme-secondary-900: #7c2d12; 33 | 34 | --theme-gradient-from: #fbbf24; 35 | --theme-gradient-to: #fb923c; 36 | --theme-gradient-bg: linear-gradient(to bottom right, #fffbeb, #fff7ed, #fffbeb); 37 | --theme-sidebar-gradient: linear-gradient(to bottom, #92400e, #b45309, #c2410c); 38 | --theme-backdrop: rgba(251, 191, 36, 0.1); 39 | 40 | /* Opacity variants */ 41 | --theme-primary-700-50: rgba(180, 83, 9, 0.5); 42 | --theme-primary-700-70: rgba(180, 83, 9, 0.7); 43 | --theme-primary-700-20: rgba(180, 83, 9, 0.2); 44 | --theme-primary-600-30: rgba(217, 119, 6, 0.3); 45 | --theme-primary-500-30: rgba(245, 158, 11, 0.3); 46 | --theme-primary-800-30: rgba(146, 64, 14, 0.3); 47 | --theme-primary-700-30: rgba(180, 83, 9, 0.3); 48 | --theme-primary-300-70: rgba(252, 211, 77, 0.7); 49 | --theme-primary-50-50: rgba(255, 251, 235, 0.5); 50 | --theme-primary-100-80: rgba(254, 243, 199, 0.8); 51 | --theme-primary-200-70: rgba(253, 230, 138, 0.7); 52 | --theme-primary-300-80: rgba(252, 211, 77, 0.8); 53 | --theme-secondary-50-50: rgba(255, 247, 237, 0.5); 54 | --theme-secondary-50-80: rgba(255, 247, 237, 0.8); 55 | --theme-primary-200-80: rgba(253, 230, 138, 0.8); 56 | } 57 | 58 | [data-theme="blue"] { 59 | /* Blue Theme */ 60 | --theme-primary-50: #eff6ff; 61 | --theme-primary-100: #dbeafe; 62 | --theme-primary-200: #bfdbfe; 63 | --theme-primary-300: #93c5fd; 64 | --theme-primary-400: #60a5fa; 65 | --theme-primary-500: #3b82f6; 66 | --theme-primary-600: #2563eb; 67 | --theme-primary-700: #1d4ed8; 68 | --theme-primary-800: #1e40af; 69 | --theme-primary-900: #1e3a8a; 70 | 71 | --theme-secondary-50: #f0f9ff; 72 | --theme-secondary-100: #e0f2fe; 73 | --theme-secondary-200: #bae6fd; 74 | --theme-secondary-300: #7dd3fc; 75 | --theme-secondary-400: #38bdf8; 76 | --theme-secondary-500: #0ea5e9; 77 | --theme-secondary-600: #0284c7; 78 | --theme-secondary-700: #0369a1; 79 | --theme-secondary-800: #075985; 80 | --theme-secondary-900: #0c4a6e; 81 | 82 | --theme-gradient-from: #3b82f6; 83 | --theme-gradient-to: #0ea5e9; 84 | --theme-gradient-bg: linear-gradient(to bottom right, #eff6ff, #f0f9ff, #eff6ff); 85 | --theme-sidebar-gradient: linear-gradient(to bottom, #1e40af, #1d4ed8, #0369a1); 86 | --theme-backdrop: rgba(59, 130, 246, 0.1); 87 | 88 | /* Opacity variants */ 89 | --theme-primary-700-50: rgba(29, 78, 216, 0.5); 90 | --theme-primary-700-70: rgba(29, 78, 216, 0.7); 91 | --theme-primary-700-20: rgba(29, 78, 216, 0.2); 92 | --theme-primary-600-30: rgba(37, 99, 235, 0.3); 93 | --theme-primary-500-30: rgba(59, 130, 246, 0.3); 94 | --theme-primary-800-30: rgba(30, 64, 175, 0.3); 95 | --theme-primary-700-30: rgba(29, 78, 216, 0.3); 96 | --theme-primary-300-70: rgba(147, 197, 253, 0.7); 97 | --theme-primary-50-50: rgba(239, 246, 255, 0.5); 98 | --theme-primary-100-80: rgba(219, 234, 254, 0.8); 99 | --theme-primary-200-70: rgba(191, 219, 254, 0.7); 100 | --theme-primary-300-80: rgba(147, 197, 253, 0.8); 101 | --theme-secondary-50-50: rgba(240, 249, 255, 0.5); 102 | --theme-secondary-50-80: rgba(240, 249, 255, 0.8); 103 | --theme-primary-200-80: rgba(191, 219, 254, 0.8); 104 | } 105 | 106 | /* Theme-aware utility classes */ 107 | .theme-bg-primary-50 { background-color: var(--theme-primary-50); } 108 | .theme-bg-primary-100 { background-color: var(--theme-primary-100); } 109 | .theme-bg-primary-200 { background-color: var(--theme-primary-200); } 110 | .theme-bg-primary-300 { background-color: var(--theme-primary-300); } 111 | .theme-bg-primary-400 { background-color: var(--theme-primary-400); } 112 | .theme-bg-primary-500 { background-color: var(--theme-primary-500); } 113 | .theme-bg-primary-600 { background-color: var(--theme-primary-600); } 114 | .theme-bg-primary-700 { background-color: var(--theme-primary-700); } 115 | .theme-bg-primary-800 { background-color: var(--theme-primary-800); } 116 | .theme-bg-primary-900 { background-color: var(--theme-primary-900); } 117 | 118 | .theme-text-primary-50 { color: var(--theme-primary-50); } 119 | .theme-text-primary-100 { color: var(--theme-primary-100); } 120 | .theme-text-primary-200 { color: var(--theme-primary-200); } 121 | .theme-text-primary-300 { color: var(--theme-primary-300); } 122 | .theme-text-primary-400 { color: var(--theme-primary-400); } 123 | .theme-text-primary-500 { color: var(--theme-primary-500); } 124 | .theme-text-primary-600 { color: var(--theme-primary-600); } 125 | .theme-text-primary-700 { color: var(--theme-primary-700); } 126 | .theme-text-primary-800 { color: var(--theme-primary-800); } 127 | .theme-text-primary-900 { color: var(--theme-primary-900); } 128 | 129 | .theme-border-primary-200 { border-color: var(--theme-primary-200); } 130 | .theme-border-primary-300 { border-color: var(--theme-primary-300); } 131 | .theme-border-primary-400 { border-color: var(--theme-primary-400); } 132 | .theme-border-primary-500 { border-color: var(--theme-primary-500); } 133 | .theme-border-primary-600 { border-color: var(--theme-primary-600); } 134 | .theme-border-primary-700 { border-color: var(--theme-primary-700); } 135 | 136 | .theme-bg-secondary-50 { background-color: var(--theme-secondary-50); } 137 | .theme-bg-secondary-100 { background-color: var(--theme-secondary-100); } 138 | .theme-bg-secondary-200 { background-color: var(--theme-secondary-200); } 139 | .theme-bg-secondary-300 { background-color: var(--theme-secondary-300); } 140 | .theme-bg-secondary-400 { background-color: var(--theme-secondary-400); } 141 | .theme-bg-secondary-500 { background-color: var(--theme-secondary-500); } 142 | .theme-bg-secondary-600 { background-color: var(--theme-secondary-600); } 143 | .theme-bg-secondary-700 { background-color: var(--theme-secondary-700); } 144 | .theme-bg-secondary-800 { background-color: var(--theme-secondary-800); } 145 | .theme-bg-secondary-900 { background-color: var(--theme-secondary-900); } 146 | 147 | .theme-text-secondary-50 { color: var(--theme-secondary-50); } 148 | .theme-text-secondary-100 { color: var(--theme-secondary-100); } 149 | .theme-text-secondary-200 { color: var(--theme-secondary-200); } 150 | .theme-text-secondary-300 { color: var(--theme-secondary-300); } 151 | .theme-text-secondary-400 { color: var(--theme-secondary-400); } 152 | .theme-text-secondary-500 { color: var(--theme-secondary-500); } 153 | .theme-text-secondary-600 { color: var(--theme-secondary-600); } 154 | .theme-text-secondary-700 { color: var(--theme-secondary-700); } 155 | .theme-text-secondary-800 { color: var(--theme-secondary-800); } 156 | .theme-text-secondary-900 { color: var(--theme-secondary-900); } 157 | 158 | .theme-gradient-bg { background: var(--theme-gradient-bg); } 159 | .theme-sidebar-gradient { background: var(--theme-sidebar-gradient); } 160 | .theme-gradient-from { background-color: var(--theme-gradient-from); } 161 | .theme-gradient-to { background-color: var(--theme-gradient-to); } 162 | 163 | /* Specific theme-aware classes with opacity */ 164 | .theme-bg-primary-700-50 { background-color: var(--theme-primary-700-50); } 165 | .theme-bg-primary-700-70 { background-color: var(--theme-primary-700-70); } 166 | .theme-bg-primary-700-20 { background-color: var(--theme-primary-700-20); } 167 | .theme-bg-primary-800-30 { background-color: var(--theme-primary-800-30); } 168 | .theme-bg-primary-600-30 { background-color: var(--theme-primary-600-30); } 169 | .theme-border-primary-600-30 { border-color: var(--theme-primary-600-30); } 170 | .theme-border-primary-500-30 { border-color: var(--theme-primary-500-30); } 171 | .theme-border-primary-700-30 { border-color: var(--theme-primary-700-30); } 172 | .theme-text-primary-300-70 { color: var(--theme-primary-300-70); } 173 | .theme-bg-primary-50-50 { background-color: var(--theme-primary-50-50); } 174 | .theme-bg-primary-100-80 { background-color: var(--theme-primary-100-80); } 175 | .theme-text-primary-200-70 { color: var(--theme-primary-200-70); } 176 | .theme-text-primary-300-80 { color: var(--theme-primary-300-80); } 177 | .theme-bg-secondary-50-50 { background-color: var(--theme-secondary-50-50); } 178 | .theme-bg-secondary-50-80 { background-color: var(--theme-secondary-50-80); } 179 | .theme-text-primary-200-80 { color: var(--theme-primary-200-80); } 180 | -------------------------------------------------------------------------------- /tests/README.md: -------------------------------------------------------------------------------- 1 | # KestrelAI Test Suite 2 | 3 | This directory contains the comprehensive test suite for KestrelAI, organized by test type and functionality. 4 | 5 | ## Test Organization 6 | 7 | ### Directory Structure 8 | 9 | ``` 10 | tests/ 11 | ├── unit/ # Unit tests 12 | │ ├── test_core.py # Core functionality tests 13 | │ └── test_agents.py # Agent functionality tests 14 | ├── integration/ # Integration tests 15 | │ ├── test_model_loop.py # Model loop integration tests 16 | │ └── test_memory_store.py # Memory store integration tests 17 | ├── api/ # Backend API tests 18 | │ └── test_backend_api.py # API endpoint tests 19 | ├── frontend/ # Frontend tests 20 | │ └── test_theme_switching.py # Theme switching tests 21 | ├── e2e/ # End-to-end tests 22 | │ └── test_workflow.py # Complete workflow tests 23 | ├── performance/ # Performance tests 24 | │ └── test_regression.py # Performance regression tests 25 | ├── utils/ # Test utilities 26 | │ ├── check_services.py # Service availability checker 27 | │ ├── test_fixtures.py # Common test fixtures 28 | │ └── test_config.py # Test configuration 29 | ├── conftest.py # Test configuration and fixtures 30 | ├── run_tests.py # Test runner script 31 | └── README.md # This file 32 | ``` 33 | 34 | ### Test Categories 35 | 36 | - **Unit Tests** (`tests/unit/`): Fast, isolated tests with mocked dependencies 37 | - **Integration Tests** (`tests/integration/`): Tests that verify component interactions 38 | - **API Tests** (`tests/api/`): Backend API endpoint tests 39 | - **Frontend Tests** (`tests/frontend/`): Frontend UI and theme tests 40 | - **End-to-End Tests** (`tests/e2e/`): Complete workflow tests 41 | - **Performance Tests** (`tests/performance/`): Performance and regression tests 42 | 43 | ## Running Tests 44 | 45 | ### Using the Test Runner (Recommended) 46 | 47 | ```bash 48 | # Run all tests 49 | python tests/run_tests.py 50 | 51 | # Run specific test categories 52 | python tests/run_tests.py unit integration 53 | python tests/run_tests.py api frontend 54 | python tests/run_tests.py e2e performance 55 | 56 | # Run with coverage 57 | python tests/run_tests.py --coverage 58 | 59 | # Run in parallel 60 | python tests/run_tests.py --parallel 61 | 62 | # Skip service check 63 | python tests/run_tests.py --skip-service-check 64 | ``` 65 | 66 | ### Basic pytest Commands 67 | 68 | ```bash 69 | # Run all tests 70 | pytest 71 | 72 | # Run specific test categories 73 | pytest -m unit 74 | pytest -m integration 75 | pytest -m api 76 | pytest -m frontend 77 | pytest -m e2e 78 | pytest -m performance 79 | 80 | # Run specific test files 81 | pytest tests/unit/test_core.py 82 | pytest tests/api/test_backend_api.py 83 | 84 | # Run with verbose output 85 | pytest -v 86 | 87 | # Run with coverage 88 | pytest --cov=KestrelAI --cov-report=html 89 | ``` 90 | 91 | ### Test Markers 92 | 93 | Tests are organized using pytest markers: 94 | 95 | - `@pytest.mark.unit`: Unit tests 96 | - `@pytest.mark.integration`: Integration tests 97 | - `@pytest.mark.api`: Backend API tests 98 | - `@pytest.mark.frontend`: Frontend UI tests 99 | - `@pytest.mark.e2e`: End-to-end tests 100 | - `@pytest.mark.performance`: Performance tests 101 | - `@pytest.mark.slow`: Slow running tests 102 | - `@pytest.mark.requires_services`: Tests requiring external services 103 | 104 | ### Service Requirements 105 | 106 | Some tests require external services to be running: 107 | 108 | - **Backend API** (`http://localhost:8000`) - Required for API tests 109 | - **Frontend** (`http://localhost:5173`) - Required for frontend tests 110 | - **Ollama** (`http://localhost:11434`) - Optional for LLM tests 111 | - **SearXNG** (`http://localhost:8080`) - Optional for search tests 112 | - **Redis** (`localhost:6379`) - Optional for caching tests 113 | 114 | Check service availability: 115 | 116 | ```bash 117 | python tests/utils/check_services.py 118 | ``` 119 | 120 | ## Test Configuration 121 | 122 | ### Environment Variables 123 | 124 | Tests use the following environment variables: 125 | 126 | - `PYTHONPATH`: Set to project root 127 | - `REDIS_HOST`: Redis server host 128 | - `REDIS_PORT`: Redis server port 129 | - `OLLAMA_BASE_URL`: Ollama server URL 130 | - `SEARXNG_URL`: SearXNG server URL 131 | 132 | ### Test Configuration 133 | 134 | Test configuration is managed in `tests/utils/test_config.py`: 135 | 136 | - Targets and URLs for services 137 | - Performance thresholds 138 | - Test data constants 139 | - Service requirements by category 140 | 141 | ### Fixtures 142 | 143 | Common test fixtures are available in `tests/utils/test_fixtures.py`: 144 | 145 | - `temp_dir`: Temporary directory for test files 146 | - `sample_task`: Sample task for testing 147 | - `sample_research_plan`: Sample research plan 148 | - `mock_llm`: Mock LLM wrapper 149 | - `mock_memory_store`: Mock memory store 150 | - `mock_redis`: Mock Redis client 151 | - `mock_ollama_client`: Mock Ollama client 152 | - `mock_chromadb`: Mock ChromaDB client 153 | - `mock_sentence_transformer`: Mock SentenceTransformer 154 | - `mock_requests`: Mock HTTP requests 155 | 156 | ## Writing Tests 157 | 158 | ### Test Structure 159 | 160 | ```python 161 | import pytest 162 | from unittest.mock import Mock, patch 163 | from tests.utils.test_fixtures import TestDataFactory 164 | 165 | @pytest.mark.unit 166 | class TestMyComponent: 167 | """Test my component functionality.""" 168 | 169 | def test_basic_functionality(self, mock_llm): 170 | """Test basic functionality.""" 171 | # Test implementation 172 | assert True 173 | 174 | @pytest.mark.requires_services 175 | def test_with_external_service(self): 176 | """Test with external service.""" 177 | # Test implementation 178 | assert True 179 | ``` 180 | 181 | ### Using Test Utilities 182 | 183 | ```python 184 | from tests.utils.test_fixtures import TestData 185 | 186 | def test_with_test_data(): 187 | """Test using test data factory.""" 188 | task = TestDataFactory.create_task( 189 | name="Custom Task", 190 | description="Custom description" 191 | ) 192 | assert task.name == "Custom Task" 193 | 194 | def test_with_config(): 195 | """Test using test configuration.""" 196 | config = get_test_config() 197 | assert config["api_base_url"] == "http://localhost:8000/api/v1" 198 | ``` 199 | 200 | ### Best Practices 201 | 202 | 1. **Use descriptive test names**: Test names should clearly describe what is being tested 203 | 2. **Mock external dependencies**: Use mocks for external services in unit tests 204 | 3. **Test edge cases**: Include tests for error conditions and edge cases 205 | 4. **Use fixtures**: Leverage pytest fixtures for common setup 206 | 5. **Mark tests appropriately**: Use the correct pytest markers 207 | 6. **Keep tests fast**: Unit tests should run quickly 208 | 7. **Test one thing**: Each test should verify one specific behavior 209 | 8. **Use test utilities**: Leverage the test utilities for common patterns 210 | 211 | ## Continuous Integration 212 | 213 | Tests are automatically run in CI/CD pipelines with the following configuration: 214 | 215 | - Python 3.9+ 216 | - pytest with coverage reporting 217 | - Service availability checking 218 | - Performance regression detection 219 | 220 | ## Troubleshooting 221 | 222 | ### Common Issues 223 | 224 | 1. **Import Errors**: Ensure PYTHONPATH is set correctly 225 | 2. **Service Connection Errors**: Check that required services are running 226 | 3. **Timeout Errors**: Increase timeout values for slow tests 227 | 4. **Mock Issues**: Verify mock configurations and return values 228 | 229 | ### Debug Mode 230 | 231 | Run tests in debug mode for detailed output: 232 | 233 | ```bash 234 | pytest -v -s --tb=long 235 | ``` 236 | 237 | ### Service Debugging 238 | 239 | Check service availability: 240 | 241 | ```bash 242 | python tests/utils/check_services.py 243 | ``` 244 | 245 | ## Performance Testing 246 | 247 | Performance tests ensure that the system meets performance requirements: 248 | 249 | - **LLM Response Time**: < 10 seconds 250 | - **Planning Phase Time**: < 30 seconds 251 | - **Task Creation Time**: < 1 second 252 | - **Redis Operation Time**: < 0.1 seconds 253 | - **API Response Time**: < 2 seconds 254 | - **Frontend Load Time**: < 3 seconds 255 | 256 | Run performance tests: 257 | 258 | ```bash 259 | pytest -m performance 260 | ``` 261 | 262 | ## Coverage 263 | 264 | Test coverage is tracked and reported: 265 | 266 | ```bash 267 | pytest --cov=KestrelAI --cov-report=html 268 | ``` 269 | 270 | Coverage reports are generated in `htmlcov/` directory. 271 | 272 | ## Test Utilities 273 | 274 | ### Service Checker 275 | 276 | The service checker (`tests/utils/check_services.py`) provides: 277 | 278 | - Service availability checking 279 | - Configuration management 280 | - Test-specific service validation 281 | 282 | ### Test Data Factory 283 | 284 | The test data factory (`tests/utils/test_fixtures.py`) provides: 285 | 286 | - Sample data creation 287 | - Mock object setup 288 | - Common test patterns 289 | 290 | ### Test Configuration 291 | 292 | Test configuration (`tests/utils/test_config.py`) provides: 293 | 294 | - Service URLs and settings 295 | - Performance thresholds 296 | - Test categorization -------------------------------------------------------------------------------- /KestrelAI/agents/searxng_service.py: -------------------------------------------------------------------------------- 1 | """ 2 | SearXNG service for web search functionality. 3 | Handles SearXNG setup, container management, and search execution. 4 | """ 5 | 6 | import logging 7 | import os 8 | import subprocess 9 | import time 10 | from typing import Any 11 | 12 | import requests 13 | from bs4 import BeautifulSoup 14 | 15 | from .url_utils import clean_url 16 | 17 | logger = logging.getLogger(__name__) 18 | 19 | # Configuration 20 | SEARXNG_URL = os.getenv("SEARXNG_URL", "http://localhost:8080/search") 21 | FETCH_BYTES = 30_000 22 | MAX_SNIPPET_LENGTH = 3000 23 | DEBUG = True 24 | 25 | # Track if we've already checked/started SearchXNG 26 | _searxng_checked = False 27 | 28 | 29 | def _is_running_in_docker() -> bool: 30 | """Check if we're running inside a Docker container""" 31 | # Check for Docker-specific environment variables or files 32 | if os.path.exists("/.dockerenv"): 33 | return True 34 | if os.path.exists("/proc/self/cgroup"): 35 | try: 36 | with open("/proc/self/cgroup") as f: 37 | if "docker" in f.read(): 38 | return True 39 | except Exception: 40 | pass 41 | if os.getenv("container") == "docker": 42 | return True 43 | return False 44 | 45 | 46 | def _check_searxng_accessible(url: str, timeout: int = 2) -> bool: 47 | """Check if SearchXNG is accessible at the given URL""" 48 | try: 49 | test_url = url.replace("/search", "") # Try base URL first 50 | res = requests.get(test_url, timeout=timeout) 51 | return res.status_code == 200 52 | except Exception: 53 | return False 54 | 55 | 56 | def _check_docker_running() -> bool: 57 | """Check if Docker daemon is running""" 58 | try: 59 | result = subprocess.run( 60 | ["docker", "info"], check=False, capture_output=True, timeout=5 61 | ) 62 | return result.returncode == 0 63 | except Exception: 64 | return False 65 | 66 | 67 | def _get_docker_compose_cmd(): 68 | """Get the appropriate docker compose command (handles both 'docker compose' and 'docker-compose')""" 69 | # Try newer 'docker compose' first (Docker Compose V2) 70 | try: 71 | result = subprocess.run( 72 | ["docker", "compose", "version"], 73 | check=False, 74 | capture_output=True, 75 | timeout=5, 76 | ) 77 | if result.returncode == 0: 78 | return ["docker", "compose"] 79 | except Exception: 80 | pass 81 | 82 | # Fall back to older 'docker-compose' (Docker Compose V1) 83 | try: 84 | result = subprocess.run( 85 | ["docker-compose", "version"], check=False, capture_output=True, timeout=5 86 | ) 87 | if result.returncode == 0: 88 | return ["docker-compose"] 89 | except Exception: 90 | pass 91 | 92 | return None 93 | 94 | 95 | def ensure_searxng_running(): 96 | """Ensure SearchXNG container is running when running locally (not in Docker)""" 97 | global _searxng_checked 98 | if _searxng_checked: 99 | return # Already checked 100 | 101 | _searxng_checked = True 102 | 103 | # Check if we're running in Docker - if so, SearchXNG should already be available 104 | if _is_running_in_docker(): 105 | return # Already running in Docker, no action needed 106 | 107 | # Also skip if SEARXNG_URL points to a Docker service name (Docker networking) 108 | if "searxng:" in SEARXNG_URL or not SEARXNG_URL.startswith("http://localhost"): 109 | return # Not a localhost URL, assume it's configured correctly 110 | 111 | # Check if SearchXNG is already accessible 112 | if _check_searxng_accessible(SEARXNG_URL): 113 | return # SearchXNG is already running 114 | 115 | # Check if Docker is running 116 | if not _check_docker_running(): 117 | logger.warning("Docker daemon is not running, cannot auto-start SearchXNG") 118 | return 119 | 120 | # Get docker compose command 121 | compose_cmd = _get_docker_compose_cmd() 122 | if not compose_cmd: 123 | logger.warning("docker compose command not found, cannot auto-start SearchXNG") 124 | return 125 | 126 | # If not accessible, try to start it via docker compose 127 | try: 128 | # Find docker-compose.yml in the project root (go up from KestrelAI/agents/) 129 | project_root = os.path.abspath( 130 | os.path.join(os.path.dirname(__file__), "..", "..") 131 | ) 132 | compose_file = os.path.join(project_root, "docker-compose.yml") 133 | 134 | if not os.path.exists(compose_file): 135 | logger.warning( 136 | f"docker-compose.yml not found at {compose_file}, cannot auto-start SearchXNG" 137 | ) 138 | return 139 | 140 | logger.info( 141 | "Starting SearchXNG container for local development (this will also start Redis if needed)..." 142 | ) 143 | 144 | # Run from the project root directory to ensure relative paths work 145 | cmd = compose_cmd + ["-f", compose_file, "up", "-d", "searxng"] 146 | result = subprocess.run( 147 | cmd, 148 | cwd=project_root, 149 | check=False, 150 | capture_output=True, 151 | text=True, 152 | timeout=60, 153 | ) 154 | 155 | if result.returncode != 0: 156 | logger.warning(f"Failed to start SearchXNG container: {result.stderr}") 157 | if result.stdout: 158 | logger.debug(f"docker compose output: {result.stdout}") 159 | return 160 | 161 | # Wait for SearchXNG to become available (with retries) 162 | logger.info("Waiting for SearchXNG to be ready...") 163 | max_retries = 30 # Wait up to 30 seconds 164 | for i in range(max_retries): 165 | if _check_searxng_accessible(SEARXNG_URL, timeout=3): 166 | logger.info("SearchXNG container is ready") 167 | return 168 | time.sleep(1) 169 | 170 | logger.warning( 171 | "SearchXNG container started but not yet accessible after 30 seconds" 172 | ) 173 | except subprocess.TimeoutExpired: 174 | logger.warning("Timeout starting SearchXNG container") 175 | except FileNotFoundError: 176 | logger.warning("docker compose command not found, cannot auto-start SearchXNG") 177 | except Exception as e: 178 | logger.warning(f"Could not start SearchXNG container: {e}") 179 | 180 | 181 | class SearXNGService: 182 | """Service for executing web searches via SearXNG""" 183 | 184 | def __init__( 185 | self, searxng_url: str = None, search_results: int = 4, debug: bool = True 186 | ): 187 | self.searxng_url = searxng_url or SEARXNG_URL 188 | self.search_results = search_results 189 | self.debug = debug 190 | 191 | def search(self, query: str) -> list[dict[str, Any]]: 192 | """Execute web search via SearXNG""" 193 | # Ensure SearchXNG is running when running locally 194 | ensure_searxng_running() 195 | 196 | params = { 197 | "q": query, 198 | "format": "json", 199 | "language": "en", 200 | "safesearch": 1, 201 | "engines": "google", 202 | "categories": "general", 203 | } 204 | headers = { 205 | "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36", 206 | "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", 207 | "Accept-Language": "en-US,en;q=0.5", 208 | "Accept-Encoding": "gzip, deflate", 209 | "Connection": "keep-alive", 210 | } 211 | 212 | try: 213 | res = requests.get( 214 | self.searxng_url, params=params, headers=headers, timeout=15 215 | ) 216 | res.raise_for_status() 217 | data = res.json() 218 | 219 | return [ 220 | { 221 | "title": r.get("title", "")[:100], 222 | "href": r.get("url", ""), 223 | "body": r.get("content", "")[:300], 224 | } 225 | for r in data.get("results", [])[: self.search_results] 226 | if r.get("url") 227 | ] 228 | except Exception as e: 229 | if self.debug: 230 | print(f"Search error: {e}") 231 | return [] 232 | 233 | def extract_text(self, url: str) -> str: 234 | """Download page & return readable text (best-effort).""" 235 | try: 236 | # Clean URL before fetching 237 | clean_url_str = clean_url(url) 238 | if clean_url_str is None: 239 | return "" 240 | 241 | resp = requests.get( 242 | clean_url_str, timeout=10, headers={"User-Agent": "Mozilla/5.0"} 243 | ) 244 | resp.raise_for_status() 245 | html = resp.text[:FETCH_BYTES] 246 | soup = BeautifulSoup(html, "html.parser") 247 | 248 | # Remove non-content elements 249 | for t in soup(["script", "style", "noscript", "meta", "link"]): 250 | t.extract() 251 | 252 | text = " ".join(soup.get_text(" ", strip=True).split()) 253 | return text[:MAX_SNIPPET_LENGTH] 254 | except Exception as e: 255 | if self.debug: 256 | print(f"Failed to extract from {url}: {e}") 257 | return "" 258 | -------------------------------------------------------------------------------- /kestrel-ui/src/ChatInterface.tsx: -------------------------------------------------------------------------------- 1 | import React, { useState, useRef, useEffect } from "react"; 2 | import { ArrowUp, Copy, ThumbsUp, ThumbsDown, RotateCcw, ArrowLeft } from "lucide-react"; 3 | 4 | interface Message { 5 | id: string; 6 | content: string; 7 | role: "user" | "assistant"; 8 | timestamp: number; 9 | } 10 | 11 | const ChatInterface: React.FC = () => { 12 | const [messages, setMessages] = useState([]); 13 | const [inputValue, setInputValue] = useState(""); 14 | const [isLoading, setIsLoading] = useState(false); 15 | const messagesEndRef = useRef(null); 16 | const textareaRef = useRef(null); 17 | 18 | const scrollToBottom = () => { 19 | messagesEndRef.current?.scrollIntoView({ behavior: "smooth" }); 20 | }; 21 | 22 | useEffect(() => { 23 | scrollToBottom(); 24 | }, [messages]); 25 | 26 | const handleSend = async () => { 27 | if (!inputValue.trim() || isLoading) return; 28 | 29 | const userMessage: Message = { 30 | id: Date.now().toString(), 31 | content: inputValue.trim(), 32 | role: "user", 33 | timestamp: Date.now(), 34 | }; 35 | 36 | setMessages(prev => [...prev, userMessage]); 37 | setInputValue(""); 38 | setIsLoading(true); 39 | 40 | // Simulate AI response 41 | setTimeout(() => { 42 | const assistantMessage: Message = { 43 | id: (Date.now() + 1).toString(), 44 | content: "This is a mockup response. In a real implementation, this would be connected to your LLM backend. The interface is designed to match modern LLM chat applications like ChatGPT and Claude, with a clean and minimal design.", 45 | role: "assistant", 46 | timestamp: Date.now(), 47 | }; 48 | setMessages(prev => [...prev, assistantMessage]); 49 | setIsLoading(false); 50 | }, 1500); 51 | }; 52 | 53 | const handleKeyPress = (e: React.KeyboardEvent) => { 54 | if (e.key === "Enter" && !e.shiftKey) { 55 | e.preventDefault(); 56 | handleSend(); 57 | } 58 | }; 59 | 60 | const copyToClipboard = (content: string) => { 61 | navigator.clipboard.writeText(content); 62 | }; 63 | 64 | const adjustTextareaHeight = () => { 65 | if (textareaRef.current) { 66 | textareaRef.current.style.height = 'auto'; 67 | textareaRef.current.style.height = `${Math.min(textareaRef.current.scrollHeight, 200)}px`; 68 | } 69 | }; 70 | 71 | useEffect(() => { 72 | adjustTextareaHeight(); 73 | }, [inputValue]); 74 | 75 | return ( 76 |
77 | {/* Header */} 78 |
79 |
80 |
81 | 86 | 87 | 88 |

KestrelAI

89 |
90 | 93 |
94 |
95 | 96 | {/* Messages */} 97 |
98 |
99 | {messages.length === 0 && ( 100 |
101 |

How can I help you today?

102 |

Start a conversation with KestrelAI

103 |
104 | )} 105 | 106 | {messages.map((message) => ( 107 |
108 |
109 |
110 | {message.role === "user" ? ( 111 |
112 | U 113 |
114 | ) : ( 115 |
116 | K 117 |
118 | )} 119 |
120 | 121 |
122 |
123 | 124 | {message.role === "user" ? "You" : "KestrelAI"} 125 | 126 |
127 | 128 |
129 |
{message.content}
130 |
131 | 132 | {message.role === "assistant" && ( 133 |
134 | 141 | 147 | 153 |
154 | )} 155 |
156 |
157 |
158 | ))} 159 | 160 | {/* Loading indicator */} 161 | {isLoading && ( 162 |
163 |
164 |
165 |
166 | K 167 |
168 |
169 |
170 |
171 | KestrelAI 172 |
173 |
174 |
175 |
176 |
177 |
178 |
179 | Thinking... 180 |
181 |
182 |
183 |
184 | )} 185 | 186 |
187 |
188 |
189 | 190 | {/* Input */} 191 |
192 |
193 |
194 |