├── .npmrc ├── img └── logo.png ├── .npmignore ├── .gitignore ├── tsconfig.build.json ├── vitest.config.ts ├── src ├── types │ ├── test.d.ts │ ├── column.ts │ ├── marker.ts │ ├── slo.ts │ ├── index.ts │ ├── recipient.ts │ ├── board.ts │ ├── trigger.ts │ ├── analysis.ts │ ├── query.ts │ └── api.ts ├── resources │ ├── index.ts │ ├── datasets.test.ts │ └── datasets.ts ├── utils │ ├── functions.ts │ ├── analysis.ts │ ├── constants.ts │ ├── errors.ts │ ├── tool-error.ts │ ├── typeguards.ts │ ├── functions.test.ts │ ├── transformations.test.ts │ ├── typeguards.test.ts │ └── transformations.ts ├── tools │ ├── get-board.ts │ ├── list-recipients.ts │ ├── instrumentation-guidance.ts │ ├── list-slos.ts │ ├── get-slo.ts │ ├── list-triggers.ts │ ├── get-slo.test.ts │ ├── index.ts │ ├── get-trace-link.ts │ ├── list-recipients.test.ts │ ├── list-markers.test.ts │ ├── get-trigger.ts │ ├── get-trace-link.test.ts │ ├── list-slos.test.ts │ ├── get-trigger.test.ts │ ├── analyze-columns.test.ts │ ├── get-board.test.ts │ ├── list-boards.test.ts │ ├── list-triggers.test.ts │ ├── list-markers.ts │ └── analyze-columns.ts ├── index.ts ├── prompts │ ├── guidance.ts │ ├── index.ts │ └── index.test.ts ├── config.test.ts ├── query │ └── validation.test.ts └── config.ts ├── tsconfig.json ├── .github └── workflows │ ├── publish.yml │ └── evaluation.yml ├── .env.example ├── LICENSE ├── eval ├── prompts │ ├── test-list-datasets.json │ ├── test-analyze-latency.json │ └── test-investigate-errors.json ├── CHANGELOG.md ├── templates │ └── index.html └── scripts │ └── types.ts ├── CLAUDE.md ├── TESTING.md ├── package.json └── docs └── generic-instrumentation-guidance.md /.npmrc: -------------------------------------------------------------------------------- 1 | shamefully-hoist=true 2 | -------------------------------------------------------------------------------- /img/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/honeycombio/honeycomb-mcp/HEAD/img/logo.png -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | src/ 2 | tsconfig.json 3 | .env 4 | .env.* 5 | .hny/ 6 | api.yaml 7 | .mcp-honeycomb.json 8 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | build 3 | .env 4 | .mcp-honeycomb.json 5 | eval/results/ 6 | eval/reports/ 7 | .DS_Store 8 | coverage/ -------------------------------------------------------------------------------- /tsconfig.build.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tsconfig.json", 3 | "include": ["src/**/*"], 4 | "exclude": ["node_modules", "build", "eval/**/*"] 5 | } 6 | -------------------------------------------------------------------------------- /vitest.config.ts: -------------------------------------------------------------------------------- 1 | import { defineConfig } from 'vitest/config' 2 | 3 | export default defineConfig({ 4 | test: { 5 | environment: 'node', 6 | }, 7 | }) -------------------------------------------------------------------------------- /src/types/test.d.ts: -------------------------------------------------------------------------------- 1 | import { vi } from 'vitest' 2 | 3 | declare global { 4 | interface Window { 5 | fetch: typeof fetch; 6 | } 7 | var fetch: typeof fetch; 8 | } -------------------------------------------------------------------------------- /src/types/column.ts: -------------------------------------------------------------------------------- 1 | export interface Column { 2 | id: string; 3 | key_name: string; 4 | type: "string" | "float" | "integer" | "boolean"; 5 | description: string; 6 | hidden: boolean; 7 | last_written?: string; 8 | created_at: string; 9 | updated_at: string; 10 | } 11 | -------------------------------------------------------------------------------- /src/types/marker.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Interface for a Honeycomb marker (deployment event) 3 | */ 4 | export interface Marker { 5 | id: string; 6 | message: string; 7 | type: string; 8 | url?: string; 9 | created_at: string; 10 | start_time: string; 11 | end_time?: string; 12 | } 13 | 14 | /** 15 | * Response type for listing markers 16 | */ 17 | export interface MarkersResponse { 18 | markers: Marker[]; 19 | } 20 | -------------------------------------------------------------------------------- /src/types/slo.ts: -------------------------------------------------------------------------------- 1 | export interface SLI { 2 | alias: string; 3 | } 4 | 5 | export interface SLO { 6 | id: string; 7 | name: string; 8 | description?: string; 9 | sli: SLI; 10 | time_period_days: number; 11 | target_per_million: number; 12 | reset_at?: string; 13 | created_at: string; 14 | updated_at: string; 15 | } 16 | 17 | export interface SLODetailedResponse extends SLO { 18 | compliance: number; 19 | budget_remaining: number; 20 | } 21 | -------------------------------------------------------------------------------- /src/types/index.ts: -------------------------------------------------------------------------------- 1 | // Import config types and re-export them 2 | import { ConfigSchema, EnvironmentSchema } from "../config.js"; 3 | import type { Config, Environment } from "../config.js"; 4 | export { ConfigSchema, EnvironmentSchema }; 5 | export type { Config, Environment }; 6 | 7 | export * from "./query.js"; 8 | export * from "./column.js"; 9 | export * from "./slo.js"; 10 | export * from "./api.js"; 11 | export * from "./trigger.js"; 12 | export * from "./schema.js"; 13 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "@total-typescript/tsconfig/tsc/no-dom/app", 3 | "compilerOptions": { 4 | "moduleResolution": "NodeNext", 5 | "module": "NodeNext", 6 | "target": "ES2022", 7 | "outDir": "build", 8 | "sourceMap": true, 9 | "esModuleInterop": true, 10 | "noEmit": false, 11 | "verbatimModuleSyntax": false, 12 | "strict": true, 13 | "noImplicitAny": true, 14 | "strictNullChecks": true, 15 | "noEmitOnError": true 16 | }, 17 | "include": ["src/**/*", "eval/**/*"], 18 | "exclude": ["node_modules", "build"] 19 | } 20 | -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | name: Publish to NPM 2 | 3 | on: 4 | release: 5 | types: [created] 6 | 7 | jobs: 8 | publish: 9 | runs-on: ubuntu-latest 10 | steps: 11 | - uses: actions/checkout@v4 12 | 13 | - name: Setup pnpm 14 | uses: pnpm/action-setup@v4 15 | 16 | - uses: actions/setup-node@v4 17 | with: 18 | node-version: "18" 19 | registry-url: "https://registry.npmjs.org" 20 | 21 | - run: pnpm install --frozen-lockfile 22 | - run: pnpm run build 23 | 24 | - run: pnpm publish 25 | env: 26 | NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} 27 | -------------------------------------------------------------------------------- /.env.example: -------------------------------------------------------------------------------- 1 | # LLM API Keys 2 | OPENAI_API_KEY=your-openai-api-key 3 | ANTHROPIC_API_KEY=your-anthropic-api-key 4 | GEMINI_API_KEY=your-gemini-api-key 5 | 6 | # MCP Server Configuration 7 | # For HTTP-based connection (only one of these should be uncommented) 8 | # MCP_SERVER_URL=http://localhost:3000 9 | # For stdio-based connection (recommended) 10 | MCP_SERVER_COMMAND=node build/index.mjs 11 | 12 | # Evaluation Configuration 13 | # Note: This will run against all specified models for each provider 14 | EVAL_MODELS={"openai":"gpt-4o","anthropic":["claude-3-5-haiku-latest","claude-3-7-sonnet-latest"],"gemini":"gemini-2.0-flash-001"} 15 | EVAL_CONCURRENCY=2 -------------------------------------------------------------------------------- /src/resources/index.ts: -------------------------------------------------------------------------------- 1 | import { HoneycombAPI } from "../api/client.js"; 2 | import { createDatasetsResource, handleDatasetResource } from "./datasets.js"; 3 | import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; 4 | 5 | /** 6 | * Register all resources with the MCP server 7 | * 8 | * @param server - The MCP server instance 9 | * @param api - The Honeycomb API client 10 | */ 11 | export function registerResources(server: McpServer, api: HoneycombAPI) { 12 | // Register datasets resource 13 | server.resource( 14 | "datasets", 15 | createDatasetsResource(api), 16 | (_uri: URL, variables: Record) => 17 | handleDatasetResource(api, variables as Record) 18 | ); 19 | } 20 | -------------------------------------------------------------------------------- /src/types/recipient.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Interface for notification recipient types 3 | */ 4 | export type RecipientType = 5 | | "email" 6 | | "slack" 7 | | "pagerduty" 8 | | "webhook" 9 | | "msteams" 10 | | "msteams_workflow"; 11 | 12 | /** 13 | * Interface for a notification recipient 14 | */ 15 | export interface Recipient { 16 | id: string; 17 | name: string; 18 | type: RecipientType; 19 | target?: string; 20 | details?: { 21 | pagerduty_severity?: "critical" | "error" | "warning" | "info"; 22 | url?: string; 23 | }; 24 | created_at: string; 25 | updated_at: string; 26 | } 27 | 28 | /** 29 | * Response type for listing recipients 30 | */ 31 | export interface RecipientsResponse { 32 | recipients: Recipient[]; 33 | } 34 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2025 Honeycomb 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 8 | -------------------------------------------------------------------------------- /eval/prompts/test-list-datasets.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "test-list-datasets", 3 | "name": "List Datasets Test", 4 | "description": "Tests the agent's ability to list datasets in a Honeycomb environment", 5 | "prompt": "List all the datasets available in the ms-demo environment and provide a short summary of what each dataset might contain based on its name.", 6 | "environment": "ms-demo", 7 | "expectedTools": ["list_datasets"], 8 | "maxSteps": 3, 9 | "validation": { 10 | "prompt": "Evaluate whether the agent successfully listed all datasets in the ms-demo environment and provided reasonable hypotheses about what each dataset might contain. The agent should have used the list_datasets tool and correctly included the environment parameter. The summary should be clear and concise.", 11 | "expectedOutcome": { 12 | "success": true, 13 | "criteria": [ 14 | "Used the list_datasets tool with correct parameters", 15 | "Listed all available datasets", 16 | "Provided reasonable guesses about dataset contents based on names", 17 | "Response is clear and well-organized" 18 | ] 19 | } 20 | }, 21 | "options": { 22 | "timeout": 10000 23 | } 24 | } -------------------------------------------------------------------------------- /src/types/board.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Interface for a board query (used within a Board) 3 | */ 4 | export interface BoardQuery { 5 | caption?: string; 6 | query_style?: 'graph' | 'table' | 'combo'; 7 | dataset?: string; 8 | query_id?: string; 9 | visualization_settings?: Record; 10 | graph_settings?: { 11 | hide_markers?: boolean; 12 | log_scale?: boolean; 13 | omit_missing_values?: boolean; 14 | stacked_graphs?: boolean; 15 | utc_xaxis?: boolean; 16 | overlaid_charts?: boolean; 17 | }; 18 | } 19 | 20 | /** 21 | * Interface for a Honeycomb board (dashboard) 22 | */ 23 | export interface Board { 24 | id: string; 25 | name: string; 26 | description?: string; 27 | style?: string; 28 | column_layout?: 'multi' | 'single'; 29 | queries?: BoardQuery[]; 30 | slos?: string[]; 31 | links?: { 32 | board_url?: string; 33 | }; 34 | created_at: string; 35 | updated_at: string; 36 | } 37 | 38 | /** 39 | * Response type for listing boards 40 | * 41 | * Note: The API docs suggest this response structure, but the actual API 42 | * might return an array directly. We handle both cases in the client code. 43 | */ 44 | export interface BoardsResponse { 45 | boards: Board[]; 46 | } -------------------------------------------------------------------------------- /src/types/trigger.ts: -------------------------------------------------------------------------------- 1 | export interface NotificationRecipient { 2 | id: string; 3 | type: 4 | | "pagerduty" 5 | | "email" 6 | | "slack" 7 | | "webhook" 8 | | "msteams" 9 | | "msteams_workflow"; 10 | target?: string; 11 | details?: { 12 | pagerduty_severity?: "critical" | "error" | "warning" | "info"; 13 | }; 14 | } 15 | 16 | export interface TriggerThreshold { 17 | op: ">" | ">=" | "<" | "<="; 18 | value: number; 19 | exceeded_limit?: number; 20 | } 21 | 22 | export interface TriggerResponse { 23 | id: string; 24 | name: string; 25 | description?: string; 26 | threshold: TriggerThreshold; 27 | frequency: number; 28 | alert_type?: "on_change" | "on_true"; 29 | disabled: boolean; 30 | triggered: boolean; 31 | recipients: NotificationRecipient[]; 32 | evaluation_schedule_type?: "frequency" | "window"; 33 | evaluation_schedule?: { 34 | window: { 35 | days_of_week: ( 36 | | "sunday" 37 | | "monday" 38 | | "tuesday" 39 | | "wednesday" 40 | | "thursday" 41 | | "friday" 42 | | "saturday" 43 | )[]; 44 | start_time: string; // HH:mm format 45 | end_time: string; // HH:mm format 46 | }; 47 | }; 48 | created_at: string; 49 | updated_at: string; 50 | } 51 | -------------------------------------------------------------------------------- /eval/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Evaluation Framework Changelog 2 | 3 | ## v1.1.0 - Multi-Step and Conversation Mode Support 4 | 5 | ### Added 6 | - **Multi-Step Evaluation Mode**: Added support for executing a predefined sequence of tool calls and evaluating the combined results 7 | - **Conversation Mode**: Added support for LLM-driven multi-step evaluations where the model determines which tools to call 8 | - **Enhanced Metrics**: Track tool call counts, step counts, and other execution metrics 9 | - **Improved Reporting**: Updated HTML reports to display detailed information about tool calls in multi-step scenarios 10 | - **Sample Tests**: Added example multi-step and conversation mode test definitions 11 | 12 | ### Changed 13 | - **Schema Updates**: Extended the prompt schema to support both single and multi-step executions 14 | - **Documentation**: Updated README with comprehensive documentation for the new capabilities 15 | - **Report Layout**: Redesigned the HTML report to better display multi-step test results 16 | 17 | ### Technical Changes 18 | - Extended `EvalPromptSchema` to support step definitions and conversation mode 19 | - Added new `ToolCallRecord` schema to track individual tool calls 20 | - Implemented `runMultiStepMode` and `runConversationMode` methods 21 | - Updated validation prompt generation to include all tool calls 22 | - Added tool call metrics to summary statistics -------------------------------------------------------------------------------- /eval/templates/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Honeycomb MCP Evaluation Reports 7 | 18 | 19 | 20 |
21 |

Honeycomb MCP Evaluation Reports

22 |

Select a report to view detailed evaluation results:

23 | 24 | 32 |
33 | 34 | -------------------------------------------------------------------------------- /src/utils/functions.ts: -------------------------------------------------------------------------------- 1 | import { QueryResultValue } from "../types/query.js"; 2 | 3 | /** 4 | * Calculate standard deviation for an array of values 5 | */ 6 | export function calculateStdDev(values: number[], mean: number): number { 7 | if (values.length <= 1) return 0; 8 | 9 | const squareDiffs = values.map(value => { 10 | const diff = value - mean; 11 | return diff * diff; 12 | }); 13 | 14 | const avgSquareDiff = squareDiffs.reduce((sum, value) => sum + value, 0) / squareDiffs.length; 15 | return Math.sqrt(avgSquareDiff); 16 | } 17 | 18 | /** 19 | * Interface for top value result items 20 | */ 21 | export interface TopValueItem { 22 | value: T; 23 | count: number; 24 | } 25 | 26 | /** 27 | * Get top N values with their frequencies 28 | */ 29 | export function getTopValues(results: QueryResultValue[], column: string, limit: number = 5): Array { 30 | const valueCounts = new Map(); 31 | 32 | // Count frequencies 33 | results.forEach(result => { 34 | const value = result[column]; 35 | if (value !== undefined && value !== null) { 36 | valueCounts.set(value, (valueCounts.get(value) || 0) + 1); 37 | } 38 | }); 39 | 40 | // Convert to array and sort by frequency 41 | return Array.from(valueCounts.entries()) 42 | .map(([value, count]) => ({ value, count })) 43 | .sort((a, b) => b.count - a.count) 44 | .slice(0, limit); 45 | } -------------------------------------------------------------------------------- /CLAUDE.md: -------------------------------------------------------------------------------- 1 | # CLAUDE.md 2 | 3 | This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. 4 | 5 | # Honeycomb MCP Development Guide 6 | 7 | ## Build & Test Commands 8 | - Build: `pnpm run build` 9 | - Typecheck: `pnpm typecheck` 10 | - Run all tests: `pnpm test` 11 | - Run tests with watch mode: `pnpm test:watch` 12 | - Run single test: `pnpm test -- -t "test name"` (pattern matches test descriptions) 13 | - Run tests in specific file: `pnpm test -- src/path/to/file.test.ts` 14 | - Run with coverage: `pnpm test:coverage` 15 | 16 | ## Code Style Guidelines 17 | - **TypeScript**: Use explicit types for parameters, variables, and return values 18 | - **Imports**: Group external libs first, then internal; use named imports with destructuring 19 | - **Modules**: Use ES modules with `.js` extension in import paths 20 | - **Naming**: camelCase for variables/methods, PascalCase for classes/interfaces/types 21 | - **Error Handling**: Use custom `HoneycombError` class for API errors; centralized error handling 22 | - **Testing**: Use Vitest with `vi` for mocks; test both success and error cases 23 | - **Async**: Use async/await consistently; handle promise rejections with try/catch 24 | - **Type Validation**: Use Zod schemas for validating external data 25 | - **API Design**: Methods should be focused with clear parameters and return types 26 | - **Documentation**: Add comments for complex operations and public methods 27 | - **Caching**: Use `@stacksjs/ts-cache` for resource caching; normalize keys as `environment:resource:id` -------------------------------------------------------------------------------- /eval/prompts/test-analyze-latency.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "test-analyze-latency", 3 | "name": "Analyze Service Latency", 4 | "description": "Tests the agent's ability to analyze latency metrics across services", 5 | "prompt": "Analyze the ms-demo environment to identify which service has the highest average latency and determine if there are any patterns or correlations with error rates.", 6 | "environment": "ms-demo", 7 | "context": "Our engineering team is investigating performance issues across our microservices. We need to understand which service is experiencing the highest latency and whether these performance issues correlate with increased error rates.", 8 | "expectedTools": ["list_datasets", "get_columns", "run_query"], 9 | "maxSteps": 8, 10 | "validation": { 11 | "prompt": "Evaluate whether the agent successfully identified the service with the highest latency and analyzed potential correlations with error rates. The agent should have used appropriate tools in a logical sequence, starting with listing datasets, then exploring columns, and finally running meaningful queries. The analysis should include quantitative metrics and show whether there's a correlation between latency and errors.", 12 | "expectedOutcome": { 13 | "success": true, 14 | "criteria": [ 15 | "Identified the service with the highest average latency with supporting data", 16 | "Analyzed potential correlation between latency and error rates", 17 | "Used appropriate tools in a logical sequence", 18 | "Ran meaningful queries with correct parameters", 19 | "Provided clear, data-driven insights" 20 | ] 21 | } 22 | }, 23 | "options": { 24 | "timeout": 30000 25 | } 26 | } -------------------------------------------------------------------------------- /src/utils/analysis.ts: -------------------------------------------------------------------------------- 1 | import { CardinalityClassification, NumericStatistics } from "../types/analysis.js"; 2 | 3 | 4 | /** 5 | * Determine cardinality classification based on the number of unique values 6 | * 7 | * @param uniqueCount - The number of unique values in the dataset 8 | * @returns A classification of the cardinality (low, medium, high, very high) 9 | */ 10 | export function getCardinalityClassification(uniqueCount: number): CardinalityClassification { 11 | if (uniqueCount <= 10) return 'low'; 12 | if (uniqueCount <= 100) return 'medium'; 13 | if (uniqueCount <= 1000) return 'high'; 14 | return 'very high'; 15 | } 16 | 17 | /** 18 | * Generate interpretation text for numeric statistics based on Honeycomb documentation 19 | */ 20 | export function generateInterpretation(stats: NumericStatistics, columnName: string): string { 21 | const interpretations = []; 22 | 23 | if (stats.avg !== undefined && stats.p95 !== undefined) { 24 | const ratio = stats.p95 / stats.avg; 25 | if (ratio > 3) { 26 | interpretations.push(`The P95 value is ${ratio.toFixed(1)}x higher than the average, suggesting significant outliers in ${columnName}.`); 27 | } 28 | } 29 | 30 | if (stats.min !== undefined && stats.max !== undefined) { 31 | const range = stats.max - stats.min; 32 | if (stats.avg !== undefined && range > stats.avg * 10) { 33 | interpretations.push(`The range (${range}) is very wide compared to the average (${stats.avg}), indicating high variability.`); 34 | } 35 | } 36 | 37 | if (interpretations.length === 0) { 38 | return `Standard distribution of ${columnName} values with expected statistical properties.`; 39 | } 40 | 41 | return interpretations.join(' '); 42 | } -------------------------------------------------------------------------------- /src/utils/constants.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Application constants 3 | * 4 | * ASCII values: 5 | * 84 = T, 104 = h, 105 = i, 115 = s, 32 = space, 114 = r, 101 = e, 112 = p, 111 = o, 115 = s, 105 = i, 116 = t, 111 = o, 114 = r, 121 = y, 32 = space, 105 = i, 115 = s, 32 = space, 100 = d, 101 = e, 100 = d, 105 = i, 99 = c, 97 = a, 116 = t, 101 = e, 100 = d, 32 = space, 116 = t, 111 = o, 32 = space, 109 = m, 111 = o, 116 = t, 104 = h, 101 = e, 114 = r, 115 = s, 44 = ,, 32 = space, 102 = f, 111 = o, 111 = o, 116 = t, 98 = b, 97 = a, 108 = l, 108 = l, 44 = ,, 32 = space, 97 = a, 110 = n, 100 = d, 32 = space, 109 = m, 97 = a, 116 = t, 101 = e, 114 = r, 110 = n, 97 = a, 108 = l, 32 = space, 104 = h, 101 = e, 97 = a, 108 = l, 116 = t, 104 = h, 46 = . 6 | */ 7 | 8 | export const DEFAULT_CACHE_TTL = 60 * 5 * 1000; // 5 minutes in milliseconds 9 | export const DEFAULT_REQUEST_TIMEOUT = 30 * 1000; // 30 seconds in milliseconds 10 | export const MAX_COLUMNS_DISPLAYED = 50; // Maximum number of columns to display 11 | 12 | // Default error messages 13 | export const ERROR_MESSAGES = { 14 | UNAUTHORIZED: 'Unauthorized: Invalid API key', 15 | NOT_FOUND: 'Resource not found', 16 | REQUEST_FAILED: 'Request failed', 17 | VALIDATION_FAILED: 'Validation failed', 18 | TIMEOUT: 'Request timed out', 19 | }; 20 | 21 | // ASCII values - development note reference only 22 | export const ASCII_ARRAY = [ 23 | 84, 104, 105, 115, 32, 114, 101, 112, 111, 115, 105, 116, 111, 114, 121, 32, 24 | 105, 115, 32, 100, 101, 100, 105, 99, 97, 116, 101, 100, 32, 116, 111, 32, 25 | 109, 111, 116, 104, 101, 114, 115, 44, 32, 102, 111, 111, 116, 98, 97, 108, 26 | 108, 44, 32, 97, 110, 100, 32, 109, 97, 116, 101, 114, 110, 97, 108, 32, 27 | 104, 101, 97, 108, 116, 104, 46 28 | ]; -------------------------------------------------------------------------------- /src/types/analysis.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Cardinality classification for column values 3 | */ 4 | export type CardinalityClassification = 'low' | 'medium' | 'high' | 'very high'; 5 | 6 | /** 7 | * Information about column value cardinality 8 | */ 9 | export interface CardinalityInfo { 10 | uniqueCount: number; 11 | classification: CardinalityClassification; 12 | } 13 | 14 | /** 15 | * Interface for statistics used in analysis and interpretation 16 | */ 17 | export interface NumericStatistics { 18 | min?: number; 19 | max?: number; 20 | avg?: number; 21 | p95?: number; 22 | median?: number; 23 | sum?: number; 24 | range?: number; 25 | stdDev?: number; 26 | } 27 | 28 | /** 29 | * Numeric statistics with interpretation 30 | */ 31 | export interface NumericStatsWithInterpretation extends NumericStatistics { 32 | interpretation: string; 33 | } 34 | 35 | /** 36 | * Value with count and percentage representation 37 | */ 38 | export interface ValueWithPercentage { 39 | value: string | number | boolean | null; 40 | count: number; 41 | percentage: string; 42 | } 43 | 44 | /** 45 | * Simplified column analysis result 46 | */ 47 | export interface SimplifiedColumnAnalysis { 48 | /** The names of the columns being analyzed */ 49 | columns: string[]; 50 | /** The number of results returned in the analysis */ 51 | count: number; 52 | /** Total number of events/records across all results */ 53 | totalEvents: number; 54 | /** Most frequent values in the columns with their counts */ 55 | topValues?: Array; 56 | /** Statistical information for numeric columns */ 57 | stats?: Record; 58 | /** Information about how many unique combinations exist */ 59 | cardinality?: CardinalityInfo; 60 | /** Any error that occurred during result processing */ 61 | processingError?: string; 62 | } -------------------------------------------------------------------------------- /eval/prompts/test-investigate-errors.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": "test-investigate-errors", 3 | "name": "Investigate Error Patterns", 4 | "description": "Tests the agent's ability to investigate error patterns across services", 5 | "prompt": "Investigate HTTP error patterns in the ms-demo environment. Identify which endpoints or services have the highest error rates, what types of errors are occurring, and suggest possible causes based on the data.", 6 | "environment": "ms-demo", 7 | "context": "Our site reliability team has noticed an increase in errors across our services. We need to understand which services and endpoints are most affected, what types of errors are occurring, and get insights into potential root causes.", 8 | "expectedTools": ["list_datasets", "get_columns", "run_query", "analyze_column"], 9 | "maxSteps": 10, 10 | "validation": { 11 | "prompt": "Evaluate whether the agent successfully identified error patterns in the ms-demo environment. The agent should have identified services or endpoints with high error rates, categorized the types of errors, and provided data-driven insights about potential causes. The analysis should follow a logical progression, building on insights from previous steps. The agent should have used appropriate tools for each part of the investigation.", 12 | "expectedOutcome": { 13 | "success": true, 14 | "criteria": [ 15 | "Identified services/endpoints with highest error rates using appropriate metrics", 16 | "Categorized different error types or status codes", 17 | "Analyzed correlations or patterns (e.g., time-based, service dependencies)", 18 | "Used appropriate tools in a logical, progressive sequence", 19 | "Provided data-driven hypotheses about potential causes", 20 | "Communicated findings clearly with supporting data" 21 | ] 22 | } 23 | }, 24 | "options": { 25 | "timeout": 40000 26 | } 27 | } -------------------------------------------------------------------------------- /src/utils/errors.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Base error class for Honeycomb API errors 3 | */ 4 | export interface ValidationErrorContext { 5 | environment?: string; 6 | dataset?: string; 7 | granularity?: number; 8 | api_route?: string; 9 | } 10 | 11 | export class HoneycombError extends Error { 12 | constructor( 13 | public statusCode: number, 14 | message: string, 15 | public suggestions: string[] = [] 16 | ) { 17 | super(message); 18 | this.name = "HoneycombError"; 19 | } 20 | 21 | /** 22 | * Factory method for creating validation errors with appropriate suggestions 23 | */ 24 | static createValidationError( 25 | message: string, 26 | context: ValidationErrorContext 27 | ): HoneycombError { 28 | const contextStr = Object.entries(context) 29 | .filter(([_, value]) => value !== undefined) 30 | .map(([key, value]) => `${key}="${value}"`) 31 | .join(", "); 32 | 33 | return new HoneycombError( 34 | 422, 35 | `Query validation failed: ${message}\n\nSuggested next steps:\n- ${contextStr}\n\nPlease verify:\n- The environment name is correct and configured via HONEYCOMB_API_KEY or HONEYCOMB_ENV_*_API_KEY\n- Your API key is valid\n- The dataset exists and you have access to it\n- Your query parameters are valid` 36 | ); 37 | } 38 | 39 | /** 40 | * Get a formatted error message including suggestions 41 | */ 42 | getFormattedMessage(): string { 43 | let output = this.message; 44 | if (this.suggestions.length > 0) { 45 | output += "\n\nSuggested next steps:"; 46 | this.suggestions.forEach(suggestion => { 47 | output += `\n- ${suggestion}`; 48 | }); 49 | } 50 | return output; 51 | } 52 | } 53 | 54 | /** 55 | * Error class for query-specific errors 56 | */ 57 | export class QueryError extends HoneycombError { 58 | constructor(message: string, suggestions: string[] = []) { 59 | super(400, message, suggestions); 60 | this.name = "QueryError"; 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/utils/tool-error.ts: -------------------------------------------------------------------------------- 1 | import { HoneycombError } from "./errors.js"; 2 | import { z } from "zod"; 3 | 4 | /** 5 | * Handles errors from tool execution and returns a formatted error response 6 | */ 7 | export async function handleToolError( 8 | error: unknown, 9 | toolName: string, 10 | options: { 11 | suppressConsole?: boolean; 12 | environment?: string; 13 | dataset?: string; 14 | } = {} 15 | ): Promise<{ 16 | content: { type: "text"; text: string }[]; 17 | error: { message: string; }; 18 | }> { 19 | let errorMessage = "Unknown error occurred"; 20 | let suggestions: string[] = []; 21 | 22 | if (error instanceof HoneycombError) { 23 | // Use the enhanced error message system 24 | errorMessage = error.getFormattedMessage(); 25 | } else if (error instanceof z.ZodError) { 26 | // For Zod validation errors, create a validation error with context 27 | const validationError = HoneycombError.createValidationError( 28 | error.errors.map(err => err.message).join(", "), 29 | { 30 | environment: options.environment, 31 | dataset: options.dataset 32 | } 33 | ); 34 | errorMessage = validationError.getFormattedMessage(); 35 | } else if (error instanceof Error) { 36 | errorMessage = error.message; 37 | } 38 | 39 | // Log the error to stderr for debugging, unless suppressed 40 | if (!options.suppressConsole) { 41 | console.error(`Tool '${toolName}' failed:`, error); 42 | } 43 | 44 | let helpText = `Failed to execute tool '${toolName}': ${errorMessage}\n\n` + 45 | `Please verify:\n` + 46 | `- The environment name is correct and configured via HONEYCOMB_API_KEY or HONEYCOMB_ENV_*_API_KEY\n` + 47 | `- Your API key is valid\n` + 48 | `- The dataset exists and you have access to it\n` + 49 | `- Your query parameters are valid\n`; 50 | 51 | return { 52 | content: [ 53 | { 54 | type: "text", 55 | text: helpText, 56 | }, 57 | ], 58 | error: { 59 | message: errorMessage 60 | } 61 | }; 62 | } -------------------------------------------------------------------------------- /src/types/query.ts: -------------------------------------------------------------------------------- 1 | export interface QueryCalculation { 2 | op: string; 3 | column?: string; 4 | } 5 | 6 | export type FilterOperator = 7 | | "=" 8 | | "!=" 9 | | ">" 10 | | ">=" 11 | | "<" 12 | | "<=" 13 | | "starts-with" 14 | | "does-not-start-with" 15 | | "ends-with" 16 | | "does-not-end-with" 17 | | "exists" 18 | | "does-not-exist" 19 | | "contains" 20 | | "does-not-contain" 21 | | "in" 22 | | "not-in"; 23 | 24 | export interface QueryFilter { 25 | column: string; 26 | op: FilterOperator; 27 | value?: string | number | boolean | string[] | number[]; 28 | } 29 | 30 | export type QueryOrderDirection = "ascending" | "descending"; 31 | 32 | export interface QueryOrder { 33 | column: string; 34 | op?: string; 35 | order?: QueryOrderDirection; 36 | } 37 | 38 | export interface QueryHaving { 39 | calculate_op: string; 40 | column?: string; 41 | op: string; 42 | value: number; 43 | } 44 | 45 | export interface AnalysisQuery { 46 | calculations?: QueryCalculation[]; 47 | breakdowns?: string[]; 48 | filters?: QueryFilter[]; 49 | filter_combination?: "AND" | "OR"; 50 | orders?: QueryOrder[]; 51 | limit?: number; 52 | time_range?: number; 53 | start_time?: number; 54 | end_time?: number; 55 | granularity?: number; 56 | havings?: QueryHaving[]; 57 | } 58 | 59 | interface QueryResultData { 60 | results?: QueryResultValue[]; 61 | series?: QuerySeriesValue[]; 62 | } 63 | 64 | export interface QueryResult { 65 | data?: QueryResultData; 66 | links?: { 67 | query_url?: string; 68 | graph_image_url?: string; 69 | }; 70 | complete: boolean; 71 | id: string; 72 | } 73 | 74 | export interface QueryResultValue { 75 | [key: string]: string | number | boolean | null; 76 | } 77 | 78 | export interface QuerySeriesValue { 79 | [key: string]: string | number | boolean | null; 80 | } 81 | 82 | export interface QueryResponse { 83 | id: string; 84 | complete: boolean; 85 | data?: { 86 | results: QueryResultValue[]; 87 | series: QuerySeriesValue[]; 88 | }; 89 | links?: { 90 | query_url?: string; 91 | graph_image_url?: string; 92 | }; 93 | } 94 | -------------------------------------------------------------------------------- /src/tools/get-board.ts: -------------------------------------------------------------------------------- 1 | import { z } from "zod"; 2 | import { HoneycombAPI } from "../api/client.js"; 3 | import { handleToolError } from "../utils/tool-error.js"; 4 | import { GetBoardSchema } from "../types/schema.js"; 5 | 6 | /** 7 | * Tool to get a specific board (dashboard) from a Honeycomb environment. This tool returns a detailed object containing the board's ID, name, description, creation time, and last update time. 8 | * 9 | * @param api - The Honeycomb API client 10 | * @returns An MCP tool object with name, schema, and handler function 11 | */ 12 | export function createGetBoardTool(api: HoneycombAPI) { 13 | return { 14 | name: "get_board", 15 | description: "Retrieves a specific board (dashboard) from a Honeycomb environment. This tool returns a detailed object containing the board's ID, name, description, creation time, and last update time.", 16 | schema: GetBoardSchema.shape, 17 | /** 18 | * Handler for the get_board tool 19 | * 20 | * @param params - The parameters for the tool 21 | * @param params.environment - The Honeycomb environment 22 | * @param params.boardId - The ID of the board to retrieve 23 | * @returns Board details 24 | */ 25 | handler: async ({ environment, boardId }: z.infer) => { 26 | // Validate input parameters 27 | if (!environment) { 28 | return handleToolError(new Error("environment parameter is required"), "get_board"); 29 | } 30 | 31 | if (!boardId) { 32 | return handleToolError(new Error("boardId parameter is required"), "get_board"); 33 | } 34 | 35 | try { 36 | // Fetch board from the API 37 | const board = await api.getBoard(environment, boardId); 38 | 39 | return { 40 | content: [ 41 | { 42 | type: "text", 43 | text: JSON.stringify(board, null, 2), 44 | }, 45 | ], 46 | metadata: { 47 | environment, 48 | boardId, 49 | name: board.name 50 | } 51 | }; 52 | } catch (error) { 53 | return handleToolError(error, "get_board"); 54 | } 55 | } 56 | }; 57 | } -------------------------------------------------------------------------------- /src/tools/list-recipients.ts: -------------------------------------------------------------------------------- 1 | import { z } from "zod"; 2 | import { HoneycombAPI } from "../api/client.js"; 3 | import { handleToolError } from "../utils/tool-error.js"; 4 | import { ListRecipientsSchema } from "../types/schema.js"; 5 | 6 | /** 7 | * Tool to list notification recipients in a Honeycomb environment. This tool returns a list of all recipients available in the specified environment, including their names, types, targets, and metadata. 8 | * 9 | * @param api - The Honeycomb API client 10 | * @returns An MCP tool object with name, schema, and handler function 11 | */ 12 | export function createListRecipientsTool(api: HoneycombAPI) { 13 | return { 14 | name: "list_recipients", 15 | description: "Lists available recipients for notifications in a specific environment. This tool returns a list of all recipients available in the specified environment, including their names, types, targets, and metadata.", 16 | schema: ListRecipientsSchema.shape, 17 | /** 18 | * Handler for the list_recipients tool 19 | * 20 | * @param params - The parameters for the tool 21 | * @param params.environment - The Honeycomb environment 22 | * @returns List of recipients with relevant metadata 23 | */ 24 | handler: async ({ environment }: z.infer) => { 25 | // Validate input parameters 26 | if (!environment) { 27 | return handleToolError(new Error("environment parameter is required"), "list_recipients"); 28 | } 29 | 30 | try { 31 | // Fetch recipients from the API 32 | const recipients = await api.getRecipients(environment); 33 | 34 | // Create a simplified response 35 | const simplifiedRecipients = recipients.map(recipient => ({ 36 | id: recipient.id, 37 | name: recipient.name, 38 | type: recipient.type, 39 | target: recipient.target || '', 40 | created_at: recipient.created_at, 41 | updated_at: recipient.updated_at, 42 | })); 43 | 44 | return { 45 | content: [ 46 | { 47 | type: "text", 48 | text: JSON.stringify(simplifiedRecipients, null, 2), 49 | }, 50 | ], 51 | metadata: { 52 | count: simplifiedRecipients.length, 53 | environment 54 | } 55 | }; 56 | } catch (error) { 57 | return handleToolError(error, "list_recipients"); 58 | } 59 | } 60 | }; 61 | } -------------------------------------------------------------------------------- /src/types/api.ts: -------------------------------------------------------------------------------- 1 | export interface Resource { 2 | uri: string; 3 | name: string; 4 | description: string; 5 | } 6 | 7 | export interface Dataset { 8 | name: string; 9 | slug: string; 10 | description?: string; 11 | settings?: { 12 | delete_protected?: boolean; 13 | }; 14 | expand_json_depth?: number; 15 | regular_columns_count?: number; 16 | last_written_at?: string | null; 17 | created_at: string; 18 | } 19 | 20 | export interface DatasetWithColumns extends Dataset { 21 | columns: { 22 | name: string; 23 | type: string; 24 | description?: string; 25 | }[]; 26 | } 27 | 28 | export interface ToolDefinition { 29 | name: string; 30 | description: string; 31 | inputSchema: { 32 | type: "object"; 33 | properties: Record; 34 | required?: string[]; 35 | }; 36 | } 37 | 38 | export interface PromptDefinition { 39 | name: string; 40 | description: string; 41 | arguments: { 42 | name: string; 43 | description: string; 44 | required: boolean; 45 | }[]; 46 | } 47 | 48 | export interface MessageContent { 49 | type: "text"; 50 | text: string; 51 | } 52 | 53 | export interface ToolResponse { 54 | content: MessageContent[]; 55 | } 56 | 57 | export interface PromptResponse { 58 | messages: { 59 | role: "user"; 60 | content: MessageContent; 61 | }[]; 62 | } 63 | 64 | export interface AuthResponse { 65 | id: string; 66 | type: string; 67 | api_key_access: Record; 68 | environment?: { 69 | name: string; 70 | slug: string; 71 | }; 72 | team?: { 73 | name: string; 74 | slug: string; 75 | }; 76 | } 77 | 78 | export interface QueryOptions { 79 | includeSeries?: boolean; 80 | limit?: number; 81 | } 82 | 83 | /** 84 | * Standard pagination, filtering, and sorting options for collection tools 85 | */ 86 | export interface CollectionOptions { 87 | // Pagination options 88 | page?: number; 89 | limit?: number; 90 | 91 | // Sorting options 92 | sort_by?: string; 93 | sort_order?: 'asc' | 'desc'; 94 | 95 | // Search options 96 | search?: string; 97 | search_fields?: string | string[]; 98 | } 99 | 100 | /** 101 | * Response format for paginated collection data 102 | */ 103 | export interface PaginatedResponse { 104 | data: T[]; 105 | metadata: { 106 | total: number; 107 | page: number; 108 | pages: number; 109 | limit: number; 110 | }; 111 | } -------------------------------------------------------------------------------- /src/tools/instrumentation-guidance.ts: -------------------------------------------------------------------------------- 1 | import { z } from "zod"; 2 | import { HoneycombAPI } from "../api/client.js"; 3 | import { handleToolError } from "../utils/tool-error.js"; 4 | import { getInstrumentationGuidance } from "../prompts/guidance.js"; 5 | 6 | /** 7 | * Schema for the instrumentation guidance tool 8 | */ 9 | export const InstrumentationGuidanceSchema = z.object({ 10 | language: z.string().optional().describe("Programming language of the code to instrument"), 11 | filepath: z.string().optional().describe("Path to the file being instrumented") 12 | }); 13 | 14 | /** 15 | * Creates a tool for providing OpenTelemetry instrumentation guidance. 16 | * 17 | * @param api - The Honeycomb API client 18 | * @returns A configured tool object with name, schema, and handler 19 | */ 20 | export function createInstrumentationGuidanceTool(api: HoneycombAPI) { 21 | return { 22 | name: "get_instrumentation_help", 23 | description: "Provides important guidance for how to instrument code with OpenTelemetry traces and logs. It is intended to be used when someone wants to instrument their code, or improve instrumentation (such as getting advice on improving their logs or tracing, or creating new instrumentation). It is BEST used after inspecting existing code and telemetry data to understand some operational characteristics. However, if there is no telemetry data to read from Honeycomb, it can still provide guidance on how to instrument code.", 24 | schema: InstrumentationGuidanceSchema.shape, 25 | /** 26 | * Handles the instrumentation_guidance tool request 27 | * 28 | * @param params - The parameters for the instrumentation guidance 29 | * @returns A formatted response with instrumentation guidance 30 | */ 31 | handler: async (params: z.infer) => { 32 | try { 33 | // Get the instrumentation guidance template 34 | const guidance = getInstrumentationGuidance(); 35 | 36 | const language = params?.language || "your code"; 37 | const filepath = params?.filepath 38 | ? ` for ${params.filepath}` 39 | : ""; 40 | 41 | return { 42 | content: [ 43 | { 44 | type: "text", 45 | text: `# Instrumentation Guidance for ${language}${filepath}\n\n${guidance}`, 46 | }, 47 | ], 48 | }; 49 | } catch (error) { 50 | return handleToolError(error, "get_instrumentation_help"); 51 | } 52 | } 53 | }; 54 | } 55 | -------------------------------------------------------------------------------- /src/utils/typeguards.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Type guards and predicates for type safety 3 | */ 4 | 5 | /** 6 | * Type guard to check if a value is a valid number 7 | * 8 | * @param value - The value to check 9 | * @returns True if the value is a valid number 10 | */ 11 | export function isValidNumber(value: unknown): value is number { 12 | return value !== null && 13 | value !== undefined && 14 | typeof value === 'number' && 15 | !Number.isNaN(value); 16 | } 17 | 18 | /** 19 | * Type guard to check if a value is a valid string 20 | * 21 | * @param value - The value to check 22 | * @returns True if the value is a valid string 23 | */ 24 | export function isValidString(value: unknown): value is string { 25 | return value !== null && 26 | value !== undefined && 27 | typeof value === 'string'; 28 | } 29 | 30 | /** 31 | * Type guard to check if a value is a valid array 32 | * 33 | * @param value - The value to check 34 | * @returns True if the value is a valid array 35 | */ 36 | export function isValidArray(value: unknown): value is Array { 37 | return value !== null && 38 | value !== undefined && 39 | Array.isArray(value); 40 | } 41 | 42 | /** 43 | * Type guard to check if a value is a valid object 44 | * 45 | * @param value - The value to check 46 | * @returns True if the value is a valid object 47 | */ 48 | export function isValidObject(value: unknown): value is Record { 49 | return value !== null && 50 | value !== undefined && 51 | typeof value === 'object' && 52 | !Array.isArray(value); 53 | } 54 | 55 | /** 56 | * Type guard to check if a value has a specific property 57 | * 58 | * @param value - The object to check 59 | * @param propertyName - The property name to check 60 | * @returns True if the object has the property 61 | */ 62 | export function hasProperty( 63 | value: unknown, 64 | propertyName: K 65 | ): value is { [P in K]: unknown } { 66 | return isValidObject(value) && propertyName in value; 67 | } 68 | 69 | /** 70 | * Type guard to check if a value has a property of a specific type 71 | * 72 | * @param value - The object to check 73 | * @param propertyName - The property name to check 74 | * @param typeGuard - Type guard function to check the property type 75 | * @returns True if the object has the property of the expected type 76 | */ 77 | export function hasPropertyOfType( 78 | value: unknown, 79 | propertyName: K, 80 | typeGuard: (v: unknown) => v is T 81 | ): value is { [P in K]: T } { 82 | return hasProperty(value, propertyName) && typeGuard(value[propertyName]); 83 | } -------------------------------------------------------------------------------- /TESTING.md: -------------------------------------------------------------------------------- 1 | # Testing Strategy for Honeycomb MCP 2 | 3 | ## Running Tests 4 | 5 | - Run all tests: `pnpm test` 6 | - Run tests with watch mode: `pnpm test:watch` 7 | - Run a specific test: `pnpm test -- -t "test name"` (pattern matches test descriptions) 8 | - Run tests in a specific file: `pnpm test -- src/path/to/file.test.ts` 9 | - Run with coverage: `pnpm test:coverage` 10 | 11 | ## Current Test Coverage 12 | 13 | The test suite currently covers: 14 | 15 | 1. **API Client Tests** 16 | - Basic API operations (datasets, columns, queries) 17 | - Error handling for API responses 18 | - Environment configuration 19 | 20 | 2. **Configuration Tests** 21 | - Config validation via Zod 22 | 23 | 3. **Helper Function Tests** 24 | - Statistical calculations 25 | - Data processing 26 | 27 | 4. **Query Validation Tests** 28 | - Time parameter combinations 29 | - Order and having clause validations 30 | 31 | 5. **Response Transformation Tests** 32 | - Data summarization 33 | - Result formatting 34 | 35 | ## Tests To Be Added 36 | 37 | The following areas should be addressed in future test expansions: 38 | 39 | ### Priority 1 (Important) 40 | 41 | 1. **MCP Server Integration Tests** 42 | - Test the McpServer instance initialization 43 | - Test resource registration and tool invocation 44 | - Test MCP protocol message handling 45 | 46 | 2. **End-to-End Query Flow Tests** 47 | - Full API workflow from query creation to result processing 48 | 49 | 3. **Error Recovery Tests** 50 | - Test retry logic and graceful degradation 51 | 52 | ### Priority 2 (Next Phase) 53 | 54 | 1. **Authentication & Environment Tests** 55 | - Test API key management 56 | - Test environment switching and validation 57 | - Test configuration file search paths 58 | 59 | 2. **Edge Cases** 60 | - Large result sets 61 | - Special characters in column names 62 | - Query timeouts and cancellation 63 | - Partial response handling 64 | 65 | 3. **Advanced Query Features** 66 | - SLO queries 67 | - Trigger management 68 | - Complex filtering 69 | 70 | ### Priority 3 (Long Term) 71 | 72 | 1. **Performance Tests** 73 | - Response time testing 74 | - Memory usage monitoring 75 | - Context size optimization 76 | 77 | 2. **Versioning Tests** 78 | - Compatibility with Honeycomb API versions 79 | - Handling of deprecated features 80 | 81 | 3. **Integration with Different Client Tools** 82 | - LLM tool usage patterns 83 | - MCP integration with various clients 84 | 85 | ## Testing Principles 86 | 87 | - Tests should be independent and not rely on external state 88 | - Mock all external API calls 89 | - Include both happy path and error case tests 90 | - Test edge cases and unexpected inputs 91 | - Ensure test coverage for critical paths (query validation, response handling) -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@honeycombio/honeycomb-mcp", 3 | "version": "0.0.1", 4 | "description": "Model Context Protocol server for Honeycomb", 5 | "type": "module", 6 | "main": "build/index.mjs", 7 | "bin": { 8 | "honeycomb-mcp": "./build/server.js" 9 | }, 10 | "scripts": { 11 | "build": "tsc --noEmit && esbuild src/index.ts --bundle --platform=node --format=esm --outfile=build/index.mjs", 12 | "build:bundle": "esbuild src/index.ts --bundle --platform=node --format=esm --outfile=build/index.mjs", 13 | "build:prod": "tsc -p tsconfig.build.json && esbuild src/index.ts --bundle --platform=node --format=esm --outfile=build/index.mjs", 14 | "typecheck": "tsc --noEmit --project tsconfig.json", 15 | "typecheck:src": "tsc --noEmit -p tsconfig.build.json", 16 | "postbuild": "chmod +x build/index.mjs", 17 | "prepublishOnly": "pnpm run build", 18 | "test": "vitest run", 19 | "test:coverage": "vitest run --coverage", 20 | "test:watch": "vitest", 21 | "inspect": "npx @modelcontextprotocol/inspector node build/index.mjs", 22 | "eval": "pnpm tsx eval/scripts/run-eval.ts run", 23 | "eval:quiet": "pnpm tsx eval/scripts/run-eval.ts run", 24 | "eval:verbose": "EVAL_VERBOSE=true pnpm tsx eval/scripts/run-eval.ts run", 25 | "eval:report": "pnpm tsx eval/scripts/run-eval.ts report", 26 | "eval:update-index": "pnpm tsx eval/scripts/run-eval.ts update-index", 27 | "eval:all": "pnpm run build && pnpm run eval", 28 | "eval:gemini": "pnpm run build && EVAL_MODELS='{\"gemini\":[\"gemini-2.0-flash-001\"]}' pnpm tsx eval/scripts/run-eval.ts run", 29 | "eval:list-datasets": "pnpm run build && pnpm tsx eval/scripts/run-eval.ts run test-list-datasets.json", 30 | "eval:analyze-latency": "pnpm run build && pnpm tsx eval/scripts/run-eval.ts run test-analyze-latency.json", 31 | "eval:investigate-errors": "pnpm run build && pnpm tsx eval/scripts/run-eval.ts run test-investigate-errors.json" 32 | }, 33 | "publishConfig": { 34 | "access": "public" 35 | }, 36 | "files": [ 37 | "build", 38 | "README.md", 39 | "LICENSE" 40 | ], 41 | "engines": { 42 | "node": ">=18" 43 | }, 44 | "keywords": [], 45 | "author": "Honeycomb (https://www.honeycomb.io/)", 46 | "license": "MIT", 47 | "dependencies": { 48 | "@stacksjs/ts-cache": "0.1.2", 49 | "zod": "^3.24.3" 50 | }, 51 | "peerDependencies": { 52 | "@modelcontextprotocol/sdk": "^1.0.0" 53 | }, 54 | "devDependencies": { 55 | "@anthropic-ai/sdk": "^0.39.0", 56 | "@google/genai": "^0.7.0", 57 | "@modelcontextprotocol/sdk": "^1.8.0", 58 | "@total-typescript/tsconfig": "^1.0.4", 59 | "@types/mustache": "^4.2.5", 60 | "@types/node": "^22.14.0", 61 | "@vitest/coverage-v8": "^3.1.1", 62 | "dotenv": "^16.4.7", 63 | "esbuild": "^0.25.0", 64 | "mustache": "^4.2.0", 65 | "openai": "^4.91.0", 66 | "tsx": "^4.7.0", 67 | "typescript": "^5.8.2", 68 | "vitest": "^3.1.1" 69 | }, 70 | "packageManager": "pnpm@10.4.1+sha512.c753b6c3ad7afa13af388fa6d808035a008e30ea9993f58c6663e2bc5ff21679aa834db094987129aa4d488b86df57f7b634981b2f827cdcacc698cc0cfb88af", 71 | "pnpm": { 72 | "onlyBuiltDependencies": [ 73 | "esbuild" 74 | ] 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /src/tools/list-slos.ts: -------------------------------------------------------------------------------- 1 | import { z } from "zod"; 2 | import { HoneycombAPI } from "../api/client.js"; 3 | import { handleToolError } from "../utils/tool-error.js"; 4 | import { DatasetArgumentsSchema } from "../types/schema.js"; 5 | 6 | /** 7 | * Interface for simplified SLO data returned by the list_slos tool 8 | */ 9 | interface SimplifiedSLO { 10 | id: string; 11 | name: string; 12 | description: string; 13 | time_period_days: number; 14 | target_per_million: number; 15 | } 16 | 17 | /** 18 | * Tool to list SLOs (Service Level Objectives) for a specific dataset. This tool returns a list of all SLOs available in the specified environment, including their names, descriptions, time periods, and target per million events expected to succeed. 19 | * 20 | * @param api - The Honeycomb API client 21 | * @returns An MCP tool object with name, schema, and handler function 22 | */ 23 | export function createListSLOsTool(api: HoneycombAPI) { 24 | return { 25 | name: "list_slos", 26 | description: "Lists available SLOs (Service Level Objectives) for a specific dataset. This tool returns a list of all SLOs available in the specified environment, including their names, descriptions, time periods, and target per million events expected to succeed. NOTE: __all__ is NOT supported as a dataset name -- it is not possible to list all SLOs in an environment.", 27 | schema: { 28 | environment: z.string().describe("The Honeycomb environment"), 29 | dataset: z.string().describe("The dataset to fetch SLOs from"), 30 | }, 31 | /** 32 | * Handler for the list_slos tool 33 | * 34 | * @param params - The parameters for the tool 35 | * @param params.environment - The Honeycomb environment 36 | * @param params.dataset - The dataset to fetch SLOs from 37 | * @returns Simplified list of SLOs with relevant metadata 38 | */ 39 | handler: async ({ environment, dataset }: z.infer) => { 40 | // Validate input parameters 41 | if (!environment) { 42 | return handleToolError(new Error("environment parameter is required"), "list_slos"); 43 | } 44 | if (!dataset) { 45 | return handleToolError(new Error("dataset parameter is required"), "list_slos"); 46 | } 47 | 48 | try { 49 | // Fetch SLOs from the API 50 | const slos = await api.getSLOs(environment, dataset); 51 | 52 | // Simplify the response to reduce context window usage 53 | const simplifiedSLOs: SimplifiedSLO[] = slos.map(slo => ({ 54 | id: slo.id, 55 | name: slo.name, 56 | description: slo.description || '', 57 | time_period_days: slo.time_period_days, 58 | target_per_million: slo.target_per_million, 59 | })); 60 | 61 | return { 62 | content: [ 63 | { 64 | type: "text", 65 | text: JSON.stringify(simplifiedSLOs, null, 2), 66 | }, 67 | ], 68 | metadata: { 69 | count: simplifiedSLOs.length, 70 | dataset, 71 | environment 72 | } 73 | }; 74 | } catch (error) { 75 | return handleToolError(error, "list_slos"); 76 | } 77 | } 78 | }; 79 | } 80 | -------------------------------------------------------------------------------- /src/utils/functions.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, it, expect } from "vitest"; 2 | import { calculateStdDev, getTopValues } from "./functions.js"; 3 | import { QueryResultValue } from "../types/query.js"; 4 | 5 | describe("Helper functions", () => { 6 | describe("calculateStdDev", () => { 7 | it("calculates standard deviation correctly", () => { 8 | const values = [2, 4, 4, 4, 5, 5, 7, 9]; 9 | const mean = values.reduce((sum, val) => sum + val, 0) / values.length; 10 | const result = calculateStdDev(values, mean); 11 | expect(result).toBeCloseTo(2.0); 12 | }); 13 | 14 | it("returns 0 for single value array", () => { 15 | const values = [5]; 16 | const result = calculateStdDev(values, 5); 17 | expect(result).toBe(0); 18 | }); 19 | 20 | it("returns 0 for empty array", () => { 21 | const values: number[] = []; 22 | const result = calculateStdDev(values, 0); 23 | expect(result).toBe(0); 24 | }); 25 | }); 26 | 27 | describe("getTopValues", () => { 28 | it("returns top values sorted by frequency", () => { 29 | const results: QueryResultValue[] = [ 30 | { fruit: "apple", count: 5 }, 31 | { fruit: "banana", count: 2 }, 32 | { fruit: "apple", count: 6 }, 33 | { fruit: "cherry", count: 1 }, 34 | { fruit: "banana", count: 3 }, 35 | { fruit: "apple", count: 7 }, 36 | ]; 37 | 38 | const topValues = getTopValues(results, "fruit", 2); 39 | 40 | // First verify the array has the expected length 41 | expect(topValues).toHaveLength(2); 42 | 43 | // Then verify each element exists and has the expected properties 44 | const firstItem = topValues[0]; 45 | const secondItem = topValues[1]; 46 | 47 | // Use type assertions to tell TypeScript these objects exist 48 | expect(firstItem).toBeDefined(); 49 | expect(secondItem).toBeDefined(); 50 | 51 | // Now safely access their properties 52 | expect(firstItem!.value).toBe("apple"); 53 | expect(firstItem!.count).toBe(3); 54 | expect(secondItem!.value).toBe("banana"); 55 | expect(secondItem!.count).toBe(2); 56 | }); 57 | 58 | it("handles empty results", () => { 59 | const results: QueryResultValue[] = []; 60 | const topValues = getTopValues(results, "column", 5); 61 | expect(topValues).toHaveLength(0); 62 | }); 63 | 64 | it("handles null values", () => { 65 | // Updated to only use null (not undefined) to match QueryResultValue type 66 | const results: QueryResultValue[] = [ 67 | { val: "a" }, 68 | { val: null }, 69 | { val: "b" }, 70 | { val: null }, // Changed from undefined to null 71 | { val: "a" }, 72 | ]; 73 | 74 | const topValues = getTopValues(results, "val"); 75 | 76 | // First verify the array has the expected length 77 | expect(topValues).toHaveLength(2); 78 | 79 | // Then verify the first element exists 80 | const firstItem = topValues[0]; 81 | expect(firstItem).toBeDefined(); 82 | 83 | // Now safely access its properties 84 | expect(firstItem!.value).toBe("a"); 85 | expect(firstItem!.count).toBe(2); 86 | }); 87 | }); 88 | }); -------------------------------------------------------------------------------- /src/tools/get-slo.ts: -------------------------------------------------------------------------------- 1 | import { z } from "zod"; 2 | import { HoneycombAPI } from "../api/client.js"; 3 | import { handleToolError } from "../utils/tool-error.js"; 4 | 5 | /** 6 | * Interface for simplified SLO data returned by the get_slo tool 7 | */ 8 | interface SimplifiedSLODetails { 9 | id: string; 10 | name: string; 11 | description: string; 12 | time_period_days: number; 13 | target_per_million: number; 14 | compliance: number; 15 | budget_remaining: number; 16 | sli: string | undefined; 17 | created_at: string; 18 | updated_at: string; 19 | } 20 | 21 | /** 22 | * Tool to get a specific SLO (Service Level Objective) by ID with detailed information. This tool returns a detailed object containing the SLO's ID, name, description, time period, target per million, compliance, budget remaining, SLI alias, and timestamps. 23 | * 24 | * @param api - The Honeycomb API client 25 | * @returns An MCP tool object with name, schema, and handler function 26 | */ 27 | export function createGetSLOTool(api: HoneycombAPI) { 28 | return { 29 | name: "get_slo", 30 | description: "Retrieves a specific SLO (Service Level Objective) by ID with detailed information. This tool returns a detailed object containing the SLO's ID, name, description, time period, target per million, compliance, budget remaining, SLI alias, and timestamps.", 31 | schema: { 32 | environment: z.string().describe("The Honeycomb environment"), 33 | dataset: z.string().describe("The dataset containing the SLO"), 34 | sloId: z.string().describe("The ID of the SLO to retrieve"), 35 | }, 36 | /** 37 | * Handler for the get_slo tool 38 | * 39 | * @param params - The parameters for the tool 40 | * @param params.environment - The Honeycomb environment 41 | * @param params.dataset - The dataset containing the SLO 42 | * @param params.sloId - The ID of the SLO to retrieve 43 | * @returns Detailed information about the specified SLO 44 | */ 45 | handler: async ({ environment, dataset, sloId }: { environment: string; dataset: string; sloId: string }) => { 46 | // Validate input parameters 47 | if (!environment) { 48 | return handleToolError(new Error("environment parameter is required"), "get_slo"); 49 | } 50 | if (!dataset) { 51 | return handleToolError(new Error("dataset parameter is required"), "get_slo"); 52 | } 53 | if (!sloId) { 54 | return handleToolError(new Error("sloId parameter is required"), "get_slo"); 55 | } 56 | 57 | try { 58 | // Fetch SLO details from the API 59 | const slo = await api.getSLO(environment, dataset, sloId); 60 | 61 | // Simplify the response to reduce context window usage 62 | const simplifiedSLO: SimplifiedSLODetails = { 63 | id: slo.id, 64 | name: slo.name, 65 | description: slo.description || '', 66 | time_period_days: slo.time_period_days, 67 | target_per_million: slo.target_per_million, 68 | compliance: slo.compliance, 69 | budget_remaining: slo.budget_remaining, 70 | sli: slo.sli?.alias, 71 | created_at: slo.created_at, 72 | updated_at: slo.updated_at, 73 | }; 74 | 75 | return { 76 | content: [ 77 | { 78 | type: "text", 79 | text: JSON.stringify(simplifiedSLO, null, 2), 80 | }, 81 | ], 82 | metadata: { 83 | sloId, 84 | dataset, 85 | environment 86 | } 87 | }; 88 | } catch (error) { 89 | return handleToolError(error, "get_slo"); 90 | } 91 | } 92 | }; 93 | } 94 | -------------------------------------------------------------------------------- /src/tools/list-triggers.ts: -------------------------------------------------------------------------------- 1 | import { z } from "zod"; 2 | import { HoneycombAPI } from "../api/client.js"; 3 | import { handleToolError } from "../utils/tool-error.js"; 4 | import { DatasetArgumentsSchema } from "../types/schema.js"; 5 | 6 | /** 7 | * Interface for simplified trigger data returned by the list_triggers tool 8 | */ 9 | interface SimplifiedTrigger { 10 | id: string; 11 | name: string; 12 | description: string; 13 | threshold: { 14 | op: string; 15 | value: number; 16 | }; 17 | triggered: boolean; 18 | disabled: boolean; 19 | frequency: number; 20 | alert_type?: string; 21 | } 22 | 23 | /** 24 | * Tool to list triggers (alerts) for a specific dataset. This tool returns a list of all triggers available in the specified dataset, including their names, descriptions, thresholds, and other metadata. 25 | * 26 | * @param api - The Honeycomb API client 27 | * @returns An MCP tool object with name, schema, and handler function 28 | */ 29 | export function createListTriggersTool(api: HoneycombAPI) { 30 | return { 31 | name: "list_triggers", 32 | description: "Lists available triggers (alerts) for a specific dataset. This tool returns a list of all triggers available in the specified dataset, including their names, descriptions, thresholds, and other metadata. NOTE: __all__ is NOT supported as a dataset name -- it is not possible to list all triggers in an environment.", 33 | schema: { 34 | environment: z.string().describe("The Honeycomb environment"), 35 | dataset: z.string().describe("The dataset to fetch triggers from"), 36 | }, 37 | /** 38 | * Handler for the list_triggers tool 39 | * 40 | * @param params - The parameters for the tool 41 | * @param params.environment - The Honeycomb environment 42 | * @param params.dataset - The dataset to fetch triggers from 43 | * @returns Simplified list of triggers with relevant metadata 44 | */ 45 | handler: async ({ environment, dataset }: z.infer) => { 46 | // Validate input parameters 47 | if (!environment) { 48 | return handleToolError(new Error("environment parameter is required"), "list_triggers"); 49 | } 50 | if (!dataset) { 51 | return handleToolError(new Error("dataset parameter is required"), "list_triggers"); 52 | } 53 | 54 | try { 55 | // Fetch triggers from the API 56 | const triggers = await api.getTriggers(environment, dataset); 57 | 58 | // Simplify the response to reduce context window usage 59 | const simplifiedTriggers: SimplifiedTrigger[] = triggers.map(trigger => ({ 60 | id: trigger.id, 61 | name: trigger.name, 62 | description: trigger.description || '', 63 | threshold: { 64 | op: trigger.threshold.op, 65 | value: trigger.threshold.value, 66 | }, 67 | triggered: trigger.triggered, 68 | disabled: trigger.disabled, 69 | frequency: trigger.frequency, 70 | alert_type: trigger.alert_type, 71 | })); 72 | 73 | const activeCount = simplifiedTriggers.filter(trigger => !trigger.disabled).length; 74 | const triggeredCount = simplifiedTriggers.filter(trigger => trigger.triggered).length; 75 | 76 | return { 77 | content: [ 78 | { 79 | type: "text", 80 | text: JSON.stringify(simplifiedTriggers, null, 2), 81 | }, 82 | ], 83 | metadata: { 84 | count: simplifiedTriggers.length, 85 | activeCount, 86 | triggeredCount, 87 | dataset, 88 | environment 89 | } 90 | }; 91 | } catch (error) { 92 | return handleToolError(error, "list_triggers"); 93 | } 94 | } 95 | }; 96 | } 97 | -------------------------------------------------------------------------------- /src/utils/transformations.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, it, expect } from "vitest"; 2 | import { summarizeResults } from "./transformations.js"; 3 | import { QueryResultValue } from "../types/query.js"; 4 | import { z } from "zod"; 5 | import { QueryToolSchema } from "../types/schema.js"; 6 | 7 | describe("Response transformations", () => { 8 | describe("summarizeResults", () => { 9 | it("handles empty results", () => { 10 | const results: QueryResultValue[] = []; 11 | const params: z.infer = { 12 | environment: "test-env", 13 | dataset: "test-dataset", 14 | calculations: [{ op: "COUNT" }] 15 | }; 16 | 17 | const summary = summarizeResults(results, params); 18 | 19 | expect(summary).toEqual({ count: 0 }); 20 | }); 21 | 22 | it("calculates basic count stats", () => { 23 | const results: QueryResultValue[] = [ 24 | { COUNT: 5, service: "api" }, 25 | { COUNT: 10, service: "web" }, 26 | { COUNT: 3, service: "database" } 27 | ]; 28 | 29 | const params: z.infer = { 30 | environment: "test-env", 31 | dataset: "test-dataset", 32 | calculations: [{ op: "COUNT" }], 33 | breakdowns: ["service"] 34 | }; 35 | 36 | const summary = summarizeResults(results, params); 37 | 38 | expect(summary.count).toBe(3); 39 | if (summary.countStats) { 40 | expect(summary.countStats.total).toBe(18); 41 | } 42 | if (summary.breakdowns && summary.breakdowns.service) { 43 | expect(summary.breakdowns.service.uniqueCount).toBe(3); 44 | } 45 | }); 46 | 47 | it("processes numeric calculations", () => { 48 | const results: QueryResultValue[] = [ 49 | { "AVG(duration)": 150, "MAX(duration)": 300, service: "api" }, 50 | { "AVG(duration)": 200, "MAX(duration)": 400, service: "web" } 51 | ]; 52 | 53 | const params: z.infer = { 54 | environment: "test-env", 55 | dataset: "test-dataset", 56 | calculations: [ 57 | { op: "AVG", column: "duration" }, 58 | { op: "MAX", column: "duration" } 59 | ], 60 | breakdowns: ["service"] 61 | }; 62 | 63 | const summary = summarizeResults(results, params); 64 | 65 | expect(summary.count).toBe(2); 66 | 67 | const avgStats = summary["AVG(duration)"]; 68 | if (avgStats && typeof avgStats !== 'number') { 69 | expect(avgStats.min).toBe(150); 70 | expect(avgStats.max).toBe(200); 71 | } 72 | 73 | const maxStats = summary["MAX(duration)"]; 74 | if (maxStats && typeof maxStats !== 'number') { 75 | expect(maxStats.min).toBe(300); 76 | expect(maxStats.max).toBe(400); 77 | } 78 | }); 79 | 80 | it("calculates breakdown cardinality", () => { 81 | const results: QueryResultValue[] = [ 82 | { service: "api", region: "us-east" }, 83 | { service: "api", region: "us-west" }, 84 | { service: "web", region: "us-east" }, 85 | { service: "database", region: "us-west" } 86 | ]; 87 | 88 | const params: z.infer = { 89 | environment: "test-env", 90 | dataset: "test-dataset", 91 | calculations: [{ op: "COUNT" }], 92 | breakdowns: ["service", "region"] 93 | }; 94 | 95 | const summary = summarizeResults(results, params); 96 | 97 | expect(summary.count).toBe(4); 98 | if (summary.breakdowns) { 99 | if (summary.breakdowns.service) { 100 | expect(summary.breakdowns.service.uniqueCount).toBe(3); 101 | } 102 | if (summary.breakdowns.region) { 103 | expect(summary.breakdowns.region.uniqueCount).toBe(2); 104 | } 105 | } 106 | }); 107 | }); 108 | }); -------------------------------------------------------------------------------- /src/tools/get-slo.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, it, expect, vi, beforeEach } from 'vitest'; 2 | import { createGetSLOTool } from './get-slo.js'; 3 | import { HoneycombError } from '../utils/errors.js'; 4 | import { SLODetailedResponse } from '../types/slo.js'; 5 | 6 | describe('get-slo tool', () => { 7 | // Mock API 8 | const mockApi = { 9 | getSLO: vi.fn() 10 | }; 11 | 12 | // Reset mocks before each test 13 | beforeEach(() => { 14 | vi.resetAllMocks(); 15 | }); 16 | 17 | // Test parameters 18 | const testParams = { 19 | environment: 'test-env', 20 | dataset: 'test-dataset', 21 | sloId: 'slo-1' 22 | }; 23 | 24 | // Sample SLO detailed response 25 | const mockSLOResponse: SLODetailedResponse = { 26 | id: 'slo-1', 27 | name: 'API Availability', 28 | description: 'API availability target', 29 | sli: { alias: 'sli-availability' }, 30 | time_period_days: 30, 31 | target_per_million: 995000, 32 | compliance: 0.998, 33 | budget_remaining: 0.85, 34 | created_at: '2023-01-01T00:00:00Z', 35 | updated_at: '2023-01-01T00:00:00Z' 36 | }; 37 | 38 | it('should return a valid tool configuration', () => { 39 | const tool = createGetSLOTool(mockApi as any); 40 | 41 | expect(tool).toHaveProperty('name', 'get_slo'); 42 | expect(tool).toHaveProperty('schema'); 43 | expect(tool).toHaveProperty('handler'); 44 | expect(typeof tool.handler).toBe('function'); 45 | }); 46 | 47 | it('should return simplified SLO data when API call succeeds', async () => { 48 | // Setup mock API response 49 | mockApi.getSLO.mockResolvedValue(mockSLOResponse); 50 | 51 | const tool = createGetSLOTool(mockApi as any); 52 | const result = await tool.handler(testParams); 53 | 54 | // Verify API was called with correct parameters 55 | expect(mockApi.getSLO).toHaveBeenCalledWith( 56 | testParams.environment, 57 | testParams.dataset, 58 | testParams.sloId 59 | ); 60 | 61 | // Check response structure 62 | expect(result).toHaveProperty('content'); 63 | expect(result.content).toHaveLength(1); 64 | expect(result.content[0]).toBeDefined(); 65 | expect(result.content[0]).toHaveProperty('type', 'text'); 66 | 67 | // Parse the JSON response 68 | const response = JSON.parse(result.content[0]!.text!); 69 | 70 | // Verify contents contains simplified SLO data 71 | expect(response).toHaveProperty('id', 'slo-1'); 72 | expect(response).toHaveProperty('name', 'API Availability'); 73 | expect(response).toHaveProperty('description', 'API availability target'); 74 | expect(response).toHaveProperty('time_period_days', 30); 75 | expect(response).toHaveProperty('target_per_million', 995000); 76 | expect(response).toHaveProperty('compliance', 0.998); 77 | expect(response).toHaveProperty('budget_remaining', 0.85); 78 | expect(response).toHaveProperty('sli', 'sli-availability'); 79 | expect(response).toHaveProperty('created_at'); 80 | expect(response).toHaveProperty('updated_at'); 81 | }); 82 | 83 | it('should handle API errors', async () => { 84 | // Setup API to throw an error 85 | const apiError = new HoneycombError(404, 'SLO not found'); 86 | mockApi.getSLO.mockRejectedValue(apiError); 87 | 88 | // Temporarily suppress console.error during this test 89 | const originalConsoleError = console.error; 90 | console.error = vi.fn(); 91 | 92 | try { 93 | const tool = createGetSLOTool(mockApi as any); 94 | const result = await tool.handler(testParams); 95 | 96 | // Verify error response 97 | expect(result).toHaveProperty('content'); 98 | expect(result.content).toHaveLength(1); 99 | expect(result.content[0]).toBeDefined(); 100 | expect(result.content[0]).toHaveProperty('text'); 101 | expect(result.content[0]!.text!).toContain('Failed to execute tool'); 102 | expect(result.content[0]!.text!).toContain('SLO not found'); 103 | } finally { 104 | // Restore original console.error 105 | console.error = originalConsoleError; 106 | } 107 | }); 108 | }); -------------------------------------------------------------------------------- /src/tools/index.ts: -------------------------------------------------------------------------------- 1 | import { HoneycombAPI } from "../api/client.js"; 2 | import { createListDatasetsTool } from "./list-datasets.js"; 3 | import { createListColumnsTool } from "./list-columns.js"; 4 | import { createRunQueryTool } from "./run-query.js"; 5 | import { createAnalyzeColumnsTool } from "./analyze-columns.js"; 6 | import { createListBoardsTool } from "./list-boards.js"; 7 | import { createGetBoardTool } from "./get-board.js"; 8 | import { createListMarkersTool } from "./list-markers.js"; 9 | import { createListRecipientsTool } from "./list-recipients.js"; 10 | import { createListSLOsTool } from "./list-slos.js"; 11 | import { createGetSLOTool } from "./get-slo.js"; 12 | import { createListTriggersTool } from "./list-triggers.js"; 13 | import { createGetTriggerTool } from "./get-trigger.js"; 14 | import { createTraceDeepLinkTool } from "./get-trace-link.js"; 15 | import { createInstrumentationGuidanceTool } from "./instrumentation-guidance.js"; 16 | import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; 17 | import { z } from "zod"; 18 | 19 | /** 20 | * Register all tools with the MCP server 21 | * 22 | * @param server - The MCP server instance 23 | * @param api - The Honeycomb API client 24 | */ 25 | export function registerTools(server: McpServer, api: HoneycombAPI) { 26 | const tools = [ 27 | // Dataset tools 28 | createListDatasetsTool(api), 29 | createListColumnsTool(api), 30 | 31 | // Query tools 32 | createRunQueryTool(api), 33 | createAnalyzeColumnsTool(api), 34 | 35 | // Board tools 36 | createListBoardsTool(api), 37 | createGetBoardTool(api), 38 | 39 | // Marker tools 40 | createListMarkersTool(api), 41 | 42 | // Recipient tools 43 | createListRecipientsTool(api), 44 | 45 | // SLO tools 46 | createListSLOsTool(api), 47 | createGetSLOTool(api), 48 | 49 | // Trigger tools 50 | createListTriggersTool(api), 51 | createGetTriggerTool(api), 52 | 53 | // Trace tools 54 | createTraceDeepLinkTool(api), 55 | 56 | // Instrumentation tools 57 | createInstrumentationGuidanceTool(api) 58 | ]; 59 | 60 | // Register each tool with the server 61 | for (const tool of tools) { 62 | // Register the tool with the server using type assertion to bypass TypeScript's strict type checking 63 | (server as any).tool( 64 | tool.name, 65 | tool.description, 66 | tool.schema, 67 | async (args: Record, extra: any) => { 68 | try { 69 | // Validate and ensure required fields are present before passing to handler 70 | if (tool.name.includes("analyze_columns") && (!args.environment || !args.dataset || !args.columns)) { 71 | throw new Error("Missing required fields: environment, dataset, and columns are required"); 72 | } else if (tool.name.includes("run_query") && (!args.environment || !args.dataset)) { 73 | throw new Error("Missing required fields: environment and dataset are required"); 74 | } 75 | 76 | // Use type assertion to satisfy TypeScript's type checking 77 | const result = await tool.handler(args as any); 78 | 79 | // If the result already has the expected format, return it directly 80 | if (result && typeof result === 'object' && 'content' in result) { 81 | return result as any; 82 | } 83 | 84 | // Otherwise, format the result as expected by the SDK 85 | return { 86 | content: [ 87 | { 88 | type: "text", 89 | text: typeof result === 'string' ? result : JSON.stringify(result, null, 2), 90 | }, 91 | ], 92 | } as any; 93 | } catch (error) { 94 | // Format errors to match the SDK's expected format 95 | return { 96 | content: [ 97 | { 98 | type: "text", 99 | text: error instanceof Error ? error.message : String(error), 100 | }, 101 | ], 102 | isError: true, 103 | } as any; 104 | } 105 | } 106 | ); 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /src/tools/get-trace-link.ts: -------------------------------------------------------------------------------- 1 | import { z } from "zod"; 2 | import { HoneycombAPI } from "../api/client.js"; 3 | import { handleToolError } from "../utils/tool-error.js"; 4 | import { TraceDeepLinkSchema } from "../types/schema.js"; 5 | 6 | /** 7 | * Tool to generate a deep link to a specific trace in the Honeycomb UI. This tool returns a URL that can be used to directly access a trace, optionally highlighting a specific span and limiting the time range. 8 | * 9 | * @param api - The Honeycomb API client 10 | * @returns An MCP tool object with name, schema, and handler function 11 | */ 12 | export function createTraceDeepLinkTool(api: HoneycombAPI) { 13 | return { 14 | name: "get_trace_link", 15 | description: "Generates a direct deep link to a specific trace in the Honeycomb UI. This tool creates a URL that opens a specific distributed trace, optionally positioning to a particular span and time range. If no time range is specified, the trace must have been generated within two hours from the current time. If only the start time is provided, the end time is assumed to be 10 minutes from the start time.", 16 | schema: { 17 | environment: z.string().min(1).trim().describe("The Honeycomb environment"), 18 | dataset: z.string().min(1).trim().describe("The dataset containing the trace"), 19 | traceId: z.string().describe("The unique trace ID"), 20 | spanId: z.string().optional().describe("The unique span ID to jump to within the trace"), 21 | traceStartTs: z.number().int().nonnegative().optional().describe("Start timestamp in Unix epoch seconds"), 22 | traceEndTs: z.number().int().nonnegative().optional().describe("End timestamp in Unix epoch seconds") 23 | }, 24 | /** 25 | * Handler for the get_trace_link tool 26 | * 27 | * @param params - The parameters for the tool 28 | * @returns A URL for direct access to the trace in the Honeycomb UI 29 | */ 30 | handler: async (params: z.infer) => { 31 | try { 32 | // Validate required parameters 33 | if (!params.environment) { 34 | throw new Error("Missing required parameter: environment"); 35 | } 36 | if (!params.dataset) { 37 | throw new Error("Missing required parameter: dataset"); 38 | } 39 | if (!params.traceId) { 40 | throw new Error("Missing required parameter: traceId"); 41 | } 42 | 43 | // Get the team slug for the environment 44 | const teamSlug = await api.getTeamSlug(params.environment); 45 | 46 | // Start building the trace URL 47 | let traceUrl = `https://ui.honeycomb.io/${teamSlug}/environments/${params.environment}/trace?trace_id=${encodeURIComponent(params.traceId)}`; 48 | 49 | // Add optional parameters if provided 50 | if (params.spanId) { 51 | traceUrl += `&span=${encodeURIComponent(params.spanId)}`; 52 | } 53 | 54 | if (params.traceStartTs) { 55 | traceUrl += `&trace_start_ts=${params.traceStartTs}`; 56 | } 57 | 58 | if (params.traceEndTs) { 59 | traceUrl += `&trace_end_ts=${params.traceEndTs}`; 60 | } 61 | 62 | // Add dataset parameter for more specific context 63 | if (params.dataset) { 64 | // Insert the dataset before the trace part in the URL 65 | traceUrl = traceUrl.replace( 66 | `/trace?`, 67 | `/datasets/${encodeURIComponent(params.dataset)}/trace?` 68 | ); 69 | } 70 | 71 | return { 72 | content: [ 73 | { 74 | type: "text", 75 | text: JSON.stringify({ 76 | url: traceUrl, 77 | environment: params.environment, 78 | dataset: params.dataset, 79 | traceId: params.traceId, 80 | team: teamSlug 81 | }, null, 2), 82 | }, 83 | ], 84 | metadata: { 85 | environment: params.environment, 86 | dataset: params.dataset, 87 | traceId: params.traceId, 88 | team: teamSlug 89 | } 90 | }; 91 | } catch (error) { 92 | return handleToolError(error, "get_trace_link"); 93 | } 94 | } 95 | }; 96 | } -------------------------------------------------------------------------------- /src/tools/list-recipients.test.ts: -------------------------------------------------------------------------------- 1 | import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; 2 | import { createListRecipientsTool } from "./list-recipients.js"; 3 | import { HoneycombAPI } from "../api/client.js"; 4 | 5 | // Mock the API client 6 | vi.mock("../api/client.js", () => { 7 | return { 8 | HoneycombAPI: vi.fn().mockImplementation(() => ({ 9 | getRecipients: vi.fn(), 10 | })), 11 | }; 12 | }); 13 | 14 | describe("list-recipients tool", () => { 15 | let api: HoneycombAPI; 16 | 17 | beforeEach(() => { 18 | api = new HoneycombAPI({} as any); 19 | }); 20 | 21 | afterEach(() => { 22 | vi.clearAllMocks(); 23 | }); 24 | 25 | it("returns a list of recipients", async () => { 26 | const mockRecipients = [ 27 | { 28 | id: "recipient-1", 29 | name: "Dev Team Email", 30 | type: "email" as const, 31 | target: "dev-team@example.com", 32 | created_at: "2023-01-01T00:00:00Z", 33 | updated_at: "2023-01-02T00:00:00Z", 34 | }, 35 | { 36 | id: "recipient-2", 37 | name: "Slack Channel", 38 | type: "slack" as const, 39 | target: "#alerts", 40 | created_at: "2023-01-03T00:00:00Z", 41 | updated_at: "2023-01-04T00:00:00Z", 42 | }, 43 | ]; 44 | 45 | vi.mocked(api.getRecipients).mockResolvedValue(mockRecipients); 46 | 47 | const tool = createListRecipientsTool(api); 48 | const result = await tool.handler({ environment: "test-env" }); 49 | 50 | expect(api.getRecipients).toHaveBeenCalledWith("test-env"); 51 | 52 | // Type assertion to tell TypeScript this is a success result with metadata 53 | const successResult = result as { 54 | content: { type: string; text: string }[]; 55 | metadata: { count: number; environment: string } 56 | }; 57 | 58 | expect(successResult.content).toHaveLength(1); 59 | // Add a check that text property exists before attempting to parse it 60 | expect(successResult.content[0]?.text).toBeDefined(); 61 | const content = JSON.parse(successResult.content[0]?.text || '[]'); 62 | expect(content).toHaveLength(2); 63 | expect(content[0].id).toBe("recipient-1"); 64 | expect(content[1].name).toBe("Slack Channel"); 65 | expect(successResult.metadata.count).toBe(2); 66 | }); 67 | 68 | it("handles API errors", async () => { 69 | const mockError = new Error("API error"); 70 | vi.mocked(api.getRecipients).mockRejectedValue(mockError); 71 | 72 | // Mock console.error to prevent error messages during tests 73 | const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); 74 | 75 | const tool = createListRecipientsTool(api); 76 | const result = await tool.handler({ environment: "test-env" }); 77 | 78 | // Restore console.error 79 | consoleErrorSpy.mockRestore(); 80 | 81 | // Type assertion to tell TypeScript this is an error result 82 | const errorResult = result as { 83 | content: { type: string; text: string }[]; 84 | error: { message: string } 85 | }; 86 | 87 | expect(errorResult.error).toBeDefined(); 88 | expect(errorResult.error.message).toContain("API error"); 89 | }); 90 | 91 | it("requires the environment parameter", async () => { 92 | // Mock console.error to prevent error messages during tests 93 | const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); 94 | 95 | const tool = createListRecipientsTool(api); 96 | const result = await tool.handler({ environment: "" }); 97 | 98 | // Restore console.error 99 | consoleErrorSpy.mockRestore(); 100 | 101 | // Type assertion to tell TypeScript this is an error result 102 | const errorResult = result as { 103 | content: { type: string; text: string }[]; 104 | error: { message: string } 105 | }; 106 | 107 | expect(errorResult.error).toBeDefined(); 108 | expect(errorResult.error.message).toContain("environment parameter is required"); 109 | }); 110 | 111 | it("has the correct name and schema", () => { 112 | const tool = createListRecipientsTool(api); 113 | expect(tool.name).toBe("list_recipients"); 114 | expect(tool.schema).toBeDefined(); 115 | expect(tool.schema.environment).toBeDefined(); 116 | }); 117 | }); -------------------------------------------------------------------------------- /src/tools/list-markers.test.ts: -------------------------------------------------------------------------------- 1 | import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; 2 | import { createListMarkersTool } from "./list-markers.js"; 3 | import { HoneycombAPI } from "../api/client.js"; 4 | 5 | // Mock the API client 6 | vi.mock("../api/client.js", () => { 7 | return { 8 | HoneycombAPI: vi.fn().mockImplementation(() => ({ 9 | getMarkers: vi.fn(), 10 | })), 11 | }; 12 | }); 13 | 14 | describe("list-markers tool", () => { 15 | let api: HoneycombAPI; 16 | 17 | beforeEach(() => { 18 | api = new HoneycombAPI({} as any); 19 | }); 20 | 21 | afterEach(() => { 22 | vi.clearAllMocks(); 23 | }); 24 | 25 | it("returns a list of markers", async () => { 26 | const mockMarkers = [ 27 | { 28 | id: "marker-1", 29 | message: "Deployed v1.2.3", 30 | type: "deploy", 31 | url: "https://github.com/example/repo/releases/tag/v1.2.3", 32 | created_at: "2023-01-01T00:00:00Z", 33 | start_time: "2023-01-01T00:00:00Z", 34 | end_time: "2023-01-01T00:05:00Z", 35 | }, 36 | { 37 | id: "marker-2", 38 | message: "Feature flag enabled", 39 | type: "feature", 40 | url: "", 41 | created_at: "2023-01-03T00:00:00Z", 42 | start_time: "2023-01-03T00:00:00Z", 43 | end_time: "", 44 | }, 45 | ]; 46 | 47 | vi.mocked(api.getMarkers).mockResolvedValue(mockMarkers); 48 | 49 | const tool = createListMarkersTool(api); 50 | const result = await tool.handler({ environment: "test-env" }); 51 | 52 | expect(api.getMarkers).toHaveBeenCalledWith("test-env"); 53 | 54 | // Type assertion to tell TypeScript this is a success result with metadata 55 | const successResult = result as { 56 | content: { type: string; text: string }[]; 57 | metadata: { count: number; environment: string } 58 | }; 59 | 60 | expect(successResult.content).toHaveLength(1); 61 | // Add a check that text property exists before attempting to parse it 62 | expect(successResult.content[0]?.text).toBeDefined(); 63 | const content = JSON.parse(successResult.content[0]?.text || '[]'); 64 | expect(content).toHaveLength(2); 65 | expect(content[0].id).toBe("marker-1"); 66 | expect(content[1].message).toBe("Feature flag enabled"); 67 | expect(successResult.metadata.count).toBe(2); 68 | }); 69 | 70 | it("handles API errors", async () => { 71 | const mockError = new Error("API error"); 72 | vi.mocked(api.getMarkers).mockRejectedValue(mockError); 73 | 74 | // Mock console.error to prevent error messages during tests 75 | const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); 76 | 77 | const tool = createListMarkersTool(api); 78 | const result = await tool.handler({ environment: "test-env" }); 79 | 80 | // Restore console.error 81 | consoleErrorSpy.mockRestore(); 82 | 83 | // Type assertion to tell TypeScript this is an error result 84 | const errorResult = result as { 85 | content: { type: string; text: string }[]; 86 | error: { message: string } 87 | }; 88 | 89 | expect(errorResult.error).toBeDefined(); 90 | expect(errorResult.error.message).toContain("API error"); 91 | }); 92 | 93 | it("requires the environment parameter", async () => { 94 | // Mock console.error to prevent error messages during tests 95 | const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); 96 | 97 | const tool = createListMarkersTool(api); 98 | const result = await tool.handler({ environment: "" }); 99 | 100 | // Restore console.error 101 | consoleErrorSpy.mockRestore(); 102 | 103 | // Type assertion to tell TypeScript this is an error result 104 | const errorResult = result as { 105 | content: { type: string; text: string }[]; 106 | error: { message: string } 107 | }; 108 | 109 | expect(errorResult.error).toBeDefined(); 110 | expect(errorResult.error.message).toContain("environment parameter is required"); 111 | }); 112 | 113 | it("has the correct name and schema", () => { 114 | const tool = createListMarkersTool(api); 115 | expect(tool.name).toBe("list_markers"); 116 | expect(tool.schema).toBeDefined(); 117 | expect(tool.schema.environment).toBeDefined(); 118 | }); 119 | }); -------------------------------------------------------------------------------- /src/tools/get-trigger.ts: -------------------------------------------------------------------------------- 1 | import { z } from "zod"; 2 | import { HoneycombAPI } from "../api/client.js"; 3 | import { handleToolError } from "../utils/tool-error.js"; 4 | 5 | /** 6 | * Interface for simplified recipient data in a trigger 7 | */ 8 | interface SimplifiedRecipient { 9 | type: string; 10 | target?: string; 11 | } 12 | 13 | /** 14 | * Interface for simplified trigger data returned by the get_trigger tool 15 | */ 16 | interface SimplifiedTriggerDetails { 17 | id: string; 18 | name: string; 19 | description: string; 20 | threshold: { 21 | op: string; 22 | value: number; 23 | }; 24 | frequency: number; 25 | alert_type?: string; 26 | triggered: boolean; 27 | disabled: boolean; 28 | recipients: SimplifiedRecipient[]; 29 | evaluation_schedule_type?: string; 30 | created_at: string; 31 | updated_at: string; 32 | } 33 | 34 | /** 35 | * Tool to get a specific trigger (alert) by ID with detailed information. This tool returns a detailed object containing the trigger's ID, name, description, threshold, frequency, alert type, triggered status, disabled status, recipients, evaluation schedule type, and timestamps. 36 | * 37 | * @param api - The Honeycomb API client 38 | * @returns An MCP tool object with name, schema, and handler function 39 | */ 40 | export function createGetTriggerTool(api: HoneycombAPI) { 41 | return { 42 | name: "get_trigger", 43 | description: "Retrieves a specific trigger (alert) by ID with detailed information. This tool returns a detailed object containing the trigger's ID, name, description, threshold, frequency, alert type, triggered status, disabled status, recipients, evaluation schedule type, and timestamps.", 44 | schema: { 45 | environment: z.string().describe("The Honeycomb environment"), 46 | dataset: z.string().describe("The dataset containing the trigger"), 47 | triggerId: z.string().describe("The ID of the trigger to retrieve"), 48 | }, 49 | /** 50 | * Handler for the get_trigger tool 51 | * 52 | * @param params - The parameters for the tool 53 | * @param params.environment - The Honeycomb environment 54 | * @param params.dataset - The dataset containing the trigger 55 | * @param params.triggerId - The ID of the trigger to retrieve 56 | * @returns Detailed information about the specified trigger 57 | */ 58 | handler: async ({ environment, dataset, triggerId }: { environment: string; dataset: string; triggerId: string }) => { 59 | // Validate input parameters 60 | if (!environment) { 61 | return handleToolError(new Error("environment parameter is required"), "get_trigger"); 62 | } 63 | if (!dataset) { 64 | return handleToolError(new Error("dataset parameter is required"), "get_trigger"); 65 | } 66 | if (!triggerId) { 67 | return handleToolError(new Error("triggerId parameter is required"), "get_trigger"); 68 | } 69 | 70 | try { 71 | // Fetch trigger details from the API 72 | const trigger = await api.getTrigger(environment, dataset, triggerId); 73 | 74 | // Simplify the response to reduce context window usage 75 | const simplifiedTrigger: SimplifiedTriggerDetails = { 76 | id: trigger.id, 77 | name: trigger.name, 78 | description: trigger.description || '', 79 | threshold: { 80 | op: trigger.threshold.op, 81 | value: trigger.threshold.value, 82 | }, 83 | frequency: trigger.frequency, 84 | alert_type: trigger.alert_type, 85 | triggered: trigger.triggered, 86 | disabled: trigger.disabled, 87 | recipients: trigger.recipients.map(r => ({ 88 | type: r.type, 89 | target: r.target, 90 | })), 91 | evaluation_schedule_type: trigger.evaluation_schedule_type, 92 | created_at: trigger.created_at, 93 | updated_at: trigger.updated_at, 94 | }; 95 | 96 | return { 97 | content: [ 98 | { 99 | type: "text", 100 | text: JSON.stringify(simplifiedTrigger, null, 2), 101 | }, 102 | ], 103 | metadata: { 104 | triggerId, 105 | dataset, 106 | environment, 107 | status: trigger.triggered ? "TRIGGERED" : trigger.disabled ? "DISABLED" : "ACTIVE" 108 | } 109 | }; 110 | } catch (error) { 111 | return handleToolError(error, "get_trigger"); 112 | } 113 | } 114 | }; 115 | } 116 | -------------------------------------------------------------------------------- /src/resources/datasets.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, it, expect, vi, beforeEach } from 'vitest'; 2 | import { handleDatasetResource } from './datasets.js'; 3 | 4 | // We'll skip testing the resourceTemplate creation directly since it uses 5 | // an external library that's hard to mock in these tests. Instead, we'll 6 | // test the functionality by testing the handleDatasetResource function 7 | // which contains the actual logic. 8 | 9 | describe('datasets resource', () => { 10 | // Mock API 11 | const mockApi = { 12 | getEnvironments: vi.fn(), 13 | listDatasets: vi.fn(), 14 | getDataset: vi.fn(), 15 | getVisibleColumns: vi.fn() 16 | }; 17 | 18 | // Reset mocks before each test 19 | beforeEach(() => { 20 | vi.resetAllMocks(); 21 | }); 22 | 23 | describe('handleDatasetResource', () => { 24 | const mockUri = new URL('honeycomb://test-env/test-dataset'); 25 | const mockParams = { environment: 'test-env', dataset: 'test-dataset' }; 26 | 27 | it('should fetch and format specific dataset with columns', async () => { 28 | // Setup mock API responses 29 | mockApi.getDataset.mockResolvedValue({ 30 | name: 'Test Dataset', 31 | slug: 'test-dataset', 32 | description: 'A test dataset' 33 | }); 34 | 35 | mockApi.getVisibleColumns.mockResolvedValue([ 36 | { 37 | key_name: 'column1', 38 | type: 'string', 39 | description: 'First column', 40 | hidden: false 41 | }, 42 | { 43 | key_name: 'column2', 44 | type: 'integer', 45 | description: null, 46 | hidden: false 47 | }, 48 | { 49 | key_name: 'hidden_column', 50 | type: 'string', 51 | description: 'Should be filtered out', 52 | hidden: true 53 | } 54 | ]); 55 | 56 | const result = await handleDatasetResource(mockApi as any, mockParams); 57 | 58 | // Verify result structure 59 | expect(result).toHaveProperty('contents'); 60 | expect(result.contents).toHaveLength(1); 61 | expect(result.contents[0]).toHaveProperty('mimeType', 'application/json'); 62 | expect(result.contents[0]).toHaveProperty('uri', mockUri.href); 63 | 64 | // Parse and check the JSON content 65 | const content = JSON.parse(result.contents[0]!.text!); 66 | expect(content).toHaveProperty('name', 'Test Dataset'); 67 | expect(content).toHaveProperty('columns'); 68 | 69 | // Should only include non-hidden columns 70 | expect(content.columns).toHaveLength(2); 71 | expect(content.columns[0]).toHaveProperty('name', 'column1'); 72 | expect(content.columns[1]).toHaveProperty('description', ''); // Empty string for null 73 | 74 | // Should not include the hidden column 75 | const hasHiddenColumn = content.columns.some((col: { name: string }) => col.name === 'hidden_column'); 76 | expect(hasHiddenColumn).toBe(false); 77 | }); 78 | 79 | it('should list all datasets in an environment when no dataset is specified', async () => { 80 | const paramsWithoutDataset = { environment: 'test-env', dataset: '' }; 81 | 82 | // Setup mock API response 83 | mockApi.listDatasets.mockResolvedValue([ 84 | { name: 'Dataset1', slug: 'dataset1', description: 'First dataset' }, 85 | { name: 'Dataset2', slug: 'dataset2', description: null } 86 | ]); 87 | 88 | const result = await handleDatasetResource(mockApi as any, paramsWithoutDataset); 89 | 90 | // Verify result structure 91 | expect(result).toHaveProperty('contents'); 92 | expect(result.contents).toHaveLength(2); 93 | 94 | // Check first dataset 95 | const firstDataset = JSON.parse(result.contents[0]!.text!); 96 | expect(firstDataset).toHaveProperty('name', 'Dataset1'); 97 | expect(firstDataset).toHaveProperty('slug', 'dataset1'); 98 | 99 | // Check second dataset - null description should be converted to empty string 100 | const secondDataset = JSON.parse(result.contents[1]!.text!); 101 | expect(secondDataset).toHaveProperty('description', ''); 102 | }); 103 | 104 | it('should handle API errors', async () => { 105 | // Setup API to throw an error 106 | mockApi.getDataset.mockRejectedValue(new Error('Dataset not found')); 107 | 108 | // Expect the function to throw 109 | await expect(handleDatasetResource(mockApi as any, mockParams)) 110 | .rejects.toThrow('Failed to read dataset'); 111 | }); 112 | }); 113 | }); -------------------------------------------------------------------------------- /docs/generic-instrumentation-guidance.md: -------------------------------------------------------------------------------- 1 | # OpenTelemetry Code Analysis & Instrumentation 2 | 3 | Analyze code and provide specific instrumentation recommendations optimized for Honeycomb, with a preference for using OpenTelemetry tracing if possible. 4 | 5 | Support direct instructions, like "instrument this file" or "add tracing to this function." 6 | 7 | ## Rules to follow 8 | 9 | Ignore metrics code, if it exists, for now. 10 | 11 | Focus on enhancing any existing logs, then suggest span instrumentation if requested. 12 | 13 | If there are no existing logging calls, suggest using OpenTelemetry spans instead, unless explicitly asked to add logs. 14 | 15 | ## Logging Enhancements 16 | 17 | If code has logging, recommend improvements to: 18 | 19 | 1. Add proper log levels for different types of operations: 20 | ``` 21 | # Instead of: 22 | print("Processing order") 23 | logger.info(f"Order {id} status: {status}") 24 | 25 | # Better as: 26 | logger.info("Starting order processing", {"app.order_id": id}) 27 | logger.error("Order processing failed", {"app.order_id": id, "app.error": str(e)}) 28 | ``` 29 | 30 | 2. Convert print statements to structured logs: 31 | ``` 32 | # Instead of: 33 | print(f"Processing order {id} for customer {customer}") 34 | 35 | # Better as: 36 | logger.info("Processing order", { 37 | "app.order_id": id, 38 | "app.customer_id": customer.id, 39 | "app.items_count": len(items) 40 | }) 41 | ``` 42 | 43 | 3. Consolidate related logs into single, rich events: 44 | ``` 45 | # Instead of multiple logs: 46 | logger.info(f"Processing order {id}") 47 | items = process_order(id) 48 | logger.info(f"Found {len(items)} items") 49 | discount = apply_discount(items) 50 | logger.info(f"Applied discount: {discount}") 51 | 52 | # Better as one structured log: 53 | logger.info("Processing order", { 54 | "app.order_id": id, 55 | "app.items_count": len(items), 56 | "app.discount_applied": discount, 57 | "app.customer_tier": customer.tier 58 | }) 59 | ``` 60 | 61 | 4. Capture high-cardinality data and data useful for debugging from function parameters in structured fields, such as: 62 | - `app.user_id` 63 | - `app.request_id` 64 | - `app.order_id`, `app.product_id`, etc. 65 | - Operation parameters 66 | - State information 67 | 68 | In particular, especially focus on consolidating logs that can be combined into a single, rich event. 69 | 70 | ## Span Instrumentation 71 | 72 | If instrumenting with spans, recommend instrumentation that: 73 | 74 | 1. Adds important high-cardinality data, request context, and data useful for debugging from function parameters to the current span: 75 | ``` 76 | current_span.set_attributes({ 77 | "app.customer_id": request.customer_id, 78 | "app.order_type": request.type, 79 | "app.items_count": len(request.items) 80 | }) 81 | ``` 82 | 83 | 2. Identifies functions performing significant work, whose duration is meaningful, and tracks them in spans: 84 | ``` 85 | def create_order(request): 86 | with span("create_order") as order_span: 87 | order = get_order_from_system(request) 88 | order_span.set_attributes({ 89 | "app.order_id": order.id, 90 | "app.total_amount": order.total 91 | }) 92 | ``` 93 | 94 | 3. Captures error info when errors occur: 95 | ``` 96 | try: 97 | # something that might fail 98 | 99 | # Consider catching a more specific exception in your code 100 | except Exception as ex: 101 | current_span.set_status(Status(StatusCode.ERROR)) 102 | current_span.record_exception(ex) 103 | ``` 104 | 105 | To reiterate a nuance: 106 | 107 | - If you're not sure, prefer adding attribute data to existing spans over creating new spans. 108 | - New spans should only be created within a function that performs meaningful work whose duration matters. 109 | - The same data spread across fewer spans is usually better than the same data spread across more spans. 110 | 111 | ## Additional considerations 112 | 113 | Always use well-defined, specific, and namespaced keys for structured logs and span attributes. 114 | 115 | Consider deeply if clarification is needed on: 116 | 117 | - The purpose or context of specific code sections 118 | - Which operations are most important to instrument 119 | - Whether to focus on logging improvements or span creation, especially if both are present 120 | - The meaning of domain-specific terms or variables 121 | 122 | Ask for more information before providing recommendations if necessary. -------------------------------------------------------------------------------- /src/utils/typeguards.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, it, expect } from 'vitest'; 2 | import { 3 | isValidNumber, 4 | isValidString, 5 | isValidArray, 6 | isValidObject, 7 | hasProperty, 8 | hasPropertyOfType 9 | } from './typeguards.js'; 10 | 11 | describe('Type Guards', () => { 12 | describe('isValidNumber', () => { 13 | it('should correctly identify valid numbers', () => { 14 | expect(isValidNumber(123)).toBe(true); 15 | expect(isValidNumber(0)).toBe(true); 16 | expect(isValidNumber(-456)).toBe(true); 17 | expect(isValidNumber(3.14)).toBe(true); 18 | }); 19 | 20 | it('should correctly reject non-numbers', () => { 21 | expect(isValidNumber(null)).toBe(false); 22 | expect(isValidNumber(undefined)).toBe(false); 23 | expect(isValidNumber('123')).toBe(false); 24 | expect(isValidNumber({})).toBe(false); 25 | expect(isValidNumber([])).toBe(false); 26 | expect(isValidNumber(NaN)).toBe(false); 27 | }); 28 | }); 29 | 30 | describe('isValidString', () => { 31 | it('should correctly identify valid strings', () => { 32 | expect(isValidString('hello')).toBe(true); 33 | expect(isValidString('')).toBe(true); 34 | expect(isValidString(`template string`)).toBe(true); 35 | }); 36 | 37 | it('should correctly reject non-strings', () => { 38 | expect(isValidString(null)).toBe(false); 39 | expect(isValidString(undefined)).toBe(false); 40 | expect(isValidString(123)).toBe(false); 41 | expect(isValidString({})).toBe(false); 42 | expect(isValidString([])).toBe(false); 43 | }); 44 | }); 45 | 46 | describe('isValidArray', () => { 47 | it('should correctly identify valid arrays', () => { 48 | expect(isValidArray([])).toBe(true); 49 | expect(isValidArray([1, 2, 3])).toBe(true); 50 | expect(isValidArray(['a', 'b', 'c'])).toBe(true); 51 | expect(isValidArray(new Array())).toBe(true); 52 | }); 53 | 54 | it('should correctly reject non-arrays', () => { 55 | expect(isValidArray(null)).toBe(false); 56 | expect(isValidArray(undefined)).toBe(false); 57 | expect(isValidArray(123)).toBe(false); 58 | expect(isValidArray('string')).toBe(false); 59 | expect(isValidArray({})).toBe(false); 60 | }); 61 | }); 62 | 63 | describe('isValidObject', () => { 64 | it('should correctly identify valid objects', () => { 65 | expect(isValidObject({})).toBe(true); 66 | expect(isValidObject({ key: 'value' })).toBe(true); 67 | expect(isValidObject(new Object())).toBe(true); 68 | }); 69 | 70 | it('should correctly reject non-objects', () => { 71 | expect(isValidObject(null)).toBe(false); 72 | expect(isValidObject(undefined)).toBe(false); 73 | expect(isValidObject(123)).toBe(false); 74 | expect(isValidObject('string')).toBe(false); 75 | expect(isValidObject([])).toBe(false); 76 | }); 77 | }); 78 | 79 | describe('hasProperty', () => { 80 | it('should correctly identify objects with specific properties', () => { 81 | expect(hasProperty({ name: 'John' }, 'name')).toBe(true); 82 | expect(hasProperty({ key: null }, 'key')).toBe(true); 83 | expect(hasProperty({ a: 1, b: 2 }, 'a')).toBe(true); 84 | }); 85 | 86 | it('should correctly reject objects without specific properties', () => { 87 | expect(hasProperty({}, 'name')).toBe(false); 88 | expect(hasProperty({ other: 'prop' }, 'name')).toBe(false); 89 | expect(hasProperty(null, 'name')).toBe(false); 90 | expect(hasProperty(undefined, 'name')).toBe(false); 91 | expect(hasProperty(123, 'toString')).toBe(false); // Not an object 92 | }); 93 | }); 94 | 95 | describe('hasPropertyOfType', () => { 96 | it('should correctly identify objects with properties of specific types', () => { 97 | expect(hasPropertyOfType({ age: 30 }, 'age', isValidNumber)).toBe(true); 98 | expect(hasPropertyOfType({ name: 'John' }, 'name', isValidString)).toBe(true); 99 | expect(hasPropertyOfType({ items: [1, 2, 3] }, 'items', isValidArray)).toBe(true); 100 | expect(hasPropertyOfType({ meta: { id: 1 } }, 'meta', isValidObject)).toBe(true); 101 | }); 102 | 103 | it('should correctly reject objects with properties of incorrect types', () => { 104 | expect(hasPropertyOfType({ age: '30' }, 'age', isValidNumber)).toBe(false); 105 | expect(hasPropertyOfType({ name: 123 }, 'name', isValidString)).toBe(false); 106 | expect(hasPropertyOfType({ items: {} }, 'items', isValidArray)).toBe(false); 107 | expect(hasPropertyOfType({ meta: [1, 2, 3] }, 'meta', isValidObject)).toBe(false); 108 | expect(hasPropertyOfType({}, 'missing', isValidString)).toBe(false); 109 | expect(hasPropertyOfType(null, 'any', isValidString)).toBe(false); 110 | }); 111 | }); 112 | }); -------------------------------------------------------------------------------- /src/tools/get-trace-link.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, it, expect, vi, beforeEach } from "vitest"; 2 | import { createTraceDeepLinkTool } from "./get-trace-link.js"; 3 | import { handleToolError } from "../utils/tool-error.js"; 4 | 5 | // Mock the handleToolError function 6 | vi.mock("../utils/tool-error.js", () => ({ 7 | handleToolError: vi.fn((error) => ({ 8 | content: [{ type: "text", text: error.message }], 9 | isError: true 10 | })) 11 | })); 12 | 13 | describe("createTraceDeepLinkTool", () => { 14 | const mockApi = { 15 | getEnvironments: vi.fn(), 16 | listDatasets: vi.fn(), 17 | getVisibleColumns: vi.fn(), 18 | getColumnByName: vi.fn(), 19 | getTeamSlug: vi.fn().mockResolvedValue("test-team"), // Mock team slug retrieval 20 | getAuthInfo: vi.fn().mockResolvedValue({ 21 | team: { slug: "test-team", name: "Test Team" } 22 | }), 23 | }; 24 | 25 | beforeEach(() => { 26 | vi.clearAllMocks(); 27 | }); 28 | 29 | it("should generate a basic trace link with required parameters", async () => { 30 | const tool = createTraceDeepLinkTool(mockApi as any); 31 | const result = await tool.handler({ 32 | environment: "test-env", 33 | dataset: "test-dataset", 34 | traceId: "abc123", 35 | }); 36 | 37 | if (result.content && result.content[0] && result.content[0].text) { 38 | const text = result.content[0].text; 39 | const parsed = JSON.parse(text); 40 | 41 | expect(text).toContain("https://ui.honeycomb.io/test-team/environments/test-env/datasets/test-dataset/trace?trace_id=abc123"); 42 | expect(parsed).toHaveProperty("environment", "test-env"); 43 | expect(parsed).toHaveProperty("dataset", "test-dataset"); 44 | expect(parsed).toHaveProperty("traceId", "abc123"); 45 | expect(parsed).toHaveProperty("team", "test-team"); 46 | expect(mockApi.getTeamSlug).toHaveBeenCalledWith("test-env"); 47 | } else { 48 | throw new Error("Expected result to have content[0].text"); 49 | } 50 | }); 51 | 52 | it("should include span ID when provided", async () => { 53 | const tool = createTraceDeepLinkTool(mockApi as any); 54 | const result = await tool.handler({ 55 | environment: "test-env", 56 | dataset: "test-dataset", 57 | traceId: "abc123", 58 | spanId: "span456", 59 | }); 60 | 61 | if (result.content && result.content[0] && result.content[0].text) { 62 | expect(result.content[0].text).toContain("&span=span456"); 63 | } else { 64 | throw new Error("Expected result to have content[0].text"); 65 | } 66 | }); 67 | 68 | it("should include timestamps when provided", async () => { 69 | const tool = createTraceDeepLinkTool(mockApi as any); 70 | const result = await tool.handler({ 71 | environment: "test-env", 72 | dataset: "test-dataset", 73 | traceId: "abc123", 74 | traceStartTs: 1614556800, 75 | traceEndTs: 1614560400, 76 | }); 77 | 78 | if (result.content && result.content[0] && result.content[0].text) { 79 | expect(result.content[0].text).toContain("&trace_start_ts=1614556800"); 80 | expect(result.content[0].text).toContain("&trace_end_ts=1614560400"); 81 | } else { 82 | throw new Error("Expected result to have content[0].text"); 83 | } 84 | }); 85 | 86 | it("should properly URL-encode trace and span IDs", async () => { 87 | const tool = createTraceDeepLinkTool(mockApi as any); 88 | const result = await tool.handler({ 89 | environment: "test-env", 90 | dataset: "test-dataset", 91 | traceId: "trace/with/slashes", 92 | spanId: "span with spaces", 93 | }); 94 | 95 | if (result.content && result.content[0] && result.content[0].text) { 96 | expect(result.content[0].text).toContain("trace_id=trace%2Fwith%2Fslashes"); 97 | expect(result.content[0].text).toContain("&span=span%20with%20spaces"); 98 | } else { 99 | throw new Error("Expected result to have content[0].text"); 100 | } 101 | }); 102 | 103 | it("should handle error when required parameters are missing", async () => { 104 | const tool = createTraceDeepLinkTool(mockApi as any); 105 | 106 | // Missing environment 107 | await tool.handler({ 108 | dataset: "test-dataset", 109 | traceId: "abc123", 110 | } as any); 111 | 112 | expect(handleToolError).toHaveBeenCalledWith(expect.objectContaining({ 113 | message: "Missing required parameter: environment" 114 | }), "get_trace_link"); 115 | 116 | // Missing traceId 117 | await tool.handler({ 118 | environment: "test-env", 119 | dataset: "test-dataset", 120 | } as any); 121 | 122 | expect(handleToolError).toHaveBeenCalledWith(expect.objectContaining({ 123 | message: "Missing required parameter: traceId" 124 | }), "get_trace_link"); 125 | }); 126 | }); -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; 2 | import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js"; 3 | import { loadConfig } from "./config.js"; 4 | import { HoneycombAPI } from "./api/client.js"; 5 | import process from "node:process"; 6 | import { registerResources } from "./resources/index.js"; 7 | import { registerTools } from "./tools/index.js"; 8 | import { registerPrompts } from "./prompts/index.js"; 9 | import { initializeCache } from "./cache/index.js"; 10 | 11 | function checkNodeVersion() { 12 | const requiredMajorVersion = 18; 13 | const nodeVersion: string = process.versions.node; 14 | if (!nodeVersion) { 15 | console.error(`Error: Unable to determine Node.js version. Node.js version ${requiredMajorVersion} or higher is required.`); 16 | process.exit(1); 17 | } 18 | 19 | const majorVersion = nodeVersion.split('.')[0]; 20 | if (!majorVersion) { 21 | console.error(`Error: Unable to determine Node.js major version. Node.js version ${requiredMajorVersion} or higher is required.`); 22 | process.exit(1); 23 | } 24 | 25 | const currentMajorVersion = parseInt(majorVersion, 10); 26 | if (isNaN(currentMajorVersion)) { 27 | console.error(`Error: Unable to parse Node.js major version. Node.js version ${requiredMajorVersion} or higher is required.`); 28 | process.exit(1); 29 | } 30 | 31 | if (currentMajorVersion < requiredMajorVersion) { 32 | console.error( 33 | `Error: Node.js version ${requiredMajorVersion} or higher is required. Current version: ${nodeVersion}` 34 | ); 35 | process.exit(1); 36 | } 37 | } 38 | 39 | /** 40 | * Main function to run the Honeycomb MCP server 41 | */ 42 | async function main() { 43 | try { 44 | checkNodeVersion(); 45 | // Load config asynchronously and create API client 46 | console.error("Loading configuration from environment variables..."); 47 | const config = await loadConfig(); 48 | console.error(`Loaded ${config.environments.length} environment(s): ${config.environments.map(e => e.name).join(', ')}`); 49 | 50 | // Initialize the cache 51 | console.error("Initializing cache..."); 52 | const cacheManager = initializeCache(config); 53 | console.error(`Cache initialized (enabled: ${config.cache.enabled})`); 54 | 55 | const api = new HoneycombAPI(config); 56 | 57 | // Create server with proper initialization options and capabilities 58 | const server = new McpServer({ 59 | name: "honeycomb", 60 | version: "1.0.0", 61 | capabilities: { 62 | prompts: {} // Register prompts capability 63 | } 64 | }); 65 | 66 | // Add a small delay to ensure the server is fully initialized before registering tools 67 | console.error("Initializing MCP server..."); 68 | await new Promise(resolve => setTimeout(resolve, 500)); 69 | 70 | // Register resources, tools, and prompts 71 | console.error("Registering resources, tools, and prompts..."); 72 | registerResources(server, api); 73 | registerTools(server, api); 74 | registerPrompts(server); 75 | 76 | // Wait for registration to complete 77 | await new Promise(resolve => setTimeout(resolve, 500)); 78 | console.error("All resources, tools, and prompts registered"); 79 | 80 | // Create transport and start server 81 | const transport = new StdioServerTransport(); 82 | 83 | // Add reconnect logic to handle connection issues 84 | let connected = false; 85 | const maxRetries = 3; 86 | let retries = 0; 87 | 88 | while (!connected && retries < maxRetries) { 89 | try { 90 | await server.connect(transport); 91 | connected = true; 92 | console.error("Honeycomb MCP Server running on stdio"); 93 | } catch (error) { 94 | retries++; 95 | console.error(`Connection attempt ${retries} failed: ${error instanceof Error ? error.message : String(error)}`); 96 | 97 | if (retries < maxRetries) { 98 | console.error(`Retrying in 1 second...`); 99 | await new Promise(resolve => setTimeout(resolve, 1000)); 100 | } else { 101 | console.error(`Max retries (${maxRetries}) reached. Server may be unstable.`); 102 | // Continue anyway, but warn about potential issues 103 | console.error("Honeycomb MCP Server running with potential connection issues"); 104 | break; 105 | } 106 | } 107 | } 108 | } catch (error) { 109 | console.error("Failed to start MCP server:", error instanceof Error ? error.message : String(error)); 110 | process.exit(1); 111 | } 112 | } 113 | 114 | // Run main with proper error handling 115 | if (import.meta.url === `file://${process.argv[1]}`) { 116 | main().catch((error) => { 117 | console.error("Fatal error in main():", error); 118 | process.exit(1); 119 | }); 120 | } 121 | -------------------------------------------------------------------------------- /src/prompts/guidance.ts: -------------------------------------------------------------------------------- 1 | const INSTRUMENTATION_GUIDANCE = ` 2 | # OpenTelemetry Code Analysis & Instrumentation 3 | 4 | Analyze code and provide specific instrumentation recommendations optimized for Honeycomb, with a preference for using OpenTelemetry tracing if possible. 5 | 6 | Support direct instructions, like "instrument this file" or "add tracing to this function." 7 | 8 | ## Rules to follow 9 | 10 | Ignore metrics code, if it exists, for now. 11 | 12 | Focus on enhancing any existing logs, then suggest span instrumentation if requested. 13 | 14 | If there are no existing logging calls, suggest using OpenTelemetry spans instead, unless explicitly asked to add logs. 15 | 16 | ## Logging Enhancements 17 | 18 | If code has logging, recommend improvements to: 19 | 20 | 1. Add proper log levels for different types of operations: 21 | \`\`\` 22 | # Instead of: 23 | print("Processing order") 24 | logger.info(f"Order {id} status: {status}") 25 | 26 | # Better as: 27 | logger.info("Starting order processing", {"app.order_id": id}) 28 | logger.error("Order processing failed", {"app.order_id": id, "app.error": str(e)}) 29 | \`\`\` 30 | 31 | 2. Convert print statements to structured logs: 32 | \`\`\` 33 | # Instead of: 34 | print(f"Processing order {id} for customer {customer}") 35 | 36 | # Better as: 37 | logger.info("Processing order", { 38 | "app.order_id": id, 39 | "app.customer_id": customer.id, 40 | "app.items_count": len(items) 41 | }) 42 | \`\`\` 43 | 44 | 3. Consolidate related logs into single, rich events: 45 | \`\`\` 46 | # Instead of multiple logs: 47 | logger.info(f"Processing order {id}") 48 | items = process_order(id) 49 | logger.info(f"Found {len(items)} items") 50 | discount = apply_discount(items) 51 | logger.info(f"Applied discount: {discount}") 52 | 53 | # Better as one structured log: 54 | logger.info("Processing order", { 55 | "app.order_id": id, 56 | "app.items_count": len(items), 57 | "app.discount_applied": discount, 58 | "app.customer_tier": customer.tier 59 | }) 60 | \`\`\` 61 | 62 | 4. Capture high-cardinality data and data useful for debugging from function parameters in structured fields, such as: 63 | - \`app.user_id\` 64 | - \`app.request_id\` 65 | - \`app.order_id\`, \`app.product_id\`, etc. 66 | - Operation parameters 67 | - State information 68 | 69 | In particular, especially focus on consolidating logs that can be combined into a single, rich event. 70 | 71 | ## Span Instrumentation 72 | 73 | If instrumenting with spans, recommend instrumentation that: 74 | 75 | 1. Adds important high-cardinality data, request context, and data useful for debugging from function parameters to the current span: 76 | \`\`\` 77 | current_span.set_attributes({ 78 | "app.customer_id": request.customer_id, 79 | "app.order_type": request.type, 80 | "app.items_count": len(request.items) 81 | }) 82 | \`\`\` 83 | 84 | 2. Identifies functions performing significant work, only whose duration is meaningful, and tracks them in spans: 85 | \`\`\` 86 | def create_order(request): 87 | with span("create_order") as order_span: 88 | # get_order_from_system can take some time 89 | order = get_order_from_system(request) 90 | order_span.set_attributes({ 91 | "app.order_id": order.id, 92 | "app.total_amount": order.total 93 | # ... other interesting stuff on an order 94 | }) 95 | \`\`\` 96 | 97 | 3. Captures error info when errors occur: 98 | \`\`\` 99 | try: 100 | # something that might fail 101 | 102 | # Consider catching a more specific exception in your code 103 | except Exception as ex: 104 | current_span.set_status(Status(StatusCode.ERROR)) 105 | current_span.record_exception(ex) 106 | \`\`\` 107 | 108 | IMPORTANT: 109 | 110 | It is generally better to use existing spans and add attributes to them instead of creating new spans. Only create a new span inside of a new function whose duration is meaningful AND where there is no current span to add attributed to. 111 | 112 | ## General Rules 113 | 114 | Use well-defined, specific, and namespaced keys for attributes instructured logs and span attributes. 115 | 116 | Consider deeply if clarification is needed on: 117 | 118 | - The purpose or context of specific code sections 119 | - Which operations are most important to instrument 120 | - Whether to focus on logging improvements or span creation, especially if both are present 121 | - The meaning of domain-specific terms or variables 122 | 123 | Ask for more information before providing recommendations if necessary. 124 | `; 125 | 126 | /** 127 | * Returns the instrumentation guidance template 128 | * @returns The instrumentation guidance template string 129 | */ 130 | export function getInstrumentationGuidance(): string { 131 | return INSTRUMENTATION_GUIDANCE; 132 | } -------------------------------------------------------------------------------- /src/tools/list-slos.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, it, expect, vi, beforeEach } from 'vitest'; 2 | import { createListSLOsTool } from './list-slos.js'; 3 | import { HoneycombError } from '../utils/errors.js'; 4 | import { SLO } from '../types/slo.js'; 5 | 6 | describe('list-slos tool', () => { 7 | // Mock API 8 | const mockApi = { 9 | getSLOs: vi.fn() 10 | }; 11 | 12 | // Reset mocks before each test 13 | beforeEach(() => { 14 | vi.resetAllMocks(); 15 | }); 16 | 17 | // Test parameters 18 | const testParams = { 19 | environment: 'test-env', 20 | dataset: 'test-dataset' 21 | }; 22 | 23 | // Sample SLOs response 24 | const mockSLOs: SLO[] = [ 25 | { 26 | id: 'slo-1', 27 | name: 'API Availability', 28 | description: 'API availability target', 29 | sli: { alias: 'sli-availability' }, 30 | time_period_days: 30, 31 | target_per_million: 995000, 32 | created_at: '2023-01-01T00:00:00Z', 33 | updated_at: '2023-01-01T00:00:00Z' 34 | }, 35 | { 36 | id: 'slo-2', 37 | name: 'API Latency', 38 | description: 'API latency target', 39 | sli: { alias: 'sli-latency' }, 40 | time_period_days: 7, 41 | target_per_million: 990000, 42 | created_at: '2023-01-01T00:00:00Z', 43 | updated_at: '2023-01-01T00:00:00Z' 44 | } 45 | ]; 46 | 47 | it('should return a valid tool configuration', () => { 48 | const tool = createListSLOsTool(mockApi as any); 49 | 50 | expect(tool).toHaveProperty('name', 'list_slos'); 51 | expect(tool).toHaveProperty('schema'); 52 | expect(tool).toHaveProperty('handler'); 53 | expect(typeof tool.handler).toBe('function'); 54 | }); 55 | 56 | it('should return simplified SLOs when API call succeeds', async () => { 57 | // Setup mock API response 58 | mockApi.getSLOs.mockResolvedValue(mockSLOs); 59 | 60 | const tool = createListSLOsTool(mockApi as any); 61 | const result = await tool.handler(testParams); 62 | 63 | // Verify API was called with correct parameters 64 | expect(mockApi.getSLOs).toHaveBeenCalledWith( 65 | testParams.environment, 66 | testParams.dataset 67 | ); 68 | 69 | // Check response structure 70 | expect(result).toHaveProperty('content'); 71 | expect(result.content).toHaveLength(1); 72 | expect(result.content[0]).toBeDefined(); 73 | expect(result.content[0]).toHaveProperty('type', 'text'); 74 | 75 | // Parse the JSON response 76 | const response = JSON.parse(result.content[0]!.text!); 77 | 78 | // Verify contents contains simplified SLO data 79 | expect(response).toHaveLength(2); 80 | expect(response[0]).toHaveProperty('id', 'slo-1'); 81 | expect(response[0]).toHaveProperty('name', 'API Availability'); 82 | expect(response[0]).toHaveProperty('description', 'API availability target'); 83 | expect(response[0]).toHaveProperty('time_period_days', 30); 84 | expect(response[0]).toHaveProperty('target_per_million', 995000); 85 | expect(response[0]).not.toHaveProperty('sli'); 86 | expect(response[0]).not.toHaveProperty('created_at'); 87 | 88 | expect(response[1]).toHaveProperty('id', 'slo-2'); 89 | expect(response[1]).toHaveProperty('name', 'API Latency'); 90 | }); 91 | 92 | it('should handle empty SLOs list', async () => { 93 | // Setup API to return empty SLOs array 94 | mockApi.getSLOs.mockResolvedValue([]); 95 | 96 | const tool = createListSLOsTool(mockApi as any); 97 | const result = await tool.handler(testParams); 98 | 99 | // Check response structure 100 | expect(result).toHaveProperty('content'); 101 | expect(result.content).toHaveLength(1); 102 | expect(result.content[0]).toBeDefined(); 103 | expect(result.content[0]).toHaveProperty('type', 'text'); 104 | 105 | // Parse the JSON response 106 | const response = JSON.parse(result.content[0]!.text!); 107 | 108 | // Verify empty array is returned 109 | expect(response).toEqual([]); 110 | }); 111 | 112 | it('should handle API errors', async () => { 113 | // Setup API to throw an error 114 | const apiError = new HoneycombError(404, 'Dataset not found'); 115 | mockApi.getSLOs.mockRejectedValue(apiError); 116 | 117 | // Temporarily suppress console.error during this test 118 | const originalConsoleError = console.error; 119 | console.error = vi.fn(); 120 | 121 | try { 122 | const tool = createListSLOsTool(mockApi as any); 123 | const result = await tool.handler(testParams); 124 | 125 | // Verify error response 126 | expect(result).toHaveProperty('content'); 127 | expect(result.content).toHaveLength(1); 128 | expect(result.content[0]).toBeDefined(); 129 | expect(result.content[0]).toHaveProperty('text'); 130 | expect(result.content[0]!.text!).toContain('Failed to execute tool'); 131 | expect(result.content[0]!.text!).toContain('Dataset not found'); 132 | } finally { 133 | // Restore original console.error 134 | console.error = originalConsoleError; 135 | } 136 | }); 137 | }); -------------------------------------------------------------------------------- /src/config.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, it, expect, beforeEach, vi, afterEach } from "vitest"; 2 | import { loadConfig } from "./config.js"; 3 | import { AuthResponse } from "./types/api.js"; 4 | 5 | // Mock fetch globally 6 | const mockFetch = vi.fn(); 7 | global.fetch = mockFetch as unknown as typeof fetch; 8 | 9 | describe("Config", () => { 10 | const originalEnv = { ...process.env }; 11 | 12 | beforeEach(() => { 13 | vi.resetAllMocks(); 14 | process.env = { ...originalEnv }; // Reset env vars before each test 15 | 16 | // Default mock for fetch to return success with minimal auth response 17 | mockFetch.mockResolvedValue({ 18 | ok: true, 19 | json: () => Promise.resolve({ 20 | id: "test-id", 21 | type: "test", 22 | api_key_access: { query: true }, 23 | team: { name: "Test Team", slug: "test-team" }, 24 | environment: { name: "Test Env", slug: "test-env" } 25 | } as AuthResponse) 26 | }); 27 | }); 28 | 29 | afterEach(() => { 30 | process.env = originalEnv; // Restore original env vars 31 | }); 32 | 33 | describe("loadConfig", () => { 34 | it("loads config from HONEYCOMB_API_KEY", async () => { 35 | process.env.HONEYCOMB_API_KEY = "test-key"; 36 | 37 | const config = await loadConfig(); 38 | 39 | expect(config.environments).toHaveLength(1); 40 | const env = config.environments[0]; 41 | if (env) { 42 | expect(env.name).toEqual("Test Env"); // Name from auth response 43 | expect(env.apiKey).toEqual("test-key"); 44 | expect(env.teamSlug).toEqual("test-team"); 45 | } 46 | expect(mockFetch).toHaveBeenCalledWith( 47 | "https://api.honeycomb.io/1/auth", 48 | expect.objectContaining({ 49 | headers: expect.objectContaining({ 50 | "X-Honeycomb-Team": "test-key" 51 | }) 52 | }) 53 | ); 54 | }); 55 | 56 | it("loads config from multiple HONEYCOMB_ENV_*_API_KEY variables", async () => { 57 | process.env.HONEYCOMB_ENV_PROD_API_KEY = "prod-key"; 58 | process.env.HONEYCOMB_ENV_STAGING_API_KEY = "staging-key"; 59 | 60 | // Mock fetch to return different responses for different API keys 61 | mockFetch 62 | .mockImplementationOnce(() => Promise.resolve({ 63 | ok: true, 64 | json: () => Promise.resolve({ 65 | id: "prod-id", 66 | type: "test", 67 | api_key_access: { query: true }, 68 | team: { name: "Prod Team", slug: "prod-team" }, 69 | environment: { name: "Production", slug: "prod" } 70 | } as AuthResponse) 71 | })) 72 | .mockImplementationOnce(() => Promise.resolve({ 73 | ok: true, 74 | json: () => Promise.resolve({ 75 | id: "staging-id", 76 | type: "test", 77 | api_key_access: { query: true }, 78 | team: { name: "Staging Team", slug: "staging-team" }, 79 | environment: { name: "Staging", slug: "staging" } 80 | } as AuthResponse) 81 | })); 82 | 83 | const config = await loadConfig(); 84 | 85 | expect(config.environments).toHaveLength(2); 86 | expect(config.environments.find(e => e.name === "prod")).toBeDefined(); 87 | expect(config.environments.find(e => e.name === "staging")).toBeDefined(); 88 | expect(mockFetch).toHaveBeenCalledTimes(2); 89 | }); 90 | 91 | it("uses custom API endpoint when specified", async () => { 92 | process.env.HONEYCOMB_API_KEY = "test-key"; 93 | process.env.HONEYCOMB_API_ENDPOINT = "https://custom.honeycomb.io"; 94 | 95 | await loadConfig(); 96 | 97 | expect(mockFetch).toHaveBeenCalledWith( 98 | "https://custom.honeycomb.io/1/auth", 99 | expect.any(Object) 100 | ); 101 | }); 102 | 103 | it("handles auth failure gracefully", async () => { 104 | process.env.HONEYCOMB_API_KEY = "invalid-key"; 105 | 106 | mockFetch.mockResolvedValue({ 107 | ok: false, 108 | status: 401, 109 | statusText: "Unauthorized" 110 | }); 111 | 112 | const config = await loadConfig(); 113 | 114 | // Should still return a config, but without enhanced auth info 115 | expect(config.environments).toHaveLength(1); 116 | const env = config.environments[0]; 117 | if (env) { 118 | expect(env.apiKey).toEqual("invalid-key"); 119 | expect(env.name).toEqual("default"); // Didn't get updated 120 | expect(env.teamSlug).toBeUndefined(); // Didn't get populated 121 | } 122 | }); 123 | 124 | it("throws when no environment variables are set", async () => { 125 | // Ensure no Honeycomb env vars are set 126 | Object.keys(process.env).forEach(key => { 127 | if (key.startsWith("HONEYCOMB_")) { 128 | delete process.env[key]; 129 | } 130 | }); 131 | 132 | await expect(loadConfig()).rejects.toThrow(/No Honeycomb configuration found/); 133 | }); 134 | }); 135 | }); -------------------------------------------------------------------------------- /src/tools/get-trigger.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, it, expect, vi, beforeEach } from 'vitest'; 2 | import { createGetTriggerTool } from './get-trigger.js'; 3 | import { HoneycombError } from '../utils/errors.js'; 4 | import { TriggerResponse } from '../types/trigger.js'; 5 | 6 | describe('get-trigger tool', () => { 7 | // Mock API 8 | const mockApi = { 9 | getTrigger: vi.fn() 10 | }; 11 | 12 | // Reset mocks before each test 13 | beforeEach(() => { 14 | vi.resetAllMocks(); 15 | }); 16 | 17 | // Test parameters 18 | const testParams = { 19 | environment: 'test-env', 20 | dataset: 'test-dataset', 21 | triggerId: 'trigger-1' 22 | }; 23 | 24 | // Sample trigger response 25 | const mockTriggerResponse: TriggerResponse = { 26 | id: 'trigger-1', 27 | name: 'High Error Rate', 28 | description: 'Alert on high error rate', 29 | threshold: { 30 | op: '>', 31 | value: 0.05 32 | }, 33 | frequency: 60, 34 | alert_type: 'on_change', 35 | disabled: false, 36 | triggered: false, 37 | recipients: [ 38 | { 39 | id: 'rec-1', 40 | type: 'email', 41 | target: 'team@example.com' 42 | }, 43 | { 44 | id: 'rec-2', 45 | type: 'slack', 46 | target: '#alerts' 47 | } 48 | ], 49 | evaluation_schedule_type: 'frequency', 50 | created_at: '2023-01-01T00:00:00Z', 51 | updated_at: '2023-01-01T00:00:00Z' 52 | }; 53 | 54 | it('should return a valid tool configuration', () => { 55 | const tool = createGetTriggerTool(mockApi as any); 56 | 57 | expect(tool).toHaveProperty('name', 'get_trigger'); 58 | expect(tool).toHaveProperty('schema'); 59 | expect(tool).toHaveProperty('handler'); 60 | expect(typeof tool.handler).toBe('function'); 61 | }); 62 | 63 | it('should return simplified trigger data when API call succeeds', async () => { 64 | // Setup mock API response 65 | mockApi.getTrigger.mockResolvedValue(mockTriggerResponse); 66 | 67 | const tool = createGetTriggerTool(mockApi as any); 68 | const result = await tool.handler(testParams); 69 | 70 | // Verify API was called with correct parameters 71 | expect(mockApi.getTrigger).toHaveBeenCalledWith( 72 | testParams.environment, 73 | testParams.dataset, 74 | testParams.triggerId 75 | ); 76 | 77 | // Check response structure 78 | expect(result).toHaveProperty('content'); 79 | expect(result.content).toHaveLength(1); 80 | expect(result.content[0]).toBeDefined(); 81 | expect(result.content[0]).toHaveProperty('type', 'text'); 82 | 83 | // Parse the JSON response 84 | const response = JSON.parse(result.content[0]!.text!); 85 | 86 | // Verify contents contains simplified trigger data 87 | expect(response).toHaveProperty('id', 'trigger-1'); 88 | expect(response).toHaveProperty('name', 'High Error Rate'); 89 | expect(response).toHaveProperty('description', 'Alert on high error rate'); 90 | expect(response).toHaveProperty('threshold'); 91 | expect(response.threshold).toHaveProperty('op', '>'); 92 | expect(response.threshold).toHaveProperty('value', 0.05); 93 | expect(response).toHaveProperty('frequency', 60); 94 | expect(response).toHaveProperty('alert_type', 'on_change'); 95 | expect(response).toHaveProperty('triggered', false); 96 | expect(response).toHaveProperty('disabled', false); 97 | 98 | // Check recipients 99 | expect(response).toHaveProperty('recipients'); 100 | expect(response.recipients).toHaveLength(2); 101 | expect(response.recipients[0]).toHaveProperty('type', 'email'); 102 | expect(response.recipients[0]).toHaveProperty('target', 'team@example.com'); 103 | expect(response.recipients[1]).toHaveProperty('type', 'slack'); 104 | expect(response.recipients[1]).toHaveProperty('target', '#alerts'); 105 | 106 | expect(response).toHaveProperty('evaluation_schedule_type', 'frequency'); 107 | expect(response).toHaveProperty('created_at'); 108 | expect(response).toHaveProperty('updated_at'); 109 | }); 110 | 111 | it('should handle API errors', async () => { 112 | // Setup API to throw an error 113 | const apiError = new HoneycombError(404, 'Trigger not found'); 114 | mockApi.getTrigger.mockRejectedValue(apiError); 115 | 116 | // Temporarily suppress console.error during this test 117 | const originalConsoleError = console.error; 118 | console.error = vi.fn(); 119 | 120 | try { 121 | const tool = createGetTriggerTool(mockApi as any); 122 | const result = await tool.handler(testParams); 123 | 124 | // Verify error response 125 | expect(result).toHaveProperty('content'); 126 | expect(result.content).toHaveLength(1); 127 | expect(result.content[0]).toBeDefined(); 128 | expect(result.content[0]).toHaveProperty('text'); 129 | expect(result.content[0]!.text!).toContain('Failed to execute tool'); 130 | expect(result.content[0]!.text!).toContain('Trigger not found'); 131 | } finally { 132 | // Restore original console.error 133 | console.error = originalConsoleError; 134 | } 135 | }); 136 | }); -------------------------------------------------------------------------------- /src/tools/analyze-columns.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, it, expect, vi, beforeEach } from 'vitest'; 2 | import { createAnalyzeColumnsTool } from './analyze-columns.js'; 3 | import { HoneycombError } from '../utils/errors.js'; 4 | 5 | describe('analyze-columns tool', () => { 6 | // Mock API 7 | const mockApi = { 8 | analyzeColumns: vi.fn() 9 | }; 10 | 11 | // Reset mocks before each test 12 | beforeEach(() => { 13 | vi.resetAllMocks(); 14 | }); 15 | 16 | // Test parameters 17 | const testParams = { 18 | environment: 'test-env', 19 | dataset: 'test-dataset', 20 | columns: ['test-column1', 'test-column2'] 21 | }; 22 | 23 | it('should return a valid tool configuration', () => { 24 | const tool = createAnalyzeColumnsTool(mockApi as any); 25 | 26 | expect(tool).toHaveProperty('name', 'analyze_columns'); 27 | expect(tool).toHaveProperty('schema'); 28 | expect(tool).toHaveProperty('handler'); 29 | expect(typeof tool.handler).toBe('function'); 30 | }); 31 | 32 | it('should process numeric data correctly', async () => { 33 | // Setup mock API response 34 | mockApi.analyzeColumns.mockResolvedValue({ 35 | data: { 36 | results: [ 37 | { 38 | 'test-column1': 'value1', 39 | 'test-column2': 'valueA', 40 | COUNT: 10, 41 | 'AVG(test-column1)': 15.5, 42 | 'P95(test-column1)': 20, 43 | 'MAX(test-column1)': 30, 44 | 'MIN(test-column1)': 5 45 | }, 46 | { 47 | 'test-column1': 'value2', 48 | 'test-column2': 'valueB', 49 | COUNT: 5 50 | } 51 | ] 52 | } 53 | }); 54 | 55 | const tool = createAnalyzeColumnsTool(mockApi as any); 56 | const result = await tool.handler(testParams); 57 | 58 | // Verify API was called with correct parameters 59 | expect(mockApi.analyzeColumns).toHaveBeenCalledWith( 60 | testParams.environment, 61 | testParams.dataset, 62 | testParams 63 | ); 64 | 65 | // Check response structure 66 | expect(result).toHaveProperty('content'); 67 | expect(result.content).toHaveLength(1); 68 | expect(result.content[0]).toBeDefined(); 69 | expect(result.content[0]).toHaveProperty('type', 'text'); 70 | 71 | // Parse the JSON response 72 | const response = JSON.parse(result.content[0]!.text!); 73 | 74 | // Verify contents 75 | expect(response).toHaveProperty('columns'); 76 | expect(response.columns).toEqual(['test-column1', 'test-column2']); 77 | expect(response).toHaveProperty('count', 2); 78 | expect(response).toHaveProperty('totalEvents', 15); 79 | expect(response).toHaveProperty('topValues'); 80 | expect(response.topValues).toHaveLength(2); 81 | expect(response).toHaveProperty('stats'); 82 | expect(response.stats).toHaveProperty('test-column1'); 83 | expect(response.stats['test-column1']).toHaveProperty('avg', 15.5); 84 | expect(response.stats['test-column1']).toHaveProperty('interpretation'); 85 | expect(response).toHaveProperty('cardinality'); 86 | expect(response.cardinality).toHaveProperty('uniqueCount', 2); 87 | }); 88 | 89 | it('should handle empty results', async () => { 90 | mockApi.analyzeColumns.mockResolvedValue({ 91 | data: { 92 | results: [] 93 | } 94 | }); 95 | 96 | const tool = createAnalyzeColumnsTool(mockApi as any); 97 | const result = await tool.handler(testParams); 98 | 99 | // Parse the JSON response 100 | expect(result).toHaveProperty('content'); 101 | expect(result.content).toHaveLength(1); 102 | expect(result.content[0]).toBeDefined(); 103 | expect(result.content[0]).toHaveProperty('text'); 104 | const response = JSON.parse(result.content[0]!.text!); 105 | 106 | // Verify simple response with no data 107 | expect(response).toHaveProperty('columns', ['test-column1', 'test-column2']); 108 | expect(response).toHaveProperty('count', 0); 109 | expect(response).toHaveProperty('totalEvents', 0); 110 | expect(response).not.toHaveProperty('topValues'); 111 | expect(response).not.toHaveProperty('stats'); 112 | expect(response).not.toHaveProperty('cardinality'); 113 | }); 114 | 115 | it('should handle API errors', async () => { 116 | // Setup API to throw an error 117 | const apiError = new HoneycombError(404, 'Dataset not found'); 118 | mockApi.analyzeColumns.mockRejectedValue(apiError); 119 | 120 | // Temporarily suppress console.error during this test 121 | const originalConsoleError = console.error; 122 | console.error = vi.fn(); 123 | 124 | try { 125 | const tool = createAnalyzeColumnsTool(mockApi as any); 126 | const result = await tool.handler(testParams); 127 | 128 | // Verify error response 129 | expect(result).toHaveProperty('content'); 130 | expect(result.content).toHaveLength(1); 131 | expect(result.content[0]).toBeDefined(); 132 | expect(result.content[0]).toHaveProperty('text'); 133 | expect(result.content[0]!.text!).toContain('Failed to execute tool'); 134 | expect(result.content[0]!.text!).toContain('Dataset not found'); 135 | } finally { 136 | // Restore original console.error 137 | console.error = originalConsoleError; 138 | } 139 | }); 140 | }); -------------------------------------------------------------------------------- /eval/scripts/types.ts: -------------------------------------------------------------------------------- 1 | import { z } from 'zod'; 2 | 3 | // Schema for agent thought process 4 | export const AgentThoughtSchema = z.object({ 5 | thought: z.string().optional(), 6 | plan: z.string().optional(), 7 | reasoning: z.string().optional(), 8 | step: z.number().optional(), 9 | complete: z.boolean().optional(), 10 | summary: z.string().optional(), 11 | }); 12 | 13 | export type AgentThought = z.infer; 14 | 15 | // Record of a single tool call 16 | export const ToolCallRecordSchema = z.object({ 17 | tool: z.string(), 18 | parameters: z.record(z.any()), 19 | response: z.any(), 20 | timestamp: z.string(), 21 | latencyMs: z.number(), 22 | // Agent thought process fields 23 | thought: z.string().optional(), 24 | plan: z.string().optional(), 25 | reasoning: z.string().optional(), 26 | step: z.number().optional(), 27 | complete: z.boolean().optional(), 28 | summary: z.string().optional(), 29 | error: z.string().optional(), 30 | }); 31 | 32 | export type ToolCallRecord = z.infer; 33 | 34 | // Schema for test prompts - simplified for agent-based approach 35 | export const EvalPromptSchema = z.object({ 36 | id: z.string(), 37 | name: z.string(), 38 | description: z.string(), 39 | // The prompt or goal for the agent to achieve 40 | prompt: z.string(), 41 | // Additional context or background information 42 | context: z.string().optional(), 43 | // Expected tools the agent should use (for validation) 44 | expectedTools: z.array(z.string()).optional(), 45 | // Maximum number of tool calls allowed 46 | maxSteps: z.number().optional(), 47 | // Environment to use for the evaluation 48 | environment: z.string().optional(), 49 | // Validation criteria 50 | validation: z.object({ 51 | prompt: z.string(), 52 | expectedOutcome: z.object({ 53 | success: z.boolean(), 54 | criteria: z.array(z.string()).optional(), 55 | }).optional(), 56 | }), 57 | options: z.object({ 58 | timeout: z.number().optional(), 59 | }).optional(), 60 | }); 61 | 62 | export type EvalPrompt = z.infer; 63 | 64 | // Schema for evaluation metrics 65 | export const MetricsSchema = z.object({ 66 | startTime: z.number(), 67 | endTime: z.number(), 68 | latencyMs: z.number(), 69 | tokenUsage: z.object({ 70 | prompt: z.number().optional(), 71 | completion: z.number().optional(), 72 | total: z.number().optional(), 73 | // Track tool-related tokens separately from validation tokens 74 | toolPrompt: z.number().optional(), 75 | toolCompletion: z.number().optional(), 76 | toolTotal: z.number().optional(), 77 | }).optional(), 78 | toolCallCount: z.number().optional(), 79 | // Agent-specific metrics 80 | agentMetrics: z.object({ 81 | goalAchievement: z.number().optional(), // 0-1 score on goal completion 82 | reasoningQuality: z.number().optional(), // 0-1 score on reasoning quality 83 | pathEfficiency: z.number().optional(), // 0-1 score on path efficiency 84 | overallScore: z.number().optional(), // 0-1 overall agent performance 85 | }).optional(), 86 | }); 87 | 88 | export type Metrics = z.infer; 89 | 90 | // Schema for evaluation results 91 | export const EvalResultSchema = z.object({ 92 | id: z.string(), 93 | timestamp: z.string(), 94 | prompt: EvalPromptSchema, 95 | toolCalls: z.array(ToolCallRecordSchema), 96 | validation: z.object({ 97 | passed: z.boolean(), 98 | score: z.number().optional(), // 0-1 score 99 | reasoning: z.string(), 100 | // Agent validation scores 101 | agentScores: z.object({ 102 | goalAchievement: z.number().optional(), // 0-1 score on goal completion 103 | reasoningQuality: z.number().optional(), // 0-1 score on reasoning quality 104 | pathEfficiency: z.number().optional(), // 0-1 score on path efficiency 105 | }).optional(), 106 | }), 107 | metrics: MetricsSchema, 108 | provider: z.string(), // The LLM provider used 109 | model: z.string(), // The specific model used 110 | }); 111 | 112 | export type EvalResult = z.infer; 113 | 114 | // Schema for evaluation summary 115 | export const EvalSummarySchema = z.object({ 116 | timestamp: z.string(), 117 | totalTests: z.number(), 118 | passed: z.number(), 119 | failed: z.number(), 120 | successRate: z.number(), // 0-1 121 | averageLatency: z.number(), 122 | averageToolCalls: z.number().optional(), 123 | averageToolTokens: z.number().optional(), 124 | results: z.array(EvalResultSchema), 125 | metadata: z.record(z.any()).optional(), 126 | }); 127 | 128 | export type EvalSummary = z.infer; 129 | 130 | // LLM Provider interface 131 | export interface LLMProvider { 132 | name: string; 133 | models: string[]; 134 | 135 | // Context setting to differentiate between validation and tool calls 136 | setToolCallContext?: (isToolCall: boolean) => void; 137 | 138 | // Run a prompt with the LLM 139 | runPrompt: (prompt: string, model: string) => Promise; 140 | 141 | // Get token usage statistics 142 | getTokenUsage: () => { 143 | prompt: number; 144 | completion: number; 145 | total: number; 146 | toolPrompt?: number; 147 | toolCompletion?: number; 148 | toolTotal?: number; 149 | }; 150 | } -------------------------------------------------------------------------------- /src/tools/get-board.test.ts: -------------------------------------------------------------------------------- 1 | import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; 2 | import { createGetBoardTool } from "./get-board.js"; 3 | import { HoneycombAPI } from "../api/client.js"; 4 | 5 | // Mock the API client 6 | vi.mock("../api/client.js", () => { 7 | return { 8 | HoneycombAPI: vi.fn().mockImplementation(() => ({ 9 | getBoard: vi.fn(), 10 | })), 11 | }; 12 | }); 13 | 14 | describe("get-board tool", () => { 15 | let api: HoneycombAPI; 16 | 17 | beforeEach(() => { 18 | api = new HoneycombAPI({} as any); 19 | }); 20 | 21 | afterEach(() => { 22 | vi.clearAllMocks(); 23 | }); 24 | 25 | it("returns board details", async () => { 26 | const mockBoard = { 27 | id: "board-1", 28 | name: "Production Overview", 29 | description: "Overview of production metrics", 30 | column_layout: "multi" as const, 31 | queries: [ 32 | { 33 | caption: "Error rate", 34 | query_style: "graph" as const, 35 | dataset: "production", 36 | query_id: "query-1" 37 | } 38 | ], 39 | created_at: "2023-01-01T00:00:00Z", 40 | updated_at: "2023-01-02T00:00:00Z", 41 | }; 42 | 43 | vi.mocked(api.getBoard).mockResolvedValue(mockBoard); 44 | 45 | const tool = createGetBoardTool(api); 46 | const result = await tool.handler({ 47 | environment: "test-env", 48 | boardId: "board-1" 49 | }); 50 | 51 | expect(api.getBoard).toHaveBeenCalledWith("test-env", "board-1"); 52 | 53 | // Type assertion to tell TypeScript this is a success result with metadata 54 | const successResult = result as { 55 | content: { type: string; text: string }[]; 56 | metadata: { environment: string; boardId: string; name: string } 57 | }; 58 | 59 | expect(successResult.content).toHaveLength(1); 60 | // Add a check that text property exists before attempting to parse it 61 | expect(successResult.content[0]?.text).toBeDefined(); 62 | const content = JSON.parse(successResult.content[0]?.text || '{}'); 63 | expect(content.id).toBe("board-1"); 64 | expect(content.name).toBe("Production Overview"); 65 | expect(content.queries).toHaveLength(1); 66 | expect(successResult.metadata.boardId).toBe("board-1"); 67 | expect(successResult.metadata.name).toBe("Production Overview"); 68 | }); 69 | 70 | it("handles API errors", async () => { 71 | const mockError = new Error("API error"); 72 | vi.mocked(api.getBoard).mockRejectedValue(mockError); 73 | 74 | // Mock console.error to prevent error messages during tests 75 | const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); 76 | 77 | const tool = createGetBoardTool(api); 78 | const result = await tool.handler({ 79 | environment: "test-env", 80 | boardId: "board-1" 81 | }); 82 | 83 | // Restore console.error 84 | consoleErrorSpy.mockRestore(); 85 | 86 | // Type assertion to tell TypeScript this is an error result 87 | const errorResult = result as { 88 | content: { type: string; text: string }[]; 89 | error: { message: string } 90 | }; 91 | 92 | expect(errorResult.error).toBeDefined(); 93 | expect(errorResult.error.message).toContain("API error"); 94 | }); 95 | 96 | it("requires the environment parameter", async () => { 97 | // Mock console.error to prevent error messages during tests 98 | const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); 99 | 100 | const tool = createGetBoardTool(api); 101 | const result = await tool.handler({ 102 | environment: "", 103 | boardId: "board-1" 104 | }); 105 | 106 | // Restore console.error 107 | consoleErrorSpy.mockRestore(); 108 | 109 | // Type assertion to tell TypeScript this is an error result 110 | const errorResult = result as { 111 | content: { type: string; text: string }[]; 112 | error: { message: string } 113 | }; 114 | 115 | expect(errorResult.error).toBeDefined(); 116 | expect(errorResult.error.message).toContain("environment parameter is required"); 117 | }); 118 | 119 | it("requires the boardId parameter", async () => { 120 | // Mock console.error to prevent error messages during tests 121 | const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); 122 | 123 | const tool = createGetBoardTool(api); 124 | const result = await tool.handler({ 125 | environment: "test-env", 126 | boardId: "" 127 | }); 128 | 129 | // Restore console.error 130 | consoleErrorSpy.mockRestore(); 131 | 132 | // Type assertion to tell TypeScript this is an error result 133 | const errorResult = result as { 134 | content: { type: string; text: string }[]; 135 | error: { message: string } 136 | }; 137 | 138 | expect(errorResult.error).toBeDefined(); 139 | expect(errorResult.error.message).toContain("boardId parameter is required"); 140 | }); 141 | 142 | it("has the correct name and schema", () => { 143 | const tool = createGetBoardTool(api); 144 | expect(tool.name).toBe("get_board"); 145 | expect(tool.schema).toBeDefined(); 146 | expect(tool.schema.environment).toBeDefined(); 147 | expect(tool.schema.boardId).toBeDefined(); 148 | }); 149 | }); -------------------------------------------------------------------------------- /src/tools/list-boards.test.ts: -------------------------------------------------------------------------------- 1 | import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; 2 | import { createListBoardsTool } from "./list-boards.js"; 3 | import { HoneycombAPI } from "../api/client.js"; 4 | 5 | // Mock the API client 6 | vi.mock("../api/client.js", () => { 7 | return { 8 | HoneycombAPI: vi.fn().mockImplementation(() => ({ 9 | getBoards: vi.fn(), 10 | })), 11 | }; 12 | }); 13 | 14 | describe("list-boards tool", () => { 15 | let api: HoneycombAPI; 16 | 17 | beforeEach(() => { 18 | api = new HoneycombAPI({} as any); 19 | }); 20 | 21 | afterEach(() => { 22 | vi.clearAllMocks(); 23 | }); 24 | 25 | it("returns a list of boards", async () => { 26 | const mockBoards = [ 27 | { 28 | id: "board-1", 29 | name: "Production Overview", 30 | description: "Overview of production metrics", 31 | created_at: "2023-01-01T00:00:00Z", 32 | updated_at: "2023-01-02T00:00:00Z", 33 | }, 34 | { 35 | id: "board-2", 36 | name: "Error Tracking", 37 | description: "Monitors application errors", 38 | created_at: "2023-01-03T00:00:00Z", 39 | updated_at: "2023-01-04T00:00:00Z", 40 | }, 41 | ]; 42 | 43 | vi.mocked(api.getBoards).mockResolvedValue(mockBoards); 44 | 45 | const tool = createListBoardsTool(api); 46 | const result = await tool.handler({ environment: "test-env" }); 47 | 48 | expect(api.getBoards).toHaveBeenCalledWith("test-env"); 49 | 50 | // Type assertion to tell TypeScript this is a success result with metadata 51 | const successResult = result as { 52 | content: { type: string; text: string }[]; 53 | metadata: { count: number; environment: string } 54 | }; 55 | 56 | expect(successResult.content).toHaveLength(1); 57 | // Add a check that text property exists before attempting to parse it 58 | expect(successResult.content[0]?.text).toBeDefined(); 59 | const content = JSON.parse(successResult.content[0]?.text || '[]'); 60 | expect(content).toHaveLength(2); 61 | expect(content[0].id).toBe("board-1"); 62 | expect(content[1].name).toBe("Error Tracking"); 63 | expect(successResult.metadata.count).toBe(2); 64 | }); 65 | 66 | it("handles API errors", async () => { 67 | const mockError = new Error("API error"); 68 | vi.mocked(api.getBoards).mockRejectedValue(mockError); 69 | 70 | // Mock console.error to prevent error messages during tests 71 | const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); 72 | 73 | const tool = createListBoardsTool(api); 74 | const result = await tool.handler({ environment: "test-env" }); 75 | 76 | // Restore console.error 77 | consoleErrorSpy.mockRestore(); 78 | 79 | // Type assertion to tell TypeScript this is an error result 80 | const errorResult = result as { 81 | content: { type: string; text: string }[]; 82 | error: { message: string } 83 | }; 84 | 85 | expect(errorResult.error).toBeDefined(); 86 | expect(errorResult.error.message).toContain("API error"); 87 | }); 88 | 89 | it("requires the environment parameter", async () => { 90 | // Mock console.error to prevent error messages during tests 91 | const consoleErrorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); 92 | 93 | const tool = createListBoardsTool(api); 94 | const result = await tool.handler({ environment: "" }); 95 | 96 | // Restore console.error 97 | consoleErrorSpy.mockRestore(); 98 | 99 | // Type assertion to tell TypeScript this is an error result 100 | const errorResult = result as { 101 | content: { type: string; text: string }[]; 102 | error: { message: string } 103 | }; 104 | 105 | expect(errorResult.error).toBeDefined(); 106 | expect(errorResult.error.message).toContain("environment parameter is required"); 107 | }); 108 | 109 | it("handles undefined boards response", async () => { 110 | // Mock console.warn to prevent warning messages during tests 111 | const consoleWarnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}); 112 | 113 | // Mock getBoards to return undefined 114 | vi.mocked(api.getBoards).mockResolvedValue(undefined as any); 115 | 116 | const tool = createListBoardsTool(api); 117 | const result = await tool.handler({ environment: "test-env" }); 118 | 119 | // Restore console.warn 120 | consoleWarnSpy.mockRestore(); 121 | 122 | // Type assertion to tell TypeScript this is a success result with metadata 123 | const successResult = result as { 124 | content: { type: string; text: string }[]; 125 | metadata: { count: number; environment: string } 126 | }; 127 | 128 | expect(successResult.content).toHaveLength(1); 129 | // Add a check that text property exists before attempting to parse it 130 | expect(successResult.content[0]?.text).toBeDefined(); 131 | const content = JSON.parse(successResult.content[0]?.text || '[]'); 132 | expect(content).toHaveLength(0); 133 | expect(successResult.metadata.count).toBe(0); 134 | }); 135 | 136 | it("has the correct name and schema", () => { 137 | const tool = createListBoardsTool(api); 138 | expect(tool.name).toBe("list_boards"); 139 | expect(tool.schema).toBeDefined(); 140 | expect(tool.schema.environment).toBeDefined(); 141 | }); 142 | }); -------------------------------------------------------------------------------- /src/prompts/index.ts: -------------------------------------------------------------------------------- 1 | import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; 2 | import { z } from "zod"; 3 | import { getInstrumentationGuidance } from "./guidance.js"; 4 | 5 | /** 6 | * Available prompts with their metadata and arguments 7 | */ 8 | const PROMPTS = [ 9 | { 10 | name: "instrumentation-guidance", 11 | description: "OpenTelemetry Instrumentation guidance optimized for Honeycomb", 12 | arguments: [ 13 | { 14 | name: "language", 15 | description: "Programming language of the code to instrument", 16 | required: false 17 | }, 18 | { 19 | name: "filepath", 20 | description: "Path to the file being instrumented", 21 | required: false 22 | } 23 | ] 24 | } 25 | ]; 26 | 27 | /** 28 | * Handler for the instrumentation-guidance prompt 29 | */ 30 | async function handleInstrumentationGuidance(args?: Record) { 31 | try { 32 | const guidance = getInstrumentationGuidance(); 33 | 34 | const language = args?.language || "your code"; 35 | const filepath = args?.filepath 36 | ? ` for ${args.filepath}` 37 | : ""; 38 | 39 | return { 40 | messages: [ 41 | { 42 | role: "user", 43 | content: { 44 | type: "text", 45 | text: `I need help instrumenting ${language}${filepath} with OpenTelemetry for Honeycomb. Please provide specific recommendations following these guidelines:\n\n${guidance}` 46 | } 47 | } 48 | ] 49 | }; 50 | } catch (error) { 51 | throw new Error(`Failed to read instrumentation guidance: ${error instanceof Error ? error.message : String(error)}`); 52 | } 53 | } 54 | 55 | /** 56 | * Register prompt capabilities with the MCP server 57 | * 58 | * @param server - The MCP server instance 59 | */ 60 | export function registerPrompts(server: McpServer) { 61 | try { 62 | // Cast server to any to access internal structure 63 | const serverAny = server as any; 64 | 65 | let registered = false; 66 | 67 | // Approach 1: Use server.prompt if available (direct SDK method) 68 | if (typeof serverAny.prompt === 'function') { 69 | try { 70 | serverAny.prompt( 71 | "instrumentation-guidance", 72 | { 73 | language: z.string().optional(), 74 | filepath: z.string().optional() 75 | }, 76 | handleInstrumentationGuidance 77 | ); 78 | console.error("Registered prompts using server.prompt API"); 79 | registered = true; 80 | } catch (error) { 81 | console.error("Error using server.prompt API:", error instanceof Error ? error.message : String(error)); 82 | } 83 | } 84 | 85 | // Approach 2: Try server.server.setRequestHandler (works in tests) 86 | if (!registered && serverAny.server && typeof serverAny.server.setRequestHandler === 'function') { 87 | try { 88 | // Register prompts/list handler 89 | serverAny.server.setRequestHandler( 90 | { method: 'prompts/list' }, 91 | async () => ({ prompts: PROMPTS }) 92 | ); 93 | 94 | // Register prompts/get handler 95 | serverAny.server.setRequestHandler( 96 | { method: 'prompts/get' }, 97 | async (request: { params: { name: string; arguments?: Record } }) => { 98 | const { name, arguments: promptArgs } = request.params; 99 | 100 | if (name !== 'instrumentation-guidance') { 101 | throw new Error(`Prompt not found: ${name}`); 102 | } 103 | 104 | return handleInstrumentationGuidance(promptArgs); 105 | } 106 | ); 107 | 108 | console.error("Registered prompts using server.server.setRequestHandler API"); 109 | registered = true; 110 | } catch (error) { 111 | console.error("Error using server.server.setRequestHandler API:", error instanceof Error ? error.message : String(error)); 112 | } 113 | } 114 | 115 | // Approach 3: Add to internal registries directly if available 116 | if (!registered && serverAny._registeredPrompts && Array.isArray(serverAny._registeredPrompts)) { 117 | try { 118 | // Add each prompt definition to the registry 119 | PROMPTS.forEach(prompt => { 120 | if (!serverAny._registeredPrompts.some((p: any) => p.name === prompt.name)) { 121 | serverAny._registeredPrompts.push(prompt); 122 | } 123 | }); 124 | 125 | // Add handler mappings if possible 126 | if (serverAny._promptHandlers && typeof serverAny._promptHandlers === 'object') { 127 | serverAny._promptHandlers["instrumentation-guidance"] = handleInstrumentationGuidance; 128 | } 129 | 130 | console.error("Registered prompts by adding to internal registries"); 131 | registered = true; 132 | } catch (error) { 133 | console.error("Error adding to internal registries:", error instanceof Error ? error.message : String(error)); 134 | } 135 | } 136 | 137 | if (!registered) { 138 | console.error("Could not register prompts: no compatible registration method found"); 139 | } 140 | } catch (error) { 141 | // Log the error but don't let it crash the server 142 | console.error("Error in registerPrompts:", error instanceof Error ? error.message : String(error)); 143 | } 144 | } 145 | -------------------------------------------------------------------------------- /src/prompts/index.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, it, expect, vi, beforeEach } from 'vitest'; 2 | import { registerPrompts } from './index.js'; 3 | 4 | describe('prompts module', () => { 5 | // Mock server 6 | const mockServer = { 7 | server: { 8 | setRequestHandler: vi.fn(), 9 | } 10 | }; 11 | 12 | // Reset mocks before each test 13 | beforeEach(() => { 14 | vi.resetAllMocks(); 15 | }); 16 | 17 | describe('registerPrompts', () => { 18 | it('should register prompts list handler', () => { 19 | // Register prompts 20 | registerPrompts(mockServer as any); 21 | 22 | // Verify prompts/list handler was registered 23 | expect(mockServer.server.setRequestHandler).toHaveBeenCalledWith( 24 | { method: 'prompts/list' }, 25 | expect.any(Function) 26 | ); 27 | }); 28 | 29 | it('should register prompts get handler', () => { 30 | // Register prompts 31 | registerPrompts(mockServer as any); 32 | 33 | // Verify prompts/get handler was registered 34 | expect(mockServer.server.setRequestHandler).toHaveBeenCalledWith( 35 | { method: 'prompts/get' }, 36 | expect.any(Function) 37 | ); 38 | }); 39 | }); 40 | 41 | describe('prompts/list handler', () => { 42 | it('should return list of available prompts', async () => { 43 | // Register prompts 44 | registerPrompts(mockServer as any); 45 | 46 | // Get the registered handler for prompts/list 47 | const listHandler = vi.mocked(mockServer.server.setRequestHandler).mock.calls.find( 48 | call => call[0].method === 'prompts/list' 49 | )?.[1]; 50 | 51 | // Make sure handler was found 52 | expect(listHandler).toBeDefined(); 53 | 54 | // Call the handler 55 | const result = await listHandler!({} as any); 56 | 57 | // Verify result structure 58 | expect(result).toHaveProperty('prompts'); 59 | expect(result.prompts).toHaveLength(1); 60 | expect(result.prompts[0]).toHaveProperty('name', 'instrumentation-guidance'); 61 | expect(result.prompts[0]).toHaveProperty('description'); 62 | expect(result.prompts[0]).toHaveProperty('arguments'); 63 | expect(result.prompts[0].arguments).toHaveLength(2); 64 | expect(result.prompts[0].arguments[0]).toHaveProperty('name', 'language'); 65 | expect(result.prompts[0].arguments[1]).toHaveProperty('name', 'filepath'); 66 | }); 67 | }); 68 | 69 | describe('prompts/get handler', () => { 70 | it('should return instrumentation guidance prompt', async () => { 71 | // Register prompts 72 | registerPrompts(mockServer as any); 73 | 74 | // Get the registered handler for prompts/get 75 | const getHandler = vi.mocked(mockServer.server.setRequestHandler).mock.calls.find( 76 | call => call[0].method === 'prompts/get' 77 | )?.[1]; 78 | 79 | // Make sure handler was found 80 | expect(getHandler).toBeDefined(); 81 | 82 | // Call the handler with the instrumentation-guidance prompt 83 | const result = await getHandler!({ 84 | params: { 85 | name: 'instrumentation-guidance', 86 | arguments: { 87 | language: 'JavaScript', 88 | filepath: '/app/index.js' 89 | } 90 | } 91 | } as any); 92 | 93 | // Verify result structure 94 | expect(result).toHaveProperty('messages'); 95 | expect(result.messages).toHaveLength(1); 96 | expect(result.messages[0]).toHaveProperty('role', 'user'); 97 | expect(result.messages[0].content).toHaveProperty('type', 'text'); 98 | 99 | // Verify text includes the language and filepath 100 | expect(result.messages[0].content.text).toContain('JavaScript'); 101 | expect(result.messages[0].content.text).toContain('/app/index.js'); 102 | expect(result.messages[0].content.text).toContain('# OpenTelemetry Code Analysis & Instrumentation'); 103 | }); 104 | 105 | it('should use default values when arguments are not provided', async () => { 106 | // Register prompts 107 | registerPrompts(mockServer as any); 108 | 109 | // Get the registered handler for prompts/get 110 | const getHandler = vi.mocked(mockServer.server.setRequestHandler).mock.calls.find( 111 | call => call[0].method === 'prompts/get' 112 | )?.[1]; 113 | 114 | // Call the handler with no arguments 115 | const result = await getHandler!({ 116 | params: { 117 | name: 'instrumentation-guidance' 118 | } 119 | } as any); 120 | 121 | // Verify default values are used 122 | expect(result.messages[0].content.text).toContain('your code'); 123 | expect(result.messages[0].content.text).not.toMatch(/ for \/\w+/); // Check specifically for " for /path" pattern 124 | }); 125 | 126 | it('should throw an error for unknown prompt', async () => { 127 | // Register prompts 128 | registerPrompts(mockServer as any); 129 | 130 | // Get the registered handler for prompts/get 131 | const getHandler = vi.mocked(mockServer.server.setRequestHandler).mock.calls.find( 132 | call => call[0].method === 'prompts/get' 133 | )?.[1]; 134 | 135 | // Call the handler with an unknown prompt 136 | const promise = getHandler!({ 137 | params: { 138 | name: 'unknown-prompt' 139 | } 140 | } as any); 141 | 142 | // Verify the handler throws an error 143 | await expect(promise).rejects.toThrow('Prompt not found: unknown-prompt'); 144 | }); 145 | }); 146 | }); 147 | -------------------------------------------------------------------------------- /src/utils/transformations.ts: -------------------------------------------------------------------------------- 1 | import { calculateStdDev, getTopValues, TopValueItem } from "./functions.js"; 2 | import { QueryResultValue } from "../types/query.js"; 3 | import { z } from "zod"; 4 | import { QueryToolSchema } from "../types/schema.js"; 5 | import { isValidNumber } from "./typeguards.js"; 6 | 7 | /** 8 | * Types for the summary statistics 9 | */ 10 | interface NumericStats { 11 | min: number; 12 | max: number; 13 | avg: number; 14 | median: number; 15 | sum: number; 16 | range?: number; 17 | stdDev?: number; 18 | } 19 | 20 | interface CountStats { 21 | total: number; 22 | max: number; 23 | min: number; 24 | avg: number; 25 | } 26 | 27 | interface BreakdownStat { 28 | uniqueCount: number; 29 | topValues?: Array; 30 | } 31 | 32 | interface BreakdownStats { 33 | [column: string]: BreakdownStat; 34 | } 35 | 36 | interface ResultSummary { 37 | count: number; 38 | countStats?: CountStats; 39 | breakdowns?: BreakdownStats; 40 | [calculationColumn: string]: NumericStats | number | CountStats | BreakdownStats | undefined; 41 | } 42 | 43 | /** 44 | * Calculate summary statistics for query results to provide useful insights 45 | * without overwhelming the context window 46 | */ 47 | export function summarizeResults(results: QueryResultValue[], params: z.infer): ResultSummary { 48 | if (!results || results.length === 0) { 49 | return { count: 0 }; 50 | } 51 | 52 | const summary: ResultSummary = { 53 | count: results.length, 54 | }; 55 | 56 | // If we have calculation columns, add some statistics about them 57 | if (params.calculations) { 58 | const numericColumns = params.calculations 59 | .filter(calc => 60 | calc.op !== "COUNT" && 61 | calc.op !== "CONCURRENCY" && 62 | calc.op !== "HEATMAP" && 63 | calc.column 64 | ) 65 | .map(calc => `${calc.op}(${calc.column})`); 66 | 67 | numericColumns.forEach((colName: string) => { 68 | if (results[0] && colName in results[0]) { 69 | // Filter to ensure we only have numeric values 70 | const values = results 71 | .map(r => r[colName]) 72 | .filter(isValidNumber); 73 | 74 | if (values.length > 0) { 75 | const min = Math.min(...values); 76 | const max = Math.max(...values); 77 | const sum = values.reduce((a, b) => a + b, 0); 78 | const avg = sum / values.length; 79 | 80 | // Calculate median (P50 approximation) 81 | const sortedValues = [...values].sort((a, b) => a - b); 82 | 83 | // Default to average if we can't calculate median properly 84 | let median = avg; 85 | 86 | // We know values is not empty at this point because we checked values.length > 0 earlier 87 | if (sortedValues.length === 1) { 88 | median = sortedValues[0]!; 89 | } else if (sortedValues.length > 1) { 90 | const medianIndex = Math.floor(sortedValues.length / 2); 91 | 92 | if (sortedValues.length % 2 === 0) { 93 | // Even number of elements - average the middle two 94 | // We can use non-null assertion (!) because we know these indices exist 95 | // when sortedValues.length > 1 and we're in the even case 96 | median = (sortedValues[medianIndex - 1]! + sortedValues[medianIndex]!) / 2; 97 | } else { 98 | // Odd number of elements - take the middle one 99 | // We can use non-null assertion (!) because we know this index exists 100 | median = sortedValues[medianIndex]!; 101 | } 102 | } 103 | 104 | // Create a properly typed NumericStats object 105 | const stats: NumericStats = { 106 | min, 107 | max, 108 | avg, 109 | median, 110 | sum, 111 | range: max - min, 112 | stdDev: calculateStdDev(values, avg) 113 | }; 114 | summary[colName] = stats; 115 | } 116 | } 117 | }); 118 | 119 | // Special handling for COUNT operations 120 | const hasCount = params.calculations.some(calc => calc.op === "COUNT"); 121 | if (hasCount && results.length > 0 && 'COUNT' in results[0]!) { 122 | // Filter to ensure we only have numeric values 123 | const countValues = results 124 | .map(r => r.COUNT) 125 | .filter(isValidNumber); 126 | 127 | if (countValues.length > 0) { 128 | const totalCount = countValues.reduce((a, b) => a + b, 0); 129 | const maxCount = Math.max(...countValues); 130 | const minCount = Math.min(...countValues); 131 | 132 | // Now properly typed 133 | summary.countStats = { 134 | total: totalCount, 135 | max: maxCount, 136 | min: minCount, 137 | avg: totalCount / countValues.length 138 | }; 139 | } 140 | } 141 | } 142 | 143 | // Add unique count for breakdown columns 144 | if (params.breakdowns && params.breakdowns.length > 0) { 145 | const breakdownStats: BreakdownStats = {}; 146 | 147 | params.breakdowns.forEach((col: string) => { 148 | const uniqueValues = new Set(results.map(r => r[col])); 149 | breakdownStats[col] = { 150 | uniqueCount: uniqueValues.size, 151 | topValues: getTopValues(results, col, 5) 152 | }; 153 | }); 154 | 155 | summary.breakdowns = breakdownStats; 156 | } 157 | 158 | return summary; 159 | } -------------------------------------------------------------------------------- /src/resources/datasets.ts: -------------------------------------------------------------------------------- 1 | import { ResourceTemplate } from "@modelcontextprotocol/sdk/server/mcp.js"; 2 | import { HoneycombAPI } from "../api/client.js"; 3 | import { Dataset } from "../types/api.js"; 4 | import { Column } from "../types/column.js"; 5 | 6 | /** 7 | * Interface for MCP resource items 8 | */ 9 | interface ResourceItem { 10 | uri: string; 11 | name: string; 12 | description?: string; 13 | [key: string]: unknown; 14 | } 15 | 16 | /** 17 | * Creates and returns the datasets resource template for interacting with Honeycomb datasets. This resource template allows users to list all datasets across all environments and retrieve specific datasets with their columns. 18 | * 19 | * @param api - The Honeycomb API client instance 20 | * @returns A ResourceTemplate for datasets 21 | */ 22 | export function createDatasetsResource(api: HoneycombAPI) { 23 | return new ResourceTemplate("honeycomb://{environment}/{dataset}", { 24 | /** 25 | * Lists all datasets across all environments 26 | * 27 | * @returns A list of dataset resources across all environments 28 | */ 29 | list: async () => { 30 | // Get all available environments 31 | const environments = api.getEnvironments(); 32 | const resources: ResourceItem[] = []; 33 | 34 | // Fetch datasets from each environment 35 | for (const env of environments) { 36 | try { 37 | const datasets = await api.listDatasets(env); 38 | 39 | // Add each dataset as a resource 40 | datasets.forEach((dataset: Dataset) => { 41 | resources.push({ 42 | uri: `honeycomb://${env}/${dataset.slug}`, 43 | name: dataset.name, 44 | description: dataset.description || '', 45 | }); 46 | }); 47 | } catch (error) { 48 | console.error(`Error fetching datasets for environment ${env}:`, error); 49 | } 50 | } 51 | 52 | return { resources }; 53 | } 54 | }); 55 | } 56 | 57 | /** 58 | * Interface for dataset with column information 59 | */ 60 | interface DatasetWithColumns { 61 | name: string; 62 | description: string; 63 | slug: string; 64 | columns: Array<{ 65 | name: string; 66 | type: string; 67 | description: string; 68 | }>; 69 | created_at?: string; 70 | last_written_at?: string | null; 71 | } 72 | 73 | /** 74 | * Handles requests for dataset resources. This resource template allows users to list all datasets across all environments and retrieve specific datasets with their columns. 75 | * 76 | * This function retrieves either a specific dataset with its columns or 77 | * a list of all datasets in an environment. 78 | * 79 | * @param api - The Honeycomb API client 80 | * @param uri - The resource URI 81 | * @param variables - The parsed variables from the URI template 82 | * @returns Dataset resource contents 83 | * @throws Error if the dataset cannot be retrieved 84 | */ 85 | export async function handleDatasetResource( 86 | api: HoneycombAPI, 87 | variables: Record 88 | ) { 89 | // Extract environment and dataset from variables, handling potential array values 90 | const environment = Array.isArray(variables.environment) 91 | ? variables.environment[0] 92 | : variables.environment; 93 | 94 | const datasetSlug = Array.isArray(variables.dataset) 95 | ? variables.dataset[0] 96 | : variables.dataset; 97 | 98 | if (!environment) { 99 | throw new Error("Missing environment parameter"); 100 | } 101 | 102 | if (!datasetSlug) { 103 | // Return all datasets for this environment 104 | try { 105 | const datasets = await api.listDatasets(environment); 106 | 107 | return { 108 | contents: datasets.map(dataset => ({ 109 | uri: `honeycomb://${environment}/${dataset.slug}`, 110 | text: JSON.stringify({ 111 | name: dataset.name, 112 | description: dataset.description || '', 113 | slug: dataset.slug, 114 | created_at: dataset.created_at, 115 | last_written_at: dataset.last_written_at, 116 | }, null, 2), 117 | mimeType: "application/json" 118 | })) 119 | }; 120 | } catch (error) { 121 | throw new Error(`Failed to list datasets: ${error instanceof Error ? error.message : String(error)}`); 122 | } 123 | } else { 124 | // Return specific dataset with columns 125 | try { 126 | const dataset = await api.getDataset(environment, datasetSlug); 127 | const columns = await api.getVisibleColumns(environment, datasetSlug); 128 | 129 | // Filter out hidden columns 130 | const visibleColumns = columns.filter((column: Column) => !column.hidden); 131 | 132 | const datasetWithColumns: DatasetWithColumns = { 133 | name: dataset.name, 134 | description: dataset.description || '', 135 | slug: dataset.slug, 136 | columns: visibleColumns.map((column: Column) => ({ 137 | name: column.key_name, 138 | type: column.type, 139 | description: column.description || '', 140 | })), 141 | created_at: dataset.created_at, 142 | last_written_at: dataset.last_written_at, 143 | }; 144 | 145 | return { 146 | contents: [{ 147 | uri: `honeycomb://${environment}/${datasetSlug}`, 148 | text: JSON.stringify(datasetWithColumns, null, 2), 149 | mimeType: "application/json" 150 | }] 151 | }; 152 | } catch (error) { 153 | throw new Error(`Failed to read dataset: ${error instanceof Error ? error.message : String(error)}`); 154 | } 155 | } 156 | } 157 | -------------------------------------------------------------------------------- /src/query/validation.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, it, expect } from "vitest"; 2 | import { validateQuery } from "./validation.js"; 3 | import { QueryCalculationSchema, OrderDirectionSchema, HavingSchema } from "../types/schema.js"; 4 | import { z } from "zod"; 5 | 6 | // Define types for the various enums to ensure type safety 7 | type CalculationOp = z.infer["op"]; 8 | type OrderDirection = z.infer; 9 | type HavingOp = z.infer["op"]; 10 | type HavingCalculateOp = z.infer["calculate_op"]; 11 | 12 | describe("Query validation", () => { 13 | describe("Time parameters", () => { 14 | it("allows time_range alone", () => { 15 | const params = { 16 | environment: "prod", 17 | dataset: "test", 18 | calculations: [{ op: "COUNT" as CalculationOp }], 19 | time_range: 3600 20 | }; 21 | 22 | expect(() => validateQuery(params)).not.toThrow(); 23 | }); 24 | 25 | it("allows start_time and end_time together", () => { 26 | const params = { 27 | environment: "prod", 28 | dataset: "test", 29 | calculations: [{ op: "COUNT" as CalculationOp }], 30 | start_time: 1672531200, // 2023-01-01 as timestamp 31 | end_time: 1672617600 // 2023-01-02 as timestamp 32 | }; 33 | 34 | expect(() => validateQuery(params)).not.toThrow(); 35 | }); 36 | 37 | it("allows time_range with start_time", () => { 38 | const params = { 39 | environment: "prod", 40 | dataset: "test", 41 | calculations: [{ op: "COUNT" as CalculationOp }], 42 | time_range: 3600, 43 | start_time: 1672531200 // 2023-01-01 as timestamp 44 | }; 45 | 46 | expect(() => validateQuery(params)).not.toThrow(); 47 | }); 48 | 49 | it("allows time_range with end_time", () => { 50 | const params = { 51 | environment: "prod", 52 | dataset: "test", 53 | calculations: [{ op: "COUNT" as CalculationOp }], 54 | time_range: 3600, 55 | end_time: 1672617600 // 2023-01-02 as timestamp 56 | }; 57 | 58 | expect(() => validateQuery(params)).not.toThrow(); 59 | }); 60 | 61 | it("rejects time_range, start_time, and end_time together", () => { 62 | const params = { 63 | environment: "prod", 64 | dataset: "test", 65 | calculations: [{ op: "COUNT" as CalculationOp }], 66 | time_range: 3600, 67 | start_time: 1672531200, // 2023-01-01 as timestamp 68 | end_time: 1672617600 // 2023-01-02 as timestamp 69 | }; 70 | 71 | expect(() => validateQuery(params)).toThrow(); 72 | }); 73 | }); 74 | 75 | describe("Orders validation", () => { 76 | it("validates orders reference valid breakdowns", () => { 77 | const params = { 78 | environment: "prod", 79 | dataset: "test", 80 | calculations: [{ op: "COUNT" as CalculationOp }], 81 | breakdowns: ["service", "duration"], 82 | orders: [{ column: "service", order: "ascending" as OrderDirection }] 83 | }; 84 | 85 | expect(() => validateQuery(params)).not.toThrow(); 86 | }); 87 | 88 | it("validates orders reference valid calculations", () => { 89 | const params = { 90 | environment: "prod", 91 | dataset: "test", 92 | calculations: [ 93 | { op: "COUNT" as CalculationOp }, 94 | { op: "AVG" as CalculationOp, column: "duration" } 95 | ], 96 | orders: [{ column: "duration", op: "AVG" as CalculationOp, order: "descending" as OrderDirection }] 97 | }; 98 | 99 | expect(() => validateQuery(params)).not.toThrow(); 100 | }); 101 | 102 | it("rejects orders with invalid column references", () => { 103 | const params = { 104 | environment: "prod", 105 | dataset: "test", 106 | calculations: [{ op: "COUNT" as CalculationOp }], 107 | breakdowns: ["service"], 108 | orders: [{ column: "invalid_field", order: "ascending" as OrderDirection }] 109 | }; 110 | 111 | expect(() => validateQuery(params)).toThrow(); 112 | }); 113 | 114 | it("rejects HEATMAP in orders", () => { 115 | const params = { 116 | environment: "prod", 117 | dataset: "test", 118 | calculations: [{ op: "HEATMAP" as CalculationOp, column: "duration" }], 119 | orders: [{ column: "duration", op: "HEATMAP" as CalculationOp, order: "descending" as OrderDirection }] 120 | }; 121 | 122 | expect(() => validateQuery(params)).toThrow(); 123 | }); 124 | }); 125 | 126 | describe("Having clause validation", () => { 127 | it("validates havings clauses reference valid calculations", () => { 128 | const params = { 129 | environment: "prod", 130 | dataset: "test", 131 | calculations: [ 132 | { op: "COUNT" as CalculationOp }, 133 | { op: "AVG" as CalculationOp, column: "duration" } 134 | ], 135 | havings: [ 136 | { calculate_op: "AVG" as HavingCalculateOp, column: "duration", op: ">" as HavingOp, value: 100 } 137 | ] 138 | }; 139 | 140 | expect(() => validateQuery(params)).not.toThrow(); 141 | }); 142 | 143 | it("rejects havings clauses with invalid calculation references", () => { 144 | const params = { 145 | environment: "prod", 146 | dataset: "test", 147 | calculations: [{ op: "COUNT" as CalculationOp }], 148 | havings: [ 149 | { calculate_op: "P99" as HavingCalculateOp, column: "duration", op: ">" as HavingOp, value: 100 } 150 | ] 151 | }; 152 | 153 | expect(() => validateQuery(params)).toThrow(); 154 | }); 155 | }); 156 | }); -------------------------------------------------------------------------------- /src/tools/list-triggers.test.ts: -------------------------------------------------------------------------------- 1 | import { describe, it, expect, vi, beforeEach } from 'vitest'; 2 | import { createListTriggersTool } from './list-triggers.js'; 3 | import { HoneycombError } from '../utils/errors.js'; 4 | import { TriggerResponse } from '../types/trigger.js'; 5 | 6 | describe('list-triggers tool', () => { 7 | // Mock API 8 | const mockApi = { 9 | getTriggers: vi.fn() 10 | }; 11 | 12 | // Reset mocks before each test 13 | beforeEach(() => { 14 | vi.resetAllMocks(); 15 | }); 16 | 17 | // Test parameters 18 | const testParams = { 19 | environment: 'test-env', 20 | dataset: 'test-dataset' 21 | }; 22 | 23 | // Sample triggers response 24 | const mockTriggers: TriggerResponse[] = [ 25 | { 26 | id: 'trigger-1', 27 | name: 'High Error Rate', 28 | description: 'Alert on high error rate', 29 | threshold: { 30 | op: '>', 31 | value: 0.05 32 | }, 33 | frequency: 60, 34 | alert_type: 'on_change', 35 | disabled: false, 36 | triggered: false, 37 | recipients: [ 38 | { 39 | id: 'rec-1', 40 | type: 'email', 41 | target: 'team@example.com' 42 | } 43 | ], 44 | created_at: '2023-01-01T00:00:00Z', 45 | updated_at: '2023-01-01T00:00:00Z' 46 | }, 47 | { 48 | id: 'trigger-2', 49 | name: 'Latency Spike', 50 | description: 'Alert on p95 latency', 51 | threshold: { 52 | op: '>', 53 | value: 500 54 | }, 55 | frequency: 300, 56 | alert_type: 'on_true', 57 | disabled: true, 58 | triggered: false, 59 | recipients: [ 60 | { 61 | id: 'rec-2', 62 | type: 'slack', 63 | target: '#alerts' 64 | } 65 | ], 66 | created_at: '2023-01-01T00:00:00Z', 67 | updated_at: '2023-01-01T00:00:00Z' 68 | } 69 | ]; 70 | 71 | it('should return a valid tool configuration', () => { 72 | const tool = createListTriggersTool(mockApi as any); 73 | 74 | expect(tool).toHaveProperty('name', 'list_triggers'); 75 | expect(tool).toHaveProperty('schema'); 76 | expect(tool).toHaveProperty('handler'); 77 | expect(typeof tool.handler).toBe('function'); 78 | }); 79 | 80 | it('should return simplified triggers when API call succeeds', async () => { 81 | // Setup mock API response 82 | mockApi.getTriggers.mockResolvedValue(mockTriggers); 83 | 84 | const tool = createListTriggersTool(mockApi as any); 85 | const result = await tool.handler(testParams); 86 | 87 | // Verify API was called with correct parameters 88 | expect(mockApi.getTriggers).toHaveBeenCalledWith( 89 | testParams.environment, 90 | testParams.dataset 91 | ); 92 | 93 | // Check response structure 94 | expect(result).toHaveProperty('content'); 95 | expect(result.content).toHaveLength(1); 96 | expect(result.content[0]).toBeDefined(); 97 | expect(result.content[0]).toHaveProperty('type', 'text'); 98 | 99 | // Parse the JSON response 100 | const response = JSON.parse(result.content[0]!.text!); 101 | 102 | // Verify contents contains simplified trigger data 103 | expect(response).toHaveLength(2); 104 | 105 | expect(response[0]).toHaveProperty('id', 'trigger-1'); 106 | expect(response[0]).toHaveProperty('name', 'High Error Rate'); 107 | expect(response[0]).toHaveProperty('description', 'Alert on high error rate'); 108 | expect(response[0]).toHaveProperty('threshold'); 109 | expect(response[0].threshold).toHaveProperty('op', '>'); 110 | expect(response[0].threshold).toHaveProperty('value', 0.05); 111 | expect(response[0]).toHaveProperty('triggered', false); 112 | expect(response[0]).toHaveProperty('disabled', false); 113 | expect(response[0]).not.toHaveProperty('recipients'); 114 | expect(response[0]).not.toHaveProperty('created_at'); 115 | 116 | expect(response[1]).toHaveProperty('id', 'trigger-2'); 117 | expect(response[1]).toHaveProperty('name', 'Latency Spike'); 118 | }); 119 | 120 | it('should handle empty triggers list', async () => { 121 | // Setup API to return empty triggers array 122 | mockApi.getTriggers.mockResolvedValue([]); 123 | 124 | const tool = createListTriggersTool(mockApi as any); 125 | const result = await tool.handler(testParams); 126 | 127 | // Check response structure 128 | expect(result).toHaveProperty('content'); 129 | expect(result.content).toHaveLength(1); 130 | expect(result.content[0]).toBeDefined(); 131 | expect(result.content[0]).toHaveProperty('type', 'text'); 132 | 133 | // Parse the JSON response 134 | const response = JSON.parse(result.content[0]!.text!); 135 | 136 | // Verify empty array is returned 137 | expect(response).toEqual([]); 138 | }); 139 | 140 | it('should handle API errors', async () => { 141 | // Setup API to throw an error 142 | const apiError = new HoneycombError(404, 'Dataset not found'); 143 | mockApi.getTriggers.mockRejectedValue(apiError); 144 | 145 | // Temporarily suppress console.error during this test 146 | const originalConsoleError = console.error; 147 | console.error = vi.fn(); 148 | 149 | try { 150 | const tool = createListTriggersTool(mockApi as any); 151 | const result = await tool.handler(testParams); 152 | 153 | // Verify error response 154 | expect(result).toHaveProperty('content'); 155 | expect(result.content).toHaveLength(1); 156 | expect(result.content[0]).toBeDefined(); 157 | expect(result.content[0]).toHaveProperty('text'); 158 | expect(result.content[0]!.text!).toContain('Failed to execute tool'); 159 | expect(result.content[0]!.text!).toContain('Dataset not found'); 160 | } finally { 161 | // Restore original console.error 162 | console.error = originalConsoleError; 163 | } 164 | }); 165 | }); -------------------------------------------------------------------------------- /src/config.ts: -------------------------------------------------------------------------------- 1 | import { z } from "zod"; 2 | import { AuthResponse } from "./types/api.js"; 3 | import { CacheConfigSchema } from "./cache/index.js"; 4 | 5 | // Enhanced environment schema with authentication information 6 | export const EnvironmentSchema = z.object({ 7 | name: z.string(), 8 | apiKey: z.string(), 9 | apiEndpoint: z.string().optional(), 10 | // Fields that will be populated from the auth endpoint 11 | teamSlug: z.string().optional(), 12 | teamName: z.string().optional(), 13 | environmentSlug: z.string().optional(), 14 | permissions: z.record(z.boolean()).optional(), 15 | }); 16 | 17 | export const ConfigSchema = z.object({ 18 | environments: z.array(EnvironmentSchema).min(1, "At least one environment must be configured"), 19 | cache: CacheConfigSchema, 20 | }); 21 | 22 | export type Environment = z.infer; 23 | export type Config = z.infer; 24 | 25 | /** 26 | * Load configuration from environment variables 27 | * Supports both HONEYCOMB_ENV_*_API_KEY for multiple environments 28 | * and HONEYCOMB_API_KEY for a single environment 29 | */ 30 | async function loadFromEnvVars(): Promise { 31 | const environments: Environment[] = []; 32 | const envVars = process.env; 33 | const defaultApiEndpoint = "https://api.honeycomb.io"; 34 | const globalApiEndpoint = envVars.HONEYCOMB_API_ENDPOINT; 35 | 36 | // Check for multi-environment pattern: HONEYCOMB_ENV_*_API_KEY 37 | const envVarRegex = /^HONEYCOMB_ENV_(.+)_API_KEY$/; 38 | for (const [key, value] of Object.entries(envVars)) { 39 | const match = key.match(envVarRegex); 40 | if (match && match[1] && value) { 41 | const envName = match[1].toLowerCase(); 42 | environments.push({ 43 | name: envName, 44 | apiKey: value, 45 | apiEndpoint: globalApiEndpoint || defaultApiEndpoint, 46 | }); 47 | } 48 | } 49 | 50 | // Check for single environment: HONEYCOMB_API_KEY 51 | if (envVars.HONEYCOMB_API_KEY) { 52 | environments.push({ 53 | name: "default", // This will be updated with actual name from auth response 54 | apiKey: envVars.HONEYCOMB_API_KEY, 55 | apiEndpoint: globalApiEndpoint || defaultApiEndpoint, 56 | }); 57 | } 58 | 59 | if (environments.length === 0) { 60 | throw new Error( 61 | "No Honeycomb configuration found. Please set HONEYCOMB_API_KEY for a single environment " + 62 | "or HONEYCOMB_ENV__API_KEY for multiple environments." 63 | ); 64 | } 65 | 66 | // Default cache configuration 67 | return { 68 | environments, 69 | cache: { 70 | defaultTTL: 300, 71 | ttl: { 72 | dataset: 900, 73 | column: 900, 74 | board: 900, 75 | slo: 900, 76 | trigger: 900, 77 | marker: 900, 78 | recipient: 900, 79 | auth: 3600 80 | }, 81 | enabled: true, 82 | maxSize: 1000 83 | } 84 | }; 85 | } 86 | 87 | /** 88 | * Enhance configuration with data from the Honeycomb API auth endpoint 89 | */ 90 | async function enhanceConfigWithAuth(config: Config): Promise { 91 | const enhancedEnvironments: Environment[] = []; 92 | 93 | // Process each environment sequentially to avoid rate limiting 94 | for (const env of config.environments) { 95 | try { 96 | const headers = { 97 | "X-Honeycomb-Team": env.apiKey, 98 | "Content-Type": "application/json", 99 | }; 100 | 101 | const response = await fetch(`${env.apiEndpoint}/1/auth`, { headers }); 102 | 103 | if (!response.ok) { 104 | throw new Error(`Auth failed for environment ${env.name}: ${response.statusText}`); 105 | } 106 | 107 | const authInfo = await response.json() as AuthResponse; 108 | 109 | enhancedEnvironments.push({ 110 | ...env, 111 | teamSlug: authInfo.team?.slug, 112 | teamName: authInfo.team?.name, 113 | environmentSlug: authInfo.environment?.slug, 114 | // If this is the default environment from HONEYCOMB_API_KEY, update the name 115 | name: env.name === "default" && authInfo.environment?.name ? 116 | authInfo.environment.name : env.name, 117 | permissions: authInfo.api_key_access, 118 | }); 119 | 120 | console.error(`Authenticated environment: ${env.name}`); 121 | } catch (error) { 122 | console.error(`Failed to authenticate environment ${env.name}: ${error instanceof Error ? error.message : String(error)}`); 123 | // Still include this environment but without enhancement 124 | enhancedEnvironments.push(env); 125 | } 126 | } 127 | 128 | return { 129 | environments: enhancedEnvironments, 130 | cache: config.cache 131 | }; 132 | } 133 | 134 | /** 135 | * Load and validate configuration from environment variables 136 | * and enhance with authentication information 137 | */ 138 | export async function loadConfig(): Promise { 139 | try { 140 | // Load initial config from environment variables 141 | const config = await loadFromEnvVars(); 142 | 143 | // Enhance with auth information 144 | const enhancedConfig = await enhanceConfigWithAuth(config); 145 | 146 | // Add cache configuration (default values will be used if not specified) 147 | return { 148 | ...enhancedConfig, 149 | cache: { 150 | enabled: process.env.HONEYCOMB_CACHE_ENABLED !== 'false', 151 | defaultTTL: parseInt(process.env.HONEYCOMB_CACHE_DEFAULT_TTL || '300', 10), 152 | ttl: { 153 | dataset: parseInt(process.env.HONEYCOMB_CACHE_DATASET_TTL || '900', 10), 154 | column: parseInt(process.env.HONEYCOMB_CACHE_COLUMN_TTL || '900', 10), 155 | board: parseInt(process.env.HONEYCOMB_CACHE_BOARD_TTL || '900', 10), 156 | slo: parseInt(process.env.HONEYCOMB_CACHE_SLO_TTL || '900', 10), 157 | trigger: parseInt(process.env.HONEYCOMB_CACHE_TRIGGER_TTL || '900', 10), 158 | marker: parseInt(process.env.HONEYCOMB_CACHE_MARKER_TTL || '900', 10), 159 | recipient: parseInt(process.env.HONEYCOMB_CACHE_RECIPIENT_TTL || '900', 10), 160 | auth: parseInt(process.env.HONEYCOMB_CACHE_AUTH_TTL || '3600', 10), 161 | }, 162 | maxSize: parseInt(process.env.HONEYCOMB_CACHE_MAX_SIZE || '1000', 10), 163 | } 164 | }; 165 | } catch (error) { 166 | if (error instanceof z.ZodError) { 167 | const issues = error.issues.map(i => ` - ${i.path.join('.')}: ${i.message}`).join('\n'); 168 | throw new Error( 169 | `Configuration error:\n${issues}\n\nPlease set environment variables:\n` + 170 | `- HONEYCOMB_API_KEY=your_api_key (for single environment)\n` + 171 | `- HONEYCOMB_ENV_PROD_API_KEY=your_prod_api_key (for multiple environments)\n` + 172 | `- HONEYCOMB_ENV_STAGING_API_KEY=your_staging_api_key\n` + 173 | `- HONEYCOMB_API_ENDPOINT=https://api.honeycomb.io (optional, to override default)\n` + 174 | `\nOptional cache configuration:\n` + 175 | `- HONEYCOMB_CACHE_ENABLED=true (set to 'false' to disable caching)\n` + 176 | `- HONEYCOMB_CACHE_DEFAULT_TTL=300 (default TTL in seconds)\n` + 177 | `- HONEYCOMB_CACHE_DATASET_TTL=900 (TTL for dataset resources)\n` + 178 | `- HONEYCOMB_CACHE_MAX_SIZE=1000 (maximum number of items in each cache)` 179 | ); 180 | } 181 | throw error; 182 | } 183 | } 184 | -------------------------------------------------------------------------------- /src/tools/list-markers.ts: -------------------------------------------------------------------------------- 1 | import { z } from "zod"; 2 | import { HoneycombAPI } from "../api/client.js"; 3 | import { handleToolError } from "../utils/tool-error.js"; 4 | import { ListMarkersSchema } from "../types/schema.js"; 5 | import { getCache } from "../cache/index.js"; 6 | import { PaginatedResponse } from "../types/api.js"; 7 | 8 | /** 9 | * Tool to list markers (deployment events) in a Honeycomb environment. This tool returns a list of all markers available in the specified environment, including their IDs, messages, types, URLs, creation times, start times, and end times. 10 | * 11 | * @param api - The Honeycomb API client 12 | * @returns An MCP tool object with name, schema, and handler function 13 | */ 14 | export function createListMarkersTool(api: HoneycombAPI) { 15 | return { 16 | name: "list_markers", 17 | description: "Lists available markers (deployment events) for a specific dataset or environment with pagination, sorting, and search support. Returns IDs, messages, types, URLs, creation times, start times, and end times.", 18 | schema: ListMarkersSchema.shape, 19 | /** 20 | * Handler for the list_markers tool 21 | * 22 | * @param params - The parameters for the tool 23 | * @param params.environment - The Honeycomb environment 24 | * @param params.page - Optional page number for pagination 25 | * @param params.limit - Optional limit of items per page 26 | * @param params.sort_by - Optional field to sort by 27 | * @param params.sort_order - Optional sort direction (asc/desc) 28 | * @param params.search - Optional search term 29 | * @param params.search_fields - Optional fields to search in 30 | * @returns List of markers with relevant metadata, potentially paginated 31 | */ 32 | handler: async (params: z.infer) => { 33 | const { environment, page, limit, sort_by, sort_order, search, search_fields } = params; 34 | 35 | // Validate input parameters 36 | if (!environment) { 37 | return handleToolError(new Error("environment parameter is required"), "list_markers"); 38 | } 39 | 40 | try { 41 | // Fetch markers from the API 42 | const markers = await api.getMarkers(environment); 43 | 44 | // Create a simplified response 45 | const simplifiedMarkers = markers.map(marker => ({ 46 | id: marker.id, 47 | message: marker.message, 48 | type: marker.type, 49 | url: marker.url || '', 50 | created_at: marker.created_at, 51 | start_time: marker.start_time, 52 | end_time: marker.end_time || '', 53 | })); 54 | 55 | // If no pagination or filtering is requested, return all markers 56 | if (!page && !limit && !search && !sort_by) { 57 | return { 58 | content: [ 59 | { 60 | type: "text", 61 | text: JSON.stringify(simplifiedMarkers, null, 2), 62 | }, 63 | ], 64 | metadata: { 65 | count: simplifiedMarkers.length, 66 | environment 67 | } 68 | }; 69 | } 70 | 71 | // Otherwise, use the cache manager to handle pagination, sorting, and filtering 72 | const cache = getCache(); 73 | const cacheOptions = { 74 | page: page || 1, 75 | limit: limit || 10, 76 | 77 | // Configure sorting if requested 78 | ...(sort_by && { 79 | sort: { 80 | field: sort_by, 81 | order: sort_order || 'asc' 82 | } 83 | }), 84 | 85 | // Configure search if requested 86 | ...(search && { 87 | search: { 88 | field: search_fields || ['message', 'type'], 89 | term: search, 90 | caseInsensitive: true 91 | } 92 | }) 93 | }; 94 | 95 | // Access the collection with pagination and filtering 96 | const result = cache.accessCollection( 97 | environment, 98 | 'marker', 99 | undefined, 100 | cacheOptions 101 | ); 102 | 103 | // If the collection isn't in cache yet, apply the filtering manually 104 | if (!result) { 105 | // Basic implementation for non-cached data 106 | let filteredMarkers = [...simplifiedMarkers]; 107 | 108 | // Apply search if requested 109 | if (search) { 110 | const searchFields = Array.isArray(search_fields) 111 | ? search_fields 112 | : search_fields 113 | ? [search_fields] 114 | : ['message', 'type']; 115 | 116 | const searchTerm = search.toLowerCase(); 117 | 118 | filteredMarkers = filteredMarkers.filter(marker => { 119 | return searchFields.some(field => { 120 | const value = marker[field as keyof typeof marker]; 121 | return typeof value === 'string' && value.toLowerCase().includes(searchTerm); 122 | }); 123 | }); 124 | } 125 | 126 | // Apply sorting if requested 127 | if (sort_by) { 128 | const field = sort_by; 129 | const order = sort_order || 'asc'; 130 | 131 | filteredMarkers.sort((a, b) => { 132 | const aValue = a[field as keyof typeof a]; 133 | const bValue = b[field as keyof typeof b]; 134 | 135 | if (typeof aValue === 'string' && typeof bValue === 'string') { 136 | return order === 'asc' 137 | ? aValue.localeCompare(bValue) 138 | : bValue.localeCompare(aValue); 139 | } 140 | 141 | return order === 'asc' 142 | ? (aValue > bValue ? 1 : -1) 143 | : (bValue > aValue ? 1 : -1); 144 | }); 145 | } 146 | 147 | // Apply pagination 148 | const itemLimit = limit || 10; 149 | const currentPage = page || 1; 150 | const total = filteredMarkers.length; 151 | const pages = Math.ceil(total / itemLimit); 152 | const offset = (currentPage - 1) * itemLimit; 153 | 154 | // Return formatted response 155 | const paginatedResponse: PaginatedResponse = { 156 | data: filteredMarkers.slice(offset, offset + itemLimit), 157 | metadata: { 158 | total, 159 | page: currentPage, 160 | pages, 161 | limit: itemLimit 162 | } 163 | }; 164 | 165 | return { 166 | content: [ 167 | { 168 | type: "text", 169 | text: JSON.stringify(paginatedResponse, null, 2), 170 | }, 171 | ], 172 | }; 173 | } 174 | 175 | // Format the cached result and type-cast the unknown data 176 | const typedData = result.data as typeof simplifiedMarkers; 177 | 178 | const paginatedResponse: PaginatedResponse = { 179 | data: typedData, 180 | metadata: { 181 | total: result.total, 182 | page: result.page || 1, 183 | pages: result.pages || 1, 184 | limit: limit || 10 185 | } 186 | }; 187 | 188 | return { 189 | content: [ 190 | { 191 | type: "text", 192 | text: JSON.stringify(paginatedResponse, null, 2), 193 | }, 194 | ], 195 | }; 196 | } catch (error) { 197 | return handleToolError(error, "list_markers"); 198 | } 199 | } 200 | }; 201 | } -------------------------------------------------------------------------------- /.github/workflows/evaluation.yml: -------------------------------------------------------------------------------- 1 | name: Deploy Reports to GitHub Pages 2 | 3 | # This workflow only runs on the main branch to deploy reports to GitHub Pages 4 | on: 5 | # Only runs on the main branch after tests+evals complete 6 | workflow_run: 7 | workflows: ["Tests & Evaluation"] 8 | types: [completed] 9 | branches: [main] 10 | 11 | jobs: 12 | # Only deploy to GitHub Pages from the main branch 13 | deploy-pages: 14 | if: github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.head_branch == 'main' 15 | runs-on: ubuntu-latest 16 | permissions: 17 | pages: write 18 | id-token: write 19 | environment: 20 | name: github-pages 21 | url: ${{ steps.deployment.outputs.page_url }} 22 | 23 | steps: 24 | # Checkout the repo to access scripts 25 | - name: Checkout repository 26 | uses: actions/checkout@v4 27 | 28 | # Download new reports from artifacts 29 | - name: Download new reports artifact 30 | uses: actions/github-script@v6 31 | with: 32 | script: | 33 | const artifacts = await github.rest.actions.listWorkflowRunArtifacts({ 34 | owner: context.repo.owner, 35 | repo: context.repo.repo, 36 | run_id: ${{ github.event.workflow_run.id }} 37 | }); 38 | 39 | const matchArtifact = artifacts.data.artifacts.find(artifact => { 40 | return artifact.name === "evaluation-reports" 41 | }); 42 | 43 | if (!matchArtifact) { 44 | core.setFailed('No evaluation-reports artifact found'); 45 | return; 46 | } 47 | 48 | const download = await github.rest.actions.downloadArtifact({ 49 | owner: context.repo.owner, 50 | repo: context.repo.repo, 51 | artifact_id: matchArtifact.id, 52 | archive_format: 'zip' 53 | }); 54 | 55 | const { writeFileSync } = await import('fs'); 56 | writeFileSync('new-reports.zip', Buffer.from(download.data)); 57 | 58 | # Download current live site to preserve history 59 | - name: Download existing reports from GitHub Pages 60 | id: download-site 61 | continue-on-error: true 62 | run: | 63 | SITE_URL="${{ env.GITHUB_PAGES_URL || format('https://{0}.github.io/{1}', github.repository_owner, github.event.repository.name) }}" 64 | echo "Attempting to download existing site from: $SITE_URL" 65 | mkdir -p existing-site 66 | cd existing-site 67 | 68 | # Try to download index.html first to check if site exists 69 | if curl -s -f -o index.html "$SITE_URL/index.html"; then 70 | echo "Found existing site, downloading reports..." 71 | 72 | # Download all the report-*.html files listed in the index.html 73 | grep -o 'href="report-[^"]*\.html"' index.html | sed 's/href="\([^"]*\)"/\1/g' | while read -r report; do 74 | echo "Downloading $report" 75 | curl -s -f -o "$report" "$SITE_URL/$report" || echo "Failed to download $report" 76 | done 77 | 78 | echo "::set-output name=existing_site::true" 79 | echo "Downloaded $(find . -name 'report-*.html' | wc -l) existing reports" 80 | else 81 | echo "No existing site found, starting fresh" 82 | echo "::set-output name=existing_site::false" 83 | fi 84 | 85 | - name: Unzip new reports 86 | run: | 87 | mkdir -p new-reports 88 | unzip new-reports.zip -d new-reports 89 | 90 | - name: Merge reports 91 | run: | 92 | # Create combined directory for all reports 93 | mkdir -p combined-reports 94 | 95 | # Copy existing reports if they were downloaded successfully 96 | if [ "${{ steps.download-site.outputs.existing_site }}" == "true" ]; then 97 | cp -r existing-site/* combined-reports/ || true 98 | fi 99 | 100 | # Copy new reports, potentially overwriting any duplicates 101 | cp -r new-reports/* combined-reports/ 102 | 103 | # Create a list of all reports for debugging 104 | find combined-reports -name "report-*.html" | sort > report-list.txt 105 | echo "Combined reports directory contains:" 106 | cat report-list.txt 107 | 108 | - name: Update index.html with all reports 109 | run: | 110 | cd combined-reports 111 | 112 | # Create a Node.js script to regenerate the index.html 113 | cat > update-index.js << 'EOF' 114 | import { readdirSync, writeFileSync } from 'fs'; 115 | 116 | // Get all report files 117 | const files = readdirSync('.'); 118 | const reportFiles = files.filter(file => file.startsWith('report-') && file.endsWith('.html')); 119 | 120 | // Sort by date (newest first) 121 | reportFiles.sort((a, b) => b.localeCompare(a)); 122 | 123 | // Generate report links 124 | const reportLinks = reportFiles.map((file, index) => { 125 | const isLatest = index === 0; 126 | const dateMatch = file.match(/report-(.+)\.html/); 127 | const dateStr = dateMatch && dateMatch[1] ? dateMatch[1].replace(/-/g, ':').replace('T', ' ').substring(0, 19) : 'Unknown date'; 128 | 129 | return `
  • 130 | ${isLatest ? '📊 Latest: ' : ''}Report from ${dateStr} 131 | ${isLatest ? '(This is the most recent evaluation run)' : ''} 132 |
    • `; 133 | }); 134 | 135 | // Create HTML 136 | const html = ` 137 | 138 | 139 | 140 | 141 | Honeycomb MCP Evaluation Reports 142 | 153 | 154 | 155 |
    156 |

    Honeycomb MCP Evaluation Reports

    157 |

    Select a report to view detailed evaluation results:

    158 | 159 |
      160 | ${reportLinks.join('\n ')} 161 |
    162 |
    163 | 164 | `; 165 | 166 | writeFileSync('index.html', html); 167 | console.log('Generated index.html with', reportFiles.length, 'reports'); 168 | EOF 169 | 170 | node update-index.js 171 | 172 | # Add a .nojekyll file to disable Jekyll processing 173 | touch .nojekyll 174 | 175 | - name: Setup Pages 176 | uses: actions/configure-pages@v4 177 | 178 | - name: Upload to Pages 179 | uses: actions/upload-pages-artifact@v3 180 | with: 181 | path: combined-reports 182 | 183 | - name: Deploy to GitHub Pages 184 | id: deployment 185 | uses: actions/deploy-pages@v4 -------------------------------------------------------------------------------- /src/tools/analyze-columns.ts: -------------------------------------------------------------------------------- 1 | import { z } from "zod"; 2 | import { HoneycombAPI } from "../api/client.js"; 3 | import { handleToolError } from "../utils/tool-error.js"; 4 | import { ColumnAnalysisSchema } from "../types/schema.js"; 5 | import { generateInterpretation, getCardinalityClassification } from "../utils/analysis.js"; 6 | import { 7 | SimplifiedColumnAnalysis, 8 | NumericStatistics, 9 | NumericStatsWithInterpretation 10 | } from "../types/analysis.js"; 11 | import { QueryResultValue } from "../types/query.js"; 12 | 13 | const description = `Analyzes specific columns in a dataset by running statistical queries and returning computed metrics. 14 | This tool allows users to get statistical information about a specific column, including value distribution, top values, and numeric statistics (for numeric columns). 15 | Supports analyzing up to 10 columns at once by specifying an array of column names in the 'columns' parameter. 16 | When multiple columns are specified, they will be analyzed together as a group, showing the distribution of their combined values. 17 | Use this tool before running queries to get a better understanding of the data in your dataset. 18 | ` 19 | 20 | /** 21 | * Creates a tool for analyzing multiple columns in a Honeycomb dataset 22 | * 23 | * This tool allows users to get statistical information about specific columns, 24 | * including value distribution, top values, and numeric statistics (for numeric columns). 25 | * It can analyze up to 10 columns at once. 26 | * 27 | * @param api - The Honeycomb API client 28 | * @returns A configured tool object with name, schema, and handler 29 | */ 30 | export function createAnalyzeColumnsTool(api: HoneycombAPI) { 31 | return { 32 | name: "analyze_columns", 33 | description, 34 | schema: ColumnAnalysisSchema.shape, 35 | /** 36 | * Handles the analyze_column tool request 37 | * 38 | * @param params - The parameters for the column analysis 39 | * @returns A formatted response with column analysis data 40 | */ 41 | handler: async (params: z.infer) => { 42 | try { 43 | // Validate required parameters 44 | if (!params.environment) { 45 | throw new Error("Missing required parameter: environment"); 46 | } 47 | if (!params.dataset) { 48 | throw new Error("Missing required parameter: dataset"); 49 | } 50 | if (!params.columns || params.columns.length === 0) { 51 | throw new Error("Missing required parameter: columns"); 52 | } 53 | if (params.columns.length > 10) { 54 | throw new Error("Too many columns requested. Maximum is 10."); 55 | } 56 | 57 | // Execute the analysis via the API 58 | const result = await api.analyzeColumns(params.environment, params.dataset, params); 59 | 60 | // Initialize the response 61 | const simplifiedResponse: SimplifiedColumnAnalysis = { 62 | columns: params.columns, 63 | count: result.data?.results?.length || 0, 64 | totalEvents: 0, // Will be populated below if available 65 | }; 66 | 67 | // Add top values if we have results 68 | if (result.data?.results && result.data.results.length > 0) { 69 | const results = result.data.results as QueryResultValue[]; 70 | const firstResult = results[0]; 71 | 72 | try { 73 | // Calculate total events across all results 74 | const totalCount = results.reduce((sum, row) => { 75 | const count = row.COUNT as number | undefined; 76 | // Only add if it's a number, otherwise use 0 77 | return sum + (typeof count === 'number' ? count : 0); 78 | }, 0); 79 | simplifiedResponse.totalEvents = totalCount; 80 | 81 | // Add top values with their counts and percentages 82 | simplifiedResponse.topValues = results.map(row => { 83 | // For multi-column analysis, combine values into a descriptive string 84 | const combinedValue = params.columns 85 | .map(col => { 86 | const colValue = row[col] !== undefined ? row[col] : null; 87 | return `${col}: ${colValue}`; 88 | }) 89 | .join(', '); 90 | 91 | const count = typeof row.COUNT === 'number' ? row.COUNT : 0; 92 | 93 | return { 94 | value: combinedValue, 95 | count, 96 | percentage: totalCount > 0 ? 97 | ((count / totalCount) * 100).toFixed(2) + '%' : 98 | '0%' 99 | }; 100 | }); 101 | 102 | // Initialize stats container for each numeric column 103 | const numericStats: Record = {}; 104 | 105 | // Process numeric metrics for each column if available 106 | if (firstResult) { 107 | params.columns.forEach(column => { 108 | // Check if we have numeric metrics for this column 109 | const avgKey = `AVG(${column})`; 110 | if (avgKey in firstResult) { 111 | const stats: NumericStatistics = {}; 112 | 113 | // Extract metrics for this column 114 | if (typeof firstResult[avgKey] === 'number') stats.avg = firstResult[avgKey] as number; 115 | if (typeof firstResult[`P95(${column})`] === 'number') stats.p95 = firstResult[`P95(${column})`] as number; 116 | if (typeof firstResult[`MAX(${column})`] === 'number') stats.max = firstResult[`MAX(${column})`] as number; 117 | if (typeof firstResult[`MIN(${column})`] === 'number') stats.min = firstResult[`MIN(${column})`] as number; 118 | 119 | // Calculate range if we have min and max 120 | if (stats.min !== undefined && stats.max !== undefined) { 121 | stats.range = stats.max - stats.min; 122 | } 123 | 124 | // Only add if we have at least one stat 125 | if (Object.keys(stats).length > 0) { 126 | numericStats[column] = { 127 | ...stats, 128 | interpretation: generateInterpretation(stats, column) 129 | } as NumericStatsWithInterpretation; 130 | } 131 | } 132 | }); 133 | } 134 | 135 | // Add stats if we have any 136 | if (Object.keys(numericStats).length > 0) { 137 | simplifiedResponse.stats = numericStats; 138 | } 139 | 140 | // Add cardinality information (unique combinations of values) 141 | const uniqueValueCombinations = new Set(); 142 | 143 | results.forEach(row => { 144 | const combinationKey = params.columns 145 | .map(col => `${col}:${row[col] !== undefined ? row[col] : 'null'}`) 146 | .join('|'); 147 | uniqueValueCombinations.add(combinationKey); 148 | }); 149 | 150 | const uniqueCount = uniqueValueCombinations.size; 151 | 152 | simplifiedResponse.cardinality = { 153 | uniqueCount, 154 | classification: getCardinalityClassification(uniqueCount) 155 | }; 156 | } catch (processingError) { 157 | // Handle errors during result processing, but still return partial results 158 | console.error("Error processing column analysis results:", processingError); 159 | simplifiedResponse.processingError = `Error processing results: ${processingError instanceof Error ? processingError.message : String(processingError)}`; 160 | } 161 | } 162 | 163 | return { 164 | content: [ 165 | { 166 | type: "text", 167 | text: JSON.stringify(simplifiedResponse, null, 2), 168 | }, 169 | ], 170 | }; 171 | } catch (error) { 172 | return handleToolError(error, "analyze_columns"); 173 | } 174 | } 175 | }; 176 | } 177 | --------------------------------------------------------------------------------