├── .gitignore ├── .npmignore ├── LICENSE ├── README.md ├── __tests__ ├── README.md ├── fixtures │ ├── mock-server.ts │ └── test-configs │ │ ├── basic-test.ts │ │ ├── basic-test.yaml │ │ ├── error-test.yaml │ │ └── invalid-config.yaml ├── integration │ └── basic-integration.test.ts ├── test-env.js └── unit │ ├── cli-validation.test.ts │ ├── mock-server.test.ts │ └── yaml-loader.test.ts ├── action.yml ├── docker-compose.yml ├── example-server ├── evals │ ├── evals.ts │ ├── evals.yaml │ └── weatherTool.ts └── index.ts ├── grafana └── provisioning │ ├── dashboards │ ├── dashboards.yml │ └── mcp-dashboard.json │ └── datasources │ └── datasource.yml ├── jest.config.cjs ├── otel └── otel-collector-config.yaml ├── package.json ├── prometheus └── prometheus.yml ├── src ├── cli.ts ├── index.ts ├── metrics.ts ├── types.ts └── yaml-loader.ts └── tsconfig.json /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | .env 3 | dist -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mclenhard/mcp-evals/HEAD/.npmignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mclenhard/mcp-evals/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mclenhard/mcp-evals/HEAD/README.md -------------------------------------------------------------------------------- /__tests__/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mclenhard/mcp-evals/HEAD/__tests__/README.md -------------------------------------------------------------------------------- /__tests__/fixtures/mock-server.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mclenhard/mcp-evals/HEAD/__tests__/fixtures/mock-server.ts -------------------------------------------------------------------------------- /__tests__/fixtures/test-configs/basic-test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mclenhard/mcp-evals/HEAD/__tests__/fixtures/test-configs/basic-test.ts -------------------------------------------------------------------------------- /__tests__/fixtures/test-configs/basic-test.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mclenhard/mcp-evals/HEAD/__tests__/fixtures/test-configs/basic-test.yaml -------------------------------------------------------------------------------- /__tests__/fixtures/test-configs/error-test.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mclenhard/mcp-evals/HEAD/__tests__/fixtures/test-configs/error-test.yaml -------------------------------------------------------------------------------- /__tests__/fixtures/test-configs/invalid-config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mclenhard/mcp-evals/HEAD/__tests__/fixtures/test-configs/invalid-config.yaml -------------------------------------------------------------------------------- /__tests__/integration/basic-integration.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mclenhard/mcp-evals/HEAD/__tests__/integration/basic-integration.test.ts -------------------------------------------------------------------------------- /__tests__/test-env.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mclenhard/mcp-evals/HEAD/__tests__/test-env.js -------------------------------------------------------------------------------- /__tests__/unit/cli-validation.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mclenhard/mcp-evals/HEAD/__tests__/unit/cli-validation.test.ts -------------------------------------------------------------------------------- /__tests__/unit/mock-server.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mclenhard/mcp-evals/HEAD/__tests__/unit/mock-server.test.ts -------------------------------------------------------------------------------- /__tests__/unit/yaml-loader.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mclenhard/mcp-evals/HEAD/__tests__/unit/yaml-loader.test.ts -------------------------------------------------------------------------------- /action.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mclenhard/mcp-evals/HEAD/action.yml -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mclenhard/mcp-evals/HEAD/docker-compose.yml -------------------------------------------------------------------------------- /example-server/evals/evals.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mclenhard/mcp-evals/HEAD/example-server/evals/evals.ts -------------------------------------------------------------------------------- /example-server/evals/evals.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mclenhard/mcp-evals/HEAD/example-server/evals/evals.yaml -------------------------------------------------------------------------------- /example-server/evals/weatherTool.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mclenhard/mcp-evals/HEAD/example-server/evals/weatherTool.ts -------------------------------------------------------------------------------- /example-server/index.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mclenhard/mcp-evals/HEAD/example-server/index.ts -------------------------------------------------------------------------------- /grafana/provisioning/dashboards/dashboards.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mclenhard/mcp-evals/HEAD/grafana/provisioning/dashboards/dashboards.yml -------------------------------------------------------------------------------- /grafana/provisioning/dashboards/mcp-dashboard.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mclenhard/mcp-evals/HEAD/grafana/provisioning/dashboards/mcp-dashboard.json -------------------------------------------------------------------------------- /grafana/provisioning/datasources/datasource.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mclenhard/mcp-evals/HEAD/grafana/provisioning/datasources/datasource.yml -------------------------------------------------------------------------------- /jest.config.cjs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mclenhard/mcp-evals/HEAD/jest.config.cjs -------------------------------------------------------------------------------- /otel/otel-collector-config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mclenhard/mcp-evals/HEAD/otel/otel-collector-config.yaml -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mclenhard/mcp-evals/HEAD/package.json -------------------------------------------------------------------------------- /prometheus/prometheus.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mclenhard/mcp-evals/HEAD/prometheus/prometheus.yml -------------------------------------------------------------------------------- /src/cli.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mclenhard/mcp-evals/HEAD/src/cli.ts -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mclenhard/mcp-evals/HEAD/src/index.ts -------------------------------------------------------------------------------- /src/metrics.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mclenhard/mcp-evals/HEAD/src/metrics.ts -------------------------------------------------------------------------------- /src/types.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mclenhard/mcp-evals/HEAD/src/types.ts -------------------------------------------------------------------------------- /src/yaml-loader.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mclenhard/mcp-evals/HEAD/src/yaml-loader.ts -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mclenhard/mcp-evals/HEAD/tsconfig.json --------------------------------------------------------------------------------