├── .changeset └── config.json ├── .github └── workflows │ ├── ci.yml │ ├── claude-code-review.yml │ ├── claude.yml │ └── release.yml ├── .gitignore ├── .husky └── pre-commit ├── CHANGELOG.md ├── LICENSE ├── README.md ├── eslint.config.mjs ├── examples ├── README.md ├── advanced │ └── llm-judge-eval.config.json ├── basic │ ├── simple-server-eval.config.json │ └── simple-server-eval.config.ts ├── integrations │ ├── github-server-eval.config.json │ ├── sse-server-eval.config.json │ └── weather-server-eval.config.json ├── package.json ├── remote │ └── remote-server-eval.config.json ├── run-evaluation.ts ├── servers │ ├── simple-mcp-server.js │ └── simple-sse-server.js ├── tools │ └── tool-health-eval.config.json └── vitest-examples │ ├── README.md │ ├── basic-tool-test.test.ts │ ├── eval-suite-example.test.ts │ ├── server │ ├── calculator-server.js │ └── filesystem-server.js │ ├── simple-server-mock.cjs │ └── simple-test.test.ts ├── package.json ├── pnpm-lock.yaml ├── src ├── cli │ ├── eval.ts │ └── index.ts ├── eval │ ├── core │ │ ├── config.ts │ │ ├── index.ts │ │ ├── runner.ts │ │ └── trace.ts │ ├── evaluators │ │ ├── deterministic.ts │ │ ├── llm-judge.ts │ │ └── tool-health.ts │ ├── infrastructure │ │ ├── json-schema-to-zod.ts │ │ ├── performance.ts │ │ ├── resilience.ts │ │ └── validation.ts │ └── reporters │ │ └── console.ts ├── index.ts ├── types │ ├── evaluation.ts │ ├── performance.ts │ ├── resilience.ts │ ├── server.ts │ ├── tool.ts │ ├── trace.ts │ └── validation.ts └── vitest │ ├── index.ts │ ├── matchers.ts │ ├── scorers │ ├── content.ts │ ├── index.ts │ ├── latency.ts │ ├── toolcall.ts │ └── workflow.ts │ ├── types.ts │ └── vitest-eval.ts ├── tests ├── README.md ├── cli │ ├── cli.test.ts │ └── eval.test.ts ├── e2e │ └── integration.test.ts ├── integration │ ├── evaluate.test.ts │ ├── runner.test.ts │ ├── tool-health.test.ts │ └── tool-schema-conversion.test.ts └── unit │ ├── core │ ├── config.test.ts │ └── trace.test.ts │ ├── evaluators │ ├── deterministic.test.ts │ └── llm-judge.test.ts │ ├── infrastructure │ └── sse-validation.test.ts │ ├── reporters │ └── reporter-console.test.ts │ └── vitest │ ├── matchers.test.ts │ ├── scorers.test.ts │ └── vitest-eval.test.ts ├── tsconfig.json └── tsup.config.ts /.changeset/config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/.changeset/config.json -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/.github/workflows/ci.yml -------------------------------------------------------------------------------- /.github/workflows/claude-code-review.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/.github/workflows/claude-code-review.yml -------------------------------------------------------------------------------- /.github/workflows/claude.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/.github/workflows/claude.yml -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/.github/workflows/release.yml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | dist 2 | node_modules 3 | .env -------------------------------------------------------------------------------- /.husky/pre-commit: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env sh 2 | 3 | pnpm exec lint-staged -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/CHANGELOG.md -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/README.md -------------------------------------------------------------------------------- /eslint.config.mjs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/eslint.config.mjs -------------------------------------------------------------------------------- /examples/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/examples/README.md -------------------------------------------------------------------------------- /examples/advanced/llm-judge-eval.config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/examples/advanced/llm-judge-eval.config.json -------------------------------------------------------------------------------- /examples/basic/simple-server-eval.config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/examples/basic/simple-server-eval.config.json -------------------------------------------------------------------------------- /examples/basic/simple-server-eval.config.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/examples/basic/simple-server-eval.config.ts -------------------------------------------------------------------------------- /examples/integrations/github-server-eval.config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/examples/integrations/github-server-eval.config.json -------------------------------------------------------------------------------- /examples/integrations/sse-server-eval.config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/examples/integrations/sse-server-eval.config.json -------------------------------------------------------------------------------- /examples/integrations/weather-server-eval.config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/examples/integrations/weather-server-eval.config.json -------------------------------------------------------------------------------- /examples/package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/examples/package.json -------------------------------------------------------------------------------- /examples/remote/remote-server-eval.config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/examples/remote/remote-server-eval.config.json -------------------------------------------------------------------------------- /examples/run-evaluation.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/examples/run-evaluation.ts -------------------------------------------------------------------------------- /examples/servers/simple-mcp-server.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/examples/servers/simple-mcp-server.js -------------------------------------------------------------------------------- /examples/servers/simple-sse-server.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/examples/servers/simple-sse-server.js -------------------------------------------------------------------------------- /examples/tools/tool-health-eval.config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/examples/tools/tool-health-eval.config.json -------------------------------------------------------------------------------- /examples/vitest-examples/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/examples/vitest-examples/README.md -------------------------------------------------------------------------------- /examples/vitest-examples/basic-tool-test.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/examples/vitest-examples/basic-tool-test.test.ts -------------------------------------------------------------------------------- /examples/vitest-examples/eval-suite-example.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/examples/vitest-examples/eval-suite-example.test.ts -------------------------------------------------------------------------------- /examples/vitest-examples/server/calculator-server.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/examples/vitest-examples/server/calculator-server.js -------------------------------------------------------------------------------- /examples/vitest-examples/server/filesystem-server.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/examples/vitest-examples/server/filesystem-server.js -------------------------------------------------------------------------------- /examples/vitest-examples/simple-server-mock.cjs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/examples/vitest-examples/simple-server-mock.cjs -------------------------------------------------------------------------------- /examples/vitest-examples/simple-test.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/examples/vitest-examples/simple-test.test.ts -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/package.json -------------------------------------------------------------------------------- /pnpm-lock.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/pnpm-lock.yaml -------------------------------------------------------------------------------- /src/cli/eval.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/src/cli/eval.ts -------------------------------------------------------------------------------- /src/cli/index.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/src/cli/index.ts -------------------------------------------------------------------------------- /src/eval/core/config.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/src/eval/core/config.ts -------------------------------------------------------------------------------- /src/eval/core/index.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/src/eval/core/index.ts -------------------------------------------------------------------------------- /src/eval/core/runner.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/src/eval/core/runner.ts -------------------------------------------------------------------------------- /src/eval/core/trace.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/src/eval/core/trace.ts -------------------------------------------------------------------------------- /src/eval/evaluators/deterministic.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/src/eval/evaluators/deterministic.ts -------------------------------------------------------------------------------- /src/eval/evaluators/llm-judge.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/src/eval/evaluators/llm-judge.ts -------------------------------------------------------------------------------- /src/eval/evaluators/tool-health.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/src/eval/evaluators/tool-health.ts -------------------------------------------------------------------------------- /src/eval/infrastructure/json-schema-to-zod.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/src/eval/infrastructure/json-schema-to-zod.ts -------------------------------------------------------------------------------- /src/eval/infrastructure/performance.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/src/eval/infrastructure/performance.ts -------------------------------------------------------------------------------- /src/eval/infrastructure/resilience.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/src/eval/infrastructure/resilience.ts -------------------------------------------------------------------------------- /src/eval/infrastructure/validation.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/src/eval/infrastructure/validation.ts -------------------------------------------------------------------------------- /src/eval/reporters/console.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/src/eval/reporters/console.ts -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/src/index.ts -------------------------------------------------------------------------------- /src/types/evaluation.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/src/types/evaluation.ts -------------------------------------------------------------------------------- /src/types/performance.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/src/types/performance.ts -------------------------------------------------------------------------------- /src/types/resilience.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/src/types/resilience.ts -------------------------------------------------------------------------------- /src/types/server.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/src/types/server.ts -------------------------------------------------------------------------------- /src/types/tool.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/src/types/tool.ts -------------------------------------------------------------------------------- /src/types/trace.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/src/types/trace.ts -------------------------------------------------------------------------------- /src/types/validation.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/src/types/validation.ts -------------------------------------------------------------------------------- /src/vitest/index.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/src/vitest/index.ts -------------------------------------------------------------------------------- /src/vitest/matchers.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/src/vitest/matchers.ts -------------------------------------------------------------------------------- /src/vitest/scorers/content.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/src/vitest/scorers/content.ts -------------------------------------------------------------------------------- /src/vitest/scorers/index.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/src/vitest/scorers/index.ts -------------------------------------------------------------------------------- /src/vitest/scorers/latency.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/src/vitest/scorers/latency.ts -------------------------------------------------------------------------------- /src/vitest/scorers/toolcall.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/src/vitest/scorers/toolcall.ts -------------------------------------------------------------------------------- /src/vitest/scorers/workflow.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/src/vitest/scorers/workflow.ts -------------------------------------------------------------------------------- /src/vitest/types.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/src/vitest/types.ts -------------------------------------------------------------------------------- /src/vitest/vitest-eval.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/src/vitest/vitest-eval.ts -------------------------------------------------------------------------------- /tests/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/tests/README.md -------------------------------------------------------------------------------- /tests/cli/cli.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/tests/cli/cli.test.ts -------------------------------------------------------------------------------- /tests/cli/eval.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/tests/cli/eval.test.ts -------------------------------------------------------------------------------- /tests/e2e/integration.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/tests/e2e/integration.test.ts -------------------------------------------------------------------------------- /tests/integration/evaluate.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/tests/integration/evaluate.test.ts -------------------------------------------------------------------------------- /tests/integration/runner.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/tests/integration/runner.test.ts -------------------------------------------------------------------------------- /tests/integration/tool-health.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/tests/integration/tool-health.test.ts -------------------------------------------------------------------------------- /tests/integration/tool-schema-conversion.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/tests/integration/tool-schema-conversion.test.ts -------------------------------------------------------------------------------- /tests/unit/core/config.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/tests/unit/core/config.test.ts -------------------------------------------------------------------------------- /tests/unit/core/trace.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/tests/unit/core/trace.test.ts -------------------------------------------------------------------------------- /tests/unit/evaluators/deterministic.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/tests/unit/evaluators/deterministic.test.ts -------------------------------------------------------------------------------- /tests/unit/evaluators/llm-judge.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/tests/unit/evaluators/llm-judge.test.ts -------------------------------------------------------------------------------- /tests/unit/infrastructure/sse-validation.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/tests/unit/infrastructure/sse-validation.test.ts -------------------------------------------------------------------------------- /tests/unit/reporters/reporter-console.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/tests/unit/reporters/reporter-console.test.ts -------------------------------------------------------------------------------- /tests/unit/vitest/matchers.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/tests/unit/vitest/matchers.test.ts -------------------------------------------------------------------------------- /tests/unit/vitest/scorers.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/tests/unit/vitest/scorers.test.ts -------------------------------------------------------------------------------- /tests/unit/vitest/vitest-eval.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/tests/unit/vitest/vitest-eval.test.ts -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/tsconfig.json -------------------------------------------------------------------------------- /tsup.config.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kylejeong2/mcpvals/HEAD/tsup.config.ts --------------------------------------------------------------------------------