├── .env.example ├── .envrc ├── .eslintrc.js ├── .flake8 ├── .github └── workflows │ ├── eval.yaml │ ├── js.yaml │ ├── lint.yaml │ └── python.yaml ├── .gitignore ├── .isort.cfg ├── .npmignore ├── .pre-commit-config.yaml ├── .prettierrc ├── .tool-versions ├── .vscode └── settings.json ├── CHANGELOG.md ├── LICENSE ├── MANIFEST.in ├── Makefile ├── README.md ├── env.sh ├── evals ├── .eslintrc.js ├── .prettierrc ├── datasets │ ├── coqa-closed-qa.json │ ├── coqa-context-relevancy.json │ └── coqa-factuality.json ├── package.json ├── src │ ├── autoevals.eval.ts │ ├── datasets.ts │ ├── duckdb.ts │ └── sync_datasets.ts └── tsconfig.json ├── js ├── embeddings.test.ts ├── index.ts ├── json.test.ts ├── json.ts ├── list.ts ├── llm.fixtures.ts ├── llm.test.ts ├── llm.ts ├── manifest.ts ├── moderation.test.ts ├── moderation.ts ├── number.ts ├── oai.test.ts ├── oai.ts ├── partial.test.ts ├── partial.ts ├── ragas.test.ts ├── ragas.ts ├── render-messages.test.ts ├── render-messages.ts ├── score.ts ├── string.ts ├── templates.ts ├── value.test.ts ├── value.ts └── yaml.d.ts ├── mise.toml ├── package.json ├── pnpm-lock.yaml ├── pnpm-workspace.yaml ├── py └── autoevals │ ├── __init__.py │ ├── json.py │ ├── list.py │ ├── llm.py │ ├── moderation.py │ ├── number.py │ ├── oai.py │ ├── partial.py │ ├── ragas.py │ ├── score.py │ ├── serializable_data_class.py │ ├── string.py │ ├── templates │ ├── test_embeddings.py │ ├── test_json.py │ ├── test_llm.py │ ├── test_moderation.py │ ├── test_oai.py │ ├── test_partial.py │ ├── test_ragas.py │ ├── test_serializable_data_class.py │ ├── test_values.py │ ├── value.py │ └── version.py ├── pyproject.toml ├── pyrightconfig.json ├── scripts └── prepare_readme.py ├── setup.py ├── templates ├── battle.yaml ├── closed_q_a.yaml ├── factuality.yaml ├── humor.yaml ├── possible.yaml ├── security.yaml ├── sql.yaml ├── summary.yaml └── translation.yaml ├── tsconfig.json ├── tsup.config.js ├── turbo.json └── vitest.config.ts /.env.example: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/.env.example -------------------------------------------------------------------------------- /.envrc: -------------------------------------------------------------------------------- 1 | source_up 2 | dotenv 3 | -------------------------------------------------------------------------------- /.eslintrc.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/.eslintrc.js -------------------------------------------------------------------------------- /.flake8: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/.flake8 -------------------------------------------------------------------------------- /.github/workflows/eval.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/.github/workflows/eval.yaml -------------------------------------------------------------------------------- /.github/workflows/js.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/.github/workflows/js.yaml -------------------------------------------------------------------------------- /.github/workflows/lint.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/.github/workflows/lint.yaml -------------------------------------------------------------------------------- /.github/workflows/python.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/.github/workflows/python.yaml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/.gitignore -------------------------------------------------------------------------------- /.isort.cfg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/.isort.cfg -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/.npmignore -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/.pre-commit-config.yaml -------------------------------------------------------------------------------- /.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "singleQuote": false 3 | } 4 | -------------------------------------------------------------------------------- /.tool-versions: -------------------------------------------------------------------------------- 1 | python 3.13.5 2 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/.vscode/settings.json -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/CHANGELOG.md -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/LICENSE -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | recursive-include py/autoevals/templates * 2 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/Makefile -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/README.md -------------------------------------------------------------------------------- /env.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/env.sh -------------------------------------------------------------------------------- /evals/.eslintrc.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/evals/.eslintrc.js -------------------------------------------------------------------------------- /evals/.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "singleQuote": false 3 | } 4 | -------------------------------------------------------------------------------- /evals/datasets/coqa-closed-qa.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/evals/datasets/coqa-closed-qa.json -------------------------------------------------------------------------------- /evals/datasets/coqa-context-relevancy.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/evals/datasets/coqa-context-relevancy.json -------------------------------------------------------------------------------- /evals/datasets/coqa-factuality.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/evals/datasets/coqa-factuality.json -------------------------------------------------------------------------------- /evals/package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/evals/package.json -------------------------------------------------------------------------------- /evals/src/autoevals.eval.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/evals/src/autoevals.eval.ts -------------------------------------------------------------------------------- /evals/src/datasets.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/evals/src/datasets.ts -------------------------------------------------------------------------------- /evals/src/duckdb.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/evals/src/duckdb.ts -------------------------------------------------------------------------------- /evals/src/sync_datasets.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/evals/src/sync_datasets.ts -------------------------------------------------------------------------------- /evals/tsconfig.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/evals/tsconfig.json -------------------------------------------------------------------------------- /js/embeddings.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/js/embeddings.test.ts -------------------------------------------------------------------------------- /js/index.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/js/index.ts -------------------------------------------------------------------------------- /js/json.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/js/json.test.ts -------------------------------------------------------------------------------- /js/json.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/js/json.ts -------------------------------------------------------------------------------- /js/list.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/js/list.ts -------------------------------------------------------------------------------- /js/llm.fixtures.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/js/llm.fixtures.ts -------------------------------------------------------------------------------- /js/llm.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/js/llm.test.ts -------------------------------------------------------------------------------- /js/llm.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/js/llm.ts -------------------------------------------------------------------------------- /js/manifest.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/js/manifest.ts -------------------------------------------------------------------------------- /js/moderation.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/js/moderation.test.ts -------------------------------------------------------------------------------- /js/moderation.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/js/moderation.ts -------------------------------------------------------------------------------- /js/number.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/js/number.ts -------------------------------------------------------------------------------- /js/oai.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/js/oai.test.ts -------------------------------------------------------------------------------- /js/oai.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/js/oai.ts -------------------------------------------------------------------------------- /js/partial.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/js/partial.test.ts -------------------------------------------------------------------------------- /js/partial.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/js/partial.ts -------------------------------------------------------------------------------- /js/ragas.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/js/ragas.test.ts -------------------------------------------------------------------------------- /js/ragas.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/js/ragas.ts -------------------------------------------------------------------------------- /js/render-messages.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/js/render-messages.test.ts -------------------------------------------------------------------------------- /js/render-messages.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/js/render-messages.ts -------------------------------------------------------------------------------- /js/score.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/js/score.ts -------------------------------------------------------------------------------- /js/string.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/js/string.ts -------------------------------------------------------------------------------- /js/templates.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/js/templates.ts -------------------------------------------------------------------------------- /js/value.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/js/value.test.ts -------------------------------------------------------------------------------- /js/value.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/js/value.ts -------------------------------------------------------------------------------- /js/yaml.d.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/js/yaml.d.ts -------------------------------------------------------------------------------- /mise.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/mise.toml -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/package.json -------------------------------------------------------------------------------- /pnpm-lock.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/pnpm-lock.yaml -------------------------------------------------------------------------------- /pnpm-workspace.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/pnpm-workspace.yaml -------------------------------------------------------------------------------- /py/autoevals/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/py/autoevals/__init__.py -------------------------------------------------------------------------------- /py/autoevals/json.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/py/autoevals/json.py -------------------------------------------------------------------------------- /py/autoevals/list.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/py/autoevals/list.py -------------------------------------------------------------------------------- /py/autoevals/llm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/py/autoevals/llm.py -------------------------------------------------------------------------------- /py/autoevals/moderation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/py/autoevals/moderation.py -------------------------------------------------------------------------------- /py/autoevals/number.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/py/autoevals/number.py -------------------------------------------------------------------------------- /py/autoevals/oai.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/py/autoevals/oai.py -------------------------------------------------------------------------------- /py/autoevals/partial.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/py/autoevals/partial.py -------------------------------------------------------------------------------- /py/autoevals/ragas.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/py/autoevals/ragas.py -------------------------------------------------------------------------------- /py/autoevals/score.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/py/autoevals/score.py -------------------------------------------------------------------------------- /py/autoevals/serializable_data_class.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/py/autoevals/serializable_data_class.py -------------------------------------------------------------------------------- /py/autoevals/string.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/py/autoevals/string.py -------------------------------------------------------------------------------- /py/autoevals/templates: -------------------------------------------------------------------------------- 1 | ../../templates -------------------------------------------------------------------------------- /py/autoevals/test_embeddings.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/py/autoevals/test_embeddings.py -------------------------------------------------------------------------------- /py/autoevals/test_json.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/py/autoevals/test_json.py -------------------------------------------------------------------------------- /py/autoevals/test_llm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/py/autoevals/test_llm.py -------------------------------------------------------------------------------- /py/autoevals/test_moderation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/py/autoevals/test_moderation.py -------------------------------------------------------------------------------- /py/autoevals/test_oai.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/py/autoevals/test_oai.py -------------------------------------------------------------------------------- /py/autoevals/test_partial.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/py/autoevals/test_partial.py -------------------------------------------------------------------------------- /py/autoevals/test_ragas.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/py/autoevals/test_ragas.py -------------------------------------------------------------------------------- /py/autoevals/test_serializable_data_class.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/py/autoevals/test_serializable_data_class.py -------------------------------------------------------------------------------- /py/autoevals/test_values.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/py/autoevals/test_values.py -------------------------------------------------------------------------------- /py/autoevals/value.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/py/autoevals/value.py -------------------------------------------------------------------------------- /py/autoevals/version.py: -------------------------------------------------------------------------------- 1 | VERSION = "0.0.130" 2 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/pyproject.toml -------------------------------------------------------------------------------- /pyrightconfig.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/pyrightconfig.json -------------------------------------------------------------------------------- /scripts/prepare_readme.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/scripts/prepare_readme.py -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/setup.py -------------------------------------------------------------------------------- /templates/battle.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/templates/battle.yaml -------------------------------------------------------------------------------- /templates/closed_q_a.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/templates/closed_q_a.yaml -------------------------------------------------------------------------------- /templates/factuality.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/templates/factuality.yaml -------------------------------------------------------------------------------- /templates/humor.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/templates/humor.yaml -------------------------------------------------------------------------------- /templates/possible.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/templates/possible.yaml -------------------------------------------------------------------------------- /templates/security.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/templates/security.yaml -------------------------------------------------------------------------------- /templates/sql.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/templates/sql.yaml -------------------------------------------------------------------------------- /templates/summary.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/templates/summary.yaml -------------------------------------------------------------------------------- /templates/translation.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/templates/translation.yaml -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/tsconfig.json -------------------------------------------------------------------------------- /tsup.config.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/tsup.config.js -------------------------------------------------------------------------------- /turbo.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/turbo.json -------------------------------------------------------------------------------- /vitest.config.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/braintrustdata/autoevals/HEAD/vitest.config.ts --------------------------------------------------------------------------------