├── .github ├── actions │ └── uv_setup │ │ └── action.yml └── workflows │ ├── build.yml │ ├── integration_tests.yml │ ├── release.yml │ └── release_js.yml ├── .gitignore ├── LICENSE ├── README.md ├── js ├── .editorconfig ├── .eslintrc.cjs ├── .gitattributes ├── .gitignore ├── .prettierrc ├── .yarn │ └── releases │ │ └── yarn-3.5.1.cjs ├── .yarnrc.yml ├── LICENSE ├── README.md ├── langchain.config.js ├── package.json ├── scripts │ ├── bump-version.js │ ├── check-npm-version.js │ └── check-version.js ├── src │ ├── code │ │ ├── base.ts │ │ ├── e2b │ │ │ ├── execution.ts │ │ │ ├── index.ts │ │ │ ├── sandbox │ │ │ │ └── files.ts │ │ │ ├── tests │ │ │ │ ├── execution.test.ts │ │ │ │ └── typescript.test.ts │ │ │ └── typescript.ts │ │ ├── llm.ts │ │ ├── tests │ │ │ ├── extraction.test.ts │ │ │ ├── llm.test.ts │ │ │ └── typescript.test.ts │ │ └── typescript.ts │ ├── exact.ts │ ├── index.ts │ ├── json │ │ └── match.ts │ ├── llm.ts │ ├── prompts │ │ ├── code_correctness.ts │ │ ├── conciseness.ts │ │ ├── correctness.ts │ │ ├── hallucination.ts │ │ ├── index.ts │ │ ├── rag_groundedness.ts │ │ ├── rag_helpfulness.ts │ │ └── rag_retrieval_relevance.ts │ ├── simulators │ │ ├── multiturn.ts │ │ ├── prebuilts.ts │ │ └── tests │ │ │ └── multiturn.test.ts │ ├── string │ │ ├── embedding_similarity.ts │ │ ├── levenshtein.ts │ │ └── tests │ │ │ ├── embedding_similarity.test.ts │ │ │ └── levenshtein.test.ts │ ├── tests │ │ ├── conciseness.test.ts │ │ ├── correctness.test.ts │ │ ├── exact.test.ts │ │ ├── hallucination.test.ts │ │ ├── json.test.ts │ │ ├── llm.test.ts │ │ └── rag.test.ts │ ├── types.ts │ └── utils.ts ├── tsconfig.cjs.json ├── tsconfig.json ├── vitest.config.ts └── yarn.lock ├── python ├── .python-version ├── LICENSE ├── README.md ├── openevals │ ├── __init__.py │ ├── code │ │ ├── __init__.py │ │ ├── base.py │ │ ├── e2b │ │ │ ├── __init__.py │ │ │ ├── execution.py │ │ │ ├── pyright.py │ │ │ └── sandbox │ │ │ │ └── files.py │ │ ├── llm.py │ │ ├── mypy.py │ │ └── pyright.py │ ├── exact.py │ ├── json │ │ ├── __init__.py │ │ └── match.py │ ├── llm.py │ ├── prompts │ │ ├── __init__.py │ │ ├── code_correctness.py │ │ ├── conciseness.py │ │ ├── correctness.py │ │ ├── hallucination.py │ │ ├── rag_groundedness.py │ │ ├── rag_helpfulness.py │ │ └── rag_retrieval_relevance.py │ ├── py.typed │ ├── simulators │ │ ├── __init__.py │ │ ├── multiturn.py │ │ └── prebuilts.py │ ├── string │ │ ├── __init__.py │ │ ├── embedding_similarity.py │ │ └── levenshtein.py │ ├── types.py │ └── utils.py ├── pyproject.toml ├── tests │ ├── __init__.py │ ├── code │ │ ├── e2b │ │ │ ├── test_execution.py │ │ │ ├── test_execution_async.py │ │ │ ├── test_typecheck.py │ │ │ └── test_typecheck_async.py │ │ ├── test_code_llm_as_judge.py │ │ ├── test_code_llm_as_judge_async.py │ │ ├── test_code_typecheck.py │ │ ├── test_code_typecheck_async.py │ │ └── test_extraction.py │ ├── simulators │ │ ├── test_multiturn.py │ │ └── test_multiturn_async.py │ ├── test_conciseness.py │ ├── test_correctness.py │ ├── test_embedding_similarity.py │ ├── test_exact.py │ ├── test_hallucination.py │ ├── test_json.py │ ├── test_json_async.py │ ├── test_levenshtein.py │ ├── test_llm.py │ ├── test_llm_async.py │ └── test_rag.py └── uv.lock ├── sandbox └── e2b │ ├── .gitignore │ ├── package.json │ ├── python │ ├── e2b.Dockerfile │ └── e2b.toml │ └── yarn.lock ├── scripts └── generate_language_readmes.py ├── static └── img │ ├── langsmith_results.png │ └── pytest_output.png └── uv.lock /.github/actions/uv_setup/action.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/.github/actions/uv_setup/action.yml -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/.github/workflows/build.yml -------------------------------------------------------------------------------- /.github/workflows/integration_tests.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/.github/workflows/integration_tests.yml -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/.github/workflows/release.yml -------------------------------------------------------------------------------- /.github/workflows/release_js.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/.github/workflows/release_js.yml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/README.md -------------------------------------------------------------------------------- /js/.editorconfig: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/js/.editorconfig -------------------------------------------------------------------------------- /js/.eslintrc.cjs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/js/.eslintrc.cjs -------------------------------------------------------------------------------- /js/.gitattributes: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/js/.gitattributes -------------------------------------------------------------------------------- /js/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/js/.gitignore -------------------------------------------------------------------------------- /js/.prettierrc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/js/.prettierrc -------------------------------------------------------------------------------- /js/.yarn/releases/yarn-3.5.1.cjs: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/js/.yarn/releases/yarn-3.5.1.cjs -------------------------------------------------------------------------------- /js/.yarnrc.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/js/.yarnrc.yml -------------------------------------------------------------------------------- /js/LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/js/LICENSE -------------------------------------------------------------------------------- /js/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/js/README.md -------------------------------------------------------------------------------- /js/langchain.config.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/js/langchain.config.js -------------------------------------------------------------------------------- /js/package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/js/package.json -------------------------------------------------------------------------------- /js/scripts/bump-version.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/js/scripts/bump-version.js -------------------------------------------------------------------------------- /js/scripts/check-npm-version.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/js/scripts/check-npm-version.js -------------------------------------------------------------------------------- /js/scripts/check-version.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/js/scripts/check-version.js -------------------------------------------------------------------------------- /js/src/code/base.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/js/src/code/base.ts -------------------------------------------------------------------------------- /js/src/code/e2b/execution.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/js/src/code/e2b/execution.ts -------------------------------------------------------------------------------- /js/src/code/e2b/index.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/js/src/code/e2b/index.ts -------------------------------------------------------------------------------- /js/src/code/e2b/sandbox/files.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/js/src/code/e2b/sandbox/files.ts -------------------------------------------------------------------------------- /js/src/code/e2b/tests/execution.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/js/src/code/e2b/tests/execution.test.ts -------------------------------------------------------------------------------- /js/src/code/e2b/tests/typescript.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/js/src/code/e2b/tests/typescript.test.ts -------------------------------------------------------------------------------- /js/src/code/e2b/typescript.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/js/src/code/e2b/typescript.ts -------------------------------------------------------------------------------- /js/src/code/llm.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/js/src/code/llm.ts -------------------------------------------------------------------------------- /js/src/code/tests/extraction.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/js/src/code/tests/extraction.test.ts -------------------------------------------------------------------------------- /js/src/code/tests/llm.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/js/src/code/tests/llm.test.ts -------------------------------------------------------------------------------- /js/src/code/tests/typescript.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/js/src/code/tests/typescript.test.ts -------------------------------------------------------------------------------- /js/src/code/typescript.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/js/src/code/typescript.ts -------------------------------------------------------------------------------- /js/src/exact.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/js/src/exact.ts -------------------------------------------------------------------------------- /js/src/index.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/js/src/index.ts -------------------------------------------------------------------------------- /js/src/json/match.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/js/src/json/match.ts -------------------------------------------------------------------------------- /js/src/llm.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/js/src/llm.ts -------------------------------------------------------------------------------- /js/src/prompts/code_correctness.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/js/src/prompts/code_correctness.ts -------------------------------------------------------------------------------- /js/src/prompts/conciseness.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/js/src/prompts/conciseness.ts -------------------------------------------------------------------------------- /js/src/prompts/correctness.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/js/src/prompts/correctness.ts -------------------------------------------------------------------------------- /js/src/prompts/hallucination.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/js/src/prompts/hallucination.ts -------------------------------------------------------------------------------- /js/src/prompts/index.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/js/src/prompts/index.ts -------------------------------------------------------------------------------- /js/src/prompts/rag_groundedness.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/js/src/prompts/rag_groundedness.ts -------------------------------------------------------------------------------- /js/src/prompts/rag_helpfulness.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/js/src/prompts/rag_helpfulness.ts -------------------------------------------------------------------------------- /js/src/prompts/rag_retrieval_relevance.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/js/src/prompts/rag_retrieval_relevance.ts -------------------------------------------------------------------------------- /js/src/simulators/multiturn.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/js/src/simulators/multiturn.ts -------------------------------------------------------------------------------- /js/src/simulators/prebuilts.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/js/src/simulators/prebuilts.ts -------------------------------------------------------------------------------- /js/src/simulators/tests/multiturn.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/js/src/simulators/tests/multiturn.test.ts -------------------------------------------------------------------------------- /js/src/string/embedding_similarity.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/js/src/string/embedding_similarity.ts -------------------------------------------------------------------------------- /js/src/string/levenshtein.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/js/src/string/levenshtein.ts -------------------------------------------------------------------------------- /js/src/string/tests/embedding_similarity.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/js/src/string/tests/embedding_similarity.test.ts -------------------------------------------------------------------------------- /js/src/string/tests/levenshtein.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/js/src/string/tests/levenshtein.test.ts -------------------------------------------------------------------------------- /js/src/tests/conciseness.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/js/src/tests/conciseness.test.ts -------------------------------------------------------------------------------- /js/src/tests/correctness.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/js/src/tests/correctness.test.ts -------------------------------------------------------------------------------- /js/src/tests/exact.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/js/src/tests/exact.test.ts -------------------------------------------------------------------------------- /js/src/tests/hallucination.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/js/src/tests/hallucination.test.ts -------------------------------------------------------------------------------- /js/src/tests/json.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/js/src/tests/json.test.ts -------------------------------------------------------------------------------- /js/src/tests/llm.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/js/src/tests/llm.test.ts -------------------------------------------------------------------------------- /js/src/tests/rag.test.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/js/src/tests/rag.test.ts -------------------------------------------------------------------------------- /js/src/types.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/js/src/types.ts -------------------------------------------------------------------------------- /js/src/utils.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/js/src/utils.ts -------------------------------------------------------------------------------- /js/tsconfig.cjs.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/js/tsconfig.cjs.json -------------------------------------------------------------------------------- /js/tsconfig.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/js/tsconfig.json -------------------------------------------------------------------------------- /js/vitest.config.ts: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/js/vitest.config.ts -------------------------------------------------------------------------------- /js/yarn.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/js/yarn.lock -------------------------------------------------------------------------------- /python/.python-version: -------------------------------------------------------------------------------- 1 | 3.11 2 | -------------------------------------------------------------------------------- /python/LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/python/LICENSE -------------------------------------------------------------------------------- /python/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/python/README.md -------------------------------------------------------------------------------- /python/openevals/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/python/openevals/__init__.py -------------------------------------------------------------------------------- /python/openevals/code/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /python/openevals/code/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/python/openevals/code/base.py -------------------------------------------------------------------------------- /python/openevals/code/e2b/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /python/openevals/code/e2b/execution.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/python/openevals/code/e2b/execution.py -------------------------------------------------------------------------------- /python/openevals/code/e2b/pyright.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/python/openevals/code/e2b/pyright.py -------------------------------------------------------------------------------- /python/openevals/code/e2b/sandbox/files.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/python/openevals/code/e2b/sandbox/files.py -------------------------------------------------------------------------------- /python/openevals/code/llm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/python/openevals/code/llm.py -------------------------------------------------------------------------------- /python/openevals/code/mypy.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/python/openevals/code/mypy.py -------------------------------------------------------------------------------- /python/openevals/code/pyright.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/python/openevals/code/pyright.py -------------------------------------------------------------------------------- /python/openevals/exact.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/python/openevals/exact.py -------------------------------------------------------------------------------- /python/openevals/json/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/python/openevals/json/__init__.py -------------------------------------------------------------------------------- /python/openevals/json/match.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/python/openevals/json/match.py -------------------------------------------------------------------------------- /python/openevals/llm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/python/openevals/llm.py -------------------------------------------------------------------------------- /python/openevals/prompts/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/python/openevals/prompts/__init__.py -------------------------------------------------------------------------------- /python/openevals/prompts/code_correctness.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/python/openevals/prompts/code_correctness.py -------------------------------------------------------------------------------- /python/openevals/prompts/conciseness.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/python/openevals/prompts/conciseness.py -------------------------------------------------------------------------------- /python/openevals/prompts/correctness.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/python/openevals/prompts/correctness.py -------------------------------------------------------------------------------- /python/openevals/prompts/hallucination.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/python/openevals/prompts/hallucination.py -------------------------------------------------------------------------------- /python/openevals/prompts/rag_groundedness.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/python/openevals/prompts/rag_groundedness.py -------------------------------------------------------------------------------- /python/openevals/prompts/rag_helpfulness.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/python/openevals/prompts/rag_helpfulness.py -------------------------------------------------------------------------------- /python/openevals/prompts/rag_retrieval_relevance.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/python/openevals/prompts/rag_retrieval_relevance.py -------------------------------------------------------------------------------- /python/openevals/py.typed: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /python/openevals/simulators/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/python/openevals/simulators/__init__.py -------------------------------------------------------------------------------- /python/openevals/simulators/multiturn.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/python/openevals/simulators/multiturn.py -------------------------------------------------------------------------------- /python/openevals/simulators/prebuilts.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/python/openevals/simulators/prebuilts.py -------------------------------------------------------------------------------- /python/openevals/string/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/python/openevals/string/__init__.py -------------------------------------------------------------------------------- /python/openevals/string/embedding_similarity.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/python/openevals/string/embedding_similarity.py -------------------------------------------------------------------------------- /python/openevals/string/levenshtein.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/python/openevals/string/levenshtein.py -------------------------------------------------------------------------------- /python/openevals/types.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/python/openevals/types.py -------------------------------------------------------------------------------- /python/openevals/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/python/openevals/utils.py -------------------------------------------------------------------------------- /python/pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/python/pyproject.toml -------------------------------------------------------------------------------- /python/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /python/tests/code/e2b/test_execution.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/python/tests/code/e2b/test_execution.py -------------------------------------------------------------------------------- /python/tests/code/e2b/test_execution_async.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/python/tests/code/e2b/test_execution_async.py -------------------------------------------------------------------------------- /python/tests/code/e2b/test_typecheck.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/python/tests/code/e2b/test_typecheck.py -------------------------------------------------------------------------------- /python/tests/code/e2b/test_typecheck_async.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/python/tests/code/e2b/test_typecheck_async.py -------------------------------------------------------------------------------- /python/tests/code/test_code_llm_as_judge.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/python/tests/code/test_code_llm_as_judge.py -------------------------------------------------------------------------------- /python/tests/code/test_code_llm_as_judge_async.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/python/tests/code/test_code_llm_as_judge_async.py -------------------------------------------------------------------------------- /python/tests/code/test_code_typecheck.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/python/tests/code/test_code_typecheck.py -------------------------------------------------------------------------------- /python/tests/code/test_code_typecheck_async.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/python/tests/code/test_code_typecheck_async.py -------------------------------------------------------------------------------- /python/tests/code/test_extraction.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/python/tests/code/test_extraction.py -------------------------------------------------------------------------------- /python/tests/simulators/test_multiturn.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/python/tests/simulators/test_multiturn.py -------------------------------------------------------------------------------- /python/tests/simulators/test_multiturn_async.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/python/tests/simulators/test_multiturn_async.py -------------------------------------------------------------------------------- /python/tests/test_conciseness.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/python/tests/test_conciseness.py -------------------------------------------------------------------------------- /python/tests/test_correctness.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/python/tests/test_correctness.py -------------------------------------------------------------------------------- /python/tests/test_embedding_similarity.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/python/tests/test_embedding_similarity.py -------------------------------------------------------------------------------- /python/tests/test_exact.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/python/tests/test_exact.py -------------------------------------------------------------------------------- /python/tests/test_hallucination.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/python/tests/test_hallucination.py -------------------------------------------------------------------------------- /python/tests/test_json.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/python/tests/test_json.py -------------------------------------------------------------------------------- /python/tests/test_json_async.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/python/tests/test_json_async.py -------------------------------------------------------------------------------- /python/tests/test_levenshtein.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/python/tests/test_levenshtein.py -------------------------------------------------------------------------------- /python/tests/test_llm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/python/tests/test_llm.py -------------------------------------------------------------------------------- /python/tests/test_llm_async.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/python/tests/test_llm_async.py -------------------------------------------------------------------------------- /python/tests/test_rag.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/python/tests/test_rag.py -------------------------------------------------------------------------------- /python/uv.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/python/uv.lock -------------------------------------------------------------------------------- /sandbox/e2b/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | .yarn/ -------------------------------------------------------------------------------- /sandbox/e2b/package.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/sandbox/e2b/package.json -------------------------------------------------------------------------------- /sandbox/e2b/python/e2b.Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/sandbox/e2b/python/e2b.Dockerfile -------------------------------------------------------------------------------- /sandbox/e2b/python/e2b.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/sandbox/e2b/python/e2b.toml -------------------------------------------------------------------------------- /sandbox/e2b/yarn.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/sandbox/e2b/yarn.lock -------------------------------------------------------------------------------- /scripts/generate_language_readmes.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/scripts/generate_language_readmes.py -------------------------------------------------------------------------------- /static/img/langsmith_results.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/static/img/langsmith_results.png -------------------------------------------------------------------------------- /static/img/pytest_output.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/static/img/pytest_output.png -------------------------------------------------------------------------------- /uv.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/langchain-ai/openevals/HEAD/uv.lock --------------------------------------------------------------------------------