├── .gitignore ├── LICENSE ├── README.md ├── examples ├── ceval │ ├── README.md │ ├── ceval-with-answer.jsonl │ ├── chat-eval.yaml │ └── prompt-eval.yaml ├── llm_judge │ ├── README.md │ ├── ceval-llm-judge.jsonl │ └── ceval-llm-judge.yaml ├── rag-eval │ ├── .gitignore │ ├── rag-eval-zh.ipynb │ ├── rag-eval.yaml │ ├── rag.py │ ├── retrieval-eval.yaml │ └── retrieval.py └── sql-eval │ ├── dusql_sample.jsonl │ ├── sql-eval-qianfan-limiter.yaml │ ├── sql-eval-qianfan.yaml │ ├── sql-eval.yaml │ └── 洗衣机.sqlite ├── pyproject.toml ├── src └── langeval │ ├── __about__.py │ ├── __init__.py │ ├── __main__.py │ ├── cli │ ├── __init__.py │ ├── application.py │ ├── constant.py │ ├── rerun │ │ └── __init__.py │ ├── run │ │ ├── __init__.py │ │ ├── display.py │ │ └── run.py │ ├── show │ │ └── __init__.py │ └── terminal.py │ ├── config │ └── __init__.py │ ├── evaluators │ ├── __init__.py │ ├── evaluator.py │ ├── exception.py │ ├── nlp │ │ └── __init__.py │ ├── rag │ │ ├── __init__.py │ │ └── utils.py │ ├── run.py │ └── sql │ │ ├── __init__.py │ │ └── sqleval.py │ ├── models │ ├── __init__.py │ ├── embeddings.py │ ├── exception.py │ ├── llms.py │ ├── openai.py │ ├── qianfan.py │ └── types.py │ ├── providers │ ├── __init__.py │ ├── exception.py │ ├── output_parser.py │ ├── provider.py │ └── run.py │ └── tasks │ ├── __init__.py │ ├── ratelimiter.py │ ├── runner.py │ └── task.py └── tests └── __init__.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninehills/langeval/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninehills/langeval/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninehills/langeval/HEAD/README.md -------------------------------------------------------------------------------- /examples/ceval/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninehills/langeval/HEAD/examples/ceval/README.md -------------------------------------------------------------------------------- /examples/ceval/ceval-with-answer.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninehills/langeval/HEAD/examples/ceval/ceval-with-answer.jsonl -------------------------------------------------------------------------------- /examples/ceval/chat-eval.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninehills/langeval/HEAD/examples/ceval/chat-eval.yaml -------------------------------------------------------------------------------- /examples/ceval/prompt-eval.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninehills/langeval/HEAD/examples/ceval/prompt-eval.yaml -------------------------------------------------------------------------------- /examples/llm_judge/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninehills/langeval/HEAD/examples/llm_judge/README.md -------------------------------------------------------------------------------- /examples/llm_judge/ceval-llm-judge.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninehills/langeval/HEAD/examples/llm_judge/ceval-llm-judge.jsonl -------------------------------------------------------------------------------- /examples/llm_judge/ceval-llm-judge.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninehills/langeval/HEAD/examples/llm_judge/ceval-llm-judge.yaml -------------------------------------------------------------------------------- /examples/rag-eval/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninehills/langeval/HEAD/examples/rag-eval/.gitignore -------------------------------------------------------------------------------- /examples/rag-eval/rag-eval-zh.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninehills/langeval/HEAD/examples/rag-eval/rag-eval-zh.ipynb -------------------------------------------------------------------------------- /examples/rag-eval/rag-eval.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninehills/langeval/HEAD/examples/rag-eval/rag-eval.yaml -------------------------------------------------------------------------------- /examples/rag-eval/rag.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninehills/langeval/HEAD/examples/rag-eval/rag.py -------------------------------------------------------------------------------- /examples/rag-eval/retrieval-eval.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninehills/langeval/HEAD/examples/rag-eval/retrieval-eval.yaml -------------------------------------------------------------------------------- /examples/rag-eval/retrieval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninehills/langeval/HEAD/examples/rag-eval/retrieval.py -------------------------------------------------------------------------------- /examples/sql-eval/dusql_sample.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninehills/langeval/HEAD/examples/sql-eval/dusql_sample.jsonl -------------------------------------------------------------------------------- /examples/sql-eval/sql-eval-qianfan-limiter.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninehills/langeval/HEAD/examples/sql-eval/sql-eval-qianfan-limiter.yaml -------------------------------------------------------------------------------- /examples/sql-eval/sql-eval-qianfan.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninehills/langeval/HEAD/examples/sql-eval/sql-eval-qianfan.yaml -------------------------------------------------------------------------------- /examples/sql-eval/sql-eval.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninehills/langeval/HEAD/examples/sql-eval/sql-eval.yaml -------------------------------------------------------------------------------- /examples/sql-eval/洗衣机.sqlite: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninehills/langeval/HEAD/examples/sql-eval/洗衣机.sqlite -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninehills/langeval/HEAD/pyproject.toml -------------------------------------------------------------------------------- /src/langeval/__about__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninehills/langeval/HEAD/src/langeval/__about__.py -------------------------------------------------------------------------------- /src/langeval/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/langeval/__main__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninehills/langeval/HEAD/src/langeval/__main__.py -------------------------------------------------------------------------------- /src/langeval/cli/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninehills/langeval/HEAD/src/langeval/cli/__init__.py -------------------------------------------------------------------------------- /src/langeval/cli/application.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninehills/langeval/HEAD/src/langeval/cli/application.py -------------------------------------------------------------------------------- /src/langeval/cli/constant.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninehills/langeval/HEAD/src/langeval/cli/constant.py -------------------------------------------------------------------------------- /src/langeval/cli/rerun/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninehills/langeval/HEAD/src/langeval/cli/rerun/__init__.py -------------------------------------------------------------------------------- /src/langeval/cli/run/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninehills/langeval/HEAD/src/langeval/cli/run/__init__.py -------------------------------------------------------------------------------- /src/langeval/cli/run/display.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninehills/langeval/HEAD/src/langeval/cli/run/display.py -------------------------------------------------------------------------------- /src/langeval/cli/run/run.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninehills/langeval/HEAD/src/langeval/cli/run/run.py -------------------------------------------------------------------------------- /src/langeval/cli/show/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninehills/langeval/HEAD/src/langeval/cli/show/__init__.py -------------------------------------------------------------------------------- /src/langeval/cli/terminal.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninehills/langeval/HEAD/src/langeval/cli/terminal.py -------------------------------------------------------------------------------- /src/langeval/config/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninehills/langeval/HEAD/src/langeval/config/__init__.py -------------------------------------------------------------------------------- /src/langeval/evaluators/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninehills/langeval/HEAD/src/langeval/evaluators/__init__.py -------------------------------------------------------------------------------- /src/langeval/evaluators/evaluator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninehills/langeval/HEAD/src/langeval/evaluators/evaluator.py -------------------------------------------------------------------------------- /src/langeval/evaluators/exception.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninehills/langeval/HEAD/src/langeval/evaluators/exception.py -------------------------------------------------------------------------------- /src/langeval/evaluators/nlp/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninehills/langeval/HEAD/src/langeval/evaluators/nlp/__init__.py -------------------------------------------------------------------------------- /src/langeval/evaluators/rag/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninehills/langeval/HEAD/src/langeval/evaluators/rag/__init__.py -------------------------------------------------------------------------------- /src/langeval/evaluators/rag/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninehills/langeval/HEAD/src/langeval/evaluators/rag/utils.py -------------------------------------------------------------------------------- /src/langeval/evaluators/run.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninehills/langeval/HEAD/src/langeval/evaluators/run.py -------------------------------------------------------------------------------- /src/langeval/evaluators/sql/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninehills/langeval/HEAD/src/langeval/evaluators/sql/__init__.py -------------------------------------------------------------------------------- /src/langeval/evaluators/sql/sqleval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninehills/langeval/HEAD/src/langeval/evaluators/sql/sqleval.py -------------------------------------------------------------------------------- /src/langeval/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninehills/langeval/HEAD/src/langeval/models/__init__.py -------------------------------------------------------------------------------- /src/langeval/models/embeddings.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninehills/langeval/HEAD/src/langeval/models/embeddings.py -------------------------------------------------------------------------------- /src/langeval/models/exception.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninehills/langeval/HEAD/src/langeval/models/exception.py -------------------------------------------------------------------------------- /src/langeval/models/llms.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninehills/langeval/HEAD/src/langeval/models/llms.py -------------------------------------------------------------------------------- /src/langeval/models/openai.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninehills/langeval/HEAD/src/langeval/models/openai.py -------------------------------------------------------------------------------- /src/langeval/models/qianfan.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninehills/langeval/HEAD/src/langeval/models/qianfan.py -------------------------------------------------------------------------------- /src/langeval/models/types.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninehills/langeval/HEAD/src/langeval/models/types.py -------------------------------------------------------------------------------- /src/langeval/providers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninehills/langeval/HEAD/src/langeval/providers/__init__.py -------------------------------------------------------------------------------- /src/langeval/providers/exception.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninehills/langeval/HEAD/src/langeval/providers/exception.py -------------------------------------------------------------------------------- /src/langeval/providers/output_parser.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninehills/langeval/HEAD/src/langeval/providers/output_parser.py -------------------------------------------------------------------------------- /src/langeval/providers/provider.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninehills/langeval/HEAD/src/langeval/providers/provider.py -------------------------------------------------------------------------------- /src/langeval/providers/run.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninehills/langeval/HEAD/src/langeval/providers/run.py -------------------------------------------------------------------------------- /src/langeval/tasks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninehills/langeval/HEAD/src/langeval/tasks/__init__.py -------------------------------------------------------------------------------- /src/langeval/tasks/ratelimiter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninehills/langeval/HEAD/src/langeval/tasks/ratelimiter.py -------------------------------------------------------------------------------- /src/langeval/tasks/runner.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninehills/langeval/HEAD/src/langeval/tasks/runner.py -------------------------------------------------------------------------------- /src/langeval/tasks/task.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ninehills/langeval/HEAD/src/langeval/tasks/task.py -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | --------------------------------------------------------------------------------