├── .github └── workflows │ └── release.yml ├── .gitignore ├── LICENSE ├── README.md ├── benchllama ├── __init__.py ├── __main__.py ├── constants.py ├── data_io │ ├── __init__.py │ └── loader.py ├── evaluation │ ├── __init__.py │ ├── code_runner.py │ ├── evaluator.py │ ├── runners │ │ ├── __init__.py │ │ ├── cpp_runner.py │ │ ├── go_runner.py │ │ ├── java_runner.py │ │ ├── javascript_runner.py │ │ ├── python_runner.py │ │ └── utils.py │ └── score_estimator.py ├── inference │ ├── __init__.py │ ├── model_provider.py │ └── prompt_formatter.py ├── logger.py ├── main.py └── utils.py ├── examples └── custom_js_evals.jsonl ├── media └── benchllama.gif ├── poetry.lock ├── pyproject.toml └── tests └── __init__.py /.github/workflows/release.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/srikanth235/benchllama/HEAD/.github/workflows/release.yml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/srikanth235/benchllama/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/srikanth235/benchllama/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/srikanth235/benchllama/HEAD/README.md -------------------------------------------------------------------------------- /benchllama/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /benchllama/__main__.py: -------------------------------------------------------------------------------- 1 | from .main import app 2 | app(name='benchllama') -------------------------------------------------------------------------------- /benchllama/constants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/srikanth235/benchllama/HEAD/benchllama/constants.py -------------------------------------------------------------------------------- /benchllama/data_io/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/srikanth235/benchllama/HEAD/benchllama/data_io/__init__.py -------------------------------------------------------------------------------- /benchllama/data_io/loader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/srikanth235/benchllama/HEAD/benchllama/data_io/loader.py -------------------------------------------------------------------------------- /benchllama/evaluation/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/srikanth235/benchllama/HEAD/benchllama/evaluation/__init__.py -------------------------------------------------------------------------------- /benchllama/evaluation/code_runner.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/srikanth235/benchllama/HEAD/benchllama/evaluation/code_runner.py -------------------------------------------------------------------------------- /benchllama/evaluation/evaluator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/srikanth235/benchllama/HEAD/benchllama/evaluation/evaluator.py -------------------------------------------------------------------------------- /benchllama/evaluation/runners/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /benchllama/evaluation/runners/cpp_runner.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/srikanth235/benchllama/HEAD/benchllama/evaluation/runners/cpp_runner.py -------------------------------------------------------------------------------- /benchllama/evaluation/runners/go_runner.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/srikanth235/benchllama/HEAD/benchllama/evaluation/runners/go_runner.py -------------------------------------------------------------------------------- /benchllama/evaluation/runners/java_runner.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/srikanth235/benchllama/HEAD/benchllama/evaluation/runners/java_runner.py -------------------------------------------------------------------------------- /benchllama/evaluation/runners/javascript_runner.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/srikanth235/benchllama/HEAD/benchllama/evaluation/runners/javascript_runner.py -------------------------------------------------------------------------------- /benchllama/evaluation/runners/python_runner.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/srikanth235/benchllama/HEAD/benchllama/evaluation/runners/python_runner.py -------------------------------------------------------------------------------- /benchllama/evaluation/runners/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/srikanth235/benchllama/HEAD/benchllama/evaluation/runners/utils.py -------------------------------------------------------------------------------- /benchllama/evaluation/score_estimator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/srikanth235/benchllama/HEAD/benchllama/evaluation/score_estimator.py -------------------------------------------------------------------------------- /benchllama/inference/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/srikanth235/benchllama/HEAD/benchllama/inference/__init__.py -------------------------------------------------------------------------------- /benchllama/inference/model_provider.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/srikanth235/benchllama/HEAD/benchllama/inference/model_provider.py -------------------------------------------------------------------------------- /benchllama/inference/prompt_formatter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/srikanth235/benchllama/HEAD/benchllama/inference/prompt_formatter.py -------------------------------------------------------------------------------- /benchllama/logger.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/srikanth235/benchllama/HEAD/benchllama/logger.py -------------------------------------------------------------------------------- /benchllama/main.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/srikanth235/benchllama/HEAD/benchllama/main.py -------------------------------------------------------------------------------- /benchllama/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/srikanth235/benchllama/HEAD/benchllama/utils.py -------------------------------------------------------------------------------- /examples/custom_js_evals.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/srikanth235/benchllama/HEAD/examples/custom_js_evals.jsonl -------------------------------------------------------------------------------- /media/benchllama.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/srikanth235/benchllama/HEAD/media/benchllama.gif -------------------------------------------------------------------------------- /poetry.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/srikanth235/benchllama/HEAD/poetry.lock -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/srikanth235/benchllama/HEAD/pyproject.toml -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | --------------------------------------------------------------------------------