├── .dockerignore ├── .github ├── CODEOWNERS ├── actions │ └── test-template │ │ └── action.yml ├── config │ ├── .coveragerc │ └── requirements.txt ├── copy-pr-bot.yaml ├── labeler.yml └── workflows │ ├── _build_container.yml │ ├── _update_dependencies.yml │ ├── build-docs.yml │ ├── build-test-publish-wheel.yml │ ├── cherry-pick-release-commit.yml │ ├── cicd-approve-test-queue.yml │ ├── cicd-main.yml │ ├── close-inactive-issue-pr.yml │ ├── community-bot.yml │ ├── copyright-check.yml │ ├── dependabot.yml │ ├── install-test.yml │ ├── labeler.yml │ ├── release-freeze.yml │ ├── release.yaml │ └── semantic-pull-request.yml ├── .gitignore ├── .pre-commit-config.yaml ├── CHANGELOG.md ├── CONTRIBUTING.md ├── LICENSE ├── NeMo_Repo_Overview_Eval.png ├── README.md ├── codecov.yml ├── docker ├── Dockerfile.ci ├── Dockerfile.ci-integration └── common │ └── install.sh ├── docs ├── _extensions │ ├── __init__.py │ ├── ai_assistant │ │ ├── README.md │ │ ├── __init__.py │ │ ├── assets │ │ │ └── styles │ │ │ │ └── ai-assistant.css │ │ ├── core │ │ │ ├── AIClient.js │ │ │ ├── ResponseProcessor.js │ │ │ └── main.js │ │ ├── integrations │ │ │ └── search-integration.js │ │ └── ui │ │ │ ├── MarkdownProcessor.js │ │ │ └── ResponseRenderer.js │ ├── content_gating │ │ ├── README.md │ │ ├── __init__.py │ │ ├── condition_evaluator.py │ │ ├── conditional_directives.py │ │ └── document_filter.py │ ├── json_output │ │ ├── README.md │ │ ├── __init__.py │ │ ├── config.py │ │ ├── content │ │ │ ├── __init__.py │ │ │ ├── extractor.py │ │ │ ├── metadata.py │ │ │ ├── structured.py │ │ │ └── text.py │ │ ├── core │ │ │ ├── __init__.py │ │ │ ├── builder.py │ │ │ ├── document_discovery.py │ │ │ ├── hierarchy_builder.py │ │ │ ├── json_formatter.py │ │ │ └── json_writer.py │ │ ├── processing │ │ │ ├── __init__.py │ │ │ ├── cache.py │ │ │ └── processor.py │ │ └── utils.py │ ├── myst_codeblock_substitutions.py │ ├── search_assets │ │ ├── __init__.py │ │ ├── enhanced-search.css │ │ ├── main.js │ │ ├── modules │ │ │ ├── DocumentLoader.js │ │ │ ├── EventHandler.js │ │ │ ├── ResultRenderer.js │ │ │ ├── SearchEngine.js │ │ │ ├── SearchInterface.js │ │ │ ├── SearchPageManager.js │ │ │ └── Utils.js │ │ └── templates │ │ │ └── search.html │ └── toctree_substitutions.py ├── _resources │ └── tasks-table.md ├── about │ ├── concepts │ │ ├── _snippets │ │ │ ├── processed_outputs_combined.json │ │ │ ├── processed_results.json │ │ │ └── results.yaml │ │ ├── adapters.md │ │ ├── architecture.md │ │ ├── evaluation-model.md │ │ ├── evaluation-output.md │ │ ├── execution-backend.md │ │ ├── framework-definition-file.md │ │ ├── index.md │ │ └── interceptors.md │ ├── index.md │ ├── key-features.md │ └── release-notes │ │ └── index.md ├── broken_links_false_positives.json ├── conf.py ├── deployment │ ├── adapters │ │ ├── configuration.md │ │ ├── index.md │ │ ├── recipes │ │ │ ├── custom-system-prompt.md │ │ │ ├── index.md │ │ │ ├── logging-caps.md │ │ │ ├── reasoning-cleanup.md │ │ │ └── response-shaping.md │ │ └── usage.md │ ├── bring-your-own-endpoint │ │ ├── hosted-services.md │ │ ├── index.md │ │ ├── manual-deployment.md │ │ └── testing-endpoint-oai-compatibility.md │ ├── index.md │ ├── launcher-orchestrated │ │ ├── index.md │ │ ├── lepton.md │ │ ├── local.md │ │ └── slurm.md │ └── nemo-fw │ │ ├── _snippets │ │ ├── arc_challenge.py │ │ ├── arc_challenge_hf.py │ │ ├── arc_challenge_mbridge.py │ │ ├── deploy_hf.sh │ │ ├── deploy_mbridge.sh │ │ ├── deploy_pytriton.sh │ │ ├── deploy_ray.sh │ │ └── mmlu.py │ │ ├── hf.md │ │ ├── index.md │ │ ├── mbridge.md │ │ ├── pytriton.md │ │ └── ray.md ├── evaluation │ ├── _snippets │ │ ├── bfcl.py │ │ ├── bigcode.py │ │ ├── commands │ │ │ ├── list_tasks.sh │ │ │ └── list_tasks_core.sh │ │ ├── garak.py │ │ ├── piqa_hf.py │ │ ├── polemo2.py │ │ ├── safety.py │ │ └── simple_evals.py │ ├── benchmarks.md │ ├── custom-task.md │ ├── index.md │ ├── nemo-fw-optional-package.md │ ├── parameters.md │ └── run-evals │ │ ├── code-generation.md │ │ ├── function-calling.md │ │ ├── index.md │ │ ├── logprobs.md │ │ └── text-gen.md ├── get-started │ ├── _snippets │ │ ├── container_run.sh │ │ ├── core_basic.py │ │ ├── core_full_cli.sh │ │ ├── core_full_example.py │ │ ├── core_multi_benchmark.py │ │ ├── deploy.sh │ │ ├── install_containers.sh │ │ ├── install_core.sh │ │ ├── install_launcher.sh │ │ ├── launcher_basic.sh │ │ ├── launcher_full_example.sh │ │ ├── nemo_fw_basic.py │ │ ├── verify_core.sh │ │ └── verify_launcher.sh │ ├── index.md │ ├── install.md │ └── quickstart │ │ ├── container.md │ │ ├── core.md │ │ ├── index.md │ │ ├── launcher.md │ │ └── nemo-fw.md ├── index.md ├── libraries │ ├── index.md │ ├── nemo-evaluator-launcher │ │ ├── configuration │ │ │ ├── deployment │ │ │ │ ├── generic.md │ │ │ │ ├── index.md │ │ │ │ ├── nim.md │ │ │ │ ├── none.md │ │ │ │ ├── sglang.md │ │ │ │ ├── trtllm.md │ │ │ │ └── vllm.md │ │ │ ├── evaluation │ │ │ │ └── index.md │ │ │ ├── executors │ │ │ │ ├── index.md │ │ │ │ ├── lepton.md │ │ │ │ ├── local.md │ │ │ │ └── slurm.md │ │ │ └── index.md │ │ ├── exporters │ │ │ ├── gsheets.md │ │ │ ├── index.md │ │ │ ├── local.md │ │ │ ├── mlflow.md │ │ │ └── wandb.md │ │ └── index.md │ └── nemo-evaluator │ │ ├── api.md │ │ ├── containers │ │ ├── code-generation.md │ │ ├── efficiency.md │ │ ├── index.md │ │ ├── language-models.md │ │ ├── safety-security.md │ │ ├── specialized-tools.md │ │ └── vision-language.md │ │ ├── extending │ │ ├── framework-definition-file │ │ │ ├── advanced-features.md │ │ │ ├── defaults-section.md │ │ │ ├── evaluations-section.md │ │ │ ├── fdf-troubleshooting.md │ │ │ ├── framework-section.md │ │ │ ├── index.md │ │ │ └── integration.md │ │ └── index.md │ │ ├── index.md │ │ ├── interceptors │ │ ├── caching.md │ │ ├── endpoint.md │ │ ├── index.md │ │ ├── payload-modification.md │ │ ├── post-evaluation-hooks.md │ │ ├── progress-tracking.md │ │ ├── raise-client-error.md │ │ ├── reasoning.md │ │ ├── request-logging.md │ │ ├── response-logging.md │ │ ├── response-stats.md │ │ └── system-messages.md │ │ ├── logging.md │ │ └── workflows │ │ ├── cli.md │ │ ├── index.md │ │ └── python-api.md ├── project.json ├── references │ ├── api │ │ ├── nemo-evaluator-launcher │ │ │ ├── api.md │ │ │ └── cli.md │ │ └── nemo-evaluator │ │ │ ├── adapters │ │ │ ├── adapter-config.rst │ │ │ ├── adapters.rst │ │ │ ├── interceptors.rst │ │ │ └── types.rst │ │ │ ├── api │ │ │ ├── api-dataclasses.rst │ │ │ └── index.rst │ │ │ └── cli.md │ ├── evaluation-utils.md │ ├── faq.md │ └── index.md ├── troubleshooting │ ├── index.md │ ├── runtime-issues │ │ ├── configuration.md │ │ ├── index.md │ │ └── launcher.md │ └── setup-issues │ │ ├── authentication.md │ │ ├── index.md │ │ └── installation.md ├── tutorials │ ├── create-framework-definition-file.md │ ├── index.md │ ├── local-evaluation-of-existing-endpoint.md │ └── nemo-fw │ │ ├── index.md │ │ └── nemo-run.md └── versions1.json ├── makefile ├── packages ├── nemo-evaluator-launcher │ ├── .python-version │ ├── CHANGELOG.md │ ├── LICENSE │ ├── README.md │ ├── THIRD_PARTY_NOTICES.txt │ ├── examples │ │ ├── lepton_nim_llama_3_1_8b_instruct.yaml │ │ ├── lepton_none_llama_3_1_8b_instruct.yaml │ │ ├── lepton_vllm_llama_3_1_8b_instruct.yaml │ │ ├── local_auto_export_llama_3_1_8b_instruct.yaml │ │ ├── local_custom_config_seed_oss_36b_instruct.yaml │ │ ├── local_limit_samples.yaml │ │ ├── local_llama_3_1_8b_instruct.yaml │ │ ├── local_llama_3_1_8b_instruct_limit_samples.yaml │ │ ├── local_nvidia_nemotron_nano_9b_v2.yaml │ │ ├── local_vllm_llama_3_1_8b_logprobs.yaml │ │ ├── notebooks │ │ │ └── nemo-evaluator-launcher-api.ipynb │ │ ├── slurm_auto_export_llama_3_1_8b_instruct.yaml │ │ ├── slurm_llama_3_1_8b_instruct.yaml │ │ ├── slurm_llama_3_1_8b_instruct_hf.yaml │ │ ├── slurm_multinode_vllm_dp_nemotron_nano_9b_v2.yaml │ │ ├── slurm_multinode_vllm_ray_tp_pp_deepseek_r1.yaml │ │ ├── slurm_no_deployment_llama_3_1_8b_instruct.yaml │ │ └── slurm_no_deployment_llama_nemotron_super_v1_nemotron_benchmarks.yaml │ ├── mypy.ini │ ├── pyproject.toml │ ├── pytest.ini │ ├── scripts │ │ ├── add_spdx.py │ │ ├── check_logging_imports.py │ │ └── generate_configs.py │ ├── src │ │ └── nemo_evaluator_launcher │ │ │ ├── __init__.py │ │ │ ├── api │ │ │ ├── __init__.py │ │ │ ├── functional.py │ │ │ ├── types.py │ │ │ └── utils.py │ │ │ ├── cli │ │ │ ├── __init__.py │ │ │ ├── export.py │ │ │ ├── info.py │ │ │ ├── kill.py │ │ │ ├── logs.py │ │ │ ├── ls_runs.py │ │ │ ├── ls_tasks.py │ │ │ ├── main.py │ │ │ ├── run.py │ │ │ ├── status.py │ │ │ └── version.py │ │ │ ├── common │ │ │ ├── __init__.py │ │ │ ├── execdb.py │ │ │ ├── helpers.py │ │ │ ├── logging_utils.py │ │ │ ├── mapping.py │ │ │ └── printing_utils.py │ │ │ ├── configs │ │ │ ├── __init__.py │ │ │ ├── default.yaml │ │ │ ├── deployment │ │ │ │ ├── generic.yaml │ │ │ │ ├── nim.yaml │ │ │ │ ├── none.yaml │ │ │ │ ├── sglang.yaml │ │ │ │ ├── trtllm.yaml │ │ │ │ └── vllm.yaml │ │ │ └── execution │ │ │ │ ├── lepton │ │ │ │ └── default.yaml │ │ │ │ ├── local.yaml │ │ │ │ └── slurm │ │ │ │ └── default.yaml │ │ │ ├── executors │ │ │ ├── __init__.py │ │ │ ├── base.py │ │ │ ├── lepton │ │ │ │ ├── __init__.py │ │ │ │ ├── deployment_helpers.py │ │ │ │ ├── executor.py │ │ │ │ └── job_helpers.py │ │ │ ├── local │ │ │ │ ├── __init__.py │ │ │ │ ├── executor.py │ │ │ │ └── run.template.sh │ │ │ ├── registry.py │ │ │ └── slurm │ │ │ │ ├── __init__.py │ │ │ │ ├── executor.py │ │ │ │ └── proxy.cfg.template │ │ │ ├── exporters │ │ │ ├── __init__.py │ │ │ ├── base.py │ │ │ ├── gsheets.py │ │ │ ├── local.py │ │ │ ├── mlflow.py │ │ │ ├── registry.py │ │ │ ├── utils.py │ │ │ └── wandb.py │ │ │ ├── package_info.py │ │ │ └── resources │ │ │ └── mapping.toml │ ├── tests │ │ ├── .gitkeep │ │ ├── __init__.py │ │ └── unit_tests │ │ │ ├── Launch_Unit_Tests.sh │ │ │ ├── __init__.py │ │ │ ├── conftest.py │ │ │ ├── exporters │ │ │ ├── test_base_exporter.py │ │ │ ├── test_config_extraction.py │ │ │ ├── test_export_api_cli.py │ │ │ ├── test_export_results_functional.py │ │ │ ├── test_exporter_registry.py │ │ │ ├── test_gsheets_exporter.py │ │ │ ├── test_local_exporter.py │ │ │ ├── test_mlflow_exporter.py │ │ │ ├── test_utils.py │ │ │ └── test_wandb_exporter.py │ │ │ ├── test_api_module.py │ │ │ ├── test_cli_export_extended.py │ │ │ ├── test_cli_info.py │ │ │ ├── test_cli_integration.py │ │ │ ├── test_cli_logs.py │ │ │ ├── test_cli_ls_integration.py │ │ │ ├── test_cli_ls_runs_extended.py │ │ │ ├── test_cli_ls_tasks.py │ │ │ ├── test_cli_version.py │ │ │ ├── test_example_configs.py │ │ │ ├── test_execdb.py │ │ │ ├── test_get_eval_factory_command.py │ │ │ ├── test_helpers.py │ │ │ ├── test_init.py │ │ │ ├── test_lepton_deployment_helpers.py │ │ │ ├── test_lepton_executor.py │ │ │ ├── test_lepton_job_helpers.py │ │ │ ├── test_local_executor.py │ │ │ ├── test_logging_utils.py │ │ │ ├── test_mapping.py │ │ │ ├── test_mapping_extended.py │ │ │ ├── test_mapping_url.py │ │ │ ├── test_printing_utils.py │ │ │ └── test_slurm_executor.py │ └── uv.lock └── nemo-evaluator │ ├── .python-version │ ├── CHANGELOG.md │ ├── LICENSE │ ├── README.md │ ├── examples │ └── notebooks │ │ └── contamination_detection_demo.ipynb │ ├── pyproject.toml │ ├── src │ └── nemo_evaluator │ │ ├── __init__.py │ │ ├── adapters │ │ ├── __init__.py │ │ ├── adapter_config.py │ │ ├── caching │ │ │ ├── __init__.py │ │ │ └── diskcaching.py │ │ ├── decorators.py │ │ ├── interceptors │ │ │ ├── __init__.py │ │ │ ├── caching_interceptor.py │ │ │ ├── endpoint_interceptor.py │ │ │ ├── logging_interceptor.py │ │ │ ├── payload_modifier_interceptor.py │ │ │ ├── progress_tracking_interceptor.py │ │ │ ├── raise_client_error_interceptor.py │ │ │ ├── reasoning_interceptor.py │ │ │ ├── response_stats_interceptor.py │ │ │ └── system_message_interceptor.py │ │ ├── registry.py │ │ ├── reports │ │ │ ├── __init__.py │ │ │ ├── post_eval_report_hook.py │ │ │ └── templates │ │ │ │ ├── __init__.py │ │ │ │ └── simple_template.py │ │ ├── server.py │ │ └── types.py │ │ ├── api │ │ ├── __init__.py │ │ └── api_dataclasses.py │ │ ├── cli │ │ ├── __init__.py │ │ └── example.py │ │ ├── core │ │ ├── __init__.py │ │ ├── entrypoint.py │ │ ├── evaluate.py │ │ ├── input.py │ │ ├── resources.py │ │ └── utils.py │ │ ├── logging │ │ ├── __init__.py │ │ ├── config.py │ │ ├── context.py │ │ └── utils.py │ │ ├── package_info.py │ │ └── resources │ │ ├── framework_tpl.yml │ │ ├── init_tpl.py │ │ └── output_tpl.py │ ├── tests │ ├── __init__.py │ ├── functional_tests │ │ ├── Launch_Functional_Tests.sh │ │ ├── e2e │ │ │ ├── __init__.py │ │ │ ├── conftest.py │ │ │ ├── test_eval_factory_fake_endpoint.py │ │ │ └── test_interceptor_integration.py │ │ ├── installed_modules │ │ │ └── 1 │ │ │ │ └── core_evals │ │ │ │ ├── __init__.py │ │ │ │ └── dummy_framework │ │ │ │ ├── __init__.py │ │ │ │ ├── dummy_entrypoint.py │ │ │ │ ├── framework.yml │ │ │ │ └── output.py │ │ ├── test_cli_example.py │ │ └── test_entrypoint.py │ └── unit_tests │ │ ├── Launch_Unit_Tests.sh │ │ ├── __init__.py │ │ ├── adapters │ │ ├── __init__.py │ │ ├── caching │ │ │ ├── __init__.py │ │ │ └── test_diskcaching.py │ │ ├── conftest.py │ │ ├── interceptors │ │ │ ├── __init__.py │ │ │ ├── test_logging_interceptor.py │ │ │ ├── test_payload_modifier.py │ │ │ ├── test_progress_tracking_interceptor.py │ │ │ ├── test_raise_client_error_interceptor.py │ │ │ ├── test_reasoning.py │ │ │ ├── test_response_stats_interceptor.py │ │ │ └── test_system_message.py │ │ ├── test_adapter_config.py │ │ ├── test_adapter_server.py │ │ ├── test_caching.py │ │ ├── test_logging_integration.py │ │ ├── test_multiple_instances.py │ │ ├── test_post_eval_hooks.py │ │ ├── test_post_eval_report_hook.py │ │ ├── test_registry.py │ │ ├── test_yaml_interceptor_config.py │ │ └── testing_utils.py │ │ ├── core │ │ ├── __init__.py │ │ ├── test_evaluate.py │ │ ├── test_input.py │ │ └── test_metadata.py │ │ ├── logging │ │ ├── __init__.py │ │ └── test_utils.py │ │ └── utils │ │ ├── __init__.py │ │ ├── test_cache_utils.py │ │ ├── test_input_utils.py │ │ └── test_run_command.py │ └── uv.lock ├── scripts ├── evaluation_with_nemo_run.py ├── helpers.py └── validate_doc_snippets.py ├── tests └── integration_tests │ ├── Launch_Integration_Tests.sh │ └── nemo_fw │ ├── test_deployment.py │ ├── test_hf_ray.py │ ├── test_nemo2_ray.py │ ├── test_nemo2_triton.py │ └── test_notebooks.py └── tutorials ├── README.md ├── mmlu.ipynb ├── simple-evals.ipynb └── wikitext.ipynb /.dockerignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/.dockerignore -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/.github/CODEOWNERS -------------------------------------------------------------------------------- /.github/actions/test-template/action.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/.github/actions/test-template/action.yml -------------------------------------------------------------------------------- /.github/config/.coveragerc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/.github/config/.coveragerc -------------------------------------------------------------------------------- /.github/config/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/.github/config/requirements.txt -------------------------------------------------------------------------------- /.github/copy-pr-bot.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/.github/copy-pr-bot.yaml -------------------------------------------------------------------------------- /.github/labeler.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/.github/labeler.yml -------------------------------------------------------------------------------- /.github/workflows/_build_container.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/.github/workflows/_build_container.yml -------------------------------------------------------------------------------- /.github/workflows/_update_dependencies.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/.github/workflows/_update_dependencies.yml -------------------------------------------------------------------------------- /.github/workflows/build-docs.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/.github/workflows/build-docs.yml -------------------------------------------------------------------------------- /.github/workflows/build-test-publish-wheel.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/.github/workflows/build-test-publish-wheel.yml -------------------------------------------------------------------------------- /.github/workflows/cherry-pick-release-commit.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/.github/workflows/cherry-pick-release-commit.yml -------------------------------------------------------------------------------- /.github/workflows/cicd-approve-test-queue.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/.github/workflows/cicd-approve-test-queue.yml -------------------------------------------------------------------------------- /.github/workflows/cicd-main.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/.github/workflows/cicd-main.yml -------------------------------------------------------------------------------- /.github/workflows/close-inactive-issue-pr.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/.github/workflows/close-inactive-issue-pr.yml -------------------------------------------------------------------------------- /.github/workflows/community-bot.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/.github/workflows/community-bot.yml -------------------------------------------------------------------------------- /.github/workflows/copyright-check.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/.github/workflows/copyright-check.yml -------------------------------------------------------------------------------- /.github/workflows/dependabot.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/.github/workflows/dependabot.yml -------------------------------------------------------------------------------- /.github/workflows/install-test.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/.github/workflows/install-test.yml -------------------------------------------------------------------------------- /.github/workflows/labeler.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/.github/workflows/labeler.yml -------------------------------------------------------------------------------- /.github/workflows/release-freeze.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/.github/workflows/release-freeze.yml -------------------------------------------------------------------------------- /.github/workflows/release.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/.github/workflows/release.yaml -------------------------------------------------------------------------------- /.github/workflows/semantic-pull-request.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/.github/workflows/semantic-pull-request.yml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/.gitignore -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/.pre-commit-config.yaml -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/CHANGELOG.md -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/CONTRIBUTING.md -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/LICENSE -------------------------------------------------------------------------------- /NeMo_Repo_Overview_Eval.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/NeMo_Repo_Overview_Eval.png -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/README.md -------------------------------------------------------------------------------- /codecov.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/codecov.yml -------------------------------------------------------------------------------- /docker/Dockerfile.ci: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docker/Dockerfile.ci -------------------------------------------------------------------------------- /docker/Dockerfile.ci-integration: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docker/Dockerfile.ci-integration -------------------------------------------------------------------------------- /docker/common/install.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docker/common/install.sh -------------------------------------------------------------------------------- /docs/_extensions/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/_extensions/__init__.py -------------------------------------------------------------------------------- /docs/_extensions/ai_assistant/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/_extensions/ai_assistant/README.md -------------------------------------------------------------------------------- /docs/_extensions/ai_assistant/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/_extensions/ai_assistant/__init__.py -------------------------------------------------------------------------------- /docs/_extensions/ai_assistant/assets/styles/ai-assistant.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/_extensions/ai_assistant/assets/styles/ai-assistant.css -------------------------------------------------------------------------------- /docs/_extensions/ai_assistant/core/AIClient.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/_extensions/ai_assistant/core/AIClient.js -------------------------------------------------------------------------------- /docs/_extensions/ai_assistant/core/ResponseProcessor.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/_extensions/ai_assistant/core/ResponseProcessor.js -------------------------------------------------------------------------------- /docs/_extensions/ai_assistant/core/main.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/_extensions/ai_assistant/core/main.js -------------------------------------------------------------------------------- /docs/_extensions/ai_assistant/integrations/search-integration.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/_extensions/ai_assistant/integrations/search-integration.js -------------------------------------------------------------------------------- /docs/_extensions/ai_assistant/ui/MarkdownProcessor.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/_extensions/ai_assistant/ui/MarkdownProcessor.js -------------------------------------------------------------------------------- /docs/_extensions/ai_assistant/ui/ResponseRenderer.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/_extensions/ai_assistant/ui/ResponseRenderer.js -------------------------------------------------------------------------------- /docs/_extensions/content_gating/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/_extensions/content_gating/README.md -------------------------------------------------------------------------------- /docs/_extensions/content_gating/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/_extensions/content_gating/__init__.py -------------------------------------------------------------------------------- /docs/_extensions/content_gating/condition_evaluator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/_extensions/content_gating/condition_evaluator.py -------------------------------------------------------------------------------- /docs/_extensions/content_gating/conditional_directives.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/_extensions/content_gating/conditional_directives.py -------------------------------------------------------------------------------- /docs/_extensions/content_gating/document_filter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/_extensions/content_gating/document_filter.py -------------------------------------------------------------------------------- /docs/_extensions/json_output/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/_extensions/json_output/README.md -------------------------------------------------------------------------------- /docs/_extensions/json_output/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/_extensions/json_output/__init__.py -------------------------------------------------------------------------------- /docs/_extensions/json_output/config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/_extensions/json_output/config.py -------------------------------------------------------------------------------- /docs/_extensions/json_output/content/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/_extensions/json_output/content/__init__.py -------------------------------------------------------------------------------- /docs/_extensions/json_output/content/extractor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/_extensions/json_output/content/extractor.py -------------------------------------------------------------------------------- /docs/_extensions/json_output/content/metadata.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/_extensions/json_output/content/metadata.py -------------------------------------------------------------------------------- /docs/_extensions/json_output/content/structured.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/_extensions/json_output/content/structured.py -------------------------------------------------------------------------------- /docs/_extensions/json_output/content/text.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/_extensions/json_output/content/text.py -------------------------------------------------------------------------------- /docs/_extensions/json_output/core/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/_extensions/json_output/core/__init__.py -------------------------------------------------------------------------------- /docs/_extensions/json_output/core/builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/_extensions/json_output/core/builder.py -------------------------------------------------------------------------------- /docs/_extensions/json_output/core/document_discovery.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/_extensions/json_output/core/document_discovery.py -------------------------------------------------------------------------------- /docs/_extensions/json_output/core/hierarchy_builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/_extensions/json_output/core/hierarchy_builder.py -------------------------------------------------------------------------------- /docs/_extensions/json_output/core/json_formatter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/_extensions/json_output/core/json_formatter.py -------------------------------------------------------------------------------- /docs/_extensions/json_output/core/json_writer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/_extensions/json_output/core/json_writer.py -------------------------------------------------------------------------------- /docs/_extensions/json_output/processing/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/_extensions/json_output/processing/__init__.py -------------------------------------------------------------------------------- /docs/_extensions/json_output/processing/cache.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/_extensions/json_output/processing/cache.py -------------------------------------------------------------------------------- /docs/_extensions/json_output/processing/processor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/_extensions/json_output/processing/processor.py -------------------------------------------------------------------------------- /docs/_extensions/json_output/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/_extensions/json_output/utils.py -------------------------------------------------------------------------------- /docs/_extensions/myst_codeblock_substitutions.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/_extensions/myst_codeblock_substitutions.py -------------------------------------------------------------------------------- /docs/_extensions/search_assets/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/_extensions/search_assets/__init__.py -------------------------------------------------------------------------------- /docs/_extensions/search_assets/enhanced-search.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/_extensions/search_assets/enhanced-search.css -------------------------------------------------------------------------------- /docs/_extensions/search_assets/main.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/_extensions/search_assets/main.js -------------------------------------------------------------------------------- /docs/_extensions/search_assets/modules/DocumentLoader.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/_extensions/search_assets/modules/DocumentLoader.js -------------------------------------------------------------------------------- /docs/_extensions/search_assets/modules/EventHandler.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/_extensions/search_assets/modules/EventHandler.js -------------------------------------------------------------------------------- /docs/_extensions/search_assets/modules/ResultRenderer.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/_extensions/search_assets/modules/ResultRenderer.js -------------------------------------------------------------------------------- /docs/_extensions/search_assets/modules/SearchEngine.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/_extensions/search_assets/modules/SearchEngine.js -------------------------------------------------------------------------------- /docs/_extensions/search_assets/modules/SearchInterface.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/_extensions/search_assets/modules/SearchInterface.js -------------------------------------------------------------------------------- /docs/_extensions/search_assets/modules/SearchPageManager.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/_extensions/search_assets/modules/SearchPageManager.js -------------------------------------------------------------------------------- /docs/_extensions/search_assets/modules/Utils.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/_extensions/search_assets/modules/Utils.js -------------------------------------------------------------------------------- /docs/_extensions/search_assets/templates/search.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/_extensions/search_assets/templates/search.html -------------------------------------------------------------------------------- /docs/_extensions/toctree_substitutions.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/_extensions/toctree_substitutions.py -------------------------------------------------------------------------------- /docs/_resources/tasks-table.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/_resources/tasks-table.md -------------------------------------------------------------------------------- /docs/about/concepts/_snippets/processed_outputs_combined.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/about/concepts/_snippets/processed_outputs_combined.json -------------------------------------------------------------------------------- /docs/about/concepts/_snippets/processed_results.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/about/concepts/_snippets/processed_results.json -------------------------------------------------------------------------------- /docs/about/concepts/_snippets/results.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/about/concepts/_snippets/results.yaml -------------------------------------------------------------------------------- /docs/about/concepts/adapters.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/about/concepts/adapters.md -------------------------------------------------------------------------------- /docs/about/concepts/architecture.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/about/concepts/architecture.md -------------------------------------------------------------------------------- /docs/about/concepts/evaluation-model.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/about/concepts/evaluation-model.md -------------------------------------------------------------------------------- /docs/about/concepts/evaluation-output.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/about/concepts/evaluation-output.md -------------------------------------------------------------------------------- /docs/about/concepts/execution-backend.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/about/concepts/execution-backend.md -------------------------------------------------------------------------------- /docs/about/concepts/framework-definition-file.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/about/concepts/framework-definition-file.md -------------------------------------------------------------------------------- /docs/about/concepts/index.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/about/concepts/index.md -------------------------------------------------------------------------------- /docs/about/concepts/interceptors.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/about/concepts/interceptors.md -------------------------------------------------------------------------------- /docs/about/index.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/about/index.md -------------------------------------------------------------------------------- /docs/about/key-features.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/about/key-features.md -------------------------------------------------------------------------------- /docs/about/release-notes/index.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/about/release-notes/index.md -------------------------------------------------------------------------------- /docs/broken_links_false_positives.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/broken_links_false_positives.json -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/conf.py -------------------------------------------------------------------------------- /docs/deployment/adapters/configuration.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/deployment/adapters/configuration.md -------------------------------------------------------------------------------- /docs/deployment/adapters/index.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/deployment/adapters/index.md -------------------------------------------------------------------------------- /docs/deployment/adapters/recipes/custom-system-prompt.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/deployment/adapters/recipes/custom-system-prompt.md -------------------------------------------------------------------------------- /docs/deployment/adapters/recipes/index.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/deployment/adapters/recipes/index.md -------------------------------------------------------------------------------- /docs/deployment/adapters/recipes/logging-caps.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/deployment/adapters/recipes/logging-caps.md -------------------------------------------------------------------------------- /docs/deployment/adapters/recipes/reasoning-cleanup.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/deployment/adapters/recipes/reasoning-cleanup.md -------------------------------------------------------------------------------- /docs/deployment/adapters/recipes/response-shaping.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/deployment/adapters/recipes/response-shaping.md -------------------------------------------------------------------------------- /docs/deployment/adapters/usage.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/deployment/adapters/usage.md -------------------------------------------------------------------------------- /docs/deployment/bring-your-own-endpoint/hosted-services.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/deployment/bring-your-own-endpoint/hosted-services.md -------------------------------------------------------------------------------- /docs/deployment/bring-your-own-endpoint/index.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/deployment/bring-your-own-endpoint/index.md -------------------------------------------------------------------------------- /docs/deployment/bring-your-own-endpoint/manual-deployment.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/deployment/bring-your-own-endpoint/manual-deployment.md -------------------------------------------------------------------------------- /docs/deployment/bring-your-own-endpoint/testing-endpoint-oai-compatibility.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/deployment/bring-your-own-endpoint/testing-endpoint-oai-compatibility.md -------------------------------------------------------------------------------- /docs/deployment/index.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/deployment/index.md -------------------------------------------------------------------------------- /docs/deployment/launcher-orchestrated/index.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/deployment/launcher-orchestrated/index.md -------------------------------------------------------------------------------- /docs/deployment/launcher-orchestrated/lepton.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/deployment/launcher-orchestrated/lepton.md -------------------------------------------------------------------------------- /docs/deployment/launcher-orchestrated/local.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/deployment/launcher-orchestrated/local.md -------------------------------------------------------------------------------- /docs/deployment/launcher-orchestrated/slurm.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/deployment/launcher-orchestrated/slurm.md -------------------------------------------------------------------------------- /docs/deployment/nemo-fw/_snippets/arc_challenge.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/deployment/nemo-fw/_snippets/arc_challenge.py -------------------------------------------------------------------------------- /docs/deployment/nemo-fw/_snippets/arc_challenge_hf.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/deployment/nemo-fw/_snippets/arc_challenge_hf.py -------------------------------------------------------------------------------- /docs/deployment/nemo-fw/_snippets/arc_challenge_mbridge.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/deployment/nemo-fw/_snippets/arc_challenge_mbridge.py -------------------------------------------------------------------------------- /docs/deployment/nemo-fw/_snippets/deploy_hf.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/deployment/nemo-fw/_snippets/deploy_hf.sh -------------------------------------------------------------------------------- /docs/deployment/nemo-fw/_snippets/deploy_mbridge.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/deployment/nemo-fw/_snippets/deploy_mbridge.sh -------------------------------------------------------------------------------- /docs/deployment/nemo-fw/_snippets/deploy_pytriton.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/deployment/nemo-fw/_snippets/deploy_pytriton.sh -------------------------------------------------------------------------------- /docs/deployment/nemo-fw/_snippets/deploy_ray.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/deployment/nemo-fw/_snippets/deploy_ray.sh -------------------------------------------------------------------------------- /docs/deployment/nemo-fw/_snippets/mmlu.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/deployment/nemo-fw/_snippets/mmlu.py -------------------------------------------------------------------------------- /docs/deployment/nemo-fw/hf.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/deployment/nemo-fw/hf.md -------------------------------------------------------------------------------- /docs/deployment/nemo-fw/index.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/deployment/nemo-fw/index.md -------------------------------------------------------------------------------- /docs/deployment/nemo-fw/mbridge.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/deployment/nemo-fw/mbridge.md -------------------------------------------------------------------------------- /docs/deployment/nemo-fw/pytriton.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/deployment/nemo-fw/pytriton.md -------------------------------------------------------------------------------- /docs/deployment/nemo-fw/ray.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/deployment/nemo-fw/ray.md -------------------------------------------------------------------------------- /docs/evaluation/_snippets/bfcl.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/evaluation/_snippets/bfcl.py -------------------------------------------------------------------------------- /docs/evaluation/_snippets/bigcode.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/evaluation/_snippets/bigcode.py -------------------------------------------------------------------------------- /docs/evaluation/_snippets/commands/list_tasks.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/evaluation/_snippets/commands/list_tasks.sh -------------------------------------------------------------------------------- /docs/evaluation/_snippets/commands/list_tasks_core.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/evaluation/_snippets/commands/list_tasks_core.sh -------------------------------------------------------------------------------- /docs/evaluation/_snippets/garak.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/evaluation/_snippets/garak.py -------------------------------------------------------------------------------- /docs/evaluation/_snippets/piqa_hf.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/evaluation/_snippets/piqa_hf.py -------------------------------------------------------------------------------- /docs/evaluation/_snippets/polemo2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/evaluation/_snippets/polemo2.py -------------------------------------------------------------------------------- /docs/evaluation/_snippets/safety.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/evaluation/_snippets/safety.py -------------------------------------------------------------------------------- /docs/evaluation/_snippets/simple_evals.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/evaluation/_snippets/simple_evals.py -------------------------------------------------------------------------------- /docs/evaluation/benchmarks.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/evaluation/benchmarks.md -------------------------------------------------------------------------------- /docs/evaluation/custom-task.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/evaluation/custom-task.md -------------------------------------------------------------------------------- /docs/evaluation/index.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/evaluation/index.md -------------------------------------------------------------------------------- /docs/evaluation/nemo-fw-optional-package.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/evaluation/nemo-fw-optional-package.md -------------------------------------------------------------------------------- /docs/evaluation/parameters.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/evaluation/parameters.md -------------------------------------------------------------------------------- /docs/evaluation/run-evals/code-generation.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/evaluation/run-evals/code-generation.md -------------------------------------------------------------------------------- /docs/evaluation/run-evals/function-calling.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/evaluation/run-evals/function-calling.md -------------------------------------------------------------------------------- /docs/evaluation/run-evals/index.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/evaluation/run-evals/index.md -------------------------------------------------------------------------------- /docs/evaluation/run-evals/logprobs.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/evaluation/run-evals/logprobs.md -------------------------------------------------------------------------------- /docs/evaluation/run-evals/text-gen.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/evaluation/run-evals/text-gen.md -------------------------------------------------------------------------------- /docs/get-started/_snippets/container_run.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/get-started/_snippets/container_run.sh -------------------------------------------------------------------------------- /docs/get-started/_snippets/core_basic.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/get-started/_snippets/core_basic.py -------------------------------------------------------------------------------- /docs/get-started/_snippets/core_full_cli.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/get-started/_snippets/core_full_cli.sh -------------------------------------------------------------------------------- /docs/get-started/_snippets/core_full_example.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/get-started/_snippets/core_full_example.py -------------------------------------------------------------------------------- /docs/get-started/_snippets/core_multi_benchmark.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/get-started/_snippets/core_multi_benchmark.py -------------------------------------------------------------------------------- /docs/get-started/_snippets/deploy.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/get-started/_snippets/deploy.sh -------------------------------------------------------------------------------- /docs/get-started/_snippets/install_containers.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/get-started/_snippets/install_containers.sh -------------------------------------------------------------------------------- /docs/get-started/_snippets/install_core.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/get-started/_snippets/install_core.sh -------------------------------------------------------------------------------- /docs/get-started/_snippets/install_launcher.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/get-started/_snippets/install_launcher.sh -------------------------------------------------------------------------------- /docs/get-started/_snippets/launcher_basic.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/get-started/_snippets/launcher_basic.sh -------------------------------------------------------------------------------- /docs/get-started/_snippets/launcher_full_example.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/get-started/_snippets/launcher_full_example.sh -------------------------------------------------------------------------------- /docs/get-started/_snippets/nemo_fw_basic.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/get-started/_snippets/nemo_fw_basic.py -------------------------------------------------------------------------------- /docs/get-started/_snippets/verify_core.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/get-started/_snippets/verify_core.sh -------------------------------------------------------------------------------- /docs/get-started/_snippets/verify_launcher.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/get-started/_snippets/verify_launcher.sh -------------------------------------------------------------------------------- /docs/get-started/index.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/get-started/index.md -------------------------------------------------------------------------------- /docs/get-started/install.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/get-started/install.md -------------------------------------------------------------------------------- /docs/get-started/quickstart/container.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/get-started/quickstart/container.md -------------------------------------------------------------------------------- /docs/get-started/quickstart/core.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/get-started/quickstart/core.md -------------------------------------------------------------------------------- /docs/get-started/quickstart/index.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/get-started/quickstart/index.md -------------------------------------------------------------------------------- /docs/get-started/quickstart/launcher.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/get-started/quickstart/launcher.md -------------------------------------------------------------------------------- /docs/get-started/quickstart/nemo-fw.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/get-started/quickstart/nemo-fw.md -------------------------------------------------------------------------------- /docs/index.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/index.md -------------------------------------------------------------------------------- /docs/libraries/index.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/libraries/index.md -------------------------------------------------------------------------------- /docs/libraries/nemo-evaluator-launcher/configuration/deployment/generic.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/libraries/nemo-evaluator-launcher/configuration/deployment/generic.md -------------------------------------------------------------------------------- /docs/libraries/nemo-evaluator-launcher/configuration/deployment/index.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/libraries/nemo-evaluator-launcher/configuration/deployment/index.md -------------------------------------------------------------------------------- /docs/libraries/nemo-evaluator-launcher/configuration/deployment/nim.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/libraries/nemo-evaluator-launcher/configuration/deployment/nim.md -------------------------------------------------------------------------------- /docs/libraries/nemo-evaluator-launcher/configuration/deployment/none.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/libraries/nemo-evaluator-launcher/configuration/deployment/none.md -------------------------------------------------------------------------------- /docs/libraries/nemo-evaluator-launcher/configuration/deployment/sglang.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/libraries/nemo-evaluator-launcher/configuration/deployment/sglang.md -------------------------------------------------------------------------------- /docs/libraries/nemo-evaluator-launcher/configuration/deployment/trtllm.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/libraries/nemo-evaluator-launcher/configuration/deployment/trtllm.md -------------------------------------------------------------------------------- /docs/libraries/nemo-evaluator-launcher/configuration/deployment/vllm.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/libraries/nemo-evaluator-launcher/configuration/deployment/vllm.md -------------------------------------------------------------------------------- /docs/libraries/nemo-evaluator-launcher/configuration/evaluation/index.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/libraries/nemo-evaluator-launcher/configuration/evaluation/index.md -------------------------------------------------------------------------------- /docs/libraries/nemo-evaluator-launcher/configuration/executors/index.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/libraries/nemo-evaluator-launcher/configuration/executors/index.md -------------------------------------------------------------------------------- /docs/libraries/nemo-evaluator-launcher/configuration/executors/lepton.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/libraries/nemo-evaluator-launcher/configuration/executors/lepton.md -------------------------------------------------------------------------------- /docs/libraries/nemo-evaluator-launcher/configuration/executors/local.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/libraries/nemo-evaluator-launcher/configuration/executors/local.md -------------------------------------------------------------------------------- /docs/libraries/nemo-evaluator-launcher/configuration/executors/slurm.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/libraries/nemo-evaluator-launcher/configuration/executors/slurm.md -------------------------------------------------------------------------------- /docs/libraries/nemo-evaluator-launcher/configuration/index.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/libraries/nemo-evaluator-launcher/configuration/index.md -------------------------------------------------------------------------------- /docs/libraries/nemo-evaluator-launcher/exporters/gsheets.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/libraries/nemo-evaluator-launcher/exporters/gsheets.md -------------------------------------------------------------------------------- /docs/libraries/nemo-evaluator-launcher/exporters/index.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/libraries/nemo-evaluator-launcher/exporters/index.md -------------------------------------------------------------------------------- /docs/libraries/nemo-evaluator-launcher/exporters/local.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/libraries/nemo-evaluator-launcher/exporters/local.md -------------------------------------------------------------------------------- /docs/libraries/nemo-evaluator-launcher/exporters/mlflow.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/libraries/nemo-evaluator-launcher/exporters/mlflow.md -------------------------------------------------------------------------------- /docs/libraries/nemo-evaluator-launcher/exporters/wandb.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/libraries/nemo-evaluator-launcher/exporters/wandb.md -------------------------------------------------------------------------------- /docs/libraries/nemo-evaluator-launcher/index.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/libraries/nemo-evaluator-launcher/index.md -------------------------------------------------------------------------------- /docs/libraries/nemo-evaluator/api.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/libraries/nemo-evaluator/api.md -------------------------------------------------------------------------------- /docs/libraries/nemo-evaluator/containers/code-generation.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/libraries/nemo-evaluator/containers/code-generation.md -------------------------------------------------------------------------------- /docs/libraries/nemo-evaluator/containers/efficiency.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/libraries/nemo-evaluator/containers/efficiency.md -------------------------------------------------------------------------------- /docs/libraries/nemo-evaluator/containers/index.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/libraries/nemo-evaluator/containers/index.md -------------------------------------------------------------------------------- /docs/libraries/nemo-evaluator/containers/language-models.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/libraries/nemo-evaluator/containers/language-models.md -------------------------------------------------------------------------------- /docs/libraries/nemo-evaluator/containers/safety-security.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/libraries/nemo-evaluator/containers/safety-security.md -------------------------------------------------------------------------------- /docs/libraries/nemo-evaluator/containers/specialized-tools.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/libraries/nemo-evaluator/containers/specialized-tools.md -------------------------------------------------------------------------------- /docs/libraries/nemo-evaluator/containers/vision-language.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/libraries/nemo-evaluator/containers/vision-language.md -------------------------------------------------------------------------------- /docs/libraries/nemo-evaluator/extending/framework-definition-file/advanced-features.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/libraries/nemo-evaluator/extending/framework-definition-file/advanced-features.md -------------------------------------------------------------------------------- /docs/libraries/nemo-evaluator/extending/framework-definition-file/defaults-section.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/libraries/nemo-evaluator/extending/framework-definition-file/defaults-section.md -------------------------------------------------------------------------------- /docs/libraries/nemo-evaluator/extending/framework-definition-file/evaluations-section.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/libraries/nemo-evaluator/extending/framework-definition-file/evaluations-section.md -------------------------------------------------------------------------------- /docs/libraries/nemo-evaluator/extending/framework-definition-file/fdf-troubleshooting.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/libraries/nemo-evaluator/extending/framework-definition-file/fdf-troubleshooting.md -------------------------------------------------------------------------------- /docs/libraries/nemo-evaluator/extending/framework-definition-file/framework-section.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/libraries/nemo-evaluator/extending/framework-definition-file/framework-section.md -------------------------------------------------------------------------------- /docs/libraries/nemo-evaluator/extending/framework-definition-file/index.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/libraries/nemo-evaluator/extending/framework-definition-file/index.md -------------------------------------------------------------------------------- /docs/libraries/nemo-evaluator/extending/framework-definition-file/integration.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/libraries/nemo-evaluator/extending/framework-definition-file/integration.md -------------------------------------------------------------------------------- /docs/libraries/nemo-evaluator/extending/index.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/libraries/nemo-evaluator/extending/index.md -------------------------------------------------------------------------------- /docs/libraries/nemo-evaluator/index.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/libraries/nemo-evaluator/index.md -------------------------------------------------------------------------------- /docs/libraries/nemo-evaluator/interceptors/caching.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/libraries/nemo-evaluator/interceptors/caching.md -------------------------------------------------------------------------------- /docs/libraries/nemo-evaluator/interceptors/endpoint.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/libraries/nemo-evaluator/interceptors/endpoint.md -------------------------------------------------------------------------------- /docs/libraries/nemo-evaluator/interceptors/index.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/libraries/nemo-evaluator/interceptors/index.md -------------------------------------------------------------------------------- /docs/libraries/nemo-evaluator/interceptors/payload-modification.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/libraries/nemo-evaluator/interceptors/payload-modification.md -------------------------------------------------------------------------------- /docs/libraries/nemo-evaluator/interceptors/post-evaluation-hooks.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/libraries/nemo-evaluator/interceptors/post-evaluation-hooks.md -------------------------------------------------------------------------------- /docs/libraries/nemo-evaluator/interceptors/progress-tracking.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/libraries/nemo-evaluator/interceptors/progress-tracking.md -------------------------------------------------------------------------------- /docs/libraries/nemo-evaluator/interceptors/raise-client-error.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/libraries/nemo-evaluator/interceptors/raise-client-error.md -------------------------------------------------------------------------------- /docs/libraries/nemo-evaluator/interceptors/reasoning.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/libraries/nemo-evaluator/interceptors/reasoning.md -------------------------------------------------------------------------------- /docs/libraries/nemo-evaluator/interceptors/request-logging.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/libraries/nemo-evaluator/interceptors/request-logging.md -------------------------------------------------------------------------------- /docs/libraries/nemo-evaluator/interceptors/response-logging.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/libraries/nemo-evaluator/interceptors/response-logging.md -------------------------------------------------------------------------------- /docs/libraries/nemo-evaluator/interceptors/response-stats.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/libraries/nemo-evaluator/interceptors/response-stats.md -------------------------------------------------------------------------------- /docs/libraries/nemo-evaluator/interceptors/system-messages.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/libraries/nemo-evaluator/interceptors/system-messages.md -------------------------------------------------------------------------------- /docs/libraries/nemo-evaluator/logging.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/libraries/nemo-evaluator/logging.md -------------------------------------------------------------------------------- /docs/libraries/nemo-evaluator/workflows/cli.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/libraries/nemo-evaluator/workflows/cli.md -------------------------------------------------------------------------------- /docs/libraries/nemo-evaluator/workflows/index.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/libraries/nemo-evaluator/workflows/index.md -------------------------------------------------------------------------------- /docs/libraries/nemo-evaluator/workflows/python-api.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/libraries/nemo-evaluator/workflows/python-api.md -------------------------------------------------------------------------------- /docs/project.json: -------------------------------------------------------------------------------- 1 | {"name": "nemo-evaluator", "version": "0.1.0"} 2 | -------------------------------------------------------------------------------- /docs/references/api/nemo-evaluator-launcher/api.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/references/api/nemo-evaluator-launcher/api.md -------------------------------------------------------------------------------- /docs/references/api/nemo-evaluator-launcher/cli.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/references/api/nemo-evaluator-launcher/cli.md -------------------------------------------------------------------------------- /docs/references/api/nemo-evaluator/adapters/adapter-config.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/references/api/nemo-evaluator/adapters/adapter-config.rst -------------------------------------------------------------------------------- /docs/references/api/nemo-evaluator/adapters/adapters.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/references/api/nemo-evaluator/adapters/adapters.rst -------------------------------------------------------------------------------- /docs/references/api/nemo-evaluator/adapters/interceptors.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/references/api/nemo-evaluator/adapters/interceptors.rst -------------------------------------------------------------------------------- /docs/references/api/nemo-evaluator/adapters/types.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/references/api/nemo-evaluator/adapters/types.rst -------------------------------------------------------------------------------- /docs/references/api/nemo-evaluator/api/api-dataclasses.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/references/api/nemo-evaluator/api/api-dataclasses.rst -------------------------------------------------------------------------------- /docs/references/api/nemo-evaluator/api/index.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/references/api/nemo-evaluator/api/index.rst -------------------------------------------------------------------------------- /docs/references/api/nemo-evaluator/cli.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/references/api/nemo-evaluator/cli.md -------------------------------------------------------------------------------- /docs/references/evaluation-utils.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/references/evaluation-utils.md -------------------------------------------------------------------------------- /docs/references/faq.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/references/faq.md -------------------------------------------------------------------------------- /docs/references/index.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/references/index.md -------------------------------------------------------------------------------- /docs/troubleshooting/index.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/troubleshooting/index.md -------------------------------------------------------------------------------- /docs/troubleshooting/runtime-issues/configuration.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/troubleshooting/runtime-issues/configuration.md -------------------------------------------------------------------------------- /docs/troubleshooting/runtime-issues/index.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/troubleshooting/runtime-issues/index.md -------------------------------------------------------------------------------- /docs/troubleshooting/runtime-issues/launcher.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/troubleshooting/runtime-issues/launcher.md -------------------------------------------------------------------------------- /docs/troubleshooting/setup-issues/authentication.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/troubleshooting/setup-issues/authentication.md -------------------------------------------------------------------------------- /docs/troubleshooting/setup-issues/index.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/troubleshooting/setup-issues/index.md -------------------------------------------------------------------------------- /docs/troubleshooting/setup-issues/installation.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/troubleshooting/setup-issues/installation.md -------------------------------------------------------------------------------- /docs/tutorials/create-framework-definition-file.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/tutorials/create-framework-definition-file.md -------------------------------------------------------------------------------- /docs/tutorials/index.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/tutorials/index.md -------------------------------------------------------------------------------- /docs/tutorials/local-evaluation-of-existing-endpoint.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/tutorials/local-evaluation-of-existing-endpoint.md -------------------------------------------------------------------------------- /docs/tutorials/nemo-fw/index.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/tutorials/nemo-fw/index.md -------------------------------------------------------------------------------- /docs/tutorials/nemo-fw/nemo-run.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/tutorials/nemo-fw/nemo-run.md -------------------------------------------------------------------------------- /docs/versions1.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/docs/versions1.json -------------------------------------------------------------------------------- /makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/makefile -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/.python-version: -------------------------------------------------------------------------------- 1 | 3.12 -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/LICENSE -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/README.md -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/THIRD_PARTY_NOTICES.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/THIRD_PARTY_NOTICES.txt -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/examples/lepton_nim_llama_3_1_8b_instruct.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/examples/lepton_nim_llama_3_1_8b_instruct.yaml -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/examples/lepton_none_llama_3_1_8b_instruct.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/examples/lepton_none_llama_3_1_8b_instruct.yaml -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/examples/lepton_vllm_llama_3_1_8b_instruct.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/examples/lepton_vllm_llama_3_1_8b_instruct.yaml -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/examples/local_auto_export_llama_3_1_8b_instruct.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/examples/local_auto_export_llama_3_1_8b_instruct.yaml -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/examples/local_custom_config_seed_oss_36b_instruct.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/examples/local_custom_config_seed_oss_36b_instruct.yaml -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/examples/local_limit_samples.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/examples/local_limit_samples.yaml -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/examples/local_llama_3_1_8b_instruct.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/examples/local_llama_3_1_8b_instruct.yaml -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/examples/local_llama_3_1_8b_instruct_limit_samples.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/examples/local_llama_3_1_8b_instruct_limit_samples.yaml -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/examples/local_nvidia_nemotron_nano_9b_v2.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/examples/local_nvidia_nemotron_nano_9b_v2.yaml -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/examples/local_vllm_llama_3_1_8b_logprobs.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/examples/local_vllm_llama_3_1_8b_logprobs.yaml -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/examples/notebooks/nemo-evaluator-launcher-api.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/examples/notebooks/nemo-evaluator-launcher-api.ipynb -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/examples/slurm_auto_export_llama_3_1_8b_instruct.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/examples/slurm_auto_export_llama_3_1_8b_instruct.yaml -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/examples/slurm_llama_3_1_8b_instruct.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/examples/slurm_llama_3_1_8b_instruct.yaml -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/examples/slurm_llama_3_1_8b_instruct_hf.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/examples/slurm_llama_3_1_8b_instruct_hf.yaml -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/examples/slurm_multinode_vllm_dp_nemotron_nano_9b_v2.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/examples/slurm_multinode_vllm_dp_nemotron_nano_9b_v2.yaml -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/examples/slurm_multinode_vllm_ray_tp_pp_deepseek_r1.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/examples/slurm_multinode_vllm_ray_tp_pp_deepseek_r1.yaml -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/examples/slurm_no_deployment_llama_3_1_8b_instruct.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/examples/slurm_no_deployment_llama_3_1_8b_instruct.yaml -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/examples/slurm_no_deployment_llama_nemotron_super_v1_nemotron_benchmarks.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/examples/slurm_no_deployment_llama_nemotron_super_v1_nemotron_benchmarks.yaml -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/mypy.ini: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/mypy.ini -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/pyproject.toml -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/pytest.ini: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/pytest.ini -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/scripts/add_spdx.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/scripts/add_spdx.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/scripts/check_logging_imports.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/scripts/check_logging_imports.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/scripts/generate_configs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/scripts/generate_configs.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/__init__.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/api/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/api/__init__.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/api/functional.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/api/functional.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/api/types.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/api/types.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/api/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/api/utils.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/cli/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/cli/__init__.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/cli/export.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/cli/export.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/cli/info.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/cli/info.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/cli/kill.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/cli/kill.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/cli/logs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/cli/logs.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/cli/ls_runs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/cli/ls_runs.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/cli/ls_tasks.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/cli/ls_tasks.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/cli/main.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/cli/main.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/cli/run.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/cli/run.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/cli/status.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/cli/status.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/cli/version.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/cli/version.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/common/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/common/__init__.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/common/execdb.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/common/execdb.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/common/helpers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/common/helpers.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/common/logging_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/common/logging_utils.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/common/mapping.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/common/mapping.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/common/printing_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/common/printing_utils.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/configs/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/configs/__init__.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/configs/default.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/configs/default.yaml -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/configs/deployment/generic.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/configs/deployment/generic.yaml -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/configs/deployment/nim.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/configs/deployment/nim.yaml -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/configs/deployment/none.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/configs/deployment/none.yaml -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/configs/deployment/sglang.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/configs/deployment/sglang.yaml -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/configs/deployment/trtllm.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/configs/deployment/trtllm.yaml -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/configs/deployment/vllm.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/configs/deployment/vllm.yaml -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/configs/execution/lepton/default.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/configs/execution/lepton/default.yaml -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/configs/execution/local.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/configs/execution/local.yaml -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/configs/execution/slurm/default.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/configs/execution/slurm/default.yaml -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/executors/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/executors/__init__.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/executors/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/executors/base.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/executors/lepton/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/executors/lepton/__init__.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/executors/lepton/deployment_helpers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/executors/lepton/deployment_helpers.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/executors/lepton/executor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/executors/lepton/executor.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/executors/lepton/job_helpers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/executors/lepton/job_helpers.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/executors/local/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/executors/local/__init__.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/executors/local/executor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/executors/local/executor.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/executors/local/run.template.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/executors/local/run.template.sh -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/executors/registry.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/executors/registry.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/executors/slurm/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/executors/slurm/__init__.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/executors/slurm/executor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/executors/slurm/executor.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/executors/slurm/proxy.cfg.template: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/executors/slurm/proxy.cfg.template -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/exporters/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/exporters/__init__.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/exporters/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/exporters/base.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/exporters/gsheets.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/exporters/gsheets.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/exporters/local.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/exporters/local.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/exporters/mlflow.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/exporters/mlflow.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/exporters/registry.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/exporters/registry.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/exporters/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/exporters/utils.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/exporters/wandb.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/exporters/wandb.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/package_info.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/package_info.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/resources/mapping.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/src/nemo_evaluator_launcher/resources/mapping.toml -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/tests/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/tests/unit_tests/Launch_Unit_Tests.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/tests/unit_tests/Launch_Unit_Tests.sh -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/tests/unit_tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/tests/unit_tests/__init__.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/tests/unit_tests/conftest.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/tests/unit_tests/conftest.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/tests/unit_tests/exporters/test_base_exporter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/tests/unit_tests/exporters/test_base_exporter.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/tests/unit_tests/exporters/test_config_extraction.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/tests/unit_tests/exporters/test_config_extraction.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/tests/unit_tests/exporters/test_export_api_cli.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/tests/unit_tests/exporters/test_export_api_cli.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/tests/unit_tests/exporters/test_export_results_functional.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/tests/unit_tests/exporters/test_export_results_functional.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/tests/unit_tests/exporters/test_exporter_registry.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/tests/unit_tests/exporters/test_exporter_registry.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/tests/unit_tests/exporters/test_gsheets_exporter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/tests/unit_tests/exporters/test_gsheets_exporter.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/tests/unit_tests/exporters/test_local_exporter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/tests/unit_tests/exporters/test_local_exporter.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/tests/unit_tests/exporters/test_mlflow_exporter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/tests/unit_tests/exporters/test_mlflow_exporter.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/tests/unit_tests/exporters/test_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/tests/unit_tests/exporters/test_utils.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/tests/unit_tests/exporters/test_wandb_exporter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/tests/unit_tests/exporters/test_wandb_exporter.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/tests/unit_tests/test_api_module.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/tests/unit_tests/test_api_module.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/tests/unit_tests/test_cli_export_extended.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/tests/unit_tests/test_cli_export_extended.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/tests/unit_tests/test_cli_info.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/tests/unit_tests/test_cli_info.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/tests/unit_tests/test_cli_integration.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/tests/unit_tests/test_cli_integration.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/tests/unit_tests/test_cli_logs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/tests/unit_tests/test_cli_logs.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/tests/unit_tests/test_cli_ls_integration.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/tests/unit_tests/test_cli_ls_integration.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/tests/unit_tests/test_cli_ls_runs_extended.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/tests/unit_tests/test_cli_ls_runs_extended.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/tests/unit_tests/test_cli_ls_tasks.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/tests/unit_tests/test_cli_ls_tasks.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/tests/unit_tests/test_cli_version.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/tests/unit_tests/test_cli_version.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/tests/unit_tests/test_example_configs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/tests/unit_tests/test_example_configs.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/tests/unit_tests/test_execdb.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/tests/unit_tests/test_execdb.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/tests/unit_tests/test_get_eval_factory_command.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/tests/unit_tests/test_get_eval_factory_command.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/tests/unit_tests/test_helpers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/tests/unit_tests/test_helpers.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/tests/unit_tests/test_init.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/tests/unit_tests/test_init.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/tests/unit_tests/test_lepton_deployment_helpers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/tests/unit_tests/test_lepton_deployment_helpers.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/tests/unit_tests/test_lepton_executor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/tests/unit_tests/test_lepton_executor.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/tests/unit_tests/test_lepton_job_helpers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/tests/unit_tests/test_lepton_job_helpers.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/tests/unit_tests/test_local_executor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/tests/unit_tests/test_local_executor.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/tests/unit_tests/test_logging_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/tests/unit_tests/test_logging_utils.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/tests/unit_tests/test_mapping.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/tests/unit_tests/test_mapping.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/tests/unit_tests/test_mapping_extended.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/tests/unit_tests/test_mapping_extended.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/tests/unit_tests/test_mapping_url.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/tests/unit_tests/test_mapping_url.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/tests/unit_tests/test_printing_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/tests/unit_tests/test_printing_utils.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/tests/unit_tests/test_slurm_executor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/tests/unit_tests/test_slurm_executor.py -------------------------------------------------------------------------------- /packages/nemo-evaluator-launcher/uv.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator-launcher/uv.lock -------------------------------------------------------------------------------- /packages/nemo-evaluator/.python-version: -------------------------------------------------------------------------------- 1 | 3.12 -------------------------------------------------------------------------------- /packages/nemo-evaluator/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /packages/nemo-evaluator/LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/LICENSE -------------------------------------------------------------------------------- /packages/nemo-evaluator/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/README.md -------------------------------------------------------------------------------- /packages/nemo-evaluator/examples/notebooks/contamination_detection_demo.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/examples/notebooks/contamination_detection_demo.ipynb -------------------------------------------------------------------------------- /packages/nemo-evaluator/pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/pyproject.toml -------------------------------------------------------------------------------- /packages/nemo-evaluator/src/nemo_evaluator/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/src/nemo_evaluator/__init__.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/src/nemo_evaluator/adapters/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/src/nemo_evaluator/adapters/__init__.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/src/nemo_evaluator/adapters/adapter_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/src/nemo_evaluator/adapters/adapter_config.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/src/nemo_evaluator/adapters/caching/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /packages/nemo-evaluator/src/nemo_evaluator/adapters/caching/diskcaching.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/src/nemo_evaluator/adapters/caching/diskcaching.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/src/nemo_evaluator/adapters/decorators.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/src/nemo_evaluator/adapters/decorators.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/src/nemo_evaluator/adapters/interceptors/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/src/nemo_evaluator/adapters/interceptors/__init__.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/src/nemo_evaluator/adapters/interceptors/caching_interceptor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/src/nemo_evaluator/adapters/interceptors/caching_interceptor.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/src/nemo_evaluator/adapters/interceptors/endpoint_interceptor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/src/nemo_evaluator/adapters/interceptors/endpoint_interceptor.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/src/nemo_evaluator/adapters/interceptors/logging_interceptor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/src/nemo_evaluator/adapters/interceptors/logging_interceptor.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/src/nemo_evaluator/adapters/interceptors/payload_modifier_interceptor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/src/nemo_evaluator/adapters/interceptors/payload_modifier_interceptor.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/src/nemo_evaluator/adapters/interceptors/progress_tracking_interceptor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/src/nemo_evaluator/adapters/interceptors/progress_tracking_interceptor.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/src/nemo_evaluator/adapters/interceptors/raise_client_error_interceptor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/src/nemo_evaluator/adapters/interceptors/raise_client_error_interceptor.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/src/nemo_evaluator/adapters/interceptors/reasoning_interceptor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/src/nemo_evaluator/adapters/interceptors/reasoning_interceptor.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/src/nemo_evaluator/adapters/interceptors/response_stats_interceptor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/src/nemo_evaluator/adapters/interceptors/response_stats_interceptor.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/src/nemo_evaluator/adapters/interceptors/system_message_interceptor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/src/nemo_evaluator/adapters/interceptors/system_message_interceptor.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/src/nemo_evaluator/adapters/registry.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/src/nemo_evaluator/adapters/registry.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/src/nemo_evaluator/adapters/reports/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/src/nemo_evaluator/adapters/reports/__init__.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/src/nemo_evaluator/adapters/reports/post_eval_report_hook.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/src/nemo_evaluator/adapters/reports/post_eval_report_hook.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/src/nemo_evaluator/adapters/reports/templates/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /packages/nemo-evaluator/src/nemo_evaluator/adapters/reports/templates/simple_template.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/src/nemo_evaluator/adapters/reports/templates/simple_template.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/src/nemo_evaluator/adapters/server.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/src/nemo_evaluator/adapters/server.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/src/nemo_evaluator/adapters/types.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/src/nemo_evaluator/adapters/types.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/src/nemo_evaluator/api/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/src/nemo_evaluator/api/__init__.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/src/nemo_evaluator/api/api_dataclasses.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/src/nemo_evaluator/api/api_dataclasses.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/src/nemo_evaluator/cli/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /packages/nemo-evaluator/src/nemo_evaluator/cli/example.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/src/nemo_evaluator/cli/example.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/src/nemo_evaluator/core/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/src/nemo_evaluator/core/__init__.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/src/nemo_evaluator/core/entrypoint.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/src/nemo_evaluator/core/entrypoint.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/src/nemo_evaluator/core/evaluate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/src/nemo_evaluator/core/evaluate.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/src/nemo_evaluator/core/input.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/src/nemo_evaluator/core/input.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/src/nemo_evaluator/core/resources.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/src/nemo_evaluator/core/resources.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/src/nemo_evaluator/core/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/src/nemo_evaluator/core/utils.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/src/nemo_evaluator/logging/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/src/nemo_evaluator/logging/__init__.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/src/nemo_evaluator/logging/config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/src/nemo_evaluator/logging/config.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/src/nemo_evaluator/logging/context.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/src/nemo_evaluator/logging/context.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/src/nemo_evaluator/logging/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/src/nemo_evaluator/logging/utils.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/src/nemo_evaluator/package_info.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/src/nemo_evaluator/package_info.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/src/nemo_evaluator/resources/framework_tpl.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/src/nemo_evaluator/resources/framework_tpl.yml -------------------------------------------------------------------------------- /packages/nemo-evaluator/src/nemo_evaluator/resources/init_tpl.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/src/nemo_evaluator/resources/init_tpl.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/src/nemo_evaluator/resources/output_tpl.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/src/nemo_evaluator/resources/output_tpl.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /packages/nemo-evaluator/tests/functional_tests/Launch_Functional_Tests.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/tests/functional_tests/Launch_Functional_Tests.sh -------------------------------------------------------------------------------- /packages/nemo-evaluator/tests/functional_tests/e2e/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/tests/functional_tests/e2e/__init__.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/tests/functional_tests/e2e/conftest.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/tests/functional_tests/e2e/conftest.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/tests/functional_tests/e2e/test_eval_factory_fake_endpoint.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/tests/functional_tests/e2e/test_eval_factory_fake_endpoint.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/tests/functional_tests/e2e/test_interceptor_integration.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/tests/functional_tests/e2e/test_interceptor_integration.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/tests/functional_tests/installed_modules/1/core_evals/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /packages/nemo-evaluator/tests/functional_tests/installed_modules/1/core_evals/dummy_framework/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/tests/functional_tests/installed_modules/1/core_evals/dummy_framework/__init__.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/tests/functional_tests/installed_modules/1/core_evals/dummy_framework/dummy_entrypoint.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/tests/functional_tests/installed_modules/1/core_evals/dummy_framework/dummy_entrypoint.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/tests/functional_tests/installed_modules/1/core_evals/dummy_framework/framework.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/tests/functional_tests/installed_modules/1/core_evals/dummy_framework/framework.yml -------------------------------------------------------------------------------- /packages/nemo-evaluator/tests/functional_tests/installed_modules/1/core_evals/dummy_framework/output.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/tests/functional_tests/installed_modules/1/core_evals/dummy_framework/output.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/tests/functional_tests/test_cli_example.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/tests/functional_tests/test_cli_example.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/tests/functional_tests/test_entrypoint.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/tests/functional_tests/test_entrypoint.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/tests/unit_tests/Launch_Unit_Tests.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/tests/unit_tests/Launch_Unit_Tests.sh -------------------------------------------------------------------------------- /packages/nemo-evaluator/tests/unit_tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /packages/nemo-evaluator/tests/unit_tests/adapters/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /packages/nemo-evaluator/tests/unit_tests/adapters/caching/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /packages/nemo-evaluator/tests/unit_tests/adapters/caching/test_diskcaching.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/tests/unit_tests/adapters/caching/test_diskcaching.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/tests/unit_tests/adapters/conftest.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/tests/unit_tests/adapters/conftest.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/tests/unit_tests/adapters/interceptors/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /packages/nemo-evaluator/tests/unit_tests/adapters/interceptors/test_logging_interceptor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/tests/unit_tests/adapters/interceptors/test_logging_interceptor.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/tests/unit_tests/adapters/interceptors/test_payload_modifier.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/tests/unit_tests/adapters/interceptors/test_payload_modifier.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/tests/unit_tests/adapters/interceptors/test_progress_tracking_interceptor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/tests/unit_tests/adapters/interceptors/test_progress_tracking_interceptor.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/tests/unit_tests/adapters/interceptors/test_raise_client_error_interceptor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/tests/unit_tests/adapters/interceptors/test_raise_client_error_interceptor.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/tests/unit_tests/adapters/interceptors/test_reasoning.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/tests/unit_tests/adapters/interceptors/test_reasoning.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/tests/unit_tests/adapters/interceptors/test_response_stats_interceptor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/tests/unit_tests/adapters/interceptors/test_response_stats_interceptor.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/tests/unit_tests/adapters/interceptors/test_system_message.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/tests/unit_tests/adapters/interceptors/test_system_message.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/tests/unit_tests/adapters/test_adapter_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/tests/unit_tests/adapters/test_adapter_config.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/tests/unit_tests/adapters/test_adapter_server.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/tests/unit_tests/adapters/test_adapter_server.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/tests/unit_tests/adapters/test_caching.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/tests/unit_tests/adapters/test_caching.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/tests/unit_tests/adapters/test_logging_integration.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/tests/unit_tests/adapters/test_logging_integration.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/tests/unit_tests/adapters/test_multiple_instances.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/tests/unit_tests/adapters/test_multiple_instances.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/tests/unit_tests/adapters/test_post_eval_hooks.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/tests/unit_tests/adapters/test_post_eval_hooks.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/tests/unit_tests/adapters/test_post_eval_report_hook.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/tests/unit_tests/adapters/test_post_eval_report_hook.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/tests/unit_tests/adapters/test_registry.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/tests/unit_tests/adapters/test_registry.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/tests/unit_tests/adapters/test_yaml_interceptor_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/tests/unit_tests/adapters/test_yaml_interceptor_config.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/tests/unit_tests/adapters/testing_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/tests/unit_tests/adapters/testing_utils.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/tests/unit_tests/core/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /packages/nemo-evaluator/tests/unit_tests/core/test_evaluate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/tests/unit_tests/core/test_evaluate.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/tests/unit_tests/core/test_input.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/tests/unit_tests/core/test_input.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/tests/unit_tests/core/test_metadata.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/tests/unit_tests/core/test_metadata.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/tests/unit_tests/logging/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/tests/unit_tests/logging/__init__.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/tests/unit_tests/logging/test_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/tests/unit_tests/logging/test_utils.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/tests/unit_tests/utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /packages/nemo-evaluator/tests/unit_tests/utils/test_cache_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/tests/unit_tests/utils/test_cache_utils.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/tests/unit_tests/utils/test_input_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/tests/unit_tests/utils/test_input_utils.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/tests/unit_tests/utils/test_run_command.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/tests/unit_tests/utils/test_run_command.py -------------------------------------------------------------------------------- /packages/nemo-evaluator/uv.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/packages/nemo-evaluator/uv.lock -------------------------------------------------------------------------------- /scripts/evaluation_with_nemo_run.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/scripts/evaluation_with_nemo_run.py -------------------------------------------------------------------------------- /scripts/helpers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/scripts/helpers.py -------------------------------------------------------------------------------- /scripts/validate_doc_snippets.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/scripts/validate_doc_snippets.py -------------------------------------------------------------------------------- /tests/integration_tests/Launch_Integration_Tests.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/tests/integration_tests/Launch_Integration_Tests.sh -------------------------------------------------------------------------------- /tests/integration_tests/nemo_fw/test_deployment.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/tests/integration_tests/nemo_fw/test_deployment.py -------------------------------------------------------------------------------- /tests/integration_tests/nemo_fw/test_hf_ray.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/tests/integration_tests/nemo_fw/test_hf_ray.py -------------------------------------------------------------------------------- /tests/integration_tests/nemo_fw/test_nemo2_ray.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/tests/integration_tests/nemo_fw/test_nemo2_ray.py -------------------------------------------------------------------------------- /tests/integration_tests/nemo_fw/test_nemo2_triton.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/tests/integration_tests/nemo_fw/test_nemo2_triton.py -------------------------------------------------------------------------------- /tests/integration_tests/nemo_fw/test_notebooks.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/tests/integration_tests/nemo_fw/test_notebooks.py -------------------------------------------------------------------------------- /tutorials/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/tutorials/README.md -------------------------------------------------------------------------------- /tutorials/mmlu.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/tutorials/mmlu.ipynb -------------------------------------------------------------------------------- /tutorials/simple-evals.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/tutorials/simple-evals.ipynb -------------------------------------------------------------------------------- /tutorials/wikitext.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVIDIA-NeMo/Evaluator/HEAD/tutorials/wikitext.ipynb --------------------------------------------------------------------------------