├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ └── feature_request.md ├── dependabot.yml └── workflows │ ├── ci.yaml │ ├── documentation.yaml │ ├── linting.yml │ └── update_version_json.py ├── .gitignore ├── .pre-commit-config.yaml ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── assets ├── README_PYPI.md └── images │ ├── COPYRIGHT.md │ ├── black_box_graphic.png │ ├── black_box_graphic_dark.png │ ├── black_box_output4.png │ ├── cranberry_jam.png │ ├── horizontal_logo.png │ ├── horizontal_logo_large.png │ ├── judges_graphic.png │ ├── judges_graphic_dark.png │ ├── panel_output2.png │ ├── stacked_logo.png │ ├── uqensemble_generate_score.png │ ├── uqensemble_generate_score_dark.png │ ├── uqensemble_output2.png │ ├── uqensemble_tune.png │ ├── uqlm_flow_ds.png │ ├── uqlm_flow_ds_dark.png │ ├── white_box_graphic.png │ ├── white_box_graphic_dark.png │ └── white_box_output2.png ├── docs ├── Makefile ├── make.bat └── source │ ├── _notebooks │ ├── examples │ │ ├── black_box_demo.ipynb │ │ ├── ensemble_off_the_shelf_demo.ipynb │ │ ├── ensemble_tuning_demo.ipynb │ │ ├── judges_demo.ipynb │ │ ├── multimodal_demo.ipynb │ │ ├── score_calibration_demo.ipynb │ │ ├── semantic_density_demo.ipynb │ │ ├── semantic_entropy_demo.ipynb │ │ ├── white_box_multi_generation_demo.ipynb │ │ └── white_box_single_generation_demo.ipynb │ └── index.rst │ ├── _static │ ├── custom.css │ ├── images │ │ ├── COPYRIGHT.md │ │ ├── black_box_graphic.png │ │ ├── black_box_graphic_dark.png │ │ ├── black_box_output4.png │ │ ├── favicon │ │ │ ├── android-chrome-192x192.png │ │ │ ├── android-chrome-512x512.png │ │ │ ├── apple-touch-icon.png │ │ │ ├── favicon-16x16.png │ │ │ ├── favicon-32x32.png │ │ │ └── favicon.ico │ │ ├── horizontal_logo.png │ │ ├── horizontal_logo_large.png │ │ ├── horizontal_logo_no_bg.png │ │ ├── judges_graphic.png │ │ ├── judges_graphic_dark.png │ │ ├── no_image.png │ │ ├── panel_output2.png │ │ ├── uqensemble_generate_score.png │ │ ├── uqensemble_generate_score_dark.png │ │ ├── uqensemble_output2.png │ │ ├── uqlm_flow_ds.png │ │ ├── uqlm_flow_ds_dark.png │ │ ├── white_box_graphic.png │ │ ├── white_box_graphic_dark.png │ │ └── white_box_output2.png │ └── research │ │ └── logo.png │ ├── _templates │ ├── base.rst │ ├── class.rst │ └── module.rst │ ├── api.rst │ ├── conf.py │ ├── contribute.rst │ ├── faqs.rst │ ├── getstarted.rst │ ├── index.rst │ └── refs.bib ├── examples ├── black_box_demo.ipynb ├── ensemble_off_the_shelf_demo.ipynb ├── ensemble_tuning_demo.ipynb ├── judges_demo.ipynb ├── multimodal_demo.ipynb ├── score_calibration_demo.ipynb ├── semantic_density_demo.ipynb ├── semantic_entropy_demo.ipynb ├── uqe_config_tuned.json ├── white_box_multi_generation_demo.ipynb └── white_box_single_generation_demo.ipynb ├── poetry.lock ├── pyproject.toml ├── tests ├── __init__.py ├── data │ ├── scorers │ │ ├── DATA_COPYRIGHT.md │ │ ├── blackbox_results_file.json │ │ ├── bsdetector_results_file.json │ │ ├── ensemble_results_file.json │ │ ├── generate_data_blackbox.py │ │ ├── generate_data_ensemble.py │ │ ├── generate_data_llmjudge.py │ │ ├── generate_data_semanticentropy.py │ │ ├── generate_data_whitebox.py │ │ ├── llmjudge_results_file.json │ │ ├── semanticdensity_results_file.json │ │ ├── semanticentropy_results_file.json │ │ ├── test_data_panelquantifier.json │ │ └── whitebox_results_file.json │ └── similarity │ │ ├── DATA_COPYRIGHT.md │ │ ├── generate_data_similarity.py │ │ └── similarity_results_file.json ├── test_blackboxuq.py ├── test_calibrators.py ├── test_ensemble.py ├── test_grader.py ├── test_llm_config.py ├── test_llmjudge.py ├── test_llmpanel.py ├── test_load_example_dataset.py ├── test_logprobs_scorer.py ├── test_nli.py ├── test_p_true.py ├── test_postprocessor.py ├── test_responsegenerator.py ├── test_sampled_logprobs.py ├── test_semanticdensity.py ├── test_semanticentropy.py ├── test_similarity.py ├── test_top_logprobs.py ├── test_tuner.py ├── test_uncertainty.py ├── test_utils_plots.py └── test_whiteboxuq.py └── uqlm ├── __init__.py ├── black_box ├── __init__.py ├── baseclass │ ├── __init__ .py │ └── similarity_scorer.py ├── bert.py ├── consistency.py ├── cosine.py └── match.py ├── calibration ├── __init__.py ├── evaluate.py └── score_calibrator.py ├── judges ├── __init__.py └── judge.py ├── nli ├── __init__.py ├── cluster.py └── nli.py ├── resources └── __init__.py ├── scorers ├── __init__.py ├── baseclass │ ├── __init__.py │ └── uncertainty.py ├── black_box.py ├── density.py ├── ensemble.py ├── entropy.py ├── panel.py └── white_box.py ├── utils ├── __init__.py ├── dataloader.py ├── display.py ├── grader.py ├── llm_config.py ├── plots.py ├── postprocessors.py ├── prompts.py ├── response_generator.py ├── results.py ├── tuner.py └── warn.py └── white_box ├── __init__.py ├── baseclass ├── __init__.py └── logprobs_scorer.py ├── p_true.py ├── sampled_logprobs.py ├── single_logprobs.py └── top_logprobs.py /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/.github/ISSUE_TEMPLATE/bug_report.md -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/.github/ISSUE_TEMPLATE/feature_request.md -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/.github/dependabot.yml -------------------------------------------------------------------------------- /.github/workflows/ci.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/.github/workflows/ci.yaml -------------------------------------------------------------------------------- /.github/workflows/documentation.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/.github/workflows/documentation.yaml -------------------------------------------------------------------------------- /.github/workflows/linting.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/.github/workflows/linting.yml -------------------------------------------------------------------------------- /.github/workflows/update_version_json.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/.github/workflows/update_version_json.py -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/.gitignore -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/.pre-commit-config.yaml -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/CODE_OF_CONDUCT.md -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/CONTRIBUTING.md -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/README.md -------------------------------------------------------------------------------- /assets/README_PYPI.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/assets/README_PYPI.md -------------------------------------------------------------------------------- /assets/images/COPYRIGHT.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/assets/images/COPYRIGHT.md -------------------------------------------------------------------------------- /assets/images/black_box_graphic.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/assets/images/black_box_graphic.png -------------------------------------------------------------------------------- /assets/images/black_box_graphic_dark.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/assets/images/black_box_graphic_dark.png -------------------------------------------------------------------------------- /assets/images/black_box_output4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/assets/images/black_box_output4.png -------------------------------------------------------------------------------- /assets/images/cranberry_jam.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/assets/images/cranberry_jam.png -------------------------------------------------------------------------------- /assets/images/horizontal_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/assets/images/horizontal_logo.png -------------------------------------------------------------------------------- /assets/images/horizontal_logo_large.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/assets/images/horizontal_logo_large.png -------------------------------------------------------------------------------- /assets/images/judges_graphic.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/assets/images/judges_graphic.png -------------------------------------------------------------------------------- /assets/images/judges_graphic_dark.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/assets/images/judges_graphic_dark.png -------------------------------------------------------------------------------- /assets/images/panel_output2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/assets/images/panel_output2.png -------------------------------------------------------------------------------- /assets/images/stacked_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/assets/images/stacked_logo.png -------------------------------------------------------------------------------- /assets/images/uqensemble_generate_score.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/assets/images/uqensemble_generate_score.png -------------------------------------------------------------------------------- /assets/images/uqensemble_generate_score_dark.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/assets/images/uqensemble_generate_score_dark.png -------------------------------------------------------------------------------- /assets/images/uqensemble_output2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/assets/images/uqensemble_output2.png -------------------------------------------------------------------------------- /assets/images/uqensemble_tune.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/assets/images/uqensemble_tune.png -------------------------------------------------------------------------------- /assets/images/uqlm_flow_ds.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/assets/images/uqlm_flow_ds.png -------------------------------------------------------------------------------- /assets/images/uqlm_flow_ds_dark.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/assets/images/uqlm_flow_ds_dark.png -------------------------------------------------------------------------------- /assets/images/white_box_graphic.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/assets/images/white_box_graphic.png -------------------------------------------------------------------------------- /assets/images/white_box_graphic_dark.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/assets/images/white_box_graphic_dark.png -------------------------------------------------------------------------------- /assets/images/white_box_output2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/assets/images/white_box_output2.png -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/docs/Makefile -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/docs/make.bat -------------------------------------------------------------------------------- /docs/source/_notebooks/examples/black_box_demo.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/docs/source/_notebooks/examples/black_box_demo.ipynb -------------------------------------------------------------------------------- /docs/source/_notebooks/examples/ensemble_off_the_shelf_demo.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/docs/source/_notebooks/examples/ensemble_off_the_shelf_demo.ipynb -------------------------------------------------------------------------------- /docs/source/_notebooks/examples/ensemble_tuning_demo.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/docs/source/_notebooks/examples/ensemble_tuning_demo.ipynb -------------------------------------------------------------------------------- /docs/source/_notebooks/examples/judges_demo.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/docs/source/_notebooks/examples/judges_demo.ipynb -------------------------------------------------------------------------------- /docs/source/_notebooks/examples/multimodal_demo.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/docs/source/_notebooks/examples/multimodal_demo.ipynb -------------------------------------------------------------------------------- /docs/source/_notebooks/examples/score_calibration_demo.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/docs/source/_notebooks/examples/score_calibration_demo.ipynb -------------------------------------------------------------------------------- /docs/source/_notebooks/examples/semantic_density_demo.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/docs/source/_notebooks/examples/semantic_density_demo.ipynb -------------------------------------------------------------------------------- /docs/source/_notebooks/examples/semantic_entropy_demo.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/docs/source/_notebooks/examples/semantic_entropy_demo.ipynb -------------------------------------------------------------------------------- /docs/source/_notebooks/examples/white_box_multi_generation_demo.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/docs/source/_notebooks/examples/white_box_multi_generation_demo.ipynb -------------------------------------------------------------------------------- /docs/source/_notebooks/examples/white_box_single_generation_demo.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/docs/source/_notebooks/examples/white_box_single_generation_demo.ipynb -------------------------------------------------------------------------------- /docs/source/_notebooks/index.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/docs/source/_notebooks/index.rst -------------------------------------------------------------------------------- /docs/source/_static/custom.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/docs/source/_static/custom.css -------------------------------------------------------------------------------- /docs/source/_static/images/COPYRIGHT.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/docs/source/_static/images/COPYRIGHT.md -------------------------------------------------------------------------------- /docs/source/_static/images/black_box_graphic.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/docs/source/_static/images/black_box_graphic.png -------------------------------------------------------------------------------- /docs/source/_static/images/black_box_graphic_dark.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/docs/source/_static/images/black_box_graphic_dark.png -------------------------------------------------------------------------------- /docs/source/_static/images/black_box_output4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/docs/source/_static/images/black_box_output4.png -------------------------------------------------------------------------------- /docs/source/_static/images/favicon/android-chrome-192x192.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/docs/source/_static/images/favicon/android-chrome-192x192.png -------------------------------------------------------------------------------- /docs/source/_static/images/favicon/android-chrome-512x512.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/docs/source/_static/images/favicon/android-chrome-512x512.png -------------------------------------------------------------------------------- /docs/source/_static/images/favicon/apple-touch-icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/docs/source/_static/images/favicon/apple-touch-icon.png -------------------------------------------------------------------------------- /docs/source/_static/images/favicon/favicon-16x16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/docs/source/_static/images/favicon/favicon-16x16.png -------------------------------------------------------------------------------- /docs/source/_static/images/favicon/favicon-32x32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/docs/source/_static/images/favicon/favicon-32x32.png -------------------------------------------------------------------------------- /docs/source/_static/images/favicon/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/docs/source/_static/images/favicon/favicon.ico -------------------------------------------------------------------------------- /docs/source/_static/images/horizontal_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/docs/source/_static/images/horizontal_logo.png -------------------------------------------------------------------------------- /docs/source/_static/images/horizontal_logo_large.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/docs/source/_static/images/horizontal_logo_large.png -------------------------------------------------------------------------------- /docs/source/_static/images/horizontal_logo_no_bg.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/docs/source/_static/images/horizontal_logo_no_bg.png -------------------------------------------------------------------------------- /docs/source/_static/images/judges_graphic.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/docs/source/_static/images/judges_graphic.png -------------------------------------------------------------------------------- /docs/source/_static/images/judges_graphic_dark.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/docs/source/_static/images/judges_graphic_dark.png -------------------------------------------------------------------------------- /docs/source/_static/images/no_image.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/docs/source/_static/images/no_image.png -------------------------------------------------------------------------------- /docs/source/_static/images/panel_output2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/docs/source/_static/images/panel_output2.png -------------------------------------------------------------------------------- /docs/source/_static/images/uqensemble_generate_score.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/docs/source/_static/images/uqensemble_generate_score.png -------------------------------------------------------------------------------- /docs/source/_static/images/uqensemble_generate_score_dark.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/docs/source/_static/images/uqensemble_generate_score_dark.png -------------------------------------------------------------------------------- /docs/source/_static/images/uqensemble_output2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/docs/source/_static/images/uqensemble_output2.png -------------------------------------------------------------------------------- /docs/source/_static/images/uqlm_flow_ds.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/docs/source/_static/images/uqlm_flow_ds.png -------------------------------------------------------------------------------- /docs/source/_static/images/uqlm_flow_ds_dark.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/docs/source/_static/images/uqlm_flow_ds_dark.png -------------------------------------------------------------------------------- /docs/source/_static/images/white_box_graphic.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/docs/source/_static/images/white_box_graphic.png -------------------------------------------------------------------------------- /docs/source/_static/images/white_box_graphic_dark.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/docs/source/_static/images/white_box_graphic_dark.png -------------------------------------------------------------------------------- /docs/source/_static/images/white_box_output2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/docs/source/_static/images/white_box_output2.png -------------------------------------------------------------------------------- /docs/source/_static/research/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/docs/source/_static/research/logo.png -------------------------------------------------------------------------------- /docs/source/_templates/base.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/docs/source/_templates/base.rst -------------------------------------------------------------------------------- /docs/source/_templates/class.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/docs/source/_templates/class.rst -------------------------------------------------------------------------------- /docs/source/_templates/module.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/docs/source/_templates/module.rst -------------------------------------------------------------------------------- /docs/source/api.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/docs/source/api.rst -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/docs/source/conf.py -------------------------------------------------------------------------------- /docs/source/contribute.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/docs/source/contribute.rst -------------------------------------------------------------------------------- /docs/source/faqs.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/docs/source/faqs.rst -------------------------------------------------------------------------------- /docs/source/getstarted.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/docs/source/getstarted.rst -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/docs/source/index.rst -------------------------------------------------------------------------------- /docs/source/refs.bib: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/docs/source/refs.bib -------------------------------------------------------------------------------- /examples/black_box_demo.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/examples/black_box_demo.ipynb -------------------------------------------------------------------------------- /examples/ensemble_off_the_shelf_demo.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/examples/ensemble_off_the_shelf_demo.ipynb -------------------------------------------------------------------------------- /examples/ensemble_tuning_demo.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/examples/ensemble_tuning_demo.ipynb -------------------------------------------------------------------------------- /examples/judges_demo.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/examples/judges_demo.ipynb -------------------------------------------------------------------------------- /examples/multimodal_demo.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/examples/multimodal_demo.ipynb -------------------------------------------------------------------------------- /examples/score_calibration_demo.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/examples/score_calibration_demo.ipynb -------------------------------------------------------------------------------- /examples/semantic_density_demo.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/examples/semantic_density_demo.ipynb -------------------------------------------------------------------------------- /examples/semantic_entropy_demo.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/examples/semantic_entropy_demo.ipynb -------------------------------------------------------------------------------- /examples/uqe_config_tuned.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/examples/uqe_config_tuned.json -------------------------------------------------------------------------------- /examples/white_box_multi_generation_demo.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/examples/white_box_multi_generation_demo.ipynb -------------------------------------------------------------------------------- /examples/white_box_single_generation_demo.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/examples/white_box_single_generation_demo.ipynb -------------------------------------------------------------------------------- /poetry.lock: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/poetry.lock -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/pyproject.toml -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/tests/__init__.py -------------------------------------------------------------------------------- /tests/data/scorers/DATA_COPYRIGHT.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/tests/data/scorers/DATA_COPYRIGHT.md -------------------------------------------------------------------------------- /tests/data/scorers/blackbox_results_file.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/tests/data/scorers/blackbox_results_file.json -------------------------------------------------------------------------------- /tests/data/scorers/bsdetector_results_file.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/tests/data/scorers/bsdetector_results_file.json -------------------------------------------------------------------------------- /tests/data/scorers/ensemble_results_file.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/tests/data/scorers/ensemble_results_file.json -------------------------------------------------------------------------------- /tests/data/scorers/generate_data_blackbox.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/tests/data/scorers/generate_data_blackbox.py -------------------------------------------------------------------------------- /tests/data/scorers/generate_data_ensemble.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/tests/data/scorers/generate_data_ensemble.py -------------------------------------------------------------------------------- /tests/data/scorers/generate_data_llmjudge.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/tests/data/scorers/generate_data_llmjudge.py -------------------------------------------------------------------------------- /tests/data/scorers/generate_data_semanticentropy.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/tests/data/scorers/generate_data_semanticentropy.py -------------------------------------------------------------------------------- /tests/data/scorers/generate_data_whitebox.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/tests/data/scorers/generate_data_whitebox.py -------------------------------------------------------------------------------- /tests/data/scorers/llmjudge_results_file.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/tests/data/scorers/llmjudge_results_file.json -------------------------------------------------------------------------------- /tests/data/scorers/semanticdensity_results_file.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/tests/data/scorers/semanticdensity_results_file.json -------------------------------------------------------------------------------- /tests/data/scorers/semanticentropy_results_file.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/tests/data/scorers/semanticentropy_results_file.json -------------------------------------------------------------------------------- /tests/data/scorers/test_data_panelquantifier.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/tests/data/scorers/test_data_panelquantifier.json -------------------------------------------------------------------------------- /tests/data/scorers/whitebox_results_file.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/tests/data/scorers/whitebox_results_file.json -------------------------------------------------------------------------------- /tests/data/similarity/DATA_COPYRIGHT.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/tests/data/similarity/DATA_COPYRIGHT.md -------------------------------------------------------------------------------- /tests/data/similarity/generate_data_similarity.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/tests/data/similarity/generate_data_similarity.py -------------------------------------------------------------------------------- /tests/data/similarity/similarity_results_file.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/tests/data/similarity/similarity_results_file.json -------------------------------------------------------------------------------- /tests/test_blackboxuq.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/tests/test_blackboxuq.py -------------------------------------------------------------------------------- /tests/test_calibrators.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/tests/test_calibrators.py -------------------------------------------------------------------------------- /tests/test_ensemble.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/tests/test_ensemble.py -------------------------------------------------------------------------------- /tests/test_grader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/tests/test_grader.py -------------------------------------------------------------------------------- /tests/test_llm_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/tests/test_llm_config.py -------------------------------------------------------------------------------- /tests/test_llmjudge.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/tests/test_llmjudge.py -------------------------------------------------------------------------------- /tests/test_llmpanel.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/tests/test_llmpanel.py -------------------------------------------------------------------------------- /tests/test_load_example_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/tests/test_load_example_dataset.py -------------------------------------------------------------------------------- /tests/test_logprobs_scorer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/tests/test_logprobs_scorer.py -------------------------------------------------------------------------------- /tests/test_nli.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/tests/test_nli.py -------------------------------------------------------------------------------- /tests/test_p_true.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/tests/test_p_true.py -------------------------------------------------------------------------------- /tests/test_postprocessor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/tests/test_postprocessor.py -------------------------------------------------------------------------------- /tests/test_responsegenerator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/tests/test_responsegenerator.py -------------------------------------------------------------------------------- /tests/test_sampled_logprobs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/tests/test_sampled_logprobs.py -------------------------------------------------------------------------------- /tests/test_semanticdensity.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/tests/test_semanticdensity.py -------------------------------------------------------------------------------- /tests/test_semanticentropy.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/tests/test_semanticentropy.py -------------------------------------------------------------------------------- /tests/test_similarity.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/tests/test_similarity.py -------------------------------------------------------------------------------- /tests/test_top_logprobs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/tests/test_top_logprobs.py -------------------------------------------------------------------------------- /tests/test_tuner.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/tests/test_tuner.py -------------------------------------------------------------------------------- /tests/test_uncertainty.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/tests/test_uncertainty.py -------------------------------------------------------------------------------- /tests/test_utils_plots.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/tests/test_utils_plots.py -------------------------------------------------------------------------------- /tests/test_whiteboxuq.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/tests/test_whiteboxuq.py -------------------------------------------------------------------------------- /uqlm/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/uqlm/__init__.py -------------------------------------------------------------------------------- /uqlm/black_box/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/uqlm/black_box/__init__.py -------------------------------------------------------------------------------- /uqlm/black_box/baseclass/__init__ .py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/uqlm/black_box/baseclass/__init__ .py -------------------------------------------------------------------------------- /uqlm/black_box/baseclass/similarity_scorer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/uqlm/black_box/baseclass/similarity_scorer.py -------------------------------------------------------------------------------- /uqlm/black_box/bert.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/uqlm/black_box/bert.py -------------------------------------------------------------------------------- /uqlm/black_box/consistency.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/uqlm/black_box/consistency.py -------------------------------------------------------------------------------- /uqlm/black_box/cosine.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/uqlm/black_box/cosine.py -------------------------------------------------------------------------------- /uqlm/black_box/match.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/uqlm/black_box/match.py -------------------------------------------------------------------------------- /uqlm/calibration/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/uqlm/calibration/__init__.py -------------------------------------------------------------------------------- /uqlm/calibration/evaluate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/uqlm/calibration/evaluate.py -------------------------------------------------------------------------------- /uqlm/calibration/score_calibrator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/uqlm/calibration/score_calibrator.py -------------------------------------------------------------------------------- /uqlm/judges/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/uqlm/judges/__init__.py -------------------------------------------------------------------------------- /uqlm/judges/judge.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/uqlm/judges/judge.py -------------------------------------------------------------------------------- /uqlm/nli/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/uqlm/nli/__init__.py -------------------------------------------------------------------------------- /uqlm/nli/cluster.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/uqlm/nli/cluster.py -------------------------------------------------------------------------------- /uqlm/nli/nli.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/uqlm/nli/nli.py -------------------------------------------------------------------------------- /uqlm/resources/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/uqlm/resources/__init__.py -------------------------------------------------------------------------------- /uqlm/scorers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/uqlm/scorers/__init__.py -------------------------------------------------------------------------------- /uqlm/scorers/baseclass/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/uqlm/scorers/baseclass/__init__.py -------------------------------------------------------------------------------- /uqlm/scorers/baseclass/uncertainty.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/uqlm/scorers/baseclass/uncertainty.py -------------------------------------------------------------------------------- /uqlm/scorers/black_box.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/uqlm/scorers/black_box.py -------------------------------------------------------------------------------- /uqlm/scorers/density.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/uqlm/scorers/density.py -------------------------------------------------------------------------------- /uqlm/scorers/ensemble.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/uqlm/scorers/ensemble.py -------------------------------------------------------------------------------- /uqlm/scorers/entropy.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/uqlm/scorers/entropy.py -------------------------------------------------------------------------------- /uqlm/scorers/panel.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/uqlm/scorers/panel.py -------------------------------------------------------------------------------- /uqlm/scorers/white_box.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/uqlm/scorers/white_box.py -------------------------------------------------------------------------------- /uqlm/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/uqlm/utils/__init__.py -------------------------------------------------------------------------------- /uqlm/utils/dataloader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/uqlm/utils/dataloader.py -------------------------------------------------------------------------------- /uqlm/utils/display.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/uqlm/utils/display.py -------------------------------------------------------------------------------- /uqlm/utils/grader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/uqlm/utils/grader.py -------------------------------------------------------------------------------- /uqlm/utils/llm_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/uqlm/utils/llm_config.py -------------------------------------------------------------------------------- /uqlm/utils/plots.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/uqlm/utils/plots.py -------------------------------------------------------------------------------- /uqlm/utils/postprocessors.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/uqlm/utils/postprocessors.py -------------------------------------------------------------------------------- /uqlm/utils/prompts.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/uqlm/utils/prompts.py -------------------------------------------------------------------------------- /uqlm/utils/response_generator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/uqlm/utils/response_generator.py -------------------------------------------------------------------------------- /uqlm/utils/results.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/uqlm/utils/results.py -------------------------------------------------------------------------------- /uqlm/utils/tuner.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/uqlm/utils/tuner.py -------------------------------------------------------------------------------- /uqlm/utils/warn.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/uqlm/utils/warn.py -------------------------------------------------------------------------------- /uqlm/white_box/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/uqlm/white_box/__init__.py -------------------------------------------------------------------------------- /uqlm/white_box/baseclass/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/uqlm/white_box/baseclass/__init__.py -------------------------------------------------------------------------------- /uqlm/white_box/baseclass/logprobs_scorer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/uqlm/white_box/baseclass/logprobs_scorer.py -------------------------------------------------------------------------------- /uqlm/white_box/p_true.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/uqlm/white_box/p_true.py -------------------------------------------------------------------------------- /uqlm/white_box/sampled_logprobs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/uqlm/white_box/sampled_logprobs.py -------------------------------------------------------------------------------- /uqlm/white_box/single_logprobs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/uqlm/white_box/single_logprobs.py -------------------------------------------------------------------------------- /uqlm/white_box/top_logprobs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvs-health/uqlm/HEAD/uqlm/white_box/top_logprobs.py --------------------------------------------------------------------------------