├── .gitignore ├── LICENSE ├── README.md ├── hparams_figures.ipynb ├── images ├── calib_l1.svg ├── crps.svg └── pce_and_rel_diags_mix_nll.svg ├── main_figures.ipynb ├── papers_vs_datasets.ipynb ├── requirements.txt ├── run.py ├── setup.cfg ├── setup.py └── uq ├── __init__.py ├── analysis ├── constants.py ├── dataframes.py ├── plot_boxplots.py ├── plot_calib_all_datasets.py ├── plot_cd_diagram.py ├── plot_cohen_d.py ├── plot_metrics.py ├── plot_metrics_regul.py ├── plot_reliability_diagrams.py └── stats.py ├── configs ├── callbacks.py ├── config.py ├── dataset_groups.py ├── experiments.py ├── general.py ├── loggers.py ├── trainer.py └── tuning.py ├── datamodules ├── base_datamodule.py ├── openml │ ├── download_openml.py │ └── openml_module.py ├── preprocessing.py ├── toy │ └── toy_module.py └── uci │ ├── download_uci.py │ └── uci_module.py ├── metrics ├── calibration.py ├── dist_metrics_computer.py ├── general.py ├── independence.py ├── metrics_collector.py ├── metrics_computer.py ├── quantile_metrics_computer.py └── quantiles.py ├── models ├── base_module.py ├── build_baseline.py ├── default_trainer.py ├── dist │ ├── base_dist_module.py │ ├── ic_module.py │ ├── marginal_regul_module.py │ └── oqr_module.py ├── general │ ├── interleaved_training.py │ ├── mlp.py │ └── post_hoc_calibration.py ├── pred_type │ ├── ecdf.py │ ├── linear_spline.py │ ├── mixture_dist.py │ ├── normal_dist.py │ ├── rational_spline.py │ ├── recalibrated_dist.py │ ├── reflected_dist.py │ ├── spline_dist.py │ └── truncated_dist.py ├── quantile │ ├── base_quantile_module.py │ ├── marginal_regul_module.py │ └── oqr_module.py └── regul │ ├── indicator.py │ ├── marginal_regul.py │ ├── oqr_regul.py │ └── temp_utils.py ├── runner.py ├── train.py └── utils ├── __init__.py ├── checkpoints.py ├── dist.py ├── fast_soft_sort ├── __init__.py ├── jax_ops.py ├── numpy_ops.py ├── pytorch_ops.py ├── tf_ops.py └── third_party │ ├── LICENSE │ ├── __init__.py │ └── isotonic.py ├── general.py ├── hparams.py ├── run_config.py ├── runs_storage.py └── torch_utils.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/README.md -------------------------------------------------------------------------------- /hparams_figures.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/hparams_figures.ipynb -------------------------------------------------------------------------------- /images/calib_l1.svg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/images/calib_l1.svg -------------------------------------------------------------------------------- /images/crps.svg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/images/crps.svg -------------------------------------------------------------------------------- /images/pce_and_rel_diags_mix_nll.svg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/images/pce_and_rel_diags_mix_nll.svg -------------------------------------------------------------------------------- /main_figures.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/main_figures.ipynb -------------------------------------------------------------------------------- /papers_vs_datasets.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/papers_vs_datasets.ipynb -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/requirements.txt -------------------------------------------------------------------------------- /run.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/run.py -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/setup.cfg -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/setup.py -------------------------------------------------------------------------------- /uq/__init__.py: -------------------------------------------------------------------------------- 1 | __version__: str = '1.0' 2 | -------------------------------------------------------------------------------- /uq/analysis/constants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/analysis/constants.py -------------------------------------------------------------------------------- /uq/analysis/dataframes.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/analysis/dataframes.py -------------------------------------------------------------------------------- /uq/analysis/plot_boxplots.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/analysis/plot_boxplots.py -------------------------------------------------------------------------------- /uq/analysis/plot_calib_all_datasets.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/analysis/plot_calib_all_datasets.py -------------------------------------------------------------------------------- /uq/analysis/plot_cd_diagram.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/analysis/plot_cd_diagram.py -------------------------------------------------------------------------------- /uq/analysis/plot_cohen_d.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/analysis/plot_cohen_d.py -------------------------------------------------------------------------------- /uq/analysis/plot_metrics.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/analysis/plot_metrics.py -------------------------------------------------------------------------------- /uq/analysis/plot_metrics_regul.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/analysis/plot_metrics_regul.py -------------------------------------------------------------------------------- /uq/analysis/plot_reliability_diagrams.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/analysis/plot_reliability_diagrams.py -------------------------------------------------------------------------------- /uq/analysis/stats.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/analysis/stats.py -------------------------------------------------------------------------------- /uq/configs/callbacks.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/configs/callbacks.py -------------------------------------------------------------------------------- /uq/configs/config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/configs/config.py -------------------------------------------------------------------------------- /uq/configs/dataset_groups.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/configs/dataset_groups.py -------------------------------------------------------------------------------- /uq/configs/experiments.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/configs/experiments.py -------------------------------------------------------------------------------- /uq/configs/general.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/configs/general.py -------------------------------------------------------------------------------- /uq/configs/loggers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/configs/loggers.py -------------------------------------------------------------------------------- /uq/configs/trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/configs/trainer.py -------------------------------------------------------------------------------- /uq/configs/tuning.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/configs/tuning.py -------------------------------------------------------------------------------- /uq/datamodules/base_datamodule.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/datamodules/base_datamodule.py -------------------------------------------------------------------------------- /uq/datamodules/openml/download_openml.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/datamodules/openml/download_openml.py -------------------------------------------------------------------------------- /uq/datamodules/openml/openml_module.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/datamodules/openml/openml_module.py -------------------------------------------------------------------------------- /uq/datamodules/preprocessing.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/datamodules/preprocessing.py -------------------------------------------------------------------------------- /uq/datamodules/toy/toy_module.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/datamodules/toy/toy_module.py -------------------------------------------------------------------------------- /uq/datamodules/uci/download_uci.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/datamodules/uci/download_uci.py -------------------------------------------------------------------------------- /uq/datamodules/uci/uci_module.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/datamodules/uci/uci_module.py -------------------------------------------------------------------------------- /uq/metrics/calibration.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/metrics/calibration.py -------------------------------------------------------------------------------- /uq/metrics/dist_metrics_computer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/metrics/dist_metrics_computer.py -------------------------------------------------------------------------------- /uq/metrics/general.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/metrics/general.py -------------------------------------------------------------------------------- /uq/metrics/independence.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/metrics/independence.py -------------------------------------------------------------------------------- /uq/metrics/metrics_collector.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/metrics/metrics_collector.py -------------------------------------------------------------------------------- /uq/metrics/metrics_computer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/metrics/metrics_computer.py -------------------------------------------------------------------------------- /uq/metrics/quantile_metrics_computer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/metrics/quantile_metrics_computer.py -------------------------------------------------------------------------------- /uq/metrics/quantiles.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/metrics/quantiles.py -------------------------------------------------------------------------------- /uq/models/base_module.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/models/base_module.py -------------------------------------------------------------------------------- /uq/models/build_baseline.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/models/build_baseline.py -------------------------------------------------------------------------------- /uq/models/default_trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/models/default_trainer.py -------------------------------------------------------------------------------- /uq/models/dist/base_dist_module.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/models/dist/base_dist_module.py -------------------------------------------------------------------------------- /uq/models/dist/ic_module.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/models/dist/ic_module.py -------------------------------------------------------------------------------- /uq/models/dist/marginal_regul_module.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/models/dist/marginal_regul_module.py -------------------------------------------------------------------------------- /uq/models/dist/oqr_module.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/models/dist/oqr_module.py -------------------------------------------------------------------------------- /uq/models/general/interleaved_training.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/models/general/interleaved_training.py -------------------------------------------------------------------------------- /uq/models/general/mlp.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/models/general/mlp.py -------------------------------------------------------------------------------- /uq/models/general/post_hoc_calibration.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/models/general/post_hoc_calibration.py -------------------------------------------------------------------------------- /uq/models/pred_type/ecdf.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/models/pred_type/ecdf.py -------------------------------------------------------------------------------- /uq/models/pred_type/linear_spline.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/models/pred_type/linear_spline.py -------------------------------------------------------------------------------- /uq/models/pred_type/mixture_dist.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/models/pred_type/mixture_dist.py -------------------------------------------------------------------------------- /uq/models/pred_type/normal_dist.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/models/pred_type/normal_dist.py -------------------------------------------------------------------------------- /uq/models/pred_type/rational_spline.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/models/pred_type/rational_spline.py -------------------------------------------------------------------------------- /uq/models/pred_type/recalibrated_dist.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/models/pred_type/recalibrated_dist.py -------------------------------------------------------------------------------- /uq/models/pred_type/reflected_dist.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/models/pred_type/reflected_dist.py -------------------------------------------------------------------------------- /uq/models/pred_type/spline_dist.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/models/pred_type/spline_dist.py -------------------------------------------------------------------------------- /uq/models/pred_type/truncated_dist.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/models/pred_type/truncated_dist.py -------------------------------------------------------------------------------- /uq/models/quantile/base_quantile_module.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/models/quantile/base_quantile_module.py -------------------------------------------------------------------------------- /uq/models/quantile/marginal_regul_module.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/models/quantile/marginal_regul_module.py -------------------------------------------------------------------------------- /uq/models/quantile/oqr_module.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/models/quantile/oqr_module.py -------------------------------------------------------------------------------- /uq/models/regul/indicator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/models/regul/indicator.py -------------------------------------------------------------------------------- /uq/models/regul/marginal_regul.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/models/regul/marginal_regul.py -------------------------------------------------------------------------------- /uq/models/regul/oqr_regul.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/models/regul/oqr_regul.py -------------------------------------------------------------------------------- /uq/models/regul/temp_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/models/regul/temp_utils.py -------------------------------------------------------------------------------- /uq/runner.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/runner.py -------------------------------------------------------------------------------- /uq/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/train.py -------------------------------------------------------------------------------- /uq/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/utils/__init__.py -------------------------------------------------------------------------------- /uq/utils/checkpoints.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/utils/checkpoints.py -------------------------------------------------------------------------------- /uq/utils/dist.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/utils/dist.py -------------------------------------------------------------------------------- /uq/utils/fast_soft_sort/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /uq/utils/fast_soft_sort/jax_ops.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/utils/fast_soft_sort/jax_ops.py -------------------------------------------------------------------------------- /uq/utils/fast_soft_sort/numpy_ops.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/utils/fast_soft_sort/numpy_ops.py -------------------------------------------------------------------------------- /uq/utils/fast_soft_sort/pytorch_ops.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/utils/fast_soft_sort/pytorch_ops.py -------------------------------------------------------------------------------- /uq/utils/fast_soft_sort/tf_ops.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/utils/fast_soft_sort/tf_ops.py -------------------------------------------------------------------------------- /uq/utils/fast_soft_sort/third_party/LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/utils/fast_soft_sort/third_party/LICENSE -------------------------------------------------------------------------------- /uq/utils/fast_soft_sort/third_party/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /uq/utils/fast_soft_sort/third_party/isotonic.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/utils/fast_soft_sort/third_party/isotonic.py -------------------------------------------------------------------------------- /uq/utils/general.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/utils/general.py -------------------------------------------------------------------------------- /uq/utils/hparams.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/utils/hparams.py -------------------------------------------------------------------------------- /uq/utils/run_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/utils/run_config.py -------------------------------------------------------------------------------- /uq/utils/runs_storage.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/utils/runs_storage.py -------------------------------------------------------------------------------- /uq/utils/torch_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Vekteur/probabilistic-calibration-study/HEAD/uq/utils/torch_utils.py --------------------------------------------------------------------------------