├── .gitignore ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── requirements.txt ├── robustness_metrics ├── __init__.py ├── bin │ ├── __init__.py │ ├── common.py │ ├── compute_report.py │ ├── compute_report_lib.py │ └── serialize.py ├── common │ ├── __init__.py │ ├── ops.py │ ├── pipeline_builder.py │ ├── pipeline_builder_test.py │ ├── registry.py │ ├── registry_test.py │ └── types.py ├── datasets │ ├── __init__.py │ ├── base.py │ ├── ood_detection.py │ ├── ood_detection_test.py │ ├── tfds.py │ └── tfds_test.py ├── import_test.py ├── metrics │ ├── __init__.py │ ├── base.py │ ├── base_test.py │ ├── diversity.py │ ├── diversity_test.py │ ├── information_criteria.py │ ├── information_criteria_test.py │ ├── retrieval.py │ ├── retrieval_test.py │ ├── serialization.py │ ├── serialization_test.py │ ├── synthetic.py │ ├── synthetic_test.py │ ├── timing.py │ ├── timing_test.py │ ├── uncertainty.py │ └── uncertainty_test.py ├── models │ ├── bit.py │ ├── clip.py │ ├── cnn_anti_alias.py │ ├── efficientnet_variants.py │ ├── imagenet.json │ ├── random_imagenet_numpy.py │ ├── rescaling.py │ ├── rescaling_test.py │ ├── resnext_wsl.py │ ├── simclr.py │ ├── taad.py │ ├── uncertainty_baselines.py │ ├── uniform_imagenet_tensorflow.py │ ├── vgg.py │ └── vit.py ├── projects │ ├── __init__.py │ └── revisiting_calibration │ │ ├── README.md │ │ ├── __init__.py │ │ ├── display.py │ │ ├── figures │ │ ├── __init__.py │ │ ├── acc_calib_regression.py │ │ ├── clean_imagenet_and_reliability_diags.py │ │ ├── clean_imagenet_temp_scaling.py │ │ ├── clean_imagenet_temp_scaling_bit_pretrain_comparison.py │ │ ├── ece_bias.py │ │ ├── imagenet_c.py │ │ ├── ood_comparison.py │ │ └── pretrain_size_steps_sweep.py │ │ ├── plotting.py │ │ ├── revisiting_calibration.ipynb │ │ └── utils.py └── reports │ ├── __init__.py │ ├── base.py │ ├── base_test.py │ ├── cifar_variants.py │ ├── cifar_variants_test.py │ ├── imagenet_variants.py │ ├── imagenet_variants_test.py │ ├── ood_detection.py │ ├── ood_detection_test.py │ ├── plex.py │ ├── plex_test.py │ └── synthetic_variants.py └── setup.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/.gitignore -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/CONTRIBUTING.md -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/README.md -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/requirements.txt -------------------------------------------------------------------------------- /robustness_metrics/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/__init__.py -------------------------------------------------------------------------------- /robustness_metrics/bin/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/bin/__init__.py -------------------------------------------------------------------------------- /robustness_metrics/bin/common.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/bin/common.py -------------------------------------------------------------------------------- /robustness_metrics/bin/compute_report.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/bin/compute_report.py -------------------------------------------------------------------------------- /robustness_metrics/bin/compute_report_lib.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/bin/compute_report_lib.py -------------------------------------------------------------------------------- /robustness_metrics/bin/serialize.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/bin/serialize.py -------------------------------------------------------------------------------- /robustness_metrics/common/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/common/__init__.py -------------------------------------------------------------------------------- /robustness_metrics/common/ops.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/common/ops.py -------------------------------------------------------------------------------- /robustness_metrics/common/pipeline_builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/common/pipeline_builder.py -------------------------------------------------------------------------------- /robustness_metrics/common/pipeline_builder_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/common/pipeline_builder_test.py -------------------------------------------------------------------------------- /robustness_metrics/common/registry.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/common/registry.py -------------------------------------------------------------------------------- /robustness_metrics/common/registry_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/common/registry_test.py -------------------------------------------------------------------------------- /robustness_metrics/common/types.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/common/types.py -------------------------------------------------------------------------------- /robustness_metrics/datasets/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/datasets/__init__.py -------------------------------------------------------------------------------- /robustness_metrics/datasets/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/datasets/base.py -------------------------------------------------------------------------------- /robustness_metrics/datasets/ood_detection.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/datasets/ood_detection.py -------------------------------------------------------------------------------- /robustness_metrics/datasets/ood_detection_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/datasets/ood_detection_test.py -------------------------------------------------------------------------------- /robustness_metrics/datasets/tfds.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/datasets/tfds.py -------------------------------------------------------------------------------- /robustness_metrics/datasets/tfds_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/datasets/tfds_test.py -------------------------------------------------------------------------------- /robustness_metrics/import_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/import_test.py -------------------------------------------------------------------------------- /robustness_metrics/metrics/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/metrics/__init__.py -------------------------------------------------------------------------------- /robustness_metrics/metrics/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/metrics/base.py -------------------------------------------------------------------------------- /robustness_metrics/metrics/base_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/metrics/base_test.py -------------------------------------------------------------------------------- /robustness_metrics/metrics/diversity.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/metrics/diversity.py -------------------------------------------------------------------------------- /robustness_metrics/metrics/diversity_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/metrics/diversity_test.py -------------------------------------------------------------------------------- /robustness_metrics/metrics/information_criteria.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/metrics/information_criteria.py -------------------------------------------------------------------------------- /robustness_metrics/metrics/information_criteria_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/metrics/information_criteria_test.py -------------------------------------------------------------------------------- /robustness_metrics/metrics/retrieval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/metrics/retrieval.py -------------------------------------------------------------------------------- /robustness_metrics/metrics/retrieval_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/metrics/retrieval_test.py -------------------------------------------------------------------------------- /robustness_metrics/metrics/serialization.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/metrics/serialization.py -------------------------------------------------------------------------------- /robustness_metrics/metrics/serialization_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/metrics/serialization_test.py -------------------------------------------------------------------------------- /robustness_metrics/metrics/synthetic.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/metrics/synthetic.py -------------------------------------------------------------------------------- /robustness_metrics/metrics/synthetic_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/metrics/synthetic_test.py -------------------------------------------------------------------------------- /robustness_metrics/metrics/timing.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/metrics/timing.py -------------------------------------------------------------------------------- /robustness_metrics/metrics/timing_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/metrics/timing_test.py -------------------------------------------------------------------------------- /robustness_metrics/metrics/uncertainty.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/metrics/uncertainty.py -------------------------------------------------------------------------------- /robustness_metrics/metrics/uncertainty_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/metrics/uncertainty_test.py -------------------------------------------------------------------------------- /robustness_metrics/models/bit.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/models/bit.py -------------------------------------------------------------------------------- /robustness_metrics/models/clip.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/models/clip.py -------------------------------------------------------------------------------- /robustness_metrics/models/cnn_anti_alias.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/models/cnn_anti_alias.py -------------------------------------------------------------------------------- /robustness_metrics/models/efficientnet_variants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/models/efficientnet_variants.py -------------------------------------------------------------------------------- /robustness_metrics/models/imagenet.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/models/imagenet.json -------------------------------------------------------------------------------- /robustness_metrics/models/random_imagenet_numpy.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/models/random_imagenet_numpy.py -------------------------------------------------------------------------------- /robustness_metrics/models/rescaling.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/models/rescaling.py -------------------------------------------------------------------------------- /robustness_metrics/models/rescaling_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/models/rescaling_test.py -------------------------------------------------------------------------------- /robustness_metrics/models/resnext_wsl.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/models/resnext_wsl.py -------------------------------------------------------------------------------- /robustness_metrics/models/simclr.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/models/simclr.py -------------------------------------------------------------------------------- /robustness_metrics/models/taad.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/models/taad.py -------------------------------------------------------------------------------- /robustness_metrics/models/uncertainty_baselines.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/models/uncertainty_baselines.py -------------------------------------------------------------------------------- /robustness_metrics/models/uniform_imagenet_tensorflow.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/models/uniform_imagenet_tensorflow.py -------------------------------------------------------------------------------- /robustness_metrics/models/vgg.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/models/vgg.py -------------------------------------------------------------------------------- /robustness_metrics/models/vit.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/models/vit.py -------------------------------------------------------------------------------- /robustness_metrics/projects/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/projects/__init__.py -------------------------------------------------------------------------------- /robustness_metrics/projects/revisiting_calibration/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/projects/revisiting_calibration/README.md -------------------------------------------------------------------------------- /robustness_metrics/projects/revisiting_calibration/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/projects/revisiting_calibration/__init__.py -------------------------------------------------------------------------------- /robustness_metrics/projects/revisiting_calibration/display.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/projects/revisiting_calibration/display.py -------------------------------------------------------------------------------- /robustness_metrics/projects/revisiting_calibration/figures/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/projects/revisiting_calibration/figures/__init__.py -------------------------------------------------------------------------------- /robustness_metrics/projects/revisiting_calibration/figures/acc_calib_regression.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/projects/revisiting_calibration/figures/acc_calib_regression.py -------------------------------------------------------------------------------- /robustness_metrics/projects/revisiting_calibration/figures/clean_imagenet_and_reliability_diags.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/projects/revisiting_calibration/figures/clean_imagenet_and_reliability_diags.py -------------------------------------------------------------------------------- /robustness_metrics/projects/revisiting_calibration/figures/clean_imagenet_temp_scaling.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/projects/revisiting_calibration/figures/clean_imagenet_temp_scaling.py -------------------------------------------------------------------------------- /robustness_metrics/projects/revisiting_calibration/figures/clean_imagenet_temp_scaling_bit_pretrain_comparison.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/projects/revisiting_calibration/figures/clean_imagenet_temp_scaling_bit_pretrain_comparison.py -------------------------------------------------------------------------------- /robustness_metrics/projects/revisiting_calibration/figures/ece_bias.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/projects/revisiting_calibration/figures/ece_bias.py -------------------------------------------------------------------------------- /robustness_metrics/projects/revisiting_calibration/figures/imagenet_c.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/projects/revisiting_calibration/figures/imagenet_c.py -------------------------------------------------------------------------------- /robustness_metrics/projects/revisiting_calibration/figures/ood_comparison.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/projects/revisiting_calibration/figures/ood_comparison.py -------------------------------------------------------------------------------- /robustness_metrics/projects/revisiting_calibration/figures/pretrain_size_steps_sweep.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/projects/revisiting_calibration/figures/pretrain_size_steps_sweep.py -------------------------------------------------------------------------------- /robustness_metrics/projects/revisiting_calibration/plotting.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/projects/revisiting_calibration/plotting.py -------------------------------------------------------------------------------- /robustness_metrics/projects/revisiting_calibration/revisiting_calibration.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/projects/revisiting_calibration/revisiting_calibration.ipynb -------------------------------------------------------------------------------- /robustness_metrics/projects/revisiting_calibration/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/projects/revisiting_calibration/utils.py -------------------------------------------------------------------------------- /robustness_metrics/reports/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/reports/__init__.py -------------------------------------------------------------------------------- /robustness_metrics/reports/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/reports/base.py -------------------------------------------------------------------------------- /robustness_metrics/reports/base_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/reports/base_test.py -------------------------------------------------------------------------------- /robustness_metrics/reports/cifar_variants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/reports/cifar_variants.py -------------------------------------------------------------------------------- /robustness_metrics/reports/cifar_variants_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/reports/cifar_variants_test.py -------------------------------------------------------------------------------- /robustness_metrics/reports/imagenet_variants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/reports/imagenet_variants.py -------------------------------------------------------------------------------- /robustness_metrics/reports/imagenet_variants_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/reports/imagenet_variants_test.py -------------------------------------------------------------------------------- /robustness_metrics/reports/ood_detection.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/reports/ood_detection.py -------------------------------------------------------------------------------- /robustness_metrics/reports/ood_detection_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/reports/ood_detection_test.py -------------------------------------------------------------------------------- /robustness_metrics/reports/plex.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/reports/plex.py -------------------------------------------------------------------------------- /robustness_metrics/reports/plex_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/reports/plex_test.py -------------------------------------------------------------------------------- /robustness_metrics/reports/synthetic_variants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/robustness_metrics/reports/synthetic_variants.py -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google-research/robustness_metrics/HEAD/setup.py --------------------------------------------------------------------------------