├── .gitignore ├── LICENSE ├── README.md ├── counterfactuals ├── configs │ ├── cifar_blackwhite.yaml │ ├── cifar_rectangular.yaml │ ├── living17_polkadots.yaml │ ├── living17_spider.yaml │ ├── waterbirds_humanface.yaml │ └── waterbirds_yellowcolor.yaml ├── eval_cifar_blackwhite.py ├── eval_cifar_rectangular.py ├── eval_living17_polkadots.py ├── eval_living17_spiderweb.py ├── eval_waterbirds_humanface.py ├── eval_waterbirds_yellowcolor.py └── metadata │ ├── polkadots.jpg │ └── spiderweb.jpg ├── modeldiff ├── __init__.py ├── differ.py └── pca.py ├── notebooks ├── README.md ├── api_example.ipynb ├── data_augmentation.ipynb ├── data_augmentation_trak.ipynb ├── imagenet_pretraining.ipynb ├── metadata │ ├── cifar10_labels.pkl │ ├── cifar_accs.pkl │ ├── cifar_dog_dataframe.pkl │ ├── cifar_truck_dataframe.pkl │ ├── living17_salamander_dataframe.pkl │ ├── living17_spider_dataframe.pkl │ ├── living17_train_metadata.df │ ├── living17_val_classification_profile.pkl │ ├── living17_val_metadata.df │ ├── living17_val_pointwise_accs.df │ ├── waterbirds_accs_pct100.pkl │ ├── waterbirds_face_dataframe.pkl │ ├── waterbirds_labels.pkl │ └── waterbirds_yellow_dataframe.pkl ├── sgd_parameters.ipynb └── src │ ├── __init__.py │ ├── data │ ├── __init__.py │ ├── datasets.py │ ├── ffcv_pipelines.py │ ├── metadata │ │ ├── train_metadata.df │ │ └── val_metadata.df │ ├── transforms.py │ └── waterbirds │ │ ├── dataset_utils.py │ │ └── generate_waterbirds_with_faces.py │ ├── models │ ├── __init__.py │ ├── cifar_resnet.py │ ├── fcn.py │ ├── multihead.py │ └── resnet9.py │ ├── train │ ├── __init__.py │ └── trainer.py │ └── utils │ ├── __init__.py │ ├── common_utils.py │ ├── data_utils.py │ ├── datamodels_utils.py │ ├── eval_utils.py │ ├── plot_utils.py │ ├── run_utils.py │ └── train_utils.py ├── requirements.txt ├── scripts ├── download_living17_checkpoints.sh └── download_scores.sh ├── setup.py ├── static ├── case_studies.jpg └── visual_summary.png └── tests ├── __init__.py ├── test_integration.py └── utils.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/README.md -------------------------------------------------------------------------------- /counterfactuals/configs/cifar_blackwhite.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/counterfactuals/configs/cifar_blackwhite.yaml -------------------------------------------------------------------------------- /counterfactuals/configs/cifar_rectangular.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/counterfactuals/configs/cifar_rectangular.yaml -------------------------------------------------------------------------------- /counterfactuals/configs/living17_polkadots.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/counterfactuals/configs/living17_polkadots.yaml -------------------------------------------------------------------------------- /counterfactuals/configs/living17_spider.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/counterfactuals/configs/living17_spider.yaml -------------------------------------------------------------------------------- /counterfactuals/configs/waterbirds_humanface.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/counterfactuals/configs/waterbirds_humanface.yaml -------------------------------------------------------------------------------- /counterfactuals/configs/waterbirds_yellowcolor.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/counterfactuals/configs/waterbirds_yellowcolor.yaml -------------------------------------------------------------------------------- /counterfactuals/eval_cifar_blackwhite.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/counterfactuals/eval_cifar_blackwhite.py -------------------------------------------------------------------------------- /counterfactuals/eval_cifar_rectangular.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/counterfactuals/eval_cifar_rectangular.py -------------------------------------------------------------------------------- /counterfactuals/eval_living17_polkadots.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/counterfactuals/eval_living17_polkadots.py -------------------------------------------------------------------------------- /counterfactuals/eval_living17_spiderweb.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/counterfactuals/eval_living17_spiderweb.py -------------------------------------------------------------------------------- /counterfactuals/eval_waterbirds_humanface.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/counterfactuals/eval_waterbirds_humanface.py -------------------------------------------------------------------------------- /counterfactuals/eval_waterbirds_yellowcolor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/counterfactuals/eval_waterbirds_yellowcolor.py -------------------------------------------------------------------------------- /counterfactuals/metadata/polkadots.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/counterfactuals/metadata/polkadots.jpg -------------------------------------------------------------------------------- /counterfactuals/metadata/spiderweb.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/counterfactuals/metadata/spiderweb.jpg -------------------------------------------------------------------------------- /modeldiff/__init__.py: -------------------------------------------------------------------------------- 1 | from .differ import ModelDiff 2 | -------------------------------------------------------------------------------- /modeldiff/differ.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/modeldiff/differ.py -------------------------------------------------------------------------------- /modeldiff/pca.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/modeldiff/pca.py -------------------------------------------------------------------------------- /notebooks/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/notebooks/README.md -------------------------------------------------------------------------------- /notebooks/api_example.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/notebooks/api_example.ipynb -------------------------------------------------------------------------------- /notebooks/data_augmentation.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/notebooks/data_augmentation.ipynb -------------------------------------------------------------------------------- /notebooks/data_augmentation_trak.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/notebooks/data_augmentation_trak.ipynb -------------------------------------------------------------------------------- /notebooks/imagenet_pretraining.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/notebooks/imagenet_pretraining.ipynb -------------------------------------------------------------------------------- /notebooks/metadata/cifar10_labels.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/notebooks/metadata/cifar10_labels.pkl -------------------------------------------------------------------------------- /notebooks/metadata/cifar_accs.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/notebooks/metadata/cifar_accs.pkl -------------------------------------------------------------------------------- /notebooks/metadata/cifar_dog_dataframe.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/notebooks/metadata/cifar_dog_dataframe.pkl -------------------------------------------------------------------------------- /notebooks/metadata/cifar_truck_dataframe.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/notebooks/metadata/cifar_truck_dataframe.pkl -------------------------------------------------------------------------------- /notebooks/metadata/living17_salamander_dataframe.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/notebooks/metadata/living17_salamander_dataframe.pkl -------------------------------------------------------------------------------- /notebooks/metadata/living17_spider_dataframe.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/notebooks/metadata/living17_spider_dataframe.pkl -------------------------------------------------------------------------------- /notebooks/metadata/living17_train_metadata.df: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/notebooks/metadata/living17_train_metadata.df -------------------------------------------------------------------------------- /notebooks/metadata/living17_val_classification_profile.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/notebooks/metadata/living17_val_classification_profile.pkl -------------------------------------------------------------------------------- /notebooks/metadata/living17_val_metadata.df: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/notebooks/metadata/living17_val_metadata.df -------------------------------------------------------------------------------- /notebooks/metadata/living17_val_pointwise_accs.df: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/notebooks/metadata/living17_val_pointwise_accs.df -------------------------------------------------------------------------------- /notebooks/metadata/waterbirds_accs_pct100.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/notebooks/metadata/waterbirds_accs_pct100.pkl -------------------------------------------------------------------------------- /notebooks/metadata/waterbirds_face_dataframe.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/notebooks/metadata/waterbirds_face_dataframe.pkl -------------------------------------------------------------------------------- /notebooks/metadata/waterbirds_labels.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/notebooks/metadata/waterbirds_labels.pkl -------------------------------------------------------------------------------- /notebooks/metadata/waterbirds_yellow_dataframe.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/notebooks/metadata/waterbirds_yellow_dataframe.pkl -------------------------------------------------------------------------------- /notebooks/sgd_parameters.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/notebooks/sgd_parameters.ipynb -------------------------------------------------------------------------------- /notebooks/src/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /notebooks/src/data/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /notebooks/src/data/datasets.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/notebooks/src/data/datasets.py -------------------------------------------------------------------------------- /notebooks/src/data/ffcv_pipelines.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/notebooks/src/data/ffcv_pipelines.py -------------------------------------------------------------------------------- /notebooks/src/data/metadata/train_metadata.df: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/notebooks/src/data/metadata/train_metadata.df -------------------------------------------------------------------------------- /notebooks/src/data/metadata/val_metadata.df: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/notebooks/src/data/metadata/val_metadata.df -------------------------------------------------------------------------------- /notebooks/src/data/transforms.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/notebooks/src/data/transforms.py -------------------------------------------------------------------------------- /notebooks/src/data/waterbirds/dataset_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/notebooks/src/data/waterbirds/dataset_utils.py -------------------------------------------------------------------------------- /notebooks/src/data/waterbirds/generate_waterbirds_with_faces.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/notebooks/src/data/waterbirds/generate_waterbirds_with_faces.py -------------------------------------------------------------------------------- /notebooks/src/models/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /notebooks/src/models/cifar_resnet.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/notebooks/src/models/cifar_resnet.py -------------------------------------------------------------------------------- /notebooks/src/models/fcn.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/notebooks/src/models/fcn.py -------------------------------------------------------------------------------- /notebooks/src/models/multihead.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/notebooks/src/models/multihead.py -------------------------------------------------------------------------------- /notebooks/src/models/resnet9.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/notebooks/src/models/resnet9.py -------------------------------------------------------------------------------- /notebooks/src/train/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /notebooks/src/train/trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/notebooks/src/train/trainer.py -------------------------------------------------------------------------------- /notebooks/src/utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /notebooks/src/utils/common_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/notebooks/src/utils/common_utils.py -------------------------------------------------------------------------------- /notebooks/src/utils/data_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/notebooks/src/utils/data_utils.py -------------------------------------------------------------------------------- /notebooks/src/utils/datamodels_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/notebooks/src/utils/datamodels_utils.py -------------------------------------------------------------------------------- /notebooks/src/utils/eval_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/notebooks/src/utils/eval_utils.py -------------------------------------------------------------------------------- /notebooks/src/utils/plot_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/notebooks/src/utils/plot_utils.py -------------------------------------------------------------------------------- /notebooks/src/utils/run_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/notebooks/src/utils/run_utils.py -------------------------------------------------------------------------------- /notebooks/src/utils/train_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/notebooks/src/utils/train_utils.py -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/requirements.txt -------------------------------------------------------------------------------- /scripts/download_living17_checkpoints.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/scripts/download_living17_checkpoints.sh -------------------------------------------------------------------------------- /scripts/download_scores.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/scripts/download_scores.sh -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/setup.py -------------------------------------------------------------------------------- /static/case_studies.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/static/case_studies.jpg -------------------------------------------------------------------------------- /static/visual_summary.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/static/visual_summary.png -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/test_integration.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/tests/test_integration.py -------------------------------------------------------------------------------- /tests/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MadryLab/modeldiff/HEAD/tests/utils.py --------------------------------------------------------------------------------