├── .gitignore ├── LICENSE ├── README.md ├── assets ├── composition_alignment.png ├── concept_alignment.png ├── everything_boxplot.png ├── human_eval.png └── human_eval_plot.png ├── configs ├── callbacks │ ├── custom.yaml │ ├── default.yaml │ ├── none.yaml │ └── wandb.yaml ├── datamodule │ ├── imagenet_vqa_json.yaml │ ├── imagenet_w_outliers.yaml │ ├── imgtxt.yaml │ ├── pacs_all_domain.yaml │ └── vqa_json.yaml ├── debug │ ├── default.yaml │ ├── limit_batches.yaml │ ├── overfit.yaml │ ├── profiler.yaml │ ├── step.yaml │ └── test_only.yaml ├── experiment │ ├── clip.yaml │ ├── erm_all_domain.yaml │ ├── erm_imagenet_w_outliers.yaml │ ├── example.yaml │ ├── imagenet_vqa_json_no_attribute.yaml │ └── vqa_json.yaml ├── external │ ├── default.yaml │ └── easyfsl.yaml ├── hparams_search │ └── mnist_optuna.yaml ├── local │ └── .gitkeep ├── log_dir │ ├── debug.yaml │ ├── default.yaml │ └── evaluation.yaml ├── logger │ ├── comet.yaml │ ├── csv.yaml │ ├── many_loggers.yaml │ ├── mlflow.yaml │ ├── neptune.yaml │ ├── tensorboard.yaml │ └── wandb.yaml ├── model │ ├── clip.yaml │ ├── erm.yaml │ ├── erm_w_outliers.yaml │ ├── imagenet_w_outliers.yaml │ └── vqa.yaml ├── test.yaml ├── train.yaml └── trainer │ ├── ddp.yaml │ └── default.yaml ├── extras ├── get_uncertainty_composition.py ├── get_uncertainty_domain.py └── get_uncertainty_objects.py ├── requirements.txt ├── scripts ├── composition_alignment │ └── test_compositions.sh ├── concept_alignment │ ├── test_domain.sh │ ├── test_imagenet.sh │ └── test_imagenet_composition.sh ├── download_checkpoints.sh └── download_data.sh ├── setup.cfg ├── src ├── __init__.py ├── callbacks │ ├── custom_evals.py │ └── wandb_callbacks.py ├── datamodules │ ├── __init__.py │ ├── components │ │ ├── __init__.py │ │ ├── laplacianofgaussianfiltering.py │ │ └── transforms.py │ ├── imagenet_datamodule_w_outliers.py │ ├── imagenet_vqa_json.py │ ├── imgtxt_datamodule.py │ ├── pacs_all_domain_datamodule.py │ └── vqa_json_datamodule.py ├── external │ └── utils.py ├── models │ ├── __init__.py │ ├── clip.py │ ├── components │ │ └── __init__.py │ ├── erm.py │ ├── erm_w_outliers.py │ ├── imagenet_w_outliers.py │ └── vqa.py ├── testing_pipeline.py ├── training_pipeline.py ├── utils │ └── __init__.py └── vendor │ └── __init__.py ├── test.py └── train.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/README.md -------------------------------------------------------------------------------- /assets/composition_alignment.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/assets/composition_alignment.png -------------------------------------------------------------------------------- /assets/concept_alignment.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/assets/concept_alignment.png -------------------------------------------------------------------------------- /assets/everything_boxplot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/assets/everything_boxplot.png -------------------------------------------------------------------------------- /assets/human_eval.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/assets/human_eval.png -------------------------------------------------------------------------------- /assets/human_eval_plot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/assets/human_eval_plot.png -------------------------------------------------------------------------------- /configs/callbacks/custom.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/configs/callbacks/custom.yaml -------------------------------------------------------------------------------- /configs/callbacks/default.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/configs/callbacks/default.yaml -------------------------------------------------------------------------------- /configs/callbacks/none.yaml: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /configs/callbacks/wandb.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/configs/callbacks/wandb.yaml -------------------------------------------------------------------------------- /configs/datamodule/imagenet_vqa_json.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/configs/datamodule/imagenet_vqa_json.yaml -------------------------------------------------------------------------------- /configs/datamodule/imagenet_w_outliers.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/configs/datamodule/imagenet_w_outliers.yaml -------------------------------------------------------------------------------- /configs/datamodule/imgtxt.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/configs/datamodule/imgtxt.yaml -------------------------------------------------------------------------------- /configs/datamodule/pacs_all_domain.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/configs/datamodule/pacs_all_domain.yaml -------------------------------------------------------------------------------- /configs/datamodule/vqa_json.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/configs/datamodule/vqa_json.yaml -------------------------------------------------------------------------------- /configs/debug/default.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/configs/debug/default.yaml -------------------------------------------------------------------------------- /configs/debug/limit_batches.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/configs/debug/limit_batches.yaml -------------------------------------------------------------------------------- /configs/debug/overfit.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/configs/debug/overfit.yaml -------------------------------------------------------------------------------- /configs/debug/profiler.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/configs/debug/profiler.yaml -------------------------------------------------------------------------------- /configs/debug/step.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/configs/debug/step.yaml -------------------------------------------------------------------------------- /configs/debug/test_only.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/configs/debug/test_only.yaml -------------------------------------------------------------------------------- /configs/experiment/clip.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/configs/experiment/clip.yaml -------------------------------------------------------------------------------- /configs/experiment/erm_all_domain.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/configs/experiment/erm_all_domain.yaml -------------------------------------------------------------------------------- /configs/experiment/erm_imagenet_w_outliers.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/configs/experiment/erm_imagenet_w_outliers.yaml -------------------------------------------------------------------------------- /configs/experiment/example.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/configs/experiment/example.yaml -------------------------------------------------------------------------------- /configs/experiment/imagenet_vqa_json_no_attribute.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/configs/experiment/imagenet_vqa_json_no_attribute.yaml -------------------------------------------------------------------------------- /configs/experiment/vqa_json.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/configs/experiment/vqa_json.yaml -------------------------------------------------------------------------------- /configs/external/default.yaml: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /configs/external/easyfsl.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/configs/external/easyfsl.yaml -------------------------------------------------------------------------------- /configs/hparams_search/mnist_optuna.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/configs/hparams_search/mnist_optuna.yaml -------------------------------------------------------------------------------- /configs/local/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /configs/log_dir/debug.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/configs/log_dir/debug.yaml -------------------------------------------------------------------------------- /configs/log_dir/default.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/configs/log_dir/default.yaml -------------------------------------------------------------------------------- /configs/log_dir/evaluation.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/configs/log_dir/evaluation.yaml -------------------------------------------------------------------------------- /configs/logger/comet.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/configs/logger/comet.yaml -------------------------------------------------------------------------------- /configs/logger/csv.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/configs/logger/csv.yaml -------------------------------------------------------------------------------- /configs/logger/many_loggers.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/configs/logger/many_loggers.yaml -------------------------------------------------------------------------------- /configs/logger/mlflow.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/configs/logger/mlflow.yaml -------------------------------------------------------------------------------- /configs/logger/neptune.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/configs/logger/neptune.yaml -------------------------------------------------------------------------------- /configs/logger/tensorboard.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/configs/logger/tensorboard.yaml -------------------------------------------------------------------------------- /configs/logger/wandb.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/configs/logger/wandb.yaml -------------------------------------------------------------------------------- /configs/model/clip.yaml: -------------------------------------------------------------------------------- 1 | _target_: src.models.clip.CLIPModule 2 | device: cuda -------------------------------------------------------------------------------- /configs/model/erm.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/configs/model/erm.yaml -------------------------------------------------------------------------------- /configs/model/erm_w_outliers.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/configs/model/erm_w_outliers.yaml -------------------------------------------------------------------------------- /configs/model/imagenet_w_outliers.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/configs/model/imagenet_w_outliers.yaml -------------------------------------------------------------------------------- /configs/model/vqa.yaml: -------------------------------------------------------------------------------- 1 | _target_: src.models.vqa.ViLTModule -------------------------------------------------------------------------------- /configs/test.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/configs/test.yaml -------------------------------------------------------------------------------- /configs/train.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/configs/train.yaml -------------------------------------------------------------------------------- /configs/trainer/ddp.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/configs/trainer/ddp.yaml -------------------------------------------------------------------------------- /configs/trainer/default.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/configs/trainer/default.yaml -------------------------------------------------------------------------------- /extras/get_uncertainty_composition.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/extras/get_uncertainty_composition.py -------------------------------------------------------------------------------- /extras/get_uncertainty_domain.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/extras/get_uncertainty_domain.py -------------------------------------------------------------------------------- /extras/get_uncertainty_objects.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/extras/get_uncertainty_objects.py -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/requirements.txt -------------------------------------------------------------------------------- /scripts/composition_alignment/test_compositions.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/scripts/composition_alignment/test_compositions.sh -------------------------------------------------------------------------------- /scripts/concept_alignment/test_domain.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/scripts/concept_alignment/test_domain.sh -------------------------------------------------------------------------------- /scripts/concept_alignment/test_imagenet.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/scripts/concept_alignment/test_imagenet.sh -------------------------------------------------------------------------------- /scripts/concept_alignment/test_imagenet_composition.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/scripts/concept_alignment/test_imagenet_composition.sh -------------------------------------------------------------------------------- /scripts/download_checkpoints.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/scripts/download_checkpoints.sh -------------------------------------------------------------------------------- /scripts/download_data.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/scripts/download_data.sh -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/setup.cfg -------------------------------------------------------------------------------- /src/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/callbacks/custom_evals.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/src/callbacks/custom_evals.py -------------------------------------------------------------------------------- /src/callbacks/wandb_callbacks.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/src/callbacks/wandb_callbacks.py -------------------------------------------------------------------------------- /src/datamodules/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/datamodules/components/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/datamodules/components/laplacianofgaussianfiltering.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/src/datamodules/components/laplacianofgaussianfiltering.py -------------------------------------------------------------------------------- /src/datamodules/components/transforms.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/src/datamodules/components/transforms.py -------------------------------------------------------------------------------- /src/datamodules/imagenet_datamodule_w_outliers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/src/datamodules/imagenet_datamodule_w_outliers.py -------------------------------------------------------------------------------- /src/datamodules/imagenet_vqa_json.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/src/datamodules/imagenet_vqa_json.py -------------------------------------------------------------------------------- /src/datamodules/imgtxt_datamodule.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/src/datamodules/imgtxt_datamodule.py -------------------------------------------------------------------------------- /src/datamodules/pacs_all_domain_datamodule.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/src/datamodules/pacs_all_domain_datamodule.py -------------------------------------------------------------------------------- /src/datamodules/vqa_json_datamodule.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/src/datamodules/vqa_json_datamodule.py -------------------------------------------------------------------------------- /src/external/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/src/external/utils.py -------------------------------------------------------------------------------- /src/models/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/models/clip.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/src/models/clip.py -------------------------------------------------------------------------------- /src/models/components/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/models/erm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/src/models/erm.py -------------------------------------------------------------------------------- /src/models/erm_w_outliers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/src/models/erm_w_outliers.py -------------------------------------------------------------------------------- /src/models/imagenet_w_outliers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/src/models/imagenet_w_outliers.py -------------------------------------------------------------------------------- /src/models/vqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/src/models/vqa.py -------------------------------------------------------------------------------- /src/testing_pipeline.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/src/testing_pipeline.py -------------------------------------------------------------------------------- /src/training_pipeline.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/src/training_pipeline.py -------------------------------------------------------------------------------- /src/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/src/utils/__init__.py -------------------------------------------------------------------------------- /src/vendor/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/src/vendor/__init__.py -------------------------------------------------------------------------------- /test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/test.py -------------------------------------------------------------------------------- /train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ConceptBed/evaluations/HEAD/train.py --------------------------------------------------------------------------------