├── .gitignore ├── .gitmodules ├── README.md ├── aro ├── .gitignore ├── LICENSE ├── README.md ├── dataset_zoo │ ├── __init__.py │ ├── aro_datasets.py │ ├── constants.py │ ├── perturbations.py │ ├── retrieval.py │ └── utils.py ├── main_aro.py ├── main_retrieval.py ├── misc │ └── __init__.py ├── model_zoo │ ├── __init__.py │ ├── blip_models.py │ ├── blip_utils │ │ ├── README.md │ │ ├── blip.py │ │ ├── blip_itm.py │ │ ├── blip_pretrain.py │ │ ├── blip_retrieval.py │ │ ├── med.py │ │ ├── utils.py │ │ └── vit.py │ ├── clip_models.py │ ├── constants.py │ ├── flava.py │ ├── xvlm_models.py │ └── xvlm_utils │ │ ├── README.md │ │ ├── box_ops.py │ │ ├── clip_vit.py │ │ ├── swin_transformer.py │ │ ├── tokenization_bert.py │ │ ├── tokenization_roberta.py │ │ ├── vit.py │ │ ├── xbert.py │ │ ├── xroberta.py │ │ └── xvlm.py ├── notebooks │ └── Replicate ARO! VG-Relation, VG-Attribution.ipynb ├── scripts │ ├── create_environment.sh │ ├── reproduce_aro.sh │ └── reproduce_retrieval.sh └── temp_data │ ├── train_neg_clip.tsv │ └── valid_neg_clip.tsv ├── checkpoints ├── hardneg-checkpoint-5000 │ ├── optimizer.bin │ ├── pytorch_lora_weights.bin │ ├── random_states_0.pkl │ ├── results.txt │ └── scheduler.bin └── noneg-checkpoint-3000 │ ├── optimizer.bin │ ├── pytorch_lora_weights.bin │ ├── random_states_0.pkl │ ├── results.txt │ └── scheduler.bin ├── data ├── clevr │ └── captions │ │ ├── CLEVR_questions.json │ │ ├── CLEVR_scenes.json │ │ ├── binding_color_shape.json │ │ ├── binding_shape_color.json │ │ ├── pair_binding_color.json │ │ ├── pair_binding_size.json │ │ ├── recognition_color.json │ │ ├── recognition_shape.json │ │ └── spatial.json ├── flickr30k │ ├── val_top10_RN50x64.json │ └── val_top10_RN50x64_text.json ├── imagecode │ ├── train_data.json │ └── valid_data.json ├── pets │ └── classes.txt ├── svo │ ├── download.py │ ├── svo.json │ └── svo_probes.csv └── winoground │ └── data.json ├── datasets_loading.py ├── diffusion_itm.py ├── hard_neg_finetuning.py ├── mainfig.jpeg ├── requirements.txt ├── setup.sh └── utils.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/.gitignore -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/.gitmodules -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/README.md -------------------------------------------------------------------------------- /aro/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/aro/.gitignore -------------------------------------------------------------------------------- /aro/LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/aro/LICENSE -------------------------------------------------------------------------------- /aro/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/aro/README.md -------------------------------------------------------------------------------- /aro/dataset_zoo/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/aro/dataset_zoo/__init__.py -------------------------------------------------------------------------------- /aro/dataset_zoo/aro_datasets.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/aro/dataset_zoo/aro_datasets.py -------------------------------------------------------------------------------- /aro/dataset_zoo/constants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/aro/dataset_zoo/constants.py -------------------------------------------------------------------------------- /aro/dataset_zoo/perturbations.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/aro/dataset_zoo/perturbations.py -------------------------------------------------------------------------------- /aro/dataset_zoo/retrieval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/aro/dataset_zoo/retrieval.py -------------------------------------------------------------------------------- /aro/dataset_zoo/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/aro/dataset_zoo/utils.py -------------------------------------------------------------------------------- /aro/main_aro.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/aro/main_aro.py -------------------------------------------------------------------------------- /aro/main_retrieval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/aro/main_retrieval.py -------------------------------------------------------------------------------- /aro/misc/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/aro/misc/__init__.py -------------------------------------------------------------------------------- /aro/model_zoo/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/aro/model_zoo/__init__.py -------------------------------------------------------------------------------- /aro/model_zoo/blip_models.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/aro/model_zoo/blip_models.py -------------------------------------------------------------------------------- /aro/model_zoo/blip_utils/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/aro/model_zoo/blip_utils/README.md -------------------------------------------------------------------------------- /aro/model_zoo/blip_utils/blip.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/aro/model_zoo/blip_utils/blip.py -------------------------------------------------------------------------------- /aro/model_zoo/blip_utils/blip_itm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/aro/model_zoo/blip_utils/blip_itm.py -------------------------------------------------------------------------------- /aro/model_zoo/blip_utils/blip_pretrain.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/aro/model_zoo/blip_utils/blip_pretrain.py -------------------------------------------------------------------------------- /aro/model_zoo/blip_utils/blip_retrieval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/aro/model_zoo/blip_utils/blip_retrieval.py -------------------------------------------------------------------------------- /aro/model_zoo/blip_utils/med.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/aro/model_zoo/blip_utils/med.py -------------------------------------------------------------------------------- /aro/model_zoo/blip_utils/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/aro/model_zoo/blip_utils/utils.py -------------------------------------------------------------------------------- /aro/model_zoo/blip_utils/vit.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/aro/model_zoo/blip_utils/vit.py -------------------------------------------------------------------------------- /aro/model_zoo/clip_models.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/aro/model_zoo/clip_models.py -------------------------------------------------------------------------------- /aro/model_zoo/constants.py: -------------------------------------------------------------------------------- 1 | CACHE_DIR="~/.cache" 2 | -------------------------------------------------------------------------------- /aro/model_zoo/flava.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/aro/model_zoo/flava.py -------------------------------------------------------------------------------- /aro/model_zoo/xvlm_models.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/aro/model_zoo/xvlm_models.py -------------------------------------------------------------------------------- /aro/model_zoo/xvlm_utils/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/aro/model_zoo/xvlm_utils/README.md -------------------------------------------------------------------------------- /aro/model_zoo/xvlm_utils/box_ops.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/aro/model_zoo/xvlm_utils/box_ops.py -------------------------------------------------------------------------------- /aro/model_zoo/xvlm_utils/clip_vit.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/aro/model_zoo/xvlm_utils/clip_vit.py -------------------------------------------------------------------------------- /aro/model_zoo/xvlm_utils/swin_transformer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/aro/model_zoo/xvlm_utils/swin_transformer.py -------------------------------------------------------------------------------- /aro/model_zoo/xvlm_utils/tokenization_bert.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/aro/model_zoo/xvlm_utils/tokenization_bert.py -------------------------------------------------------------------------------- /aro/model_zoo/xvlm_utils/tokenization_roberta.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/aro/model_zoo/xvlm_utils/tokenization_roberta.py -------------------------------------------------------------------------------- /aro/model_zoo/xvlm_utils/vit.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/aro/model_zoo/xvlm_utils/vit.py -------------------------------------------------------------------------------- /aro/model_zoo/xvlm_utils/xbert.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/aro/model_zoo/xvlm_utils/xbert.py -------------------------------------------------------------------------------- /aro/model_zoo/xvlm_utils/xroberta.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/aro/model_zoo/xvlm_utils/xroberta.py -------------------------------------------------------------------------------- /aro/model_zoo/xvlm_utils/xvlm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/aro/model_zoo/xvlm_utils/xvlm.py -------------------------------------------------------------------------------- /aro/notebooks/Replicate ARO! VG-Relation, VG-Attribution.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/aro/notebooks/Replicate ARO! VG-Relation, VG-Attribution.ipynb -------------------------------------------------------------------------------- /aro/scripts/create_environment.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/aro/scripts/create_environment.sh -------------------------------------------------------------------------------- /aro/scripts/reproduce_aro.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/aro/scripts/reproduce_aro.sh -------------------------------------------------------------------------------- /aro/scripts/reproduce_retrieval.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/aro/scripts/reproduce_retrieval.sh -------------------------------------------------------------------------------- /aro/temp_data/train_neg_clip.tsv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/aro/temp_data/train_neg_clip.tsv -------------------------------------------------------------------------------- /aro/temp_data/valid_neg_clip.tsv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/aro/temp_data/valid_neg_clip.tsv -------------------------------------------------------------------------------- /checkpoints/hardneg-checkpoint-5000/optimizer.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/checkpoints/hardneg-checkpoint-5000/optimizer.bin -------------------------------------------------------------------------------- /checkpoints/hardneg-checkpoint-5000/pytorch_lora_weights.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/checkpoints/hardneg-checkpoint-5000/pytorch_lora_weights.bin -------------------------------------------------------------------------------- /checkpoints/hardneg-checkpoint-5000/random_states_0.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/checkpoints/hardneg-checkpoint-5000/random_states_0.pkl -------------------------------------------------------------------------------- /checkpoints/hardneg-checkpoint-5000/results.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/checkpoints/hardneg-checkpoint-5000/results.txt -------------------------------------------------------------------------------- /checkpoints/hardneg-checkpoint-5000/scheduler.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/checkpoints/hardneg-checkpoint-5000/scheduler.bin -------------------------------------------------------------------------------- /checkpoints/noneg-checkpoint-3000/optimizer.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/checkpoints/noneg-checkpoint-3000/optimizer.bin -------------------------------------------------------------------------------- /checkpoints/noneg-checkpoint-3000/pytorch_lora_weights.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/checkpoints/noneg-checkpoint-3000/pytorch_lora_weights.bin -------------------------------------------------------------------------------- /checkpoints/noneg-checkpoint-3000/random_states_0.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/checkpoints/noneg-checkpoint-3000/random_states_0.pkl -------------------------------------------------------------------------------- /checkpoints/noneg-checkpoint-3000/results.txt: -------------------------------------------------------------------------------- 1 | MSCOCO Val Accuracy: 0.625 2 | Max more than once: 0 3 | Sample size 328 4 | -------------------------------------------------------------------------------- /checkpoints/noneg-checkpoint-3000/scheduler.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/checkpoints/noneg-checkpoint-3000/scheduler.bin -------------------------------------------------------------------------------- /data/clevr/captions/CLEVR_questions.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/data/clevr/captions/CLEVR_questions.json -------------------------------------------------------------------------------- /data/clevr/captions/CLEVR_scenes.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/data/clevr/captions/CLEVR_scenes.json -------------------------------------------------------------------------------- /data/clevr/captions/binding_color_shape.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/data/clevr/captions/binding_color_shape.json -------------------------------------------------------------------------------- /data/clevr/captions/binding_shape_color.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/data/clevr/captions/binding_shape_color.json -------------------------------------------------------------------------------- /data/clevr/captions/pair_binding_color.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/data/clevr/captions/pair_binding_color.json -------------------------------------------------------------------------------- /data/clevr/captions/pair_binding_size.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/data/clevr/captions/pair_binding_size.json -------------------------------------------------------------------------------- /data/clevr/captions/recognition_color.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/data/clevr/captions/recognition_color.json -------------------------------------------------------------------------------- /data/clevr/captions/recognition_shape.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/data/clevr/captions/recognition_shape.json -------------------------------------------------------------------------------- /data/clevr/captions/spatial.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/data/clevr/captions/spatial.json -------------------------------------------------------------------------------- /data/flickr30k/val_top10_RN50x64.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/data/flickr30k/val_top10_RN50x64.json -------------------------------------------------------------------------------- /data/flickr30k/val_top10_RN50x64_text.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/data/flickr30k/val_top10_RN50x64_text.json -------------------------------------------------------------------------------- /data/imagecode/train_data.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/data/imagecode/train_data.json -------------------------------------------------------------------------------- /data/imagecode/valid_data.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/data/imagecode/valid_data.json -------------------------------------------------------------------------------- /data/pets/classes.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/data/pets/classes.txt -------------------------------------------------------------------------------- /data/svo/download.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/data/svo/download.py -------------------------------------------------------------------------------- /data/svo/svo.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/data/svo/svo.json -------------------------------------------------------------------------------- /data/svo/svo_probes.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/data/svo/svo_probes.csv -------------------------------------------------------------------------------- /data/winoground/data.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/data/winoground/data.json -------------------------------------------------------------------------------- /datasets_loading.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/datasets_loading.py -------------------------------------------------------------------------------- /diffusion_itm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/diffusion_itm.py -------------------------------------------------------------------------------- /hard_neg_finetuning.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/hard_neg_finetuning.py -------------------------------------------------------------------------------- /mainfig.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/mainfig.jpeg -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/requirements.txt -------------------------------------------------------------------------------- /setup.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/setup.sh -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/McGill-NLP/diffusion-itm/HEAD/utils.py --------------------------------------------------------------------------------