├── .gitignore ├── README.md ├── base.yml ├── data ├── infoseek │ ├── cleaned_dataset_mc_Qwen2-VL-7B-Instruct.json │ ├── cleaned_dataset_mc_instructblip-vicuna-7b.json │ ├── cleaned_dataset_mc_llava-v1.6-34b-hf.json │ ├── cleaned_dataset_mc_llava-v1.6-vicuna-13b-hf.json │ ├── cleaned_dataset_mc_llava-v1.6-vicuna-7b-hf.json │ └── sampled_val_mc.json └── viquae │ ├── cleaned_dataset_mc.json │ ├── cleaned_dataset_mc_Qwen2-VL-7B-Instruct.json │ ├── cleaned_dataset_mc_instructblip-vicuna-7b.json │ ├── cleaned_dataset_mc_llava-v1.6-34b-hf.json │ ├── cleaned_dataset_mc_llava-v1.6-vicuna-13b-hf.json │ ├── cleaned_dataset_mc_llava-v1.6-vicuna-7b-hf.json │ └── multiple_choice_data.json ├── figures └── case.jpg ├── scripts ├── post_hoc_cd.sh ├── post_hoc_cd_infer.sh ├── predict.sh ├── run_cd_metric.sh └── run_post_hoc_robust.sh ├── src ├── analysis │ ├── post_hoc_contrastive_decoding.py │ ├── post_hoc_contrastive_decoding_metric.py │ ├── predict.py │ ├── predict_vllm.py │ ├── prob_shift.py │ └── run_contrastive_decoding.py ├── evaluate │ └── evaluate_mc.py ├── inference_time │ ├── post_hoc_contrastive_decoding.py │ ├── post_hoc_robust.py │ ├── predict_robust.py │ ├── predict_vllm.py │ └── prob.py ├── models │ ├── __init__.py │ ├── api.py │ ├── contrastive_decoding.py │ └── local.py ├── prompt │ └── __init__.py └── utils │ ├── data_utils.py │ ├── parser_utils.py │ └── utils.py └── vllm.yml /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__/ -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luka-group/vlm-knowledge-conflict/HEAD/README.md -------------------------------------------------------------------------------- /base.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luka-group/vlm-knowledge-conflict/HEAD/base.yml -------------------------------------------------------------------------------- /data/infoseek/cleaned_dataset_mc_Qwen2-VL-7B-Instruct.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luka-group/vlm-knowledge-conflict/HEAD/data/infoseek/cleaned_dataset_mc_Qwen2-VL-7B-Instruct.json -------------------------------------------------------------------------------- /data/infoseek/cleaned_dataset_mc_instructblip-vicuna-7b.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luka-group/vlm-knowledge-conflict/HEAD/data/infoseek/cleaned_dataset_mc_instructblip-vicuna-7b.json -------------------------------------------------------------------------------- /data/infoseek/cleaned_dataset_mc_llava-v1.6-34b-hf.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luka-group/vlm-knowledge-conflict/HEAD/data/infoseek/cleaned_dataset_mc_llava-v1.6-34b-hf.json -------------------------------------------------------------------------------- /data/infoseek/cleaned_dataset_mc_llava-v1.6-vicuna-13b-hf.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luka-group/vlm-knowledge-conflict/HEAD/data/infoseek/cleaned_dataset_mc_llava-v1.6-vicuna-13b-hf.json -------------------------------------------------------------------------------- /data/infoseek/cleaned_dataset_mc_llava-v1.6-vicuna-7b-hf.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luka-group/vlm-knowledge-conflict/HEAD/data/infoseek/cleaned_dataset_mc_llava-v1.6-vicuna-7b-hf.json -------------------------------------------------------------------------------- /data/infoseek/sampled_val_mc.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luka-group/vlm-knowledge-conflict/HEAD/data/infoseek/sampled_val_mc.json -------------------------------------------------------------------------------- /data/viquae/cleaned_dataset_mc.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luka-group/vlm-knowledge-conflict/HEAD/data/viquae/cleaned_dataset_mc.json -------------------------------------------------------------------------------- /data/viquae/cleaned_dataset_mc_Qwen2-VL-7B-Instruct.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luka-group/vlm-knowledge-conflict/HEAD/data/viquae/cleaned_dataset_mc_Qwen2-VL-7B-Instruct.json -------------------------------------------------------------------------------- /data/viquae/cleaned_dataset_mc_instructblip-vicuna-7b.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luka-group/vlm-knowledge-conflict/HEAD/data/viquae/cleaned_dataset_mc_instructblip-vicuna-7b.json -------------------------------------------------------------------------------- /data/viquae/cleaned_dataset_mc_llava-v1.6-34b-hf.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luka-group/vlm-knowledge-conflict/HEAD/data/viquae/cleaned_dataset_mc_llava-v1.6-34b-hf.json -------------------------------------------------------------------------------- /data/viquae/cleaned_dataset_mc_llava-v1.6-vicuna-13b-hf.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luka-group/vlm-knowledge-conflict/HEAD/data/viquae/cleaned_dataset_mc_llava-v1.6-vicuna-13b-hf.json -------------------------------------------------------------------------------- /data/viquae/cleaned_dataset_mc_llava-v1.6-vicuna-7b-hf.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luka-group/vlm-knowledge-conflict/HEAD/data/viquae/cleaned_dataset_mc_llava-v1.6-vicuna-7b-hf.json -------------------------------------------------------------------------------- /data/viquae/multiple_choice_data.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luka-group/vlm-knowledge-conflict/HEAD/data/viquae/multiple_choice_data.json -------------------------------------------------------------------------------- /figures/case.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luka-group/vlm-knowledge-conflict/HEAD/figures/case.jpg -------------------------------------------------------------------------------- /scripts/post_hoc_cd.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luka-group/vlm-knowledge-conflict/HEAD/scripts/post_hoc_cd.sh -------------------------------------------------------------------------------- /scripts/post_hoc_cd_infer.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luka-group/vlm-knowledge-conflict/HEAD/scripts/post_hoc_cd_infer.sh -------------------------------------------------------------------------------- /scripts/predict.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luka-group/vlm-knowledge-conflict/HEAD/scripts/predict.sh -------------------------------------------------------------------------------- /scripts/run_cd_metric.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luka-group/vlm-knowledge-conflict/HEAD/scripts/run_cd_metric.sh -------------------------------------------------------------------------------- /scripts/run_post_hoc_robust.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luka-group/vlm-knowledge-conflict/HEAD/scripts/run_post_hoc_robust.sh -------------------------------------------------------------------------------- /src/analysis/post_hoc_contrastive_decoding.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luka-group/vlm-knowledge-conflict/HEAD/src/analysis/post_hoc_contrastive_decoding.py -------------------------------------------------------------------------------- /src/analysis/post_hoc_contrastive_decoding_metric.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luka-group/vlm-knowledge-conflict/HEAD/src/analysis/post_hoc_contrastive_decoding_metric.py -------------------------------------------------------------------------------- /src/analysis/predict.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luka-group/vlm-knowledge-conflict/HEAD/src/analysis/predict.py -------------------------------------------------------------------------------- /src/analysis/predict_vllm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luka-group/vlm-knowledge-conflict/HEAD/src/analysis/predict_vllm.py -------------------------------------------------------------------------------- /src/analysis/prob_shift.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luka-group/vlm-knowledge-conflict/HEAD/src/analysis/prob_shift.py -------------------------------------------------------------------------------- /src/analysis/run_contrastive_decoding.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luka-group/vlm-knowledge-conflict/HEAD/src/analysis/run_contrastive_decoding.py -------------------------------------------------------------------------------- /src/evaluate/evaluate_mc.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luka-group/vlm-knowledge-conflict/HEAD/src/evaluate/evaluate_mc.py -------------------------------------------------------------------------------- /src/inference_time/post_hoc_contrastive_decoding.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luka-group/vlm-knowledge-conflict/HEAD/src/inference_time/post_hoc_contrastive_decoding.py -------------------------------------------------------------------------------- /src/inference_time/post_hoc_robust.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luka-group/vlm-knowledge-conflict/HEAD/src/inference_time/post_hoc_robust.py -------------------------------------------------------------------------------- /src/inference_time/predict_robust.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luka-group/vlm-knowledge-conflict/HEAD/src/inference_time/predict_robust.py -------------------------------------------------------------------------------- /src/inference_time/predict_vllm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luka-group/vlm-knowledge-conflict/HEAD/src/inference_time/predict_vllm.py -------------------------------------------------------------------------------- /src/inference_time/prob.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luka-group/vlm-knowledge-conflict/HEAD/src/inference_time/prob.py -------------------------------------------------------------------------------- /src/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luka-group/vlm-knowledge-conflict/HEAD/src/models/__init__.py -------------------------------------------------------------------------------- /src/models/api.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luka-group/vlm-knowledge-conflict/HEAD/src/models/api.py -------------------------------------------------------------------------------- /src/models/contrastive_decoding.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luka-group/vlm-knowledge-conflict/HEAD/src/models/contrastive_decoding.py -------------------------------------------------------------------------------- /src/models/local.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luka-group/vlm-knowledge-conflict/HEAD/src/models/local.py -------------------------------------------------------------------------------- /src/prompt/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luka-group/vlm-knowledge-conflict/HEAD/src/prompt/__init__.py -------------------------------------------------------------------------------- /src/utils/data_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luka-group/vlm-knowledge-conflict/HEAD/src/utils/data_utils.py -------------------------------------------------------------------------------- /src/utils/parser_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luka-group/vlm-knowledge-conflict/HEAD/src/utils/parser_utils.py -------------------------------------------------------------------------------- /src/utils/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luka-group/vlm-knowledge-conflict/HEAD/src/utils/utils.py -------------------------------------------------------------------------------- /vllm.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/luka-group/vlm-knowledge-conflict/HEAD/vllm.yml --------------------------------------------------------------------------------