├── DATA.md ├── LICENSE ├── README.md ├── benchmark ├── __init__.py ├── clip_benchmark │ ├── __init__.py │ ├── dataset.py │ ├── evaluate.py │ └── model.py ├── data │ ├── __init__.py │ ├── dataset.py │ ├── loaders.py │ └── options_generation.py ├── evaluate.py └── model │ ├── __init__.py │ ├── model.py │ ├── model_internlm_xcomposer2.py │ ├── model_mobilevlm.py │ ├── model_yivl.py │ └── my_llava.py ├── data ├── fgvc_aircraft-ViT-L-14-laion2b_s32b_b82k-4.json ├── flowers102-ViT-L-14-laion2b_s32b_b82k-4.json ├── food101-ViT-L-14-laion2b_s32b_b82k-4.json ├── geode-ViT-L-14-laion2b_s32b_b82k-4.json ├── imagenet-6k-animal-ViT-L-14-laion2b_s32b_b82k-4.json ├── imagenet-6k-artifact-ViT-L-14-laion2b_s32b_b82k-4.json ├── imagenet-6k-food-ViT-L-14-laion2b_s32b_b82k-4.json ├── imagenet-6k-plant-ViT-L-14-laion2b_s32b_b82k-4.json ├── imagenet-ViT-L-14-laion2b_s32b_b82k-4.json ├── imagenet-adversarial-ViT-L-14-laion2b_s32b_b82k-4.json ├── imagenet-rendition-ViT-L-14-laion2b_s32b_b82k-4.json ├── imagenet-sketch-ViT-L-14-laion2b_s32b_b82k-4.json ├── oxford_pet-ViT-L-14-laion2b_s32b_b82k-4.json └── stanford_cars-ViT-L-14-laion2b_s32b_b82k-4.json ├── evaluation └── evaluate.ipynb ├── media └── foci.png ├── requirements.txt ├── run_ic_bench.py ├── run_ic_bench_clip.py ├── scripts ├── blip2-flanxl.sh ├── idefics2-8b.sh ├── idefics_9b-instruct.sh ├── imagenet6k_all.sh ├── instructblip-flanxl.sh ├── instructblip-vicuna7b.sh ├── internlm2.sh ├── llava1_5-7b.sh ├── llava1_6-mistral-7b.sh ├── mobilevlmv2-1_7.sh ├── mobilevlmv2-3.sh ├── mobilevlmv2-7.sh ├── my_stability1b6-base.sh ├── my_stability1b6-generative.sh ├── my_stability1b6-imagenetqa.sh ├── my_stability1b6-large.sh ├── my_stability1b6-nopretrain.sh ├── my_stability1b6-siglip.sh ├── my_stability1b6-template.sh ├── pali_gemma3b-mix.sh └── qwen-vl-chat.sh └── util └── caption_imagenet.py /DATA.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gregor-ge/FOCI-Benchmark/HEAD/DATA.md -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gregor-ge/FOCI-Benchmark/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gregor-ge/FOCI-Benchmark/HEAD/README.md -------------------------------------------------------------------------------- /benchmark/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /benchmark/clip_benchmark/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /benchmark/clip_benchmark/dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gregor-ge/FOCI-Benchmark/HEAD/benchmark/clip_benchmark/dataset.py -------------------------------------------------------------------------------- /benchmark/clip_benchmark/evaluate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gregor-ge/FOCI-Benchmark/HEAD/benchmark/clip_benchmark/evaluate.py -------------------------------------------------------------------------------- /benchmark/clip_benchmark/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gregor-ge/FOCI-Benchmark/HEAD/benchmark/clip_benchmark/model.py -------------------------------------------------------------------------------- /benchmark/data/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /benchmark/data/dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gregor-ge/FOCI-Benchmark/HEAD/benchmark/data/dataset.py -------------------------------------------------------------------------------- /benchmark/data/loaders.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gregor-ge/FOCI-Benchmark/HEAD/benchmark/data/loaders.py -------------------------------------------------------------------------------- /benchmark/data/options_generation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gregor-ge/FOCI-Benchmark/HEAD/benchmark/data/options_generation.py -------------------------------------------------------------------------------- /benchmark/evaluate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gregor-ge/FOCI-Benchmark/HEAD/benchmark/evaluate.py -------------------------------------------------------------------------------- /benchmark/model/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /benchmark/model/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gregor-ge/FOCI-Benchmark/HEAD/benchmark/model/model.py -------------------------------------------------------------------------------- /benchmark/model/model_internlm_xcomposer2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gregor-ge/FOCI-Benchmark/HEAD/benchmark/model/model_internlm_xcomposer2.py -------------------------------------------------------------------------------- /benchmark/model/model_mobilevlm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gregor-ge/FOCI-Benchmark/HEAD/benchmark/model/model_mobilevlm.py -------------------------------------------------------------------------------- /benchmark/model/model_yivl.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gregor-ge/FOCI-Benchmark/HEAD/benchmark/model/model_yivl.py -------------------------------------------------------------------------------- /benchmark/model/my_llava.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gregor-ge/FOCI-Benchmark/HEAD/benchmark/model/my_llava.py -------------------------------------------------------------------------------- /data/fgvc_aircraft-ViT-L-14-laion2b_s32b_b82k-4.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gregor-ge/FOCI-Benchmark/HEAD/data/fgvc_aircraft-ViT-L-14-laion2b_s32b_b82k-4.json -------------------------------------------------------------------------------- /data/flowers102-ViT-L-14-laion2b_s32b_b82k-4.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gregor-ge/FOCI-Benchmark/HEAD/data/flowers102-ViT-L-14-laion2b_s32b_b82k-4.json -------------------------------------------------------------------------------- /data/food101-ViT-L-14-laion2b_s32b_b82k-4.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gregor-ge/FOCI-Benchmark/HEAD/data/food101-ViT-L-14-laion2b_s32b_b82k-4.json -------------------------------------------------------------------------------- /data/geode-ViT-L-14-laion2b_s32b_b82k-4.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gregor-ge/FOCI-Benchmark/HEAD/data/geode-ViT-L-14-laion2b_s32b_b82k-4.json -------------------------------------------------------------------------------- /data/imagenet-6k-animal-ViT-L-14-laion2b_s32b_b82k-4.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gregor-ge/FOCI-Benchmark/HEAD/data/imagenet-6k-animal-ViT-L-14-laion2b_s32b_b82k-4.json -------------------------------------------------------------------------------- /data/imagenet-6k-artifact-ViT-L-14-laion2b_s32b_b82k-4.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gregor-ge/FOCI-Benchmark/HEAD/data/imagenet-6k-artifact-ViT-L-14-laion2b_s32b_b82k-4.json -------------------------------------------------------------------------------- /data/imagenet-6k-food-ViT-L-14-laion2b_s32b_b82k-4.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gregor-ge/FOCI-Benchmark/HEAD/data/imagenet-6k-food-ViT-L-14-laion2b_s32b_b82k-4.json -------------------------------------------------------------------------------- /data/imagenet-6k-plant-ViT-L-14-laion2b_s32b_b82k-4.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gregor-ge/FOCI-Benchmark/HEAD/data/imagenet-6k-plant-ViT-L-14-laion2b_s32b_b82k-4.json -------------------------------------------------------------------------------- /data/imagenet-ViT-L-14-laion2b_s32b_b82k-4.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gregor-ge/FOCI-Benchmark/HEAD/data/imagenet-ViT-L-14-laion2b_s32b_b82k-4.json -------------------------------------------------------------------------------- /data/imagenet-adversarial-ViT-L-14-laion2b_s32b_b82k-4.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gregor-ge/FOCI-Benchmark/HEAD/data/imagenet-adversarial-ViT-L-14-laion2b_s32b_b82k-4.json -------------------------------------------------------------------------------- /data/imagenet-rendition-ViT-L-14-laion2b_s32b_b82k-4.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gregor-ge/FOCI-Benchmark/HEAD/data/imagenet-rendition-ViT-L-14-laion2b_s32b_b82k-4.json -------------------------------------------------------------------------------- /data/imagenet-sketch-ViT-L-14-laion2b_s32b_b82k-4.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gregor-ge/FOCI-Benchmark/HEAD/data/imagenet-sketch-ViT-L-14-laion2b_s32b_b82k-4.json -------------------------------------------------------------------------------- /data/oxford_pet-ViT-L-14-laion2b_s32b_b82k-4.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gregor-ge/FOCI-Benchmark/HEAD/data/oxford_pet-ViT-L-14-laion2b_s32b_b82k-4.json -------------------------------------------------------------------------------- /data/stanford_cars-ViT-L-14-laion2b_s32b_b82k-4.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gregor-ge/FOCI-Benchmark/HEAD/data/stanford_cars-ViT-L-14-laion2b_s32b_b82k-4.json -------------------------------------------------------------------------------- /evaluation/evaluate.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gregor-ge/FOCI-Benchmark/HEAD/evaluation/evaluate.ipynb -------------------------------------------------------------------------------- /media/foci.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gregor-ge/FOCI-Benchmark/HEAD/media/foci.png -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gregor-ge/FOCI-Benchmark/HEAD/requirements.txt -------------------------------------------------------------------------------- /run_ic_bench.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gregor-ge/FOCI-Benchmark/HEAD/run_ic_bench.py -------------------------------------------------------------------------------- /run_ic_bench_clip.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gregor-ge/FOCI-Benchmark/HEAD/run_ic_bench_clip.py -------------------------------------------------------------------------------- /scripts/blip2-flanxl.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gregor-ge/FOCI-Benchmark/HEAD/scripts/blip2-flanxl.sh -------------------------------------------------------------------------------- /scripts/idefics2-8b.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gregor-ge/FOCI-Benchmark/HEAD/scripts/idefics2-8b.sh -------------------------------------------------------------------------------- /scripts/idefics_9b-instruct.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gregor-ge/FOCI-Benchmark/HEAD/scripts/idefics_9b-instruct.sh -------------------------------------------------------------------------------- /scripts/imagenet6k_all.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gregor-ge/FOCI-Benchmark/HEAD/scripts/imagenet6k_all.sh -------------------------------------------------------------------------------- /scripts/instructblip-flanxl.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gregor-ge/FOCI-Benchmark/HEAD/scripts/instructblip-flanxl.sh -------------------------------------------------------------------------------- /scripts/instructblip-vicuna7b.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gregor-ge/FOCI-Benchmark/HEAD/scripts/instructblip-vicuna7b.sh -------------------------------------------------------------------------------- /scripts/internlm2.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gregor-ge/FOCI-Benchmark/HEAD/scripts/internlm2.sh -------------------------------------------------------------------------------- /scripts/llava1_5-7b.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gregor-ge/FOCI-Benchmark/HEAD/scripts/llava1_5-7b.sh -------------------------------------------------------------------------------- /scripts/llava1_6-mistral-7b.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gregor-ge/FOCI-Benchmark/HEAD/scripts/llava1_6-mistral-7b.sh -------------------------------------------------------------------------------- /scripts/mobilevlmv2-1_7.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gregor-ge/FOCI-Benchmark/HEAD/scripts/mobilevlmv2-1_7.sh -------------------------------------------------------------------------------- /scripts/mobilevlmv2-3.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gregor-ge/FOCI-Benchmark/HEAD/scripts/mobilevlmv2-3.sh -------------------------------------------------------------------------------- /scripts/mobilevlmv2-7.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gregor-ge/FOCI-Benchmark/HEAD/scripts/mobilevlmv2-7.sh -------------------------------------------------------------------------------- /scripts/my_stability1b6-base.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gregor-ge/FOCI-Benchmark/HEAD/scripts/my_stability1b6-base.sh -------------------------------------------------------------------------------- /scripts/my_stability1b6-generative.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gregor-ge/FOCI-Benchmark/HEAD/scripts/my_stability1b6-generative.sh -------------------------------------------------------------------------------- /scripts/my_stability1b6-imagenetqa.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gregor-ge/FOCI-Benchmark/HEAD/scripts/my_stability1b6-imagenetqa.sh -------------------------------------------------------------------------------- /scripts/my_stability1b6-large.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gregor-ge/FOCI-Benchmark/HEAD/scripts/my_stability1b6-large.sh -------------------------------------------------------------------------------- /scripts/my_stability1b6-nopretrain.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gregor-ge/FOCI-Benchmark/HEAD/scripts/my_stability1b6-nopretrain.sh -------------------------------------------------------------------------------- /scripts/my_stability1b6-siglip.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gregor-ge/FOCI-Benchmark/HEAD/scripts/my_stability1b6-siglip.sh -------------------------------------------------------------------------------- /scripts/my_stability1b6-template.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gregor-ge/FOCI-Benchmark/HEAD/scripts/my_stability1b6-template.sh -------------------------------------------------------------------------------- /scripts/pali_gemma3b-mix.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gregor-ge/FOCI-Benchmark/HEAD/scripts/pali_gemma3b-mix.sh -------------------------------------------------------------------------------- /scripts/qwen-vl-chat.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gregor-ge/FOCI-Benchmark/HEAD/scripts/qwen-vl-chat.sh -------------------------------------------------------------------------------- /util/caption_imagenet.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gregor-ge/FOCI-Benchmark/HEAD/util/caption_imagenet.py --------------------------------------------------------------------------------