├── .gitignore ├── LICENSE ├── README.md ├── data_filtering ├── baselines.py ├── baselines │ ├── __init__.py │ ├── additional_text_filter.py │ ├── apply_filter.py │ ├── image_based_clustering.md │ ├── image_based_clustering.py │ └── utils.py ├── requirements.txt └── resharder.py ├── data_prepare └── split_mammoth_10m.py ├── docs ├── DATA_Filter.md └── Eval.md ├── mm_sequence_packing ├── multiprocess_sequence_packing_image_to_json.sh ├── multiprocess_sequence_packing_image_to_pil.sh ├── sequence_packing_image_to_json.py └── sequence_packing_image_to_pil.py ├── prismatic-vlms ├── .gitignore ├── .pre-commit-config.yaml ├── LICENSE ├── Makefile ├── fine_tune.sh ├── fine_tune_mammoth.sh ├── prismatic │ ├── __init__.py │ ├── conf │ │ ├── __init__.py │ │ ├── datasets.py │ │ └── models.py │ ├── models │ │ ├── __init__.py │ │ ├── backbones │ │ │ ├── __init__.py │ │ │ ├── llm │ │ │ │ ├── __init__.py │ │ │ │ ├── base_llm.py │ │ │ │ ├── llama2.py │ │ │ │ ├── llama3.py │ │ │ │ ├── mistral.py │ │ │ │ ├── phi3.py │ │ │ │ ├── prompting │ │ │ │ │ ├── __init__.py │ │ │ │ │ ├── base_prompter.py │ │ │ │ │ ├── llama2_chat_prompter.py │ │ │ │ │ ├── llama3_chat_prompter.py │ │ │ │ │ ├── phi_3_prompter.py │ │ │ │ │ ├── qwen2_prompter.py │ │ │ │ │ └── vicuna_v15_prompter.py │ │ │ │ └── qwen2.py │ │ │ └── vision │ │ │ │ ├── __init__.py │ │ │ │ ├── base_vision.py │ │ │ │ ├── clip_vit.py │ │ │ │ ├── dinoclip_vit.py │ │ │ │ ├── dinosiglip_vit.py │ │ │ │ ├── dinov2_vit.py │ │ │ │ ├── in1k_vit.py │ │ │ │ └── siglip_vit.py │ │ ├── load.py │ │ ├── materialize.py │ │ ├── registry.py │ │ └── vlms │ │ │ ├── __init__.py │ │ │ ├── base_vlm.py │ │ │ └── prismatic.py │ ├── overwatch │ │ ├── __init__.py │ │ └── overwatch.py │ ├── preprocessing │ │ ├── __init__.py │ │ ├── datasets │ │ │ ├── __init__.py │ │ │ └── datasets.py │ │ ├── download.py │ │ └── materialize.py │ ├── py.typed │ ├── training │ │ ├── __init__.py │ │ ├── materialize.py │ │ ├── metrics.py │ │ └── strategies │ │ │ ├── __init__.py │ │ │ ├── base_strategy.py │ │ │ ├── ddp.py │ │ │ └── fsdp.py │ └── util │ │ ├── __init__.py │ │ ├── batching_utils.py │ │ ├── data_utils.py │ │ ├── nn_utils.py │ │ └── torch_utils.py ├── pyproject.toml ├── scripts │ ├── additional-datasets │ │ ├── lrv_instruct.py │ │ └── lvis_instruct_4v.py │ ├── generate.py │ ├── preprocess.py │ └── pretrain.py └── train.sh ├── test.py └── vlm-evaluation ├── .gitignore ├── .pre-commit-config.yaml ├── LICENSE ├── eval.sh ├── images └── 03-evaluation-suite-med-res.png ├── pyproject.toml ├── scripts ├── coco_score.py ├── datasets │ └── prepare.py ├── evaluate.py ├── interactive_demo.py └── score.py └── vlm_eval ├── __init__.py ├── conf ├── __init__.py └── datasets.py ├── models ├── __init__.py ├── instructblip.py ├── llava.py ├── prismatic.py └── qwen2vl.py ├── overwatch ├── __init__.py └── overwatch.py ├── serve ├── __init__.py ├── controller.py ├── examples │ ├── cows_in_pasture.png │ └── monkey_knives.png └── gradio_web_server.py ├── tasks ├── __init__.py ├── builders.py ├── download.py ├── harnesses │ ├── __init__.py │ ├── ai2d.py │ ├── gqa.py │ ├── mantis.py │ ├── mathvista.py │ ├── mmbench.py │ ├── mmlu.py │ ├── mmmu.py │ ├── mmstar.py │ ├── mscoco_karpathy.py │ ├── ocidref.py │ ├── okvqa.py │ ├── pope.py │ ├── refcoco.py │ ├── seedbench.py │ ├── tallyqa.py │ ├── textvqa.py │ ├── vizwiz.py │ ├── vqav2.py │ └── vsr.py └── registry.py └── util ├── __init__.py ├── evaluation ├── __init__.py ├── gqa │ ├── __init__.py │ └── eval.py ├── mmmu │ ├── __init__.py │ └── eval.py ├── nocaps │ └── metrics.py ├── textvqa │ ├── __init__.py │ └── m4c_evaluators.py ├── vizwiz │ ├── __init__.py │ └── eval.py └── vqav2 │ ├── __init__.py │ └── eval.py ├── interfaces.py ├── loading ├── __init__.py └── refer.py └── preprocessing.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/README.md -------------------------------------------------------------------------------- /data_filtering/baselines.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/data_filtering/baselines.py -------------------------------------------------------------------------------- /data_filtering/baselines/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /data_filtering/baselines/additional_text_filter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/data_filtering/baselines/additional_text_filter.py -------------------------------------------------------------------------------- /data_filtering/baselines/apply_filter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/data_filtering/baselines/apply_filter.py -------------------------------------------------------------------------------- /data_filtering/baselines/image_based_clustering.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/data_filtering/baselines/image_based_clustering.md -------------------------------------------------------------------------------- /data_filtering/baselines/image_based_clustering.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/data_filtering/baselines/image_based_clustering.py -------------------------------------------------------------------------------- /data_filtering/baselines/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/data_filtering/baselines/utils.py -------------------------------------------------------------------------------- /data_filtering/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/data_filtering/requirements.txt -------------------------------------------------------------------------------- /data_filtering/resharder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/data_filtering/resharder.py -------------------------------------------------------------------------------- /data_prepare/split_mammoth_10m.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/data_prepare/split_mammoth_10m.py -------------------------------------------------------------------------------- /docs/DATA_Filter.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/docs/DATA_Filter.md -------------------------------------------------------------------------------- /docs/Eval.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/docs/Eval.md -------------------------------------------------------------------------------- /mm_sequence_packing/multiprocess_sequence_packing_image_to_json.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/mm_sequence_packing/multiprocess_sequence_packing_image_to_json.sh -------------------------------------------------------------------------------- /mm_sequence_packing/multiprocess_sequence_packing_image_to_pil.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/mm_sequence_packing/multiprocess_sequence_packing_image_to_pil.sh -------------------------------------------------------------------------------- /mm_sequence_packing/sequence_packing_image_to_json.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/mm_sequence_packing/sequence_packing_image_to_json.py -------------------------------------------------------------------------------- /mm_sequence_packing/sequence_packing_image_to_pil.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/mm_sequence_packing/sequence_packing_image_to_pil.py -------------------------------------------------------------------------------- /prismatic-vlms/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/.gitignore -------------------------------------------------------------------------------- /prismatic-vlms/.pre-commit-config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/.pre-commit-config.yaml -------------------------------------------------------------------------------- /prismatic-vlms/LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/LICENSE -------------------------------------------------------------------------------- /prismatic-vlms/Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/Makefile -------------------------------------------------------------------------------- /prismatic-vlms/fine_tune.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/fine_tune.sh -------------------------------------------------------------------------------- /prismatic-vlms/fine_tune_mammoth.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/fine_tune_mammoth.sh -------------------------------------------------------------------------------- /prismatic-vlms/prismatic/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/prismatic/__init__.py -------------------------------------------------------------------------------- /prismatic-vlms/prismatic/conf/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/prismatic/conf/__init__.py -------------------------------------------------------------------------------- /prismatic-vlms/prismatic/conf/datasets.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/prismatic/conf/datasets.py -------------------------------------------------------------------------------- /prismatic-vlms/prismatic/conf/models.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/prismatic/conf/models.py -------------------------------------------------------------------------------- /prismatic-vlms/prismatic/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/prismatic/models/__init__.py -------------------------------------------------------------------------------- /prismatic-vlms/prismatic/models/backbones/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /prismatic-vlms/prismatic/models/backbones/llm/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/prismatic/models/backbones/llm/__init__.py -------------------------------------------------------------------------------- /prismatic-vlms/prismatic/models/backbones/llm/base_llm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/prismatic/models/backbones/llm/base_llm.py -------------------------------------------------------------------------------- /prismatic-vlms/prismatic/models/backbones/llm/llama2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/prismatic/models/backbones/llm/llama2.py -------------------------------------------------------------------------------- /prismatic-vlms/prismatic/models/backbones/llm/llama3.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/prismatic/models/backbones/llm/llama3.py -------------------------------------------------------------------------------- /prismatic-vlms/prismatic/models/backbones/llm/mistral.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/prismatic/models/backbones/llm/mistral.py -------------------------------------------------------------------------------- /prismatic-vlms/prismatic/models/backbones/llm/phi3.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/prismatic/models/backbones/llm/phi3.py -------------------------------------------------------------------------------- /prismatic-vlms/prismatic/models/backbones/llm/prompting/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/prismatic/models/backbones/llm/prompting/__init__.py -------------------------------------------------------------------------------- /prismatic-vlms/prismatic/models/backbones/llm/prompting/base_prompter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/prismatic/models/backbones/llm/prompting/base_prompter.py -------------------------------------------------------------------------------- /prismatic-vlms/prismatic/models/backbones/llm/prompting/llama2_chat_prompter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/prismatic/models/backbones/llm/prompting/llama2_chat_prompter.py -------------------------------------------------------------------------------- /prismatic-vlms/prismatic/models/backbones/llm/prompting/llama3_chat_prompter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/prismatic/models/backbones/llm/prompting/llama3_chat_prompter.py -------------------------------------------------------------------------------- /prismatic-vlms/prismatic/models/backbones/llm/prompting/phi_3_prompter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/prismatic/models/backbones/llm/prompting/phi_3_prompter.py -------------------------------------------------------------------------------- /prismatic-vlms/prismatic/models/backbones/llm/prompting/qwen2_prompter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/prismatic/models/backbones/llm/prompting/qwen2_prompter.py -------------------------------------------------------------------------------- /prismatic-vlms/prismatic/models/backbones/llm/prompting/vicuna_v15_prompter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/prismatic/models/backbones/llm/prompting/vicuna_v15_prompter.py -------------------------------------------------------------------------------- /prismatic-vlms/prismatic/models/backbones/llm/qwen2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/prismatic/models/backbones/llm/qwen2.py -------------------------------------------------------------------------------- /prismatic-vlms/prismatic/models/backbones/vision/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/prismatic/models/backbones/vision/__init__.py -------------------------------------------------------------------------------- /prismatic-vlms/prismatic/models/backbones/vision/base_vision.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/prismatic/models/backbones/vision/base_vision.py -------------------------------------------------------------------------------- /prismatic-vlms/prismatic/models/backbones/vision/clip_vit.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/prismatic/models/backbones/vision/clip_vit.py -------------------------------------------------------------------------------- /prismatic-vlms/prismatic/models/backbones/vision/dinoclip_vit.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/prismatic/models/backbones/vision/dinoclip_vit.py -------------------------------------------------------------------------------- /prismatic-vlms/prismatic/models/backbones/vision/dinosiglip_vit.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/prismatic/models/backbones/vision/dinosiglip_vit.py -------------------------------------------------------------------------------- /prismatic-vlms/prismatic/models/backbones/vision/dinov2_vit.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/prismatic/models/backbones/vision/dinov2_vit.py -------------------------------------------------------------------------------- /prismatic-vlms/prismatic/models/backbones/vision/in1k_vit.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/prismatic/models/backbones/vision/in1k_vit.py -------------------------------------------------------------------------------- /prismatic-vlms/prismatic/models/backbones/vision/siglip_vit.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/prismatic/models/backbones/vision/siglip_vit.py -------------------------------------------------------------------------------- /prismatic-vlms/prismatic/models/load.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/prismatic/models/load.py -------------------------------------------------------------------------------- /prismatic-vlms/prismatic/models/materialize.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/prismatic/models/materialize.py -------------------------------------------------------------------------------- /prismatic-vlms/prismatic/models/registry.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/prismatic/models/registry.py -------------------------------------------------------------------------------- /prismatic-vlms/prismatic/models/vlms/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/prismatic/models/vlms/__init__.py -------------------------------------------------------------------------------- /prismatic-vlms/prismatic/models/vlms/base_vlm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/prismatic/models/vlms/base_vlm.py -------------------------------------------------------------------------------- /prismatic-vlms/prismatic/models/vlms/prismatic.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/prismatic/models/vlms/prismatic.py -------------------------------------------------------------------------------- /prismatic-vlms/prismatic/overwatch/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/prismatic/overwatch/__init__.py -------------------------------------------------------------------------------- /prismatic-vlms/prismatic/overwatch/overwatch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/prismatic/overwatch/overwatch.py -------------------------------------------------------------------------------- /prismatic-vlms/prismatic/preprocessing/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/prismatic/preprocessing/__init__.py -------------------------------------------------------------------------------- /prismatic-vlms/prismatic/preprocessing/datasets/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/prismatic/preprocessing/datasets/__init__.py -------------------------------------------------------------------------------- /prismatic-vlms/prismatic/preprocessing/datasets/datasets.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/prismatic/preprocessing/datasets/datasets.py -------------------------------------------------------------------------------- /prismatic-vlms/prismatic/preprocessing/download.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/prismatic/preprocessing/download.py -------------------------------------------------------------------------------- /prismatic-vlms/prismatic/preprocessing/materialize.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/prismatic/preprocessing/materialize.py -------------------------------------------------------------------------------- /prismatic-vlms/prismatic/py.typed: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /prismatic-vlms/prismatic/training/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/prismatic/training/__init__.py -------------------------------------------------------------------------------- /prismatic-vlms/prismatic/training/materialize.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/prismatic/training/materialize.py -------------------------------------------------------------------------------- /prismatic-vlms/prismatic/training/metrics.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/prismatic/training/metrics.py -------------------------------------------------------------------------------- /prismatic-vlms/prismatic/training/strategies/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/prismatic/training/strategies/__init__.py -------------------------------------------------------------------------------- /prismatic-vlms/prismatic/training/strategies/base_strategy.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/prismatic/training/strategies/base_strategy.py -------------------------------------------------------------------------------- /prismatic-vlms/prismatic/training/strategies/ddp.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/prismatic/training/strategies/ddp.py -------------------------------------------------------------------------------- /prismatic-vlms/prismatic/training/strategies/fsdp.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/prismatic/training/strategies/fsdp.py -------------------------------------------------------------------------------- /prismatic-vlms/prismatic/util/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/prismatic/util/__init__.py -------------------------------------------------------------------------------- /prismatic-vlms/prismatic/util/batching_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/prismatic/util/batching_utils.py -------------------------------------------------------------------------------- /prismatic-vlms/prismatic/util/data_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/prismatic/util/data_utils.py -------------------------------------------------------------------------------- /prismatic-vlms/prismatic/util/nn_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/prismatic/util/nn_utils.py -------------------------------------------------------------------------------- /prismatic-vlms/prismatic/util/torch_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/prismatic/util/torch_utils.py -------------------------------------------------------------------------------- /prismatic-vlms/pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/pyproject.toml -------------------------------------------------------------------------------- /prismatic-vlms/scripts/additional-datasets/lrv_instruct.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/scripts/additional-datasets/lrv_instruct.py -------------------------------------------------------------------------------- /prismatic-vlms/scripts/additional-datasets/lvis_instruct_4v.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/scripts/additional-datasets/lvis_instruct_4v.py -------------------------------------------------------------------------------- /prismatic-vlms/scripts/generate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/scripts/generate.py -------------------------------------------------------------------------------- /prismatic-vlms/scripts/preprocess.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/scripts/preprocess.py -------------------------------------------------------------------------------- /prismatic-vlms/scripts/pretrain.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/scripts/pretrain.py -------------------------------------------------------------------------------- /prismatic-vlms/train.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/prismatic-vlms/train.sh -------------------------------------------------------------------------------- /test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/test.py -------------------------------------------------------------------------------- /vlm-evaluation/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/vlm-evaluation/.gitignore -------------------------------------------------------------------------------- /vlm-evaluation/.pre-commit-config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/vlm-evaluation/.pre-commit-config.yaml -------------------------------------------------------------------------------- /vlm-evaluation/LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/vlm-evaluation/LICENSE -------------------------------------------------------------------------------- /vlm-evaluation/eval.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/vlm-evaluation/eval.sh -------------------------------------------------------------------------------- /vlm-evaluation/images/03-evaluation-suite-med-res.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/vlm-evaluation/images/03-evaluation-suite-med-res.png -------------------------------------------------------------------------------- /vlm-evaluation/pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/vlm-evaluation/pyproject.toml -------------------------------------------------------------------------------- /vlm-evaluation/scripts/coco_score.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/vlm-evaluation/scripts/coco_score.py -------------------------------------------------------------------------------- /vlm-evaluation/scripts/datasets/prepare.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/vlm-evaluation/scripts/datasets/prepare.py -------------------------------------------------------------------------------- /vlm-evaluation/scripts/evaluate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/vlm-evaluation/scripts/evaluate.py -------------------------------------------------------------------------------- /vlm-evaluation/scripts/interactive_demo.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/vlm-evaluation/scripts/interactive_demo.py -------------------------------------------------------------------------------- /vlm-evaluation/scripts/score.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/vlm-evaluation/scripts/score.py -------------------------------------------------------------------------------- /vlm-evaluation/vlm_eval/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /vlm-evaluation/vlm_eval/conf/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/vlm-evaluation/vlm_eval/conf/__init__.py -------------------------------------------------------------------------------- /vlm-evaluation/vlm_eval/conf/datasets.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/vlm-evaluation/vlm_eval/conf/datasets.py -------------------------------------------------------------------------------- /vlm-evaluation/vlm_eval/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/vlm-evaluation/vlm_eval/models/__init__.py -------------------------------------------------------------------------------- /vlm-evaluation/vlm_eval/models/instructblip.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/vlm-evaluation/vlm_eval/models/instructblip.py -------------------------------------------------------------------------------- /vlm-evaluation/vlm_eval/models/llava.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/vlm-evaluation/vlm_eval/models/llava.py -------------------------------------------------------------------------------- /vlm-evaluation/vlm_eval/models/prismatic.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/vlm-evaluation/vlm_eval/models/prismatic.py -------------------------------------------------------------------------------- /vlm-evaluation/vlm_eval/models/qwen2vl.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/vlm-evaluation/vlm_eval/models/qwen2vl.py -------------------------------------------------------------------------------- /vlm-evaluation/vlm_eval/overwatch/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/vlm-evaluation/vlm_eval/overwatch/__init__.py -------------------------------------------------------------------------------- /vlm-evaluation/vlm_eval/overwatch/overwatch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/vlm-evaluation/vlm_eval/overwatch/overwatch.py -------------------------------------------------------------------------------- /vlm-evaluation/vlm_eval/serve/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/vlm-evaluation/vlm_eval/serve/__init__.py -------------------------------------------------------------------------------- /vlm-evaluation/vlm_eval/serve/controller.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/vlm-evaluation/vlm_eval/serve/controller.py -------------------------------------------------------------------------------- /vlm-evaluation/vlm_eval/serve/examples/cows_in_pasture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/vlm-evaluation/vlm_eval/serve/examples/cows_in_pasture.png -------------------------------------------------------------------------------- /vlm-evaluation/vlm_eval/serve/examples/monkey_knives.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/vlm-evaluation/vlm_eval/serve/examples/monkey_knives.png -------------------------------------------------------------------------------- /vlm-evaluation/vlm_eval/serve/gradio_web_server.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/vlm-evaluation/vlm_eval/serve/gradio_web_server.py -------------------------------------------------------------------------------- /vlm-evaluation/vlm_eval/tasks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/vlm-evaluation/vlm_eval/tasks/__init__.py -------------------------------------------------------------------------------- /vlm-evaluation/vlm_eval/tasks/builders.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/vlm-evaluation/vlm_eval/tasks/builders.py -------------------------------------------------------------------------------- /vlm-evaluation/vlm_eval/tasks/download.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/vlm-evaluation/vlm_eval/tasks/download.py -------------------------------------------------------------------------------- /vlm-evaluation/vlm_eval/tasks/harnesses/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/vlm-evaluation/vlm_eval/tasks/harnesses/__init__.py -------------------------------------------------------------------------------- /vlm-evaluation/vlm_eval/tasks/harnesses/ai2d.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/vlm-evaluation/vlm_eval/tasks/harnesses/ai2d.py -------------------------------------------------------------------------------- /vlm-evaluation/vlm_eval/tasks/harnesses/gqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/vlm-evaluation/vlm_eval/tasks/harnesses/gqa.py -------------------------------------------------------------------------------- /vlm-evaluation/vlm_eval/tasks/harnesses/mantis.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/vlm-evaluation/vlm_eval/tasks/harnesses/mantis.py -------------------------------------------------------------------------------- /vlm-evaluation/vlm_eval/tasks/harnesses/mathvista.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/vlm-evaluation/vlm_eval/tasks/harnesses/mathvista.py -------------------------------------------------------------------------------- /vlm-evaluation/vlm_eval/tasks/harnesses/mmbench.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/vlm-evaluation/vlm_eval/tasks/harnesses/mmbench.py -------------------------------------------------------------------------------- /vlm-evaluation/vlm_eval/tasks/harnesses/mmlu.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/vlm-evaluation/vlm_eval/tasks/harnesses/mmlu.py -------------------------------------------------------------------------------- /vlm-evaluation/vlm_eval/tasks/harnesses/mmmu.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/vlm-evaluation/vlm_eval/tasks/harnesses/mmmu.py -------------------------------------------------------------------------------- /vlm-evaluation/vlm_eval/tasks/harnesses/mmstar.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/vlm-evaluation/vlm_eval/tasks/harnesses/mmstar.py -------------------------------------------------------------------------------- /vlm-evaluation/vlm_eval/tasks/harnesses/mscoco_karpathy.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/vlm-evaluation/vlm_eval/tasks/harnesses/mscoco_karpathy.py -------------------------------------------------------------------------------- /vlm-evaluation/vlm_eval/tasks/harnesses/ocidref.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/vlm-evaluation/vlm_eval/tasks/harnesses/ocidref.py -------------------------------------------------------------------------------- /vlm-evaluation/vlm_eval/tasks/harnesses/okvqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/vlm-evaluation/vlm_eval/tasks/harnesses/okvqa.py -------------------------------------------------------------------------------- /vlm-evaluation/vlm_eval/tasks/harnesses/pope.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/vlm-evaluation/vlm_eval/tasks/harnesses/pope.py -------------------------------------------------------------------------------- /vlm-evaluation/vlm_eval/tasks/harnesses/refcoco.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/vlm-evaluation/vlm_eval/tasks/harnesses/refcoco.py -------------------------------------------------------------------------------- /vlm-evaluation/vlm_eval/tasks/harnesses/seedbench.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/vlm-evaluation/vlm_eval/tasks/harnesses/seedbench.py -------------------------------------------------------------------------------- /vlm-evaluation/vlm_eval/tasks/harnesses/tallyqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/vlm-evaluation/vlm_eval/tasks/harnesses/tallyqa.py -------------------------------------------------------------------------------- /vlm-evaluation/vlm_eval/tasks/harnesses/textvqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/vlm-evaluation/vlm_eval/tasks/harnesses/textvqa.py -------------------------------------------------------------------------------- /vlm-evaluation/vlm_eval/tasks/harnesses/vizwiz.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/vlm-evaluation/vlm_eval/tasks/harnesses/vizwiz.py -------------------------------------------------------------------------------- /vlm-evaluation/vlm_eval/tasks/harnesses/vqav2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/vlm-evaluation/vlm_eval/tasks/harnesses/vqav2.py -------------------------------------------------------------------------------- /vlm-evaluation/vlm_eval/tasks/harnesses/vsr.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/vlm-evaluation/vlm_eval/tasks/harnesses/vsr.py -------------------------------------------------------------------------------- /vlm-evaluation/vlm_eval/tasks/registry.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/vlm-evaluation/vlm_eval/tasks/registry.py -------------------------------------------------------------------------------- /vlm-evaluation/vlm_eval/util/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /vlm-evaluation/vlm_eval/util/evaluation/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /vlm-evaluation/vlm_eval/util/evaluation/gqa/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /vlm-evaluation/vlm_eval/util/evaluation/gqa/eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/vlm-evaluation/vlm_eval/util/evaluation/gqa/eval.py -------------------------------------------------------------------------------- /vlm-evaluation/vlm_eval/util/evaluation/mmmu/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /vlm-evaluation/vlm_eval/util/evaluation/mmmu/eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/vlm-evaluation/vlm_eval/util/evaluation/mmmu/eval.py -------------------------------------------------------------------------------- /vlm-evaluation/vlm_eval/util/evaluation/nocaps/metrics.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/vlm-evaluation/vlm_eval/util/evaluation/nocaps/metrics.py -------------------------------------------------------------------------------- /vlm-evaluation/vlm_eval/util/evaluation/textvqa/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /vlm-evaluation/vlm_eval/util/evaluation/textvqa/m4c_evaluators.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/vlm-evaluation/vlm_eval/util/evaluation/textvqa/m4c_evaluators.py -------------------------------------------------------------------------------- /vlm-evaluation/vlm_eval/util/evaluation/vizwiz/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /vlm-evaluation/vlm_eval/util/evaluation/vizwiz/eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/vlm-evaluation/vlm_eval/util/evaluation/vizwiz/eval.py -------------------------------------------------------------------------------- /vlm-evaluation/vlm_eval/util/evaluation/vqav2/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /vlm-evaluation/vlm_eval/util/evaluation/vqav2/eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/vlm-evaluation/vlm_eval/util/evaluation/vqav2/eval.py -------------------------------------------------------------------------------- /vlm-evaluation/vlm_eval/util/interfaces.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/vlm-evaluation/vlm_eval/util/interfaces.py -------------------------------------------------------------------------------- /vlm-evaluation/vlm_eval/util/loading/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /vlm-evaluation/vlm_eval/util/loading/refer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/vlm-evaluation/vlm_eval/util/loading/refer.py -------------------------------------------------------------------------------- /vlm-evaluation/vlm_eval/util/preprocessing.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Victorwz/Open-Qwen2VL/HEAD/vlm-evaluation/vlm_eval/util/preprocessing.py --------------------------------------------------------------------------------