├── .gitattributes ├── .gitignore ├── LICENSE ├── README.md ├── example_data └── sample_100_images.zip ├── llava ├── __init__.py ├── constants.py ├── conversation.py ├── conversation_2.py ├── dataset │ ├── check_inbatchsim.py │ ├── inbatch_clip_score.py │ └── inbatch_ssim_score.py ├── eval │ ├── eval_mathverse.py │ ├── eval_mathvision.py │ ├── eval_mathvista.py │ ├── eval_okvqa.py │ ├── eval_science_qa.py │ ├── eval_textvqa.py │ ├── eval_vqa.py │ ├── m4c_evaluator.py │ ├── model_qa.py │ ├── model_vqa.py │ ├── model_vqa_loader.py │ ├── model_vqa_loader_few_shot.py │ ├── model_vqa_loader_few_shot_mathverse.py │ ├── model_vqa_loader_few_shot_mathvision.py │ ├── model_vqa_loader_few_shot_mathvista.py │ ├── model_vqa_loader_few_shot_scienceQA.py │ ├── model_vqa_science.py │ ├── rices.py │ ├── run_llava.py │ ├── translation_husky.py │ ├── translation_tool.py │ ├── utils.py │ ├── vqa_tools │ │ ├── __init__.py │ │ ├── vqa.py │ │ └── vqa_eval.py │ └── webpage │ │ ├── figures │ │ ├── alpaca.png │ │ ├── bard.jpg │ │ ├── chatgpt.svg │ │ ├── llama.jpg │ │ ├── swords_FILL0_wght300_GRAD0_opsz48.svg │ │ └── vicuna.jpeg │ │ ├── index.html │ │ ├── script.js │ │ └── styles.css ├── mm_utils.py ├── model │ ├── __init__.py │ ├── apply_delta.py │ ├── builder.py │ ├── consolidate.py │ ├── language_model │ │ ├── __init__.py │ │ ├── internlm_chat │ │ │ ├── __init__.py │ │ │ ├── configuration_internlm2.py │ │ │ ├── modeling_internlm2.py │ │ │ └── tokenization_internlm2.py │ │ ├── llava_internlm.py │ │ ├── llava_llama.py │ │ ├── llava_mistral.py │ │ └── llava_mpt.py │ ├── llava_arch.py │ ├── make_delta.py │ ├── multimodal_encoder │ │ ├── builder.py │ │ └── clip_encoder.py │ ├── multimodal_projector │ │ └── builder.py │ └── utils.py ├── serve │ ├── __init__.py │ ├── cli.py │ ├── controller.py │ ├── examples │ │ ├── extreme_ironing.jpg │ │ └── waterview.jpg │ ├── gradio_web_server.py │ ├── model_worker.py │ ├── register_worker.py │ ├── sglang_worker.py │ └── test_message.py ├── train │ ├── datasets.py │ ├── llama_flash_attn_monkey_patch.py │ ├── llama_xformers_attn_monkey_patch.py │ ├── llava_trainer.py │ ├── train.py │ ├── train_interleaved.py │ ├── train_interleaved_mem.py │ ├── train_mem.py │ └── train_xformers.py └── utils.py ├── playground └── data │ └── eval │ └── mathvista │ └── test_mini.json ├── pyproject.toml ├── scripts ├── mathverse_fewshot.sh ├── mathvision_fewshot.sh ├── mathvista_fewshot.sh ├── meta_mathverse_fewshot_h100.sh ├── meta_mathvision_fewshot_h100.sh ├── meta_mathvista_fewshot_h100.sh ├── okvqa_fewshot.sh ├── run_training.sh ├── scienceQA_fewshot.sh └── textvqa_fewshot.sh ├── src ├── logo.png └── page_fig.png └── training_idefics ├── evaluation ├── eval_mathverse.py ├── eval_mathvision.py ├── eval_mathvista.py ├── eval_okvqa.py ├── eval_textvqa.py ├── eval_vqav2.py ├── run_evaluation.sh ├── score_OKVQA.py ├── score_TextVqa.py ├── score_VQAv2.py ├── score_mathverse.py ├── score_mathvision.py ├── score_mathvista.py ├── utils.py └── vqa_tools │ ├── __init__.py │ ├── vqa.py │ └── vqa_eval.py ├── run.sh └── training ├── random_inefics2_from_mistral.py ├── run_multi_node.sh ├── test_idefics2_model.py └── training.py /.gitattributes: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/.gitattributes -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/README.md -------------------------------------------------------------------------------- /example_data/sample_100_images.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/example_data/sample_100_images.zip -------------------------------------------------------------------------------- /llava/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/__init__.py -------------------------------------------------------------------------------- /llava/constants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/constants.py -------------------------------------------------------------------------------- /llava/conversation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/conversation.py -------------------------------------------------------------------------------- /llava/conversation_2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/conversation_2.py -------------------------------------------------------------------------------- /llava/dataset/check_inbatchsim.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/dataset/check_inbatchsim.py -------------------------------------------------------------------------------- /llava/dataset/inbatch_clip_score.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/dataset/inbatch_clip_score.py -------------------------------------------------------------------------------- /llava/dataset/inbatch_ssim_score.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/dataset/inbatch_ssim_score.py -------------------------------------------------------------------------------- /llava/eval/eval_mathverse.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/eval/eval_mathverse.py -------------------------------------------------------------------------------- /llava/eval/eval_mathvision.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/eval/eval_mathvision.py -------------------------------------------------------------------------------- /llava/eval/eval_mathvista.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/eval/eval_mathvista.py -------------------------------------------------------------------------------- /llava/eval/eval_okvqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/eval/eval_okvqa.py -------------------------------------------------------------------------------- /llava/eval/eval_science_qa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/eval/eval_science_qa.py -------------------------------------------------------------------------------- /llava/eval/eval_textvqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/eval/eval_textvqa.py -------------------------------------------------------------------------------- /llava/eval/eval_vqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/eval/eval_vqa.py -------------------------------------------------------------------------------- /llava/eval/m4c_evaluator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/eval/m4c_evaluator.py -------------------------------------------------------------------------------- /llava/eval/model_qa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/eval/model_qa.py -------------------------------------------------------------------------------- /llava/eval/model_vqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/eval/model_vqa.py -------------------------------------------------------------------------------- /llava/eval/model_vqa_loader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/eval/model_vqa_loader.py -------------------------------------------------------------------------------- /llava/eval/model_vqa_loader_few_shot.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/eval/model_vqa_loader_few_shot.py -------------------------------------------------------------------------------- /llava/eval/model_vqa_loader_few_shot_mathverse.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/eval/model_vqa_loader_few_shot_mathverse.py -------------------------------------------------------------------------------- /llava/eval/model_vqa_loader_few_shot_mathvision.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/eval/model_vqa_loader_few_shot_mathvision.py -------------------------------------------------------------------------------- /llava/eval/model_vqa_loader_few_shot_mathvista.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/eval/model_vqa_loader_few_shot_mathvista.py -------------------------------------------------------------------------------- /llava/eval/model_vqa_loader_few_shot_scienceQA.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/eval/model_vqa_loader_few_shot_scienceQA.py -------------------------------------------------------------------------------- /llava/eval/model_vqa_science.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/eval/model_vqa_science.py -------------------------------------------------------------------------------- /llava/eval/rices.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/eval/rices.py -------------------------------------------------------------------------------- /llava/eval/run_llava.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/eval/run_llava.py -------------------------------------------------------------------------------- /llava/eval/translation_husky.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/eval/translation_husky.py -------------------------------------------------------------------------------- /llava/eval/translation_tool.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/eval/translation_tool.py -------------------------------------------------------------------------------- /llava/eval/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/eval/utils.py -------------------------------------------------------------------------------- /llava/eval/vqa_tools/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/eval/vqa_tools/__init__.py -------------------------------------------------------------------------------- /llava/eval/vqa_tools/vqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/eval/vqa_tools/vqa.py -------------------------------------------------------------------------------- /llava/eval/vqa_tools/vqa_eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/eval/vqa_tools/vqa_eval.py -------------------------------------------------------------------------------- /llava/eval/webpage/figures/alpaca.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/eval/webpage/figures/alpaca.png -------------------------------------------------------------------------------- /llava/eval/webpage/figures/bard.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/eval/webpage/figures/bard.jpg -------------------------------------------------------------------------------- /llava/eval/webpage/figures/chatgpt.svg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/eval/webpage/figures/chatgpt.svg -------------------------------------------------------------------------------- /llava/eval/webpage/figures/llama.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/eval/webpage/figures/llama.jpg -------------------------------------------------------------------------------- /llava/eval/webpage/figures/swords_FILL0_wght300_GRAD0_opsz48.svg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/eval/webpage/figures/swords_FILL0_wght300_GRAD0_opsz48.svg -------------------------------------------------------------------------------- /llava/eval/webpage/figures/vicuna.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/eval/webpage/figures/vicuna.jpeg -------------------------------------------------------------------------------- /llava/eval/webpage/index.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/eval/webpage/index.html -------------------------------------------------------------------------------- /llava/eval/webpage/script.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/eval/webpage/script.js -------------------------------------------------------------------------------- /llava/eval/webpage/styles.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/eval/webpage/styles.css -------------------------------------------------------------------------------- /llava/mm_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/mm_utils.py -------------------------------------------------------------------------------- /llava/model/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/model/__init__.py -------------------------------------------------------------------------------- /llava/model/apply_delta.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/model/apply_delta.py -------------------------------------------------------------------------------- /llava/model/builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/model/builder.py -------------------------------------------------------------------------------- /llava/model/consolidate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/model/consolidate.py -------------------------------------------------------------------------------- /llava/model/language_model/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /llava/model/language_model/internlm_chat/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /llava/model/language_model/internlm_chat/configuration_internlm2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/model/language_model/internlm_chat/configuration_internlm2.py -------------------------------------------------------------------------------- /llava/model/language_model/internlm_chat/modeling_internlm2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/model/language_model/internlm_chat/modeling_internlm2.py -------------------------------------------------------------------------------- /llava/model/language_model/internlm_chat/tokenization_internlm2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/model/language_model/internlm_chat/tokenization_internlm2.py -------------------------------------------------------------------------------- /llava/model/language_model/llava_internlm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/model/language_model/llava_internlm.py -------------------------------------------------------------------------------- /llava/model/language_model/llava_llama.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/model/language_model/llava_llama.py -------------------------------------------------------------------------------- /llava/model/language_model/llava_mistral.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/model/language_model/llava_mistral.py -------------------------------------------------------------------------------- /llava/model/language_model/llava_mpt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/model/language_model/llava_mpt.py -------------------------------------------------------------------------------- /llava/model/llava_arch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/model/llava_arch.py -------------------------------------------------------------------------------- /llava/model/make_delta.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/model/make_delta.py -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/model/multimodal_encoder/builder.py -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/clip_encoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/model/multimodal_encoder/clip_encoder.py -------------------------------------------------------------------------------- /llava/model/multimodal_projector/builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/model/multimodal_projector/builder.py -------------------------------------------------------------------------------- /llava/model/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/model/utils.py -------------------------------------------------------------------------------- /llava/serve/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /llava/serve/cli.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/serve/cli.py -------------------------------------------------------------------------------- /llava/serve/controller.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/serve/controller.py -------------------------------------------------------------------------------- /llava/serve/examples/extreme_ironing.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/serve/examples/extreme_ironing.jpg -------------------------------------------------------------------------------- /llava/serve/examples/waterview.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/serve/examples/waterview.jpg -------------------------------------------------------------------------------- /llava/serve/gradio_web_server.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/serve/gradio_web_server.py -------------------------------------------------------------------------------- /llava/serve/model_worker.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/serve/model_worker.py -------------------------------------------------------------------------------- /llava/serve/register_worker.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/serve/register_worker.py -------------------------------------------------------------------------------- /llava/serve/sglang_worker.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/serve/sglang_worker.py -------------------------------------------------------------------------------- /llava/serve/test_message.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/serve/test_message.py -------------------------------------------------------------------------------- /llava/train/datasets.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/train/datasets.py -------------------------------------------------------------------------------- /llava/train/llama_flash_attn_monkey_patch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/train/llama_flash_attn_monkey_patch.py -------------------------------------------------------------------------------- /llava/train/llama_xformers_attn_monkey_patch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/train/llama_xformers_attn_monkey_patch.py -------------------------------------------------------------------------------- /llava/train/llava_trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/train/llava_trainer.py -------------------------------------------------------------------------------- /llava/train/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/train/train.py -------------------------------------------------------------------------------- /llava/train/train_interleaved.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/train/train_interleaved.py -------------------------------------------------------------------------------- /llava/train/train_interleaved_mem.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/train/train_interleaved_mem.py -------------------------------------------------------------------------------- /llava/train/train_mem.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/train/train_mem.py -------------------------------------------------------------------------------- /llava/train/train_xformers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/train/train_xformers.py -------------------------------------------------------------------------------- /llava/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/llava/utils.py -------------------------------------------------------------------------------- /playground/data/eval/mathvista/test_mini.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/playground/data/eval/mathvista/test_mini.json -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/pyproject.toml -------------------------------------------------------------------------------- /scripts/mathverse_fewshot.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/scripts/mathverse_fewshot.sh -------------------------------------------------------------------------------- /scripts/mathvision_fewshot.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/scripts/mathvision_fewshot.sh -------------------------------------------------------------------------------- /scripts/mathvista_fewshot.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/scripts/mathvista_fewshot.sh -------------------------------------------------------------------------------- /scripts/meta_mathverse_fewshot_h100.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/scripts/meta_mathverse_fewshot_h100.sh -------------------------------------------------------------------------------- /scripts/meta_mathvision_fewshot_h100.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/scripts/meta_mathvision_fewshot_h100.sh -------------------------------------------------------------------------------- /scripts/meta_mathvista_fewshot_h100.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/scripts/meta_mathvista_fewshot_h100.sh -------------------------------------------------------------------------------- /scripts/okvqa_fewshot.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/scripts/okvqa_fewshot.sh -------------------------------------------------------------------------------- /scripts/run_training.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/scripts/run_training.sh -------------------------------------------------------------------------------- /scripts/scienceQA_fewshot.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/scripts/scienceQA_fewshot.sh -------------------------------------------------------------------------------- /scripts/textvqa_fewshot.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/scripts/textvqa_fewshot.sh -------------------------------------------------------------------------------- /src/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/src/logo.png -------------------------------------------------------------------------------- /src/page_fig.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/src/page_fig.png -------------------------------------------------------------------------------- /training_idefics/evaluation/eval_mathverse.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/training_idefics/evaluation/eval_mathverse.py -------------------------------------------------------------------------------- /training_idefics/evaluation/eval_mathvision.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/training_idefics/evaluation/eval_mathvision.py -------------------------------------------------------------------------------- /training_idefics/evaluation/eval_mathvista.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/training_idefics/evaluation/eval_mathvista.py -------------------------------------------------------------------------------- /training_idefics/evaluation/eval_okvqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/training_idefics/evaluation/eval_okvqa.py -------------------------------------------------------------------------------- /training_idefics/evaluation/eval_textvqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/training_idefics/evaluation/eval_textvqa.py -------------------------------------------------------------------------------- /training_idefics/evaluation/eval_vqav2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/training_idefics/evaluation/eval_vqav2.py -------------------------------------------------------------------------------- /training_idefics/evaluation/run_evaluation.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/training_idefics/evaluation/run_evaluation.sh -------------------------------------------------------------------------------- /training_idefics/evaluation/score_OKVQA.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/training_idefics/evaluation/score_OKVQA.py -------------------------------------------------------------------------------- /training_idefics/evaluation/score_TextVqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/training_idefics/evaluation/score_TextVqa.py -------------------------------------------------------------------------------- /training_idefics/evaluation/score_VQAv2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/training_idefics/evaluation/score_VQAv2.py -------------------------------------------------------------------------------- /training_idefics/evaluation/score_mathverse.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/training_idefics/evaluation/score_mathverse.py -------------------------------------------------------------------------------- /training_idefics/evaluation/score_mathvision.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/training_idefics/evaluation/score_mathvision.py -------------------------------------------------------------------------------- /training_idefics/evaluation/score_mathvista.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/training_idefics/evaluation/score_mathvista.py -------------------------------------------------------------------------------- /training_idefics/evaluation/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/training_idefics/evaluation/utils.py -------------------------------------------------------------------------------- /training_idefics/evaluation/vqa_tools/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/training_idefics/evaluation/vqa_tools/__init__.py -------------------------------------------------------------------------------- /training_idefics/evaluation/vqa_tools/vqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/training_idefics/evaluation/vqa_tools/vqa.py -------------------------------------------------------------------------------- /training_idefics/evaluation/vqa_tools/vqa_eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/training_idefics/evaluation/vqa_tools/vqa_eval.py -------------------------------------------------------------------------------- /training_idefics/run.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/training_idefics/run.sh -------------------------------------------------------------------------------- /training_idefics/training/random_inefics2_from_mistral.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/training_idefics/training/random_inefics2_from_mistral.py -------------------------------------------------------------------------------- /training_idefics/training/run_multi_node.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/training_idefics/training/run_multi_node.sh -------------------------------------------------------------------------------- /training_idefics/training/test_idefics2_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/training_idefics/training/test_idefics2_model.py -------------------------------------------------------------------------------- /training_idefics/training/training.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/DAMO-NLP-SG/multimodal_textbook/HEAD/training_idefics/training/training.py --------------------------------------------------------------------------------