├── .gitignore ├── LICENSE ├── README.md ├── assets └── logo.png ├── curation └── Sample_Zero-Shot_Grounding_RSNA │ ├── MedKLIP_config.yaml │ ├── dataset │ └── dataset_RSNA.py │ ├── inference_attention-map_score.py │ ├── models │ ├── model_MedKLIP.py │ ├── tokenization_bert.py │ └── transformer.py │ └── observation explanation.json ├── data ├── iuxray_dpo_weighted.json ├── mimic_dpo_weighted.json ├── slake_dpo_weighted.json └── vqa-rad_dpo_weighted.json ├── eval ├── eval_report.py └── eval_vqa.py ├── inference ├── llava-med-1.5_report.py ├── llava-med-1.5_vqa.py └── utils.py ├── requirements.txt ├── scripts ├── inference_attention-map_score.sh ├── inference_llava-med_report.sh ├── inference_llava-med_vqa.sh └── train_dpo_visual-text.sh └── train └── dpo ├── LICENSE ├── cog.yaml ├── dpo_trainer_weighted.py ├── llava ├── __init__.py ├── constants.py ├── conversation.py ├── conversation_new.py ├── eval │ ├── eval_gpt_review.py │ ├── eval_gpt_review_bench.py │ ├── eval_gpt_review_visual.py │ ├── eval_pope.py │ ├── eval_science_qa.py │ ├── eval_science_qa_gpt4.py │ ├── eval_science_qa_gpt4_requery.py │ ├── eval_textvqa.py │ ├── generate_webpage_data_from_table.py │ ├── m4c_evaluator.py │ ├── model_qa.py │ ├── model_vqa.py │ ├── model_vqa_loader.py │ ├── model_vqa_mmbench.py │ ├── model_vqa_science.py │ ├── qa_baseline_gpt35.py │ ├── run_llava.py │ ├── summarize_gpt_review.py │ ├── table │ │ ├── answer │ │ │ ├── answer_alpaca-13b.jsonl │ │ │ ├── answer_bard.jsonl │ │ │ ├── answer_gpt35.jsonl │ │ │ ├── answer_llama-13b.jsonl │ │ │ └── answer_vicuna-13b.jsonl │ │ ├── caps_boxes_coco2014_val_80.jsonl │ │ ├── model.jsonl │ │ ├── prompt.jsonl │ │ ├── question.jsonl │ │ ├── results │ │ │ ├── test_sqa_llava_13b_v0.json │ │ │ └── test_sqa_llava_lcs_558k_sqa_12e_vicuna_v1_3_13b.json │ │ ├── review │ │ │ ├── review_alpaca-13b_vicuna-13b.jsonl │ │ │ ├── review_bard_vicuna-13b.jsonl │ │ │ ├── review_gpt35_vicuna-13b.jsonl │ │ │ └── review_llama-13b_vicuna-13b.jsonl │ │ ├── reviewer.jsonl │ │ └── rule.json │ └── webpage │ │ └── styles.css ├── mm_utils.py ├── model │ ├── __init__.py │ ├── apply_delta.py │ ├── builder.py │ ├── consolidate.py │ ├── language_model │ │ ├── llava_llama.py │ │ ├── llava_mistral.py │ │ └── llava_mpt.py │ ├── llava_arch.py │ ├── make_delta.py │ ├── multimodal_encoder │ │ ├── builder.py │ │ └── clip_encoder.py │ ├── multimodal_projector │ │ └── builder.py │ └── utils.py ├── serve │ ├── __init__.py │ ├── cli.py │ ├── controller.py │ ├── gradio_web_server.py │ ├── model_worker.py │ ├── register_worker.py │ ├── sglang_worker.py │ └── test_message.py ├── train │ ├── llama_flash_attn_monkey_patch.py │ ├── llama_xformers_attn_monkey_patch.py │ ├── llava_trainer.py │ ├── train.py │ ├── train_dpo.py │ ├── train_dpo_inherent.py │ ├── train_mem.py │ └── train_xformers.py └── utils.py ├── llava_trainer_weighted.py ├── povid_infer.py ├── predict.py ├── pyproject.toml ├── scripts ├── convert_gqa_for_eval.py ├── convert_mmbench_for_submission.py ├── convert_mmvet_for_eval.py ├── convert_seed_for_submission.py ├── convert_sqa_to_llava.py ├── convert_sqa_to_llava_base_prompt.py ├── convert_vizwiz_for_submission.py ├── convert_vqav2_for_submission.py ├── extract_mm_projector.py ├── finetune.sh ├── finetune_full_schedule.sh ├── finetune_lora.sh ├── finetune_qlora.sh ├── finetune_sqa.sh ├── merge_lora_weights.py ├── pretrain.sh ├── pretrain_xformers.sh ├── run_povid.sh ├── sqa_eval_batch.sh ├── sqa_eval_gather.sh ├── upload_pypi.sh ├── v1_5 │ ├── eval │ │ ├── gqa.sh │ │ ├── llavabench.sh │ │ ├── mmbench.sh │ │ ├── mmbench_cn.sh │ │ ├── mme.sh │ │ ├── mmvet.sh │ │ ├── pope.sh │ │ ├── qbench.sh │ │ ├── qbench_zh.sh │ │ ├── seed.sh │ │ ├── sqa.sh │ │ ├── textvqa.sh │ │ ├── vizwiz.sh │ │ └── vqav2.sh │ ├── finetune.sh │ ├── finetune_lora.sh │ ├── finetune_task.sh │ ├── finetune_task_lora.sh │ └── pretrain.sh ├── zero2.json ├── zero3.json └── zero3_offload.json ├── tool ├── dpo_trainer.py └── dpo_trainer_inherent.py └── train_dpo_weighted.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/README.md -------------------------------------------------------------------------------- /assets/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/assets/logo.png -------------------------------------------------------------------------------- /curation/Sample_Zero-Shot_Grounding_RSNA/MedKLIP_config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/curation/Sample_Zero-Shot_Grounding_RSNA/MedKLIP_config.yaml -------------------------------------------------------------------------------- /curation/Sample_Zero-Shot_Grounding_RSNA/dataset/dataset_RSNA.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/curation/Sample_Zero-Shot_Grounding_RSNA/dataset/dataset_RSNA.py -------------------------------------------------------------------------------- /curation/Sample_Zero-Shot_Grounding_RSNA/inference_attention-map_score.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/curation/Sample_Zero-Shot_Grounding_RSNA/inference_attention-map_score.py -------------------------------------------------------------------------------- /curation/Sample_Zero-Shot_Grounding_RSNA/models/model_MedKLIP.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/curation/Sample_Zero-Shot_Grounding_RSNA/models/model_MedKLIP.py -------------------------------------------------------------------------------- /curation/Sample_Zero-Shot_Grounding_RSNA/models/tokenization_bert.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/curation/Sample_Zero-Shot_Grounding_RSNA/models/tokenization_bert.py -------------------------------------------------------------------------------- /curation/Sample_Zero-Shot_Grounding_RSNA/models/transformer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/curation/Sample_Zero-Shot_Grounding_RSNA/models/transformer.py -------------------------------------------------------------------------------- /curation/Sample_Zero-Shot_Grounding_RSNA/observation explanation.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/curation/Sample_Zero-Shot_Grounding_RSNA/observation explanation.json -------------------------------------------------------------------------------- /data/iuxray_dpo_weighted.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/data/iuxray_dpo_weighted.json -------------------------------------------------------------------------------- /data/mimic_dpo_weighted.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/data/mimic_dpo_weighted.json -------------------------------------------------------------------------------- /data/slake_dpo_weighted.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/data/slake_dpo_weighted.json -------------------------------------------------------------------------------- /data/vqa-rad_dpo_weighted.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/data/vqa-rad_dpo_weighted.json -------------------------------------------------------------------------------- /eval/eval_report.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/eval/eval_report.py -------------------------------------------------------------------------------- /eval/eval_vqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/eval/eval_vqa.py -------------------------------------------------------------------------------- /inference/llava-med-1.5_report.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/inference/llava-med-1.5_report.py -------------------------------------------------------------------------------- /inference/llava-med-1.5_vqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/inference/llava-med-1.5_vqa.py -------------------------------------------------------------------------------- /inference/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/inference/utils.py -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/requirements.txt -------------------------------------------------------------------------------- /scripts/inference_attention-map_score.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/scripts/inference_attention-map_score.sh -------------------------------------------------------------------------------- /scripts/inference_llava-med_report.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/scripts/inference_llava-med_report.sh -------------------------------------------------------------------------------- /scripts/inference_llava-med_vqa.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/scripts/inference_llava-med_vqa.sh -------------------------------------------------------------------------------- /scripts/train_dpo_visual-text.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/scripts/train_dpo_visual-text.sh -------------------------------------------------------------------------------- /train/dpo/LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/LICENSE -------------------------------------------------------------------------------- /train/dpo/cog.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/cog.yaml -------------------------------------------------------------------------------- /train/dpo/dpo_trainer_weighted.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/dpo_trainer_weighted.py -------------------------------------------------------------------------------- /train/dpo/llava/__init__.py: -------------------------------------------------------------------------------- 1 | from .model import LlavaLlamaForCausalLM 2 | -------------------------------------------------------------------------------- /train/dpo/llava/constants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/constants.py -------------------------------------------------------------------------------- /train/dpo/llava/conversation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/conversation.py -------------------------------------------------------------------------------- /train/dpo/llava/conversation_new.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/conversation_new.py -------------------------------------------------------------------------------- /train/dpo/llava/eval/eval_gpt_review.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/eval/eval_gpt_review.py -------------------------------------------------------------------------------- /train/dpo/llava/eval/eval_gpt_review_bench.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/eval/eval_gpt_review_bench.py -------------------------------------------------------------------------------- /train/dpo/llava/eval/eval_gpt_review_visual.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/eval/eval_gpt_review_visual.py -------------------------------------------------------------------------------- /train/dpo/llava/eval/eval_pope.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/eval/eval_pope.py -------------------------------------------------------------------------------- /train/dpo/llava/eval/eval_science_qa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/eval/eval_science_qa.py -------------------------------------------------------------------------------- /train/dpo/llava/eval/eval_science_qa_gpt4.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/eval/eval_science_qa_gpt4.py -------------------------------------------------------------------------------- /train/dpo/llava/eval/eval_science_qa_gpt4_requery.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/eval/eval_science_qa_gpt4_requery.py -------------------------------------------------------------------------------- /train/dpo/llava/eval/eval_textvqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/eval/eval_textvqa.py -------------------------------------------------------------------------------- /train/dpo/llava/eval/generate_webpage_data_from_table.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/eval/generate_webpage_data_from_table.py -------------------------------------------------------------------------------- /train/dpo/llava/eval/m4c_evaluator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/eval/m4c_evaluator.py -------------------------------------------------------------------------------- /train/dpo/llava/eval/model_qa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/eval/model_qa.py -------------------------------------------------------------------------------- /train/dpo/llava/eval/model_vqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/eval/model_vqa.py -------------------------------------------------------------------------------- /train/dpo/llava/eval/model_vqa_loader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/eval/model_vqa_loader.py -------------------------------------------------------------------------------- /train/dpo/llava/eval/model_vqa_mmbench.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/eval/model_vqa_mmbench.py -------------------------------------------------------------------------------- /train/dpo/llava/eval/model_vqa_science.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/eval/model_vqa_science.py -------------------------------------------------------------------------------- /train/dpo/llava/eval/qa_baseline_gpt35.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/eval/qa_baseline_gpt35.py -------------------------------------------------------------------------------- /train/dpo/llava/eval/run_llava.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/eval/run_llava.py -------------------------------------------------------------------------------- /train/dpo/llava/eval/summarize_gpt_review.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/eval/summarize_gpt_review.py -------------------------------------------------------------------------------- /train/dpo/llava/eval/table/answer/answer_alpaca-13b.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/eval/table/answer/answer_alpaca-13b.jsonl -------------------------------------------------------------------------------- /train/dpo/llava/eval/table/answer/answer_bard.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/eval/table/answer/answer_bard.jsonl -------------------------------------------------------------------------------- /train/dpo/llava/eval/table/answer/answer_gpt35.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/eval/table/answer/answer_gpt35.jsonl -------------------------------------------------------------------------------- /train/dpo/llava/eval/table/answer/answer_llama-13b.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/eval/table/answer/answer_llama-13b.jsonl -------------------------------------------------------------------------------- /train/dpo/llava/eval/table/answer/answer_vicuna-13b.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/eval/table/answer/answer_vicuna-13b.jsonl -------------------------------------------------------------------------------- /train/dpo/llava/eval/table/caps_boxes_coco2014_val_80.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/eval/table/caps_boxes_coco2014_val_80.jsonl -------------------------------------------------------------------------------- /train/dpo/llava/eval/table/model.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/eval/table/model.jsonl -------------------------------------------------------------------------------- /train/dpo/llava/eval/table/prompt.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/eval/table/prompt.jsonl -------------------------------------------------------------------------------- /train/dpo/llava/eval/table/question.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/eval/table/question.jsonl -------------------------------------------------------------------------------- /train/dpo/llava/eval/table/results/test_sqa_llava_13b_v0.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/eval/table/results/test_sqa_llava_13b_v0.json -------------------------------------------------------------------------------- /train/dpo/llava/eval/table/results/test_sqa_llava_lcs_558k_sqa_12e_vicuna_v1_3_13b.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/eval/table/results/test_sqa_llava_lcs_558k_sqa_12e_vicuna_v1_3_13b.json -------------------------------------------------------------------------------- /train/dpo/llava/eval/table/review/review_alpaca-13b_vicuna-13b.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/eval/table/review/review_alpaca-13b_vicuna-13b.jsonl -------------------------------------------------------------------------------- /train/dpo/llava/eval/table/review/review_bard_vicuna-13b.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/eval/table/review/review_bard_vicuna-13b.jsonl -------------------------------------------------------------------------------- /train/dpo/llava/eval/table/review/review_gpt35_vicuna-13b.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/eval/table/review/review_gpt35_vicuna-13b.jsonl -------------------------------------------------------------------------------- /train/dpo/llava/eval/table/review/review_llama-13b_vicuna-13b.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/eval/table/review/review_llama-13b_vicuna-13b.jsonl -------------------------------------------------------------------------------- /train/dpo/llava/eval/table/reviewer.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/eval/table/reviewer.jsonl -------------------------------------------------------------------------------- /train/dpo/llava/eval/table/rule.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/eval/table/rule.json -------------------------------------------------------------------------------- /train/dpo/llava/eval/webpage/styles.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/eval/webpage/styles.css -------------------------------------------------------------------------------- /train/dpo/llava/mm_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/mm_utils.py -------------------------------------------------------------------------------- /train/dpo/llava/model/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/model/__init__.py -------------------------------------------------------------------------------- /train/dpo/llava/model/apply_delta.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/model/apply_delta.py -------------------------------------------------------------------------------- /train/dpo/llava/model/builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/model/builder.py -------------------------------------------------------------------------------- /train/dpo/llava/model/consolidate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/model/consolidate.py -------------------------------------------------------------------------------- /train/dpo/llava/model/language_model/llava_llama.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/model/language_model/llava_llama.py -------------------------------------------------------------------------------- /train/dpo/llava/model/language_model/llava_mistral.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/model/language_model/llava_mistral.py -------------------------------------------------------------------------------- /train/dpo/llava/model/language_model/llava_mpt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/model/language_model/llava_mpt.py -------------------------------------------------------------------------------- /train/dpo/llava/model/llava_arch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/model/llava_arch.py -------------------------------------------------------------------------------- /train/dpo/llava/model/make_delta.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/model/make_delta.py -------------------------------------------------------------------------------- /train/dpo/llava/model/multimodal_encoder/builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/model/multimodal_encoder/builder.py -------------------------------------------------------------------------------- /train/dpo/llava/model/multimodal_encoder/clip_encoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/model/multimodal_encoder/clip_encoder.py -------------------------------------------------------------------------------- /train/dpo/llava/model/multimodal_projector/builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/model/multimodal_projector/builder.py -------------------------------------------------------------------------------- /train/dpo/llava/model/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/model/utils.py -------------------------------------------------------------------------------- /train/dpo/llava/serve/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /train/dpo/llava/serve/cli.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/serve/cli.py -------------------------------------------------------------------------------- /train/dpo/llava/serve/controller.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/serve/controller.py -------------------------------------------------------------------------------- /train/dpo/llava/serve/gradio_web_server.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/serve/gradio_web_server.py -------------------------------------------------------------------------------- /train/dpo/llava/serve/model_worker.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/serve/model_worker.py -------------------------------------------------------------------------------- /train/dpo/llava/serve/register_worker.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/serve/register_worker.py -------------------------------------------------------------------------------- /train/dpo/llava/serve/sglang_worker.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/serve/sglang_worker.py -------------------------------------------------------------------------------- /train/dpo/llava/serve/test_message.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/serve/test_message.py -------------------------------------------------------------------------------- /train/dpo/llava/train/llama_flash_attn_monkey_patch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/train/llama_flash_attn_monkey_patch.py -------------------------------------------------------------------------------- /train/dpo/llava/train/llama_xformers_attn_monkey_patch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/train/llama_xformers_attn_monkey_patch.py -------------------------------------------------------------------------------- /train/dpo/llava/train/llava_trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/train/llava_trainer.py -------------------------------------------------------------------------------- /train/dpo/llava/train/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/train/train.py -------------------------------------------------------------------------------- /train/dpo/llava/train/train_dpo.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/train/train_dpo.py -------------------------------------------------------------------------------- /train/dpo/llava/train/train_dpo_inherent.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/train/train_dpo_inherent.py -------------------------------------------------------------------------------- /train/dpo/llava/train/train_mem.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/train/train_mem.py -------------------------------------------------------------------------------- /train/dpo/llava/train/train_xformers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/train/train_xformers.py -------------------------------------------------------------------------------- /train/dpo/llava/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava/utils.py -------------------------------------------------------------------------------- /train/dpo/llava_trainer_weighted.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/llava_trainer_weighted.py -------------------------------------------------------------------------------- /train/dpo/povid_infer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/povid_infer.py -------------------------------------------------------------------------------- /train/dpo/predict.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/predict.py -------------------------------------------------------------------------------- /train/dpo/pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/pyproject.toml -------------------------------------------------------------------------------- /train/dpo/scripts/convert_gqa_for_eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/scripts/convert_gqa_for_eval.py -------------------------------------------------------------------------------- /train/dpo/scripts/convert_mmbench_for_submission.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/scripts/convert_mmbench_for_submission.py -------------------------------------------------------------------------------- /train/dpo/scripts/convert_mmvet_for_eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/scripts/convert_mmvet_for_eval.py -------------------------------------------------------------------------------- /train/dpo/scripts/convert_seed_for_submission.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/scripts/convert_seed_for_submission.py -------------------------------------------------------------------------------- /train/dpo/scripts/convert_sqa_to_llava.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/scripts/convert_sqa_to_llava.py -------------------------------------------------------------------------------- /train/dpo/scripts/convert_sqa_to_llava_base_prompt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/scripts/convert_sqa_to_llava_base_prompt.py -------------------------------------------------------------------------------- /train/dpo/scripts/convert_vizwiz_for_submission.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/scripts/convert_vizwiz_for_submission.py -------------------------------------------------------------------------------- /train/dpo/scripts/convert_vqav2_for_submission.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/scripts/convert_vqav2_for_submission.py -------------------------------------------------------------------------------- /train/dpo/scripts/extract_mm_projector.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/scripts/extract_mm_projector.py -------------------------------------------------------------------------------- /train/dpo/scripts/finetune.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/scripts/finetune.sh -------------------------------------------------------------------------------- /train/dpo/scripts/finetune_full_schedule.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/scripts/finetune_full_schedule.sh -------------------------------------------------------------------------------- /train/dpo/scripts/finetune_lora.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/scripts/finetune_lora.sh -------------------------------------------------------------------------------- /train/dpo/scripts/finetune_qlora.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/scripts/finetune_qlora.sh -------------------------------------------------------------------------------- /train/dpo/scripts/finetune_sqa.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/scripts/finetune_sqa.sh -------------------------------------------------------------------------------- /train/dpo/scripts/merge_lora_weights.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/scripts/merge_lora_weights.py -------------------------------------------------------------------------------- /train/dpo/scripts/pretrain.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/scripts/pretrain.sh -------------------------------------------------------------------------------- /train/dpo/scripts/pretrain_xformers.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/scripts/pretrain_xformers.sh -------------------------------------------------------------------------------- /train/dpo/scripts/run_povid.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/scripts/run_povid.sh -------------------------------------------------------------------------------- /train/dpo/scripts/sqa_eval_batch.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/scripts/sqa_eval_batch.sh -------------------------------------------------------------------------------- /train/dpo/scripts/sqa_eval_gather.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/scripts/sqa_eval_gather.sh -------------------------------------------------------------------------------- /train/dpo/scripts/upload_pypi.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/scripts/upload_pypi.sh -------------------------------------------------------------------------------- /train/dpo/scripts/v1_5/eval/gqa.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/scripts/v1_5/eval/gqa.sh -------------------------------------------------------------------------------- /train/dpo/scripts/v1_5/eval/llavabench.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/scripts/v1_5/eval/llavabench.sh -------------------------------------------------------------------------------- /train/dpo/scripts/v1_5/eval/mmbench.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/scripts/v1_5/eval/mmbench.sh -------------------------------------------------------------------------------- /train/dpo/scripts/v1_5/eval/mmbench_cn.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/scripts/v1_5/eval/mmbench_cn.sh -------------------------------------------------------------------------------- /train/dpo/scripts/v1_5/eval/mme.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/scripts/v1_5/eval/mme.sh -------------------------------------------------------------------------------- /train/dpo/scripts/v1_5/eval/mmvet.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/scripts/v1_5/eval/mmvet.sh -------------------------------------------------------------------------------- /train/dpo/scripts/v1_5/eval/pope.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/scripts/v1_5/eval/pope.sh -------------------------------------------------------------------------------- /train/dpo/scripts/v1_5/eval/qbench.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/scripts/v1_5/eval/qbench.sh -------------------------------------------------------------------------------- /train/dpo/scripts/v1_5/eval/qbench_zh.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/scripts/v1_5/eval/qbench_zh.sh -------------------------------------------------------------------------------- /train/dpo/scripts/v1_5/eval/seed.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/scripts/v1_5/eval/seed.sh -------------------------------------------------------------------------------- /train/dpo/scripts/v1_5/eval/sqa.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/scripts/v1_5/eval/sqa.sh -------------------------------------------------------------------------------- /train/dpo/scripts/v1_5/eval/textvqa.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/scripts/v1_5/eval/textvqa.sh -------------------------------------------------------------------------------- /train/dpo/scripts/v1_5/eval/vizwiz.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/scripts/v1_5/eval/vizwiz.sh -------------------------------------------------------------------------------- /train/dpo/scripts/v1_5/eval/vqav2.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/scripts/v1_5/eval/vqav2.sh -------------------------------------------------------------------------------- /train/dpo/scripts/v1_5/finetune.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/scripts/v1_5/finetune.sh -------------------------------------------------------------------------------- /train/dpo/scripts/v1_5/finetune_lora.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/scripts/v1_5/finetune_lora.sh -------------------------------------------------------------------------------- /train/dpo/scripts/v1_5/finetune_task.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/scripts/v1_5/finetune_task.sh -------------------------------------------------------------------------------- /train/dpo/scripts/v1_5/finetune_task_lora.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/scripts/v1_5/finetune_task_lora.sh -------------------------------------------------------------------------------- /train/dpo/scripts/v1_5/pretrain.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/scripts/v1_5/pretrain.sh -------------------------------------------------------------------------------- /train/dpo/scripts/zero2.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/scripts/zero2.json -------------------------------------------------------------------------------- /train/dpo/scripts/zero3.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/scripts/zero3.json -------------------------------------------------------------------------------- /train/dpo/scripts/zero3_offload.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/scripts/zero3_offload.json -------------------------------------------------------------------------------- /train/dpo/tool/dpo_trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/tool/dpo_trainer.py -------------------------------------------------------------------------------- /train/dpo/tool/dpo_trainer_inherent.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/tool/dpo_trainer_inherent.py -------------------------------------------------------------------------------- /train/dpo/train_dpo_weighted.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aiming-lab/MMedPO/HEAD/train/dpo/train_dpo_weighted.py --------------------------------------------------------------------------------