├── .github ├── CONTRIBUTING.md └── workflows │ ├── deploy.yml │ └── lint.yml ├── .gitignore ├── .owners.yml ├── .pre-commit-config-zh-cn.yaml ├── .pre-commit-config.yaml ├── LICENSE ├── README.md ├── dataset_prepare.md ├── docs ├── en │ ├── notes │ │ └── changelog.md │ └── user_guides │ │ ├── chat.md │ │ ├── dataset_format.md │ │ ├── dataset_prepare.md │ │ ├── finetune.md │ │ ├── incremental_pretraining.md │ │ ├── intern_repo_dataset.md │ │ ├── multi_turn_conversation.md │ │ ├── prompt_template.md │ │ └── single_turn_conversation.md └── zh_cn │ ├── notes │ └── changelog.md │ └── user_guides │ ├── ceph.md │ ├── chat.md │ ├── config.md │ ├── custom_dataset │ ├── Offline.md │ └── Online.md │ ├── dataset_format.md │ ├── dataset_prepare.md │ ├── finetune.md │ ├── ftdp_dataset │ ├── Case1.md │ ├── Case2.md │ ├── Case3.md │ ├── Case4.md │ └── README.md │ ├── incremental_pretraining.md │ ├── intern_repo_dataset.md │ ├── llava_offline.md │ ├── multi_turn_conversation.md │ ├── prompt_template.md │ ├── sequence_parallel.md │ ├── single_turn_conversation.md │ └── varlen_attention.md ├── evaluation.md ├── examples ├── demo_data │ ├── multi_turn_1 │ │ ├── README.md │ │ ├── config.py │ │ ├── data.json │ │ └── map_fn.py │ ├── multi_turn_2 │ │ ├── README.md │ │ ├── config.py │ │ ├── data.json │ │ └── map_fn.py │ ├── pretrain │ │ ├── README.md │ │ ├── config.py │ │ ├── data.json │ │ └── map_fn.py │ └── single_turn │ │ ├── README.md │ │ ├── config.py │ │ ├── data.json │ │ └── map_fn.py ├── example.jpg └── huggingface_trainer │ ├── README.md │ ├── train_hf.py │ ├── train_lora_hf.py │ └── train_qlora_hf.py ├── images ├── MG.png ├── Main-results1.png ├── More_result.png ├── framework.png └── teaser.png ├── mg_llava ├── bbox_generation │ └── image_offline_to_bbox.py ├── config │ ├── Yi-34B │ │ ├── fuse_more_34B_test.py │ │ ├── fuse_more_yi34B_clip_L_14_336_pretrain_padding.py │ │ └── fuse_more_yi34B_clip_L_14_336_sft_padding.py │ ├── llama3 │ │ ├── fuse_more_llama3_clip_L_14_336_pretrain_padding.py │ │ └── fuse_more_llama3_clip_L_14_336_sft_padding.py │ ├── phi3 │ │ ├── fuse_more_phi3_clip_L_14_336_pretrain_padding.py │ │ └── fuse_more_phi3_clip_L_14_336_sft_padding.py │ └── vicuna │ │ ├── fuse_more_vicuna7b_clip_L_14_336_pretrain_padding.py │ │ ├── fuse_more_vicuna7b_clip_L_14_336_sft_padding.py │ │ ├── fuse_vicuna13b_clip_L_14_336_pretrain_padding.py │ │ └── fuse_vicuna13b_clip_L_14_336_sft_padding.py ├── dataset │ ├── __init__.py │ ├── __pycache__ │ │ ├── __init__.cpython-310.pyc │ │ ├── huggingface.cpython-310.pyc │ │ ├── llamavid_map_fn.cpython-310.pyc │ │ ├── llava.cpython-310.pyc │ │ └── utils.cpython-310.pyc │ ├── huggingface.py │ ├── llava.py │ ├── utils.py │ └── video │ │ ├── __pycache__ │ │ ├── batch_sampler.cpython-310.pyc │ │ ├── video_collate_fn.cpython-310.pyc │ │ └── video_length_grouped.cpython-310.pyc │ │ ├── batch_sampler.py │ │ ├── video_collate_fn.py │ │ └── video_length_grouped.py ├── evaluation │ ├── __init__.py │ ├── __pycache__ │ │ ├── __init__.cpython-310.pyc │ │ ├── llvid_proxy_eval_dataset.cpython-310.pyc │ │ └── qformer_proxy_eval_dataset.cpython-310.pyc │ ├── eval_gqa.py │ ├── gqa_llava_eval_dataset.py │ ├── m4c_evaluator.py │ ├── mg_llava_proxy_eval_dataset.py │ ├── video │ │ ├── hooks.py │ │ ├── video_mg_llava_proxy_eval_dataset.py │ │ └── video_qa_dataset.py │ └── vqav2_llava_eval_dataset.py └── module │ ├── __init__.py │ ├── __pycache__ │ ├── __init__.cpython-310.pyc │ ├── bert_qformer.cpython-310.pyc │ ├── box_model.cpython-310.pyc │ ├── fuse_attention_object_llava_model.cpython-310.pyc │ ├── fuse_box2object_llava_model.cpython-310.pyc │ ├── fuse_box_object_llava_model.cpython-310.pyc │ ├── fuse_llamavid_dataset.cpython-310.pyc │ ├── fuse_llamavid_model.cpython-310.pyc │ ├── fuse_mgm_box_model.cpython-310.pyc │ ├── fuse_mgm_object_llava_model.cpython-310.pyc │ ├── fuse_mlp_object_llava_model.cpython-310.pyc │ ├── fuse_model.cpython-310.pyc │ ├── fuse_object_llava_model.cpython-310.pyc │ ├── fuse_only_model.cpython-310.pyc │ ├── instruct_blip.cpython-310.pyc │ ├── mg_llava_dataset.cpython-310.pyc │ ├── multi_fuse_object_llava_model.cpython-310.pyc │ ├── openclip_encoder.cpython-310.pyc │ ├── qformer_box_object_llava.cpython-310.pyc │ ├── qformer_object_llava.cpython-310.pyc │ ├── qformer_object_llava_dataset.cpython-310.pyc │ ├── resampler.cpython-310.pyc │ └── video_llava.cpython-310.pyc │ ├── box_model.py │ ├── chat.py │ ├── chat_utils.py │ ├── fuse_model.py │ ├── mg_llava_dataset.py │ ├── openclip_encoder.py │ └── test_34B │ └── special_test.py ├── requirements.txt ├── requirements ├── deepspeed.txt ├── mgllava.txt ├── modelscope.txt └── runtime.txt ├── script ├── chat.sh ├── test.sh ├── train_llama3.sh ├── train_phi3.sh ├── train_pretrain.sh ├── train_sft.sh ├── train_vicuna13B.sh ├── train_vicuna7B.sh └── train_yi34b.sh ├── setup.cfg ├── setup.py └── xtuner ├── __init__.py ├── __pycache__ ├── __init__.cpython-310.pyc ├── entry_point.cpython-310.pyc ├── registry.cpython-310.pyc └── version.cpython-310.pyc ├── apis ├── __init__.py ├── datasets │ ├── __init__.py │ ├── alpaca.py │ ├── arxiv.py │ ├── code_alpaca.py │ ├── colorist.py │ ├── lawyer.py │ ├── medical.py │ ├── moss_003_sft.py │ ├── oasst1.py │ ├── open_orca.py │ ├── sql.py │ ├── tiny_codes.py │ └── wizardlm.py ├── model.py └── training_args.py ├── configs ├── __init__.py ├── __pycache__ │ └── __init__.cpython-310.pyc ├── baichuan │ ├── baichuan2_13b_base │ │ ├── baichuan2_13b_base_qlora_alpaca_e3.py │ │ ├── baichuan2_13b_base_qlora_alpaca_enzh_e3.py │ │ ├── baichuan2_13b_base_qlora_alpaca_enzh_oasst1_e3.py │ │ ├── baichuan2_13b_base_qlora_alpaca_zh_e3.py │ │ ├── baichuan2_13b_base_qlora_arxiv_gentitle_e3.py │ │ ├── baichuan2_13b_base_qlora_code_alpaca_e3.py │ │ ├── baichuan2_13b_base_qlora_colorist_e5.py │ │ ├── baichuan2_13b_base_qlora_lawyer_e3.py │ │ ├── baichuan2_13b_base_qlora_oasst1_512_e3.py │ │ ├── baichuan2_13b_base_qlora_oasst1_e3.py │ │ ├── baichuan2_13b_base_qlora_open_platypus_e3.py │ │ └── baichuan2_13b_base_qlora_sql_e3.py │ ├── baichuan2_13b_chat │ │ ├── baichuan2_13b_chat_qlora_alpaca_e3.py │ │ ├── baichuan2_13b_chat_qlora_alpaca_enzh_e3.py │ │ ├── baichuan2_13b_chat_qlora_alpaca_enzh_oasst1_e3.py │ │ ├── baichuan2_13b_chat_qlora_alpaca_zh_e3.py │ │ ├── baichuan2_13b_chat_qlora_code_alpaca_e3.py │ │ ├── baichuan2_13b_chat_qlora_lawyer_e3.py │ │ ├── baichuan2_13b_chat_qlora_oasst1_512_e3.py │ │ ├── baichuan2_13b_chat_qlora_oasst1_e3.py │ │ └── baichuan2_13b_chat_qlora_open_platypus_e3.py │ ├── baichuan2_7b_base │ │ ├── baichuan2_7b_base_qlora_alpaca_e3.py │ │ ├── baichuan2_7b_base_qlora_alpaca_enzh_e3.py │ │ ├── baichuan2_7b_base_qlora_alpaca_enzh_oasst1_e3.py │ │ ├── baichuan2_7b_base_qlora_alpaca_zh_e3.py │ │ ├── baichuan2_7b_base_qlora_arxiv_gentitle_e3.py │ │ ├── baichuan2_7b_base_qlora_code_alpaca_e3.py │ │ ├── baichuan2_7b_base_qlora_colorist_e5.py │ │ ├── baichuan2_7b_base_qlora_lawyer_e3.py │ │ ├── baichuan2_7b_base_qlora_oasst1_512_e3.py │ │ ├── baichuan2_7b_base_qlora_oasst1_e3.py │ │ ├── baichuan2_7b_base_qlora_open_platypus_e3.py │ │ └── baichuan2_7b_base_qlora_sql_e3.py │ ├── baichuan2_7b_chat │ │ ├── baichuan2_7b_chat_qlora_alpaca_e3.py │ │ ├── baichuan2_7b_chat_qlora_alpaca_enzh_e3.py │ │ ├── baichuan2_7b_chat_qlora_alpaca_enzh_oasst1_e3.py │ │ ├── baichuan2_7b_chat_qlora_alpaca_zh_e3.py │ │ ├── baichuan2_7b_chat_qlora_code_alpaca_e3.py │ │ ├── baichuan2_7b_chat_qlora_lawyer_e3.py │ │ ├── baichuan2_7b_chat_qlora_oasst1_512_e3.py │ │ ├── baichuan2_7b_chat_qlora_oasst1_e3.py │ │ └── baichuan2_7b_chat_qlora_open_platypus_e3.py │ ├── baichuan_13b_base │ │ ├── baichuan_13b_base_qlora_alpaca_e3.py │ │ ├── baichuan_13b_base_qlora_alpaca_enzh_e3.py │ │ ├── baichuan_13b_base_qlora_alpaca_enzh_oasst1_e3.py │ │ ├── baichuan_13b_base_qlora_alpaca_zh_e3.py │ │ ├── baichuan_13b_base_qlora_arxiv_gentitle_e3.py │ │ ├── baichuan_13b_base_qlora_code_alpaca_e3.py │ │ ├── baichuan_13b_base_qlora_colorist_e5.py │ │ ├── baichuan_13b_base_qlora_lawyer_e3.py │ │ ├── baichuan_13b_base_qlora_medical_e1.py │ │ ├── baichuan_13b_base_qlora_moss_sft_all_e1.py │ │ ├── baichuan_13b_base_qlora_moss_sft_all_e2_gpu8.py │ │ ├── baichuan_13b_base_qlora_moss_sft_plugins_e1.py │ │ ├── baichuan_13b_base_qlora_oasst1_512_e3.py │ │ ├── baichuan_13b_base_qlora_oasst1_e3.py │ │ ├── baichuan_13b_base_qlora_open_platypus_e3.py │ │ ├── baichuan_13b_base_qlora_openorca_e1.py │ │ ├── baichuan_13b_base_qlora_sql_e3.py │ │ └── baichuan_13b_base_qlora_tiny_codes_e1.py │ ├── baichuan_13b_chat │ │ ├── baichuan_13b_chat_qlora_alpaca_e3.py │ │ ├── baichuan_13b_chat_qlora_alpaca_enzh_e3.py │ │ ├── baichuan_13b_chat_qlora_alpaca_enzh_oasst1_e3.py │ │ ├── baichuan_13b_chat_qlora_alpaca_zh_e3.py │ │ ├── baichuan_13b_chat_qlora_arxiv_gentitle_e3.py │ │ ├── baichuan_13b_chat_qlora_code_alpaca_e3.py │ │ ├── baichuan_13b_chat_qlora_colorist_e5.py │ │ ├── baichuan_13b_chat_qlora_lawyer_e3.py │ │ ├── baichuan_13b_chat_qlora_medical_e1.py │ │ ├── baichuan_13b_chat_qlora_oasst1_512_e3.py │ │ ├── baichuan_13b_chat_qlora_oasst1_e3.py │ │ ├── baichuan_13b_chat_qlora_open_platypus_e3.py │ │ ├── baichuan_13b_chat_qlora_openorca_e1.py │ │ ├── baichuan_13b_chat_qlora_sql_e3.py │ │ └── baichuan_13b_chat_qlora_tiny_codes_e1.py │ └── baichuan_7b │ │ ├── baichuan_7b_qlora_alpaca_e3.py │ │ ├── baichuan_7b_qlora_alpaca_enzh_e3.py │ │ ├── baichuan_7b_qlora_alpaca_enzh_oasst1_e3.py │ │ ├── baichuan_7b_qlora_alpaca_zh_e3.py │ │ ├── baichuan_7b_qlora_arxiv_gentitle_e3.py │ │ ├── baichuan_7b_qlora_code_alpaca_e3.py │ │ ├── baichuan_7b_qlora_colorist_e5.py │ │ ├── baichuan_7b_qlora_lawyer_e3.py │ │ ├── baichuan_7b_qlora_medical_e1.py │ │ ├── baichuan_7b_qlora_moss_sft_all_e1.py │ │ ├── baichuan_7b_qlora_moss_sft_all_e2_gpu8.py │ │ ├── baichuan_7b_qlora_moss_sft_plugins_e1.py │ │ ├── baichuan_7b_qlora_oasst1_512_e3.py │ │ ├── baichuan_7b_qlora_oasst1_e3.py │ │ ├── baichuan_7b_qlora_open_platypus_e3.py │ │ ├── baichuan_7b_qlora_openorca_e1.py │ │ ├── baichuan_7b_qlora_sql_e3.py │ │ └── baichuan_7b_qlora_tiny_codes_e1.py ├── chatglm │ ├── chatglm2_6b │ │ ├── chatglm2_6b_qlora_alpaca_e3.py │ │ ├── chatglm2_6b_qlora_alpaca_enzh_e3.py │ │ ├── chatglm2_6b_qlora_alpaca_enzh_oasst1_e3.py │ │ ├── chatglm2_6b_qlora_alpaca_zh_e3.py │ │ ├── chatglm2_6b_qlora_arxiv_gentitle_e3.py │ │ ├── chatglm2_6b_qlora_code_alpaca_e3.py │ │ ├── chatglm2_6b_qlora_colorist_e5.py │ │ ├── chatglm2_6b_qlora_lawyer_e3.py │ │ ├── chatglm2_6b_qlora_medical_e1.py │ │ ├── chatglm2_6b_qlora_oasst1_512_e3.py │ │ ├── chatglm2_6b_qlora_oasst1_e3.py │ │ ├── chatglm2_6b_qlora_open_platypus_e3.py │ │ ├── chatglm2_6b_qlora_openorca_e1.py │ │ ├── chatglm2_6b_qlora_sql_e3.py │ │ └── chatglm2_6b_qlora_tiny_codes_e1.py │ ├── chatglm3_6b │ │ ├── chatglm3_6b_qlora_alpaca_e3.py │ │ ├── chatglm3_6b_qlora_alpaca_enzh_e3.py │ │ ├── chatglm3_6b_qlora_alpaca_enzh_oasst1_e3.py │ │ ├── chatglm3_6b_qlora_alpaca_zh_e3.py │ │ ├── chatglm3_6b_qlora_arxiv_gentitle_e3.py │ │ ├── chatglm3_6b_qlora_code_alpaca_e3.py │ │ ├── chatglm3_6b_qlora_colorist_e5.py │ │ ├── chatglm3_6b_qlora_lawyer_e3.py │ │ ├── chatglm3_6b_qlora_medical_e1.py │ │ ├── chatglm3_6b_qlora_oasst1_512_e3.py │ │ ├── chatglm3_6b_qlora_oasst1_e3.py │ │ ├── chatglm3_6b_qlora_open_platypus_e3.py │ │ ├── chatglm3_6b_qlora_openorca_e1.py │ │ ├── chatglm3_6b_qlora_sql_e3.py │ │ └── chatglm3_6b_qlora_tiny_codes_e1.py │ └── chatglm3_6b_base │ │ ├── chatglm3_6b_base_qlora_alpaca_e3.py │ │ ├── chatglm3_6b_base_qlora_alpaca_enzh_e3.py │ │ ├── chatglm3_6b_base_qlora_alpaca_enzh_oasst1_e3.py │ │ ├── chatglm3_6b_base_qlora_alpaca_zh_e3.py │ │ ├── chatglm3_6b_base_qlora_arxiv_gentitle_e3.py │ │ ├── chatglm3_6b_base_qlora_code_alpaca_e3.py │ │ ├── chatglm3_6b_base_qlora_colorist_e5.py │ │ ├── chatglm3_6b_base_qlora_lawyer_e3.py │ │ ├── chatglm3_6b_base_qlora_medical_e1.py │ │ ├── chatglm3_6b_base_qlora_oasst1_512_e3.py │ │ ├── chatglm3_6b_base_qlora_oasst1_e3.py │ │ ├── chatglm3_6b_base_qlora_open_platypus_e3.py │ │ ├── chatglm3_6b_base_qlora_openorca_e1.py │ │ ├── chatglm3_6b_base_qlora_sql_e3.py │ │ └── chatglm3_6b_base_qlora_tiny_codes_e1.py ├── cohere │ ├── README.md │ └── cohere_104b │ │ └── cohere_100b_128k_sp32.py ├── deepseek │ ├── deepseek_coder_6_7b_base │ │ └── deepseek_coder_6_7b_base_qlora_code_alpaca_e3.py │ ├── deepseek_coder_6_7b_instruct │ │ └── deepseekcoder_6_7b_instruct_qlora_code_alpaca_e3.py │ ├── deepseek_moe_16b_base │ │ ├── deepseek_moe_16b_base_full_oasst1_e3.py │ │ └── deepseek_moe_16b_base_qlora_oasst1_e3.py │ └── deepseek_moe_16b_chat │ │ ├── deepseek_moe_16b_chat_full_oasst1_e3.py │ │ └── deepseek_moe_16b_chat_qlora_oasst1_e3.py ├── deepspeed │ ├── deepspeed_zero1.json │ ├── deepspeed_zero2.json │ ├── deepspeed_zero2_offload.json │ ├── deepspeed_zero3.json │ ├── deepspeed_zero3_debug.json │ └── deepspeed_zero3_offload.json ├── gemma │ ├── gemma_2b │ │ ├── gemma_2b_full_alpaca_e3.py │ │ └── gemma_2b_qlora_alpaca_e3.py │ ├── gemma_2b_it │ │ ├── gemma_2b_it_full_alpaca_e3.py │ │ └── gemma_2b_it_qlora_alpaca_e3.py │ ├── gemma_7b │ │ ├── gemma_7b_full_alpaca_e3.py │ │ └── gemma_7b_qlora_alpaca_e3.py │ └── gemma_7b_it │ │ ├── gemma_7b_it_full_alpaca_e3.py │ │ └── gemma_7b_it_qlora_alpaca_e3.py ├── internlm │ ├── internlm2_1_8b │ │ ├── internlm2_1_8b_full_alpaca_e3.py │ │ └── internlm2_1_8b_qlora_alpaca_e3.py │ ├── internlm2_20b │ │ ├── internlm2_20b_full_finetune_custom_dataset_e1.py │ │ ├── internlm2_20b_qlora_alpaca_e3.py │ │ ├── internlm2_20b_qlora_arxiv_gentitle_e3.py │ │ ├── internlm2_20b_qlora_code_alpaca_e3.py │ │ ├── internlm2_20b_qlora_colorist_e5.py │ │ ├── internlm2_20b_qlora_lawyer_e3.py │ │ ├── internlm2_20b_qlora_msagent_react_e3_gpu8.py │ │ ├── internlm2_20b_qlora_oasst1_512_e3.py │ │ ├── internlm2_20b_qlora_oasst1_e3.py │ │ └── internlm2_20b_qlora_sql_e3.py │ ├── internlm2_7b │ │ ├── internlm2_7b_full_finetune_custom_dataset_e1.py │ │ ├── internlm2_7b_full_finetune_custom_dataset_e1_sequence_parallel_4.py │ │ ├── internlm2_7b_qlora_alpaca_e3.py │ │ ├── internlm2_7b_qlora_arxiv_gentitle_e3.py │ │ ├── internlm2_7b_qlora_code_alpaca_e3.py │ │ ├── internlm2_7b_qlora_colorist_e5.py │ │ ├── internlm2_7b_qlora_json_e3.py │ │ ├── internlm2_7b_qlora_lawyer_e3.py │ │ ├── internlm2_7b_qlora_msagent_react_e3_gpu8.py │ │ ├── internlm2_7b_qlora_oasst1_512_e3.py │ │ ├── internlm2_7b_qlora_oasst1_e3.py │ │ ├── internlm2_7b_qlora_sql_e3.py │ │ ├── internlm2_7b_w_tokenized_dataset.py │ │ └── internlm2_7b_w_untokenized_dataset.py │ ├── internlm2_chat_1_8b │ │ ├── internlm2_chat_1_8b_full_alpaca_e3.py │ │ └── internlm2_chat_1_8b_qlora_alpaca_e3.py │ ├── internlm2_chat_20b │ │ ├── internlm2_chat_20b_full_finetune_custom_dataset_e1.py │ │ ├── internlm2_chat_20b_qlora_alpaca_e3.py │ │ ├── internlm2_chat_20b_qlora_code_alpaca_e3.py │ │ ├── internlm2_chat_20b_qlora_lawyer_e3.py │ │ ├── internlm2_chat_20b_qlora_oasst1_512_e3.py │ │ └── internlm2_chat_20b_qlora_oasst1_e3.py │ ├── internlm2_chat_7b │ │ ├── internlm2_chat_7b_full_finetune_custom_dataset_e1.py │ │ ├── internlm2_chat_7b_qlora_alpaca_e3.py │ │ ├── internlm2_chat_7b_qlora_code_alpaca_e3.py │ │ ├── internlm2_chat_7b_qlora_lawyer_e3.py │ │ ├── internlm2_chat_7b_qlora_oasst1_512_e3.py │ │ └── internlm2_chat_7b_qlora_oasst1_e3.py │ ├── internlm_20b │ │ ├── internlm_20b_qlora_alpaca_e3.py │ │ ├── internlm_20b_qlora_alpaca_enzh_e3.py │ │ ├── internlm_20b_qlora_alpaca_enzh_oasst1_e3.py │ │ ├── internlm_20b_qlora_alpaca_zh_e3.py │ │ ├── internlm_20b_qlora_arxiv_gentitle_e3.py │ │ ├── internlm_20b_qlora_code_alpaca_e3.py │ │ ├── internlm_20b_qlora_colorist_e5.py │ │ ├── internlm_20b_qlora_lawyer_e3.py │ │ ├── internlm_20b_qlora_msagent_react_e3_gpu8.py │ │ ├── internlm_20b_qlora_oasst1_512_e3.py │ │ ├── internlm_20b_qlora_oasst1_e3.py │ │ ├── internlm_20b_qlora_open_platypus_e3.py │ │ └── internlm_20b_qlora_sql_e3.py │ ├── internlm_7b │ │ ├── internlm_7b_full_alpaca_e3.py │ │ ├── internlm_7b_full_alpaca_enzh_e3.py │ │ ├── internlm_7b_full_alpaca_enzh_oasst1_e3.py │ │ ├── internlm_7b_full_alpaca_zh_e3.py │ │ ├── internlm_7b_full_intern_repo_dataset_template.py │ │ ├── internlm_7b_full_oasst1_e3.py │ │ ├── internlm_7b_qlora_alpaca_e3.py │ │ ├── internlm_7b_qlora_alpaca_enzh_e3.py │ │ ├── internlm_7b_qlora_alpaca_enzh_oasst1_e3.py │ │ ├── internlm_7b_qlora_alpaca_zh_e3.py │ │ ├── internlm_7b_qlora_arxiv_gentitle_e3.py │ │ ├── internlm_7b_qlora_code_alpaca_e3.py │ │ ├── internlm_7b_qlora_colorist_e5.py │ │ ├── internlm_7b_qlora_json_e3.py │ │ ├── internlm_7b_qlora_lawyer_e3.py │ │ ├── internlm_7b_qlora_medical_e1.py │ │ ├── internlm_7b_qlora_moss_sft_all_e1.py │ │ ├── internlm_7b_qlora_moss_sft_all_e2_gpu8.py │ │ ├── internlm_7b_qlora_moss_sft_plugins_e1.py │ │ ├── internlm_7b_qlora_msagent_react_e3_gpu8.py │ │ ├── internlm_7b_qlora_oasst1_512_e3.py │ │ ├── internlm_7b_qlora_oasst1_e3.py │ │ ├── internlm_7b_qlora_oasst1_e3_hf.py │ │ ├── internlm_7b_qlora_oasst1_mmlu_e3.py │ │ ├── internlm_7b_qlora_open_platypus_e3.py │ │ ├── internlm_7b_qlora_openorca_e1.py │ │ ├── internlm_7b_qlora_sql_e3.py │ │ └── internlm_7b_qlora_tiny_codes_e1.py │ ├── internlm_chat_20b │ │ ├── internlm_chat_20b_qlora_alpaca_e3.py │ │ ├── internlm_chat_20b_qlora_alpaca_enzh_e3.py │ │ ├── internlm_chat_20b_qlora_alpaca_enzh_oasst1_e3.py │ │ ├── internlm_chat_20b_qlora_alpaca_zh_e3.py │ │ ├── internlm_chat_20b_qlora_code_alpaca_e3.py │ │ ├── internlm_chat_20b_qlora_lawyer_e3.py │ │ ├── internlm_chat_20b_qlora_oasst1_512_e3.py │ │ ├── internlm_chat_20b_qlora_oasst1_e3.py │ │ └── internlm_chat_20b_qlora_open_platypus_e3.py │ └── internlm_chat_7b │ │ ├── internlm_chat_7b_qlora_alpaca_e3.py │ │ ├── internlm_chat_7b_qlora_alpaca_enzh_e3.py │ │ ├── internlm_chat_7b_qlora_alpaca_enzh_oasst1_e3.py │ │ ├── internlm_chat_7b_qlora_alpaca_zh_e3.py │ │ ├── internlm_chat_7b_qlora_arxiv_gentitle_e3.py │ │ ├── internlm_chat_7b_qlora_code_alpaca_e3.py │ │ ├── internlm_chat_7b_qlora_colorist_e5.py │ │ ├── internlm_chat_7b_qlora_lawyer_e3.py │ │ ├── internlm_chat_7b_qlora_medical_e1.py │ │ ├── internlm_chat_7b_qlora_oasst1_512_e3.py │ │ ├── internlm_chat_7b_qlora_oasst1_e3.py │ │ ├── internlm_chat_7b_qlora_open_platypus_e3.py │ │ ├── internlm_chat_7b_qlora_openorca_e1.py │ │ ├── internlm_chat_7b_qlora_sql_e3.py │ │ └── internlm_chat_7b_qlora_tiny_codes_e1.py ├── llama │ ├── llama2_70b │ │ ├── llama2_70b_full_wizardlm_e1.py │ │ ├── llama2_70b_int8_lora_open_platypus_e1.py │ │ ├── llama2_70b_int8_lora_open_platypus_e1_hf.py │ │ ├── llama2_70b_qlora_open_platypus_e1.py │ │ └── llama2_70b_qlora_open_platypus_e1_hf.py │ ├── llama2_7b │ │ ├── llama2_7b_full_pgbooks_400iters_sp1.py │ │ ├── llama2_7b_full_pgbooks_400iters_sp4.py │ │ ├── llama2_7b_full_wizardlm_e1.py │ │ ├── llama2_7b_qlora_alpaca_e3.py │ │ ├── llama2_7b_qlora_alpaca_enzh_e3.py │ │ ├── llama2_7b_qlora_alpaca_enzh_oasst1_e3.py │ │ ├── llama2_7b_qlora_alpaca_zh_e3.py │ │ ├── llama2_7b_qlora_arxiv_gentitle_e3.py │ │ ├── llama2_7b_qlora_code_alpaca_e3.py │ │ ├── llama2_7b_qlora_colorist_e5.py │ │ ├── llama2_7b_qlora_lawyer_e3.py │ │ ├── llama2_7b_qlora_medical_e1.py │ │ ├── llama2_7b_qlora_moss_sft_all_e1.py │ │ ├── llama2_7b_qlora_moss_sft_all_e2_gpu8.py │ │ ├── llama2_7b_qlora_moss_sft_plugins_e1.py │ │ ├── llama2_7b_qlora_msagent_react_e3_gpu8.py │ │ ├── llama2_7b_qlora_oasst1_512_e3.py │ │ ├── llama2_7b_qlora_oasst1_e3.py │ │ ├── llama2_7b_qlora_open_platypus_e3.py │ │ ├── llama2_7b_qlora_openorca_e1.py │ │ ├── llama2_7b_qlora_sql_e3.py │ │ └── llama2_7b_qlora_tiny_codes_e1.py │ ├── llama2_7b_chat │ │ ├── llama2_7b_chat_qlora_alpaca_e3.py │ │ ├── llama2_7b_chat_qlora_alpaca_enzh_e3.py │ │ ├── llama2_7b_chat_qlora_alpaca_enzh_oasst1_e3.py │ │ ├── llama2_7b_chat_qlora_alpaca_zh_e3.py │ │ ├── llama2_7b_chat_qlora_arxiv_gentitle_e3.py │ │ ├── llama2_7b_chat_qlora_code_alpaca_e3.py │ │ ├── llama2_7b_chat_qlora_colorist_e5.py │ │ ├── llama2_7b_chat_qlora_lawyer_e3.py │ │ ├── llama2_7b_chat_qlora_medical_e1.py │ │ ├── llama2_7b_chat_qlora_oasst1_512_e3.py │ │ ├── llama2_7b_chat_qlora_oasst1_e3.py │ │ ├── llama2_7b_chat_qlora_open_platypus_e3.py │ │ ├── llama2_7b_chat_qlora_openorca_e1.py │ │ ├── llama2_7b_chat_qlora_sql_e3.py │ │ └── llama2_7b_chat_qlora_tiny_codes_e1.py │ ├── llama3_8b │ │ ├── README.md │ │ └── llama3_8b_full_alpaca_e3.py │ ├── llama3_8b_instruct │ │ ├── llama3_8b_instruct_full_alpaca_e3.py │ │ └── llama3_8b_instruct_qlora_alpaca_e3.py │ └── llama_7b │ │ ├── llama_7b_qlora_alpaca_e3.py │ │ ├── llama_7b_qlora_alpaca_enzh_e3.py │ │ ├── llama_7b_qlora_alpaca_enzh_oasst1_e3.py │ │ ├── llama_7b_qlora_alpaca_zh_e3.py │ │ ├── llama_7b_qlora_arxiv_gentitle_e3.py │ │ ├── llama_7b_qlora_code_alpaca_e3.py │ │ ├── llama_7b_qlora_colorist_e5.py │ │ ├── llama_7b_qlora_lawyer_e3.py │ │ ├── llama_7b_qlora_medical_e1.py │ │ ├── llama_7b_qlora_moss_sft_all_e1.py │ │ ├── llama_7b_qlora_moss_sft_all_e2_gpu8.py │ │ ├── llama_7b_qlora_moss_sft_plugins_e1.py │ │ ├── llama_7b_qlora_oasst1_512_e3.py │ │ ├── llama_7b_qlora_oasst1_e3.py │ │ ├── llama_7b_qlora_open_platypus_e3.py │ │ ├── llama_7b_qlora_openorca_e1.py │ │ ├── llama_7b_qlora_sql_e3.py │ │ └── llama_7b_qlora_tiny_codes_e1.py ├── llava │ ├── README.md │ ├── README_zh-CN.md │ ├── internlm2_chat_1_8b_clip_vit_large_p14_336 │ │ ├── finetune │ │ │ └── llava_internlm2_chat_1_8b_qlora_clip_vit_large_p14_336_lora_e1_gpu8_finetune.py │ │ └── pretrain │ │ │ └── llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu8_pretrain.py │ ├── internlm2_chat_20b_clip_vit_large_p14_336 │ │ ├── finetune │ │ │ └── llava_internlm2_chat_20b_qlora_clip_vit_large_p14_336_lora_e1_gpu8_finetune.py │ │ └── pretrain │ │ │ └── llava_internlm2_chat_20b_clip_vit_large_p14_336_e1_gpu8_pretrain.py │ ├── internlm2_chat_7b_clip_vit_large_p14_336 │ │ ├── finetune │ │ │ └── llava_internlm2_chat_7b_qlora_clip_vit_large_p14_336_lora_e1_gpu8_finetune.py │ │ └── pretrain │ │ │ └── llava_internlm2_chat_7b_clip_vit_large_p14_336_e1_gpu8_pretrain.py │ ├── internlm_chat_7b_clip_vit_large_p14_336 │ │ ├── finetune │ │ │ └── llava_internlm_chat_7b_qlora_clip_vit_large_p14_336_lora_e1_gpu8_finetune.py │ │ └── pretrain │ │ │ └── llava_internlm_chat_7b_clip_vit_large_p14_336_e1_gpu8_pretrain.py │ ├── llama3_70b_chat │ │ └── llava_llama3_70b_chat_clip_e1_gpu16_sharegpt4v_pretrain.py │ ├── llama3_8b_chat │ │ ├── llava_llama3_8b_chat_clip_e1_gpu8_pretrain.py │ │ ├── llava_llama3_8b_chat_clip_e1_gpu8_sharegpt4v_pretrain.py │ │ ├── llava_llama3_8b_chat_clip_lora_e1_gpu8_finetune.py │ │ └── llava_llama3_8b_chat_clip_lora_e1_gpu8_internvl_finetune.py │ ├── llama3_8b_instruct_clip_vit_large_p14_336 │ │ ├── README.md │ │ ├── finetune │ │ │ ├── llava_llama3_8b_instruct_full_clip_vit_large_p14_336_e1_gpu8_finetune.py │ │ │ ├── llava_llama3_8b_instruct_full_clip_vit_large_p14_336_lora_e1_gpu8_finetune.py │ │ │ ├── llava_llama3_8b_instruct_full_clip_vit_large_p14_336_lora_e1_gpu8_internvl_finetune.py │ │ │ └── llava_llama3_8b_instruct_qlora_clip_vit_large_p14_336_e1_gpu1_finetune.py │ │ └── pretrain │ │ │ ├── llava_llama3_8b_instruct_clip_vit_large_p14_336_e1_gpu8_pretrain.py │ │ │ ├── llava_llama3_8b_instruct_clip_vit_large_p14_336_e1_gpu8_sharegpt4v_pretrain.py │ │ │ └── llava_llama3_8b_instruct_quant_clip_vit_large_p14_336_e1_gpu1_pretrain.py │ ├── official │ │ ├── llava_v15_13b │ │ │ ├── llava_v15_13b_finetune.py │ │ │ ├── llava_v15_13b_finetune_lora.py │ │ │ └── llava_v15_13b_pretrain.py │ │ └── llava_v15_7b │ │ │ ├── llava_v15_7b_finetune.py │ │ │ ├── llava_v15_7b_finetune_lora.py │ │ │ └── llava_v15_7b_pretrain.py │ ├── phi2_2_7b_siglip_so400m_p14_384 │ │ ├── llava_phi2_2_7b_clip_p14_384_convnext_e1_gpu8_finetune.py │ │ ├── llava_phi2_2_7b_clip_p14_384_convnext_e1_gpu8_pretrain.py │ │ ├── llava_phi2_2_7b_qlora_siglip_so400m_p14_384_e1_gpu8_finetune.py │ │ ├── llava_phi2_2_7b_qlora_siglip_so400m_p14_384_loar_e1_gpu8_finetune.py │ │ ├── llava_phi2_2_7b_siglip_so400m_p14_384_e1_gpu8_all_finetune.py │ │ ├── llava_phi2_2_7b_siglip_so400m_p14_384_e1_gpu8_finetune.py │ │ ├── llava_phi2_2_7b_siglip_so400m_p14_384_e1_gpu8_pretrain.py │ │ ├── llava_phi2_2_7b_siglip_so400m_p14_384_s2_e1_gpu8_pretrain.py │ │ ├── llava_phi2_2_7b_siglip_so400m_p14_anyres_e1_gpu8_all_finetune.py │ │ ├── llava_phi2_2_7b_siglip_so400m_p14_anyres_e1_gpu8_pretrain.py │ │ └── llava_phi2_2_7b_siglip_so400m_p14_anyres_pixel_shuffle_e1_gpu8_all_finetune.py │ ├── phi3_mini_4k_instruct_clip_vit_large_p14_336 │ │ ├── finetune │ │ │ └── llava_phi3_mini_4k_instruct_full_clip_vit_large_p14_336_e1_gpu8_finetune.py │ │ └── pretrain │ │ │ ├── llava_phi3_mini_4k_instruct_clip_vit_large_p14_336_e1_gpu8_pretrain.py │ │ │ └── llava_phi3_mini_4k_instruct_clip_vit_large_p14_336_e1_gpu8_sharegpt4v_pretrain.py │ ├── phi3_mini_chat │ │ ├── llava_phi3_mini_4k_instruct_clip_vit_large_p14_336_e1_gpu8_pretrain.py │ │ ├── llava_phi3_mini_4k_instruct_full_clip_vit_large_lora_p14_336_anyres_e1_gpu8_finetune.py │ │ └── llava_phi3_mini_4k_instruct_full_clip_vit_large_lora_p14_336_e1_gpu8_finetune.py │ ├── vicuna_13b_v15_clip_vit_large_p14_336 │ │ ├── finetune │ │ │ └── llava_vicuna_13b_v15_qlora_clip_vit_large_p14_336_lora_e1_gpu8_finetune.py │ │ └── pretrain │ │ │ └── llava_vicuna_13b_v15_clip_vit_large_p14_336_e1_gpu8_pretrain.py │ └── vicuna_7b_v15_clip_vit_large_p14_336 │ │ ├── finetune │ │ ├── llava_vicuna_7b_v15_qlora_clip_vit_large_p14_336_lora_e1_gpu8_finetune.py │ │ └── llava_vicuna_7b_v15_qlora_clip_vit_large_p14_336_lora_e1_gpu8_finetune_refcoco.py │ │ └── pretrain │ │ └── llava_vicuna_7b_v15_clip_vit_large_p14_336_e1_gpu8_pretrain.py ├── mistral │ ├── mistral_7b_full_finetune_custom_dataset_e1.py │ ├── mistral_7b_qlora_skypile_pretrain_e1.py │ ├── mistral_7b_w_tokenized_dataset.py │ └── mistral_7b_w_untokenized_dataset.py ├── mixtral │ ├── README.md │ ├── mixtral_8x7b │ │ ├── mixtral_8x7b_full_oasst1_e3.py │ │ └── mixtral_8x7b_qlora_oasst1_e3.py │ └── mixtral_8x7b_instruct │ │ ├── mixtral_8x7b_instruct_full_oasst1_e3.py │ │ └── mixtral_8x7b_instruct_qlora_oasst1_e3.py ├── phi │ └── phi3 │ │ ├── phi3_mini_128k_instruct_full_alpaca_e3.py │ │ ├── phi3_mini_128k_instruct_qlora_alpaca_e3.py │ │ ├── phi3_mini_4k_instruct_full_alpaca_e3.py │ │ └── phi3_mini_4k_instruct_qlora_alpaca_e3.py ├── qwen │ ├── qwen1 │ │ ├── qwen_1_8b │ │ │ ├── qwen_1_8b_qlora_alpaca_e3.py │ │ │ ├── qwen_1_8b_qlora_alpaca_enzh_e3.py │ │ │ ├── qwen_1_8b_qlora_alpaca_enzh_oasst1_e3.py │ │ │ ├── qwen_1_8b_qlora_alpaca_zh_e3.py │ │ │ └── qwen_1_8b_qlora_code_alpaca_e3.py │ │ ├── qwen_1_8b_chat │ │ │ ├── qwen_1_8b_chat_qlora_alpaca_e3.py │ │ │ ├── qwen_1_8b_chat_qlora_alpaca_enzh_e3.py │ │ │ ├── qwen_1_8b_chat_qlora_alpaca_enzh_oasst1_e3.py │ │ │ ├── qwen_1_8b_chat_qlora_alpaca_zh_e3.py │ │ │ └── qwen_1_8b_chat_qlora_code_alpaca_e3.py │ │ ├── qwen_72b │ │ │ ├── qwen_72b_qlora_alpaca_e3.py │ │ │ ├── qwen_72b_qlora_alpaca_enzh_e3.py │ │ │ ├── qwen_72b_qlora_alpaca_enzh_oasst1_e3.py │ │ │ ├── qwen_72b_qlora_alpaca_zh_e3.py │ │ │ └── qwen_72b_qlora_code_alpaca_e3.py │ │ ├── qwen_7b │ │ │ ├── qwen_7b_qlora_alpaca_e3.py │ │ │ ├── qwen_7b_qlora_alpaca_enzh_e3.py │ │ │ ├── qwen_7b_qlora_alpaca_enzh_oasst1_e3.py │ │ │ ├── qwen_7b_qlora_alpaca_zh_e3.py │ │ │ ├── qwen_7b_qlora_arxiv_gentitle_e3.py │ │ │ ├── qwen_7b_qlora_code_alpaca_e3.py │ │ │ ├── qwen_7b_qlora_colorist_e5.py │ │ │ ├── qwen_7b_qlora_lawyer_e3.py │ │ │ ├── qwen_7b_qlora_medical_e1.py │ │ │ ├── qwen_7b_qlora_moss_sft_all_e1.py │ │ │ ├── qwen_7b_qlora_moss_sft_all_e2_gpu8.py │ │ │ ├── qwen_7b_qlora_moss_sft_plugins_e1.py │ │ │ ├── qwen_7b_qlora_oasst1_512_e3.py │ │ │ ├── qwen_7b_qlora_oasst1_e3.py │ │ │ ├── qwen_7b_qlora_open_platypus_e3.py │ │ │ ├── qwen_7b_qlora_openorca_e1.py │ │ │ ├── qwen_7b_qlora_sql_e3.py │ │ │ └── qwen_7b_qlora_tiny_codes_e1.py │ │ └── qwen_7b_chat │ │ │ ├── qwen_7b_chat_qlora_alpaca_e3.py │ │ │ ├── qwen_7b_chat_qlora_alpaca_enzh_e3.py │ │ │ ├── qwen_7b_chat_qlora_alpaca_enzh_oasst1_e3.py │ │ │ ├── qwen_7b_chat_qlora_alpaca_zh_e3.py │ │ │ ├── qwen_7b_chat_qlora_arxiv_gentitle_e3.py │ │ │ ├── qwen_7b_chat_qlora_code_alpaca_e3.py │ │ │ ├── qwen_7b_chat_qlora_colorist_e5.py │ │ │ ├── qwen_7b_chat_qlora_lawyer_e3.py │ │ │ ├── qwen_7b_chat_qlora_medical_e1.py │ │ │ ├── qwen_7b_chat_qlora_oasst1_512_e3.py │ │ │ ├── qwen_7b_chat_qlora_oasst1_e3.py │ │ │ ├── qwen_7b_chat_qlora_open_platypus_e3.py │ │ │ ├── qwen_7b_chat_qlora_openorca_e1.py │ │ │ ├── qwen_7b_chat_qlora_sql_e3.py │ │ │ └── qwen_7b_chat_qlora_tiny_codes_e1.py │ └── qwen1_5 │ │ ├── qwen1_5_0_5b │ │ ├── qwen1_5_0_5b_full_alpaca_e3.py │ │ └── qwen1_5_0_5b_qlora_alpaca_e3.py │ │ ├── qwen1_5_0_5b_chat │ │ ├── qwen1_5_0_5b_chat_full_alpaca_e3.py │ │ └── qwen1_5_0_5b_chat_qlora_alpaca_e3.py │ │ ├── qwen1_5_14b │ │ ├── qwen1_5_14b_full_alpaca_e3.py │ │ └── qwen1_5_14b_qlora_alpaca_e3.py │ │ ├── qwen1_5_14b_chat │ │ ├── qwen1_5_14b_chat_full_alpaca_e3.py │ │ └── qwen1_5_14b_chat_qlora_alpaca_e3.py │ │ ├── qwen1_5_1_8b │ │ ├── qwen1_5_1_8b_full_alpaca_e3.py │ │ └── qwen1_5_1_8b_qlora_alpaca_e3.py │ │ ├── qwen1_5_1_8b_chat │ │ ├── qwen1_5_1_8b_chat_full_alpaca_e3.py │ │ └── qwen1_5_1_8b_chat_qlora_alpaca_e3.py │ │ ├── qwen1_5_4b │ │ ├── qwen1_5_4b_full_alpaca_e3.py │ │ └── qwen1_5_4b_qlora_alpaca_e3.py │ │ ├── qwen1_5_4b_chat │ │ ├── qwen1_5_4b_chat_full_alpaca_e3.py │ │ └── qwen1_5_4b_chat_qlora_alpaca_e3.py │ │ ├── qwen1_5_72b │ │ ├── qwen1_5_72b_full_alpaca_e3.py │ │ └── qwen1_5_72b_qlora_alpaca_e3.py │ │ ├── qwen1_5_72b_chat │ │ ├── qwen1_5_72b_chat_full_alpaca_e3.py │ │ └── qwen1_5_72b_chat_qlora_alpaca_e3.py │ │ ├── qwen1_5_7b │ │ ├── qwen1_5_7b_full_alpaca_e3.py │ │ └── qwen1_5_7b_qlora_alpaca_e3.py │ │ └── qwen1_5_7b_chat │ │ ├── qwen1_5_7b_chat_full_alpaca_e3.py │ │ └── qwen1_5_7b_chat_qlora_alpaca_e3.py ├── qwen_moe │ └── qwen1_5 │ │ └── qwen1_5_moe_a2_7_b_chat │ │ └── qwen1_5_moe_a2_7_b_chat_full_alpaca_e3.py ├── starcoder │ └── starcoder_qlora_stack_exchange_example.py ├── yi │ ├── yi_34b │ │ └── yi_34b_qlora_alpaca_enzh_e3.py │ └── yi_6b │ │ └── yi_6b_qlora_alpaca_enzh_e3.py └── zephyr │ └── zephyr_7b_beta_qlora_alpaca_e3.py ├── dataset ├── __init__.py ├── __pycache__ │ ├── __init__.cpython-310.pyc │ ├── anyres_llava_proxy_eval_dataset.cpython-310.pyc │ ├── concat_dataset.cpython-310.pyc │ ├── huggingface.cpython-310.pyc │ ├── intern_repo.cpython-310.pyc │ ├── json_dataset.cpython-310.pyc │ ├── llava.cpython-310.pyc │ ├── llava_proxy_eval_dataset.cpython-310.pyc │ ├── mini_gemini_dataset.cpython-310.pyc │ ├── mini_gemini_proxy_eval_dataset.cpython-310.pyc │ ├── modelscope.cpython-310.pyc │ ├── moss_sft.cpython-310.pyc │ ├── object_llava.cpython-310.pyc │ ├── refcoco_json.cpython-310.pyc │ └── utils.cpython-310.pyc ├── anyres_llava_proxy_eval_dataset.py ├── collate_fns │ ├── __init__.py │ ├── __pycache__ │ │ ├── __init__.cpython-310.pyc │ │ ├── defalut_collate_fn.cpython-310.pyc │ │ ├── default_collate_fn.cpython-310.pyc │ │ ├── mm_collate_fn.cpython-310.pyc │ │ └── mmlu_collate_fn.cpython-310.pyc │ ├── default_collate_fn.py │ ├── mm_collate_fn.py │ └── mmlu_collate_fn.py ├── concat_dataset.py ├── evaluation │ ├── MMVP │ │ ├── gpt_eval.sh │ │ └── gpt_grader.py │ ├── __init__.py │ ├── __pycache__ │ │ ├── __init__.cpython-310.pyc │ │ ├── base_eval_dataset.cpython-310.pyc │ │ ├── chartqa_dataset.cpython-310.pyc │ │ ├── general_vqa_dataset.cpython-310.pyc │ │ ├── gqa_dataset.cpython-310.pyc │ │ ├── gqa_eval_utils.cpython-310.pyc │ │ ├── hallusion_dataset.cpython-310.pyc │ │ ├── hallusion_llava_dataset.cpython-310.pyc │ │ ├── mme_dataset.cpython-310.pyc │ │ ├── mme_llava_dataset.cpython-310.pyc │ │ ├── multiple_choice_dataset.cpython-310.pyc │ │ ├── multiple_choice_llava_dataset.cpython-310.pyc │ │ ├── pope_dataset.cpython-310.pyc │ │ ├── pope_llava_dataset.cpython-310.pyc │ │ ├── textvqa_dataset.cpython-310.pyc │ │ ├── textvqa_llava_dataset.cpython-310.pyc │ │ ├── textvqa_utils.cpython-310.pyc │ │ ├── utils.cpython-310.pyc │ │ ├── vqav2_dataset.cpython-310.pyc │ │ └── vqav2_utils.cpython-310.pyc │ ├── base_eval_dataset.py │ ├── chartqa_dataset.py │ ├── general_vqa_dataset.py │ ├── gqa_dataset.py │ ├── gqa_eval_utils.py │ ├── hallusion_dataset.py │ ├── llava_w │ │ ├── eval_gpt_review_bench.py │ │ ├── gpt_eval.sh │ │ ├── rule.json │ │ └── summarize_gpt_review.py │ ├── math_dataset.py │ ├── math_vista │ │ ├── __pycache__ │ │ │ └── utilities.cpython-310.pyc │ │ ├── calculate_score.py │ │ ├── extract_answer.py │ │ ├── gpt_eval.sh │ │ ├── prompts │ │ │ ├── __pycache__ │ │ │ │ └── ext_ans.cpython-310.pyc │ │ │ └── ext_ans.py │ │ └── utilities.py │ ├── mme_dataset.py │ ├── mmvp_dataset.py │ ├── multiple_choice_dataset.py │ ├── pope_dataset.py │ ├── textvqa_dataset.py │ ├── textvqa_utils.py │ ├── utils.py │ ├── vqa_dataset.py │ ├── vqav2_dataset.py │ └── vqav2_utils.py ├── huggingface.py ├── intern_repo.py ├── json_dataset.py ├── llava.py ├── llava_proxy_eval_dataset.py ├── map_fns │ ├── __init__.py │ ├── __pycache__ │ │ ├── __init__.cpython-310.pyc │ │ └── template_map_fn.cpython-310.pyc │ ├── dataset_map_fns │ │ ├── __init__.py │ │ ├── __pycache__ │ │ │ ├── __init__.cpython-310.pyc │ │ │ ├── alpaca_map_fn.cpython-310.pyc │ │ │ ├── alpaca_zh_map_fn.cpython-310.pyc │ │ │ ├── arxiv_map_fn.cpython-310.pyc │ │ │ ├── code_alpaca_map_fn.cpython-310.pyc │ │ │ ├── colors_map_fn.cpython-310.pyc │ │ │ ├── crime_kg_assitant_map_fn.cpython-310.pyc │ │ │ ├── default_map_fn.cpython-310.pyc │ │ │ ├── law_reference_map_fn.cpython-310.pyc │ │ │ ├── llava_map_fn.cpython-310.pyc │ │ │ ├── medical_map_fn.cpython-310.pyc │ │ │ ├── msagent_map_fn.cpython-310.pyc │ │ │ ├── oasst1_map_fn.cpython-310.pyc │ │ │ ├── openai_map_fn.cpython-310.pyc │ │ │ ├── openorca_map_fn.cpython-310.pyc │ │ │ ├── pretrain_map_fn.cpython-310.pyc │ │ │ ├── sql_map_fn.cpython-310.pyc │ │ │ ├── stack_exchange_map_fn.cpython-310.pyc │ │ │ ├── tiny_codes_map_fn.cpython-310.pyc │ │ │ └── wizardlm_map_fn.cpython-310.pyc │ │ ├── alpaca_map_fn.py │ │ ├── alpaca_zh_map_fn.py │ │ ├── arxiv_map_fn.py │ │ ├── code_alpaca_map_fn.py │ │ ├── colors_map_fn.py │ │ ├── crime_kg_assitant_map_fn.py │ │ ├── default_map_fn.py │ │ ├── law_reference_map_fn.py │ │ ├── llava_map_fn.py │ │ ├── medical_map_fn.py │ │ ├── msagent_map_fn.py │ │ ├── oasst1_map_fn.py │ │ ├── openai_map_fn.py │ │ ├── openorca_map_fn.py │ │ ├── pretrain_map_fn.py │ │ ├── sql_map_fn.py │ │ ├── stack_exchange_map_fn.py │ │ ├── tiny_codes_map_fn.py │ │ └── wizardlm_map_fn.py │ └── template_map_fn.py ├── mini_gemini_dataset.py ├── mini_gemini_proxy_eval_dataset.py ├── modelscope.py ├── moss_sft.py ├── refcoco_json.py ├── samplers │ ├── __init__.py │ ├── __pycache__ │ │ ├── __init__.cpython-310.pyc │ │ ├── intern_repo.cpython-310.pyc │ │ └── length_grouped.cpython-310.pyc │ ├── intern_repo.py │ └── length_grouped.py └── utils.py ├── engine ├── __init__.py ├── __pycache__ │ └── __init__.cpython-310.pyc ├── _strategy │ ├── __init__.py │ ├── __pycache__ │ │ ├── __init__.cpython-310.pyc │ │ └── deepspeed.cpython-310.pyc │ └── deepspeed.py ├── hooks │ ├── __init__.py │ ├── __pycache__ │ │ ├── __init__.cpython-310.pyc │ │ ├── dataset_info_hook.cpython-310.pyc │ │ ├── evaluate_chat_hook.cpython-310.pyc │ │ ├── throughput_hook.cpython-310.pyc │ │ └── varlen_attn_args_to_messagehub_hook.cpython-310.pyc │ ├── dataset_info_hook.py │ ├── evaluate_chat_hook.py │ ├── throughput_hook.py │ └── varlen_attn_args_to_messagehub_hook.py └── runner │ ├── __init__.py │ ├── __pycache__ │ ├── __init__.cpython-310.pyc │ └── loops.cpython-310.pyc │ └── loops.py ├── entry_point.py ├── evaluation ├── __init__.py └── metrics │ ├── __init__.py │ └── mmlu_metric.py ├── model ├── __init__.py ├── __pycache__ │ ├── __init__.cpython-310.pyc │ ├── anyres_llava.cpython-310.pyc │ ├── llava.cpython-310.pyc │ ├── mini_gemini.cpython-310.pyc │ ├── object_llava.cpython-310.pyc │ ├── sft.cpython-310.pyc │ └── utils.cpython-310.pyc ├── anyres_llava.py ├── llava.py ├── mini_gemini.py ├── modules │ ├── __init__.py │ ├── __pycache__ │ │ └── __init__.cpython-310.pyc │ ├── dispatch │ │ ├── __init__.py │ │ ├── __pycache__ │ │ │ ├── __init__.cpython-310.pyc │ │ │ ├── attention.cpython-310.pyc │ │ │ ├── baichuan.cpython-310.pyc │ │ │ ├── llama.cpython-310.pyc │ │ │ ├── phi3.cpython-310.pyc │ │ │ ├── utils.cpython-310.pyc │ │ │ └── yi.cpython-310.pyc │ │ ├── attention.py │ │ ├── baichuan.py │ │ ├── cohere.py │ │ ├── internlm.py │ │ ├── internlm2.py │ │ ├── llama.py │ │ ├── mistral.py │ │ ├── phi3.py │ │ ├── qwen2.py │ │ ├── triton_kernels │ │ │ ├── __init__.py │ │ │ ├── __pycache__ │ │ │ │ ├── __init__.cpython-310.pyc │ │ │ │ ├── layer_norm.cpython-310.pyc │ │ │ │ ├── rms_norm.cpython-310.pyc │ │ │ │ └── rotary.cpython-310.pyc │ │ │ ├── layer_norm.py │ │ │ ├── rms_norm.py │ │ │ └── rotary.py │ │ ├── utils.py │ │ └── yi.py │ ├── openclip_encoder.py │ └── projector │ │ ├── __init__.py │ │ ├── __pycache__ │ │ ├── __init__.cpython-310.pyc │ │ ├── configuration_projector.cpython-310.pyc │ │ └── modeling_projector.cpython-310.pyc │ │ ├── configuration_projector.py │ │ └── modeling_projector.py ├── sft.py └── utils.py ├── parallel ├── __init__.py ├── __pycache__ │ └── __init__.cpython-310.pyc └── sequence │ ├── __init__.py │ ├── __pycache__ │ ├── __init__.cpython-310.pyc │ ├── attention.cpython-310.pyc │ ├── comm.cpython-310.pyc │ ├── data_collate.cpython-310.pyc │ ├── reduce_loss.cpython-310.pyc │ ├── sampler.cpython-310.pyc │ └── setup_distributed.cpython-310.pyc │ ├── attention.py │ ├── comm.py │ ├── data_collate.py │ ├── reduce_loss.py │ ├── sampler.py │ └── setup_distributed.py ├── registry.py ├── tools ├── __pycache__ │ └── utils.cpython-310.pyc ├── chat.py ├── check_custom_dataset.py ├── copy_cfg.py ├── data_preprocess │ ├── arxiv.py │ └── convert_refcoco.py ├── eval_refcoco.py ├── get_data_order.py ├── list_cfg.py ├── list_dataset_format.py ├── log_dataset.py ├── mmbench.py ├── model_converters │ ├── merge.py │ ├── pth_to_hf.py │ └── split.py ├── plugins │ ├── __init__.py │ ├── api.py │ ├── calculate.py │ ├── search.py │ └── solve.py ├── process_untokenized_datasets.py ├── process_untokenized_datasets_legacy.py ├── process_untokenized_llava_data.py ├── test.py ├── tokenize_ftdp_datasets.py ├── train.py └── utils.py ├── utils ├── __init__.py ├── __pycache__ │ ├── __init__.cpython-310.pyc │ ├── constants.cpython-310.pyc │ ├── fileio.cpython-310.pyc │ ├── stop_criteria.cpython-310.pyc │ └── templates.cpython-310.pyc ├── constants.py ├── fileio.py ├── stop_criteria.py └── templates.py └── version.py /.github/workflows/deploy.yml: -------------------------------------------------------------------------------- 1 | name: deploy 2 | 3 | on: push 4 | 5 | concurrency: 6 | group: ${{ github.workflow }}-${{ github.ref }} 7 | cancel-in-progress: true 8 | 9 | jobs: 10 | build-n-publish: 11 | runs-on: ubuntu-latest 12 | if: startsWith(github.event.ref, 'refs/tags') 13 | steps: 14 | - uses: actions/checkout@v2 15 | - name: Set up Python 3.8 16 | uses: actions/setup-python@v2 17 | with: 18 | python-version: 3.8 19 | - name: Build XTuner 20 | run: | 21 | pip install wheel 22 | python setup.py sdist bdist_wheel 23 | - name: Publish distribution to PyPI 24 | run: | 25 | pip install twine 26 | twine upload dist/* -u __token__ -p ${{ secrets.pypi_password }} 27 | -------------------------------------------------------------------------------- /.github/workflows/lint.yml: -------------------------------------------------------------------------------- 1 | name: lint 2 | 3 | on: [push, pull_request] 4 | 5 | concurrency: 6 | group: ${{ github.workflow }}-${{ github.ref }} 7 | cancel-in-progress: true 8 | 9 | jobs: 10 | lint: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@v2 14 | - name: Set up Python 3.8 15 | uses: actions/setup-python@v2 16 | with: 17 | python-version: 3.8 18 | - name: Install pre-commit hook 19 | run: | 20 | pip install pre-commit 21 | pre-commit install 22 | - name: Linting 23 | run: pre-commit run --all-files 24 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .history 2 | -------------------------------------------------------------------------------- /.owners.yml: -------------------------------------------------------------------------------- 1 | assign: 2 | issues: disabled 3 | pull_requests: disabled 4 | strategy: 5 | random 6 | # daily-shift-based 7 | schedule: 8 | '*/1 * * * *' 9 | -------------------------------------------------------------------------------- /.pre-commit-config-zh-cn.yaml: -------------------------------------------------------------------------------- 1 | exclude: ^tests/data/ 2 | repos: 3 | - repo: https://gitee.com/openmmlab/mirrors-flake8 4 | rev: 5.0.4 5 | hooks: 6 | - id: flake8 7 | - repo: https://gitee.com/openmmlab/mirrors-isort 8 | rev: 5.11.5 9 | hooks: 10 | - id: isort 11 | - repo: https://gitee.com/openmmlab/mirrors-yapf 12 | rev: v0.32.0 13 | hooks: 14 | - id: yapf 15 | - repo: https://gitee.com/openmmlab/mirrors-pre-commit-hooks 16 | rev: v4.3.0 17 | hooks: 18 | - id: trailing-whitespace 19 | - id: check-yaml 20 | - id: end-of-file-fixer 21 | - id: requirements-txt-fixer 22 | - id: double-quote-string-fixer 23 | - id: check-merge-conflict 24 | - id: fix-encoding-pragma 25 | args: ["--remove"] 26 | - id: mixed-line-ending 27 | args: ["--fix=lf"] 28 | - repo: https://gitee.com/openmmlab/mirrors-codespell 29 | rev: v2.2.1 30 | hooks: 31 | - id: codespell 32 | - repo: https://gitee.com/openmmlab/mirrors-mdformat 33 | rev: 0.7.9 34 | hooks: 35 | - id: mdformat 36 | args: ["--number"] 37 | additional_dependencies: 38 | - mdformat-openmmlab 39 | - mdformat_frontmatter 40 | - linkify-it-py 41 | - repo: https://gitee.com/openmmlab/mirrors-docformatter 42 | rev: v1.3.1 43 | hooks: 44 | - id: docformatter 45 | args: ["--in-place", "--wrap-descriptions", "79"] 46 | - repo: https://github.com/asottile/pyupgrade 47 | rev: v3.0.0 48 | hooks: 49 | - id: pyupgrade 50 | args: ["--py36-plus"] 51 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | exclude: ^tests/data/ 2 | repos: 3 | - repo: https://github.com/PyCQA/flake8 4 | rev: 5.0.4 5 | hooks: 6 | - id: flake8 7 | - repo: https://github.com/PyCQA/isort 8 | rev: 5.11.5 9 | hooks: 10 | - id: isort 11 | - repo: https://github.com/pre-commit/mirrors-yapf 12 | rev: v0.32.0 13 | hooks: 14 | - id: yapf 15 | exclude: 'xtuner/parallel/sequence/__init__.py' 16 | - repo: https://github.com/pre-commit/pre-commit-hooks 17 | rev: v4.3.0 18 | hooks: 19 | - id: trailing-whitespace 20 | - id: check-yaml 21 | - id: end-of-file-fixer 22 | - id: requirements-txt-fixer 23 | - id: double-quote-string-fixer 24 | - id: check-merge-conflict 25 | - id: fix-encoding-pragma 26 | args: ["--remove"] 27 | - id: mixed-line-ending 28 | args: ["--fix=lf"] 29 | - repo: https://github.com/codespell-project/codespell 30 | rev: v2.2.1 31 | hooks: 32 | - id: codespell 33 | - repo: https://github.com/executablebooks/mdformat 34 | rev: 0.7.9 35 | hooks: 36 | - id: mdformat 37 | args: ["--number"] 38 | additional_dependencies: 39 | - mdformat-openmmlab 40 | - mdformat_frontmatter 41 | - linkify-it-py 42 | exclude: 'docs/zh_cn/user_guides/sequence_parallel.md' 43 | - repo: https://github.com/myint/docformatter 44 | rev: v1.3.1 45 | hooks: 46 | - id: docformatter 47 | args: ["--in-place", "--wrap-descriptions", "79"] 48 | - repo: https://github.com/asottile/pyupgrade 49 | rev: v3.0.0 50 | hooks: 51 | - id: pyupgrade 52 | args: ["--py36-plus"] 53 | -------------------------------------------------------------------------------- /docs/en/notes/changelog.md: -------------------------------------------------------------------------------- 1 | 14 | 15 | # Changelog 16 | 17 | ## v0.1.0 (2023.08.30) 18 | 19 | XTuner is released! 🔥🔥🔥 20 | 21 | ### Highlights 22 | 23 | - XTuner supports LLM fine-tuning on consumer-grade GPUs. The minimum GPU memory required for 7B LLM fine-tuning is only **8GB**. 24 | - XTuner supports various LLMs, datasets, algorithms and training pipelines. 25 | - Several fine-tuned adapters are released simultaneously, including various gameplays such as the colorist LLM, plugins-based LLM, and many more. For further details, please visit [XTuner on HuggingFace](https://huggingface.co/xtuner)! 26 | -------------------------------------------------------------------------------- /docs/zh_cn/notes/changelog.md: -------------------------------------------------------------------------------- 1 | 14 | 15 | # 变更日志 16 | 17 | ## v0.1.0 (2023.08.30) 18 | 19 | XTuner 正式发布！🔥🔥🔥 20 | 21 | ### 亮点 22 | 23 | - XTuner 支持使用消费级显卡微调大语言模型。微调 7B 大语言模型的最低显存开销仅为 **8GB**。 24 | - XTuner 支持多种大语言模型、数据集、微调算法和训练流程。 25 | - 众多微调好的 adapter 也同步发布，包括调色师、插件对话等多种玩法。更多信息，请访问 [HuggingFace 仓库](https://huggingface.co/xtuner)。 26 | -------------------------------------------------------------------------------- /docs/zh_cn/user_guides/ceph.md: -------------------------------------------------------------------------------- 1 | ## 功能说明 2 | 3 | ### 已支持的功能 4 | 5 | - 保存 DeepSpeed Checkpoint 至 CEPH 6 | - 从 Ceph 上的 DeepSpeed Checkpoint 续训 7 | - `pth_to_hf` 支持 Ceph 上的 DeepSpeed Checkpoint 8 | 9 | ### 暂不支持的功能 10 | 11 | - 训练时从 Ceph 加载 Huggingface 模型， 与 `zero3` 加载权重冲突 12 | - HuggingFace `save_pretrained` 保存至 Ceph， 逻辑过于复杂，没办法 patch 13 | 14 | ## 使用说明 15 | 16 | #### 1. 验证 ceph 环境 17 | 18 | 使用前需确保 `petrel sdk` 可用，并且要使用的 Ceph bucket 存在且可用 19 | 20 | 验证 `aws` 命令行工具 21 | 22 | ```bash 23 | # 验证 aws 命令行工具 24 | aws s3 ls $YOUR_BUCKET 25 | ``` 26 | 27 | 验证 `petrel sdk` 28 | 29 | ```python 30 | bucket = 's3://xxx' 31 | 32 | from mmengine import get_file_backend 33 | backend = get_file_backend(bucket) 34 | 35 | for f in backend.list_dir_or_file(bucket): 36 | print(f) 37 | ``` 38 | 39 | #### 2. 训练时保存 Checkpoint 至 Ceph 40 | 41 | `XTuner` 根据环境变量 `DS_CEPH_DIR` 来判断是否将 checkpoint 保存至 ceph 42 | 43 | ```bash 44 | DS_CEPH_DIR=s3://xxxx srun ${SRUN_ARGS} xtuner train $CONFIG --launcher slurm 45 | ``` 46 | 47 | #### 3. 从 Ceph 上的 Checkpoint 续训 48 | 49 | Resume 时，要填写 checkpoint 在 ceph 上的完整路径 50 | 51 | ```bash 52 | DS_CEPH_DIR=s3://xxxx srun ${SRUN_ARGS} xtuner train $CONFIG --launcher slurm --resume s3://xxx/yyy/epoch_x.pth 53 | ``` 54 | 55 | #### 4. 将 Ceph 上的 Checkpoint 转换为 HF 模型 56 | 57 | 不支持 `$HF_DIR` 为 ceph 路径 58 | 59 | 由于 Checkpoint 中存储了优化器状态，加载比较耗时，对于 ZeRO 1&2 可以直接加载 checkpoint 中的 `model_states.pt` 文件加速转换过程；ZeRO 3 必须先加载整个 checkpoint 60 | 61 | ```bash 62 | srun ${SRUN_ARGS} xtuner convert pth_to_hf $CONFIG s3://xxx/yyy/epoch_x.pth $HF_DIR 63 | 64 | ``` 65 | -------------------------------------------------------------------------------- /docs/zh_cn/user_guides/ftdp_dataset/README.md: -------------------------------------------------------------------------------- 1 | ftdp 是一个闭源的处理数据工具，开源社区用户可以忽略此文档。 2 | 3 | 本节介绍了常见的 4 种使用 ftdp 数据集训练的使用场景： 4 | 5 | - [Case 1: 使用 Processed 数据集训练 InternLM2](Case1.md) 6 | - [Case 2: 使用 Processed 数据集训练非 InternLM2 模型](Case2.md) 7 | - [Case 3: 使用 Processed 普通对话数据集训任意模型](Case3.md) 8 | - [Case 4: 使用 Tokenized 数据集训练 InternLM2](Case4.md) 9 | 10 | 请先参考下方流程图，选择自己的使用场景。 11 | 12 | ```mermaid 13 | graph TD; 14 | A{ftdp 数据} 15 | A -->|是| B{数据 tokenized} 16 | B -->|否| C{使用 Internlm2 对话模板} 17 | C -->|是| D{训练 Internlm2 } 18 | D -->|是| E[Case 1] 19 | D -->|否| F[Case 2] 20 | C -->|否| G{离线处理数据集} 21 | G -->|是| H[尚不支持] 22 | G -->|否| I[Case 3] 23 | B -->|是| J[Case 4] 24 | ``` 25 | -------------------------------------------------------------------------------- /docs/zh_cn/user_guides/llava_offline.md: -------------------------------------------------------------------------------- 1 | # 离线处理 Llava 训练数据集 2 | 3 | 当训练数据量非常大时，每次训练的时候都先在线处理数据可能会极为耗时。我们可以先对原始数据进行离线处理并保存至本地，随后的多次训练可以读入本地离线处理好的数据后直接开始训练。 4 | 5 | ## Step 1, 导出模板 config 文件 6 | 7 | 可使用以下命令查看 XTuner 中提供的 Llava 训练相关的 config： 8 | 9 | ``` 10 | xtuner list-cfg -p llava 11 | ``` 12 | 13 | 找到需要使用的 config 文件并导出至当前目录下： 14 | 15 | ``` 16 | xtuner copy-cfg ${CONFIG_NAME} . 17 | ``` 18 | 19 | ## Step 2, 离线处理数据集 20 | 21 | 使用以下命令可离线处理训练数据集中的文本数据： 22 | 23 | ``` 24 | python xtuner/tools/process_untokenized_llava_data.py \ 25 | ${CONFIG_PATH} \ 26 | --save-folder /folder/to/save/processed/dataset 27 | ``` 28 | 29 | 其中，${CONFIG_PATH} 为第一步中导出的 config 文件路径，`/folder/to/save/processed/dataset` 则需要指定为离线处理数据的保存路径。 30 | 31 | ## Step 3, 修改 config 文件 32 | 33 | 对 Step 1 中导出的 config 文件做如下修改： 34 | 35 | ```diff 36 | ####################################################################### 37 | # PART 3 Dataset & Dataloader # 38 | ####################################################################### 39 | llava_dataset = dict( 40 | - data_path=data_path, 41 | - tokenizer=tokenizer, 42 | + offline_processed_text_folder=/folder/to/save/processed/dataset 43 | ...) 44 | ``` 45 | 46 | 其中，`/folder/to/save/processed/dataset` 为 Step 2 保存的离线处理数据路径。 47 | 48 | ## Step 4，开始训练 49 | 50 | 使用 Step 3 修改得到的 config 训练即可。 51 | -------------------------------------------------------------------------------- /examples/demo_data/multi_turn_1/data.json: -------------------------------------------------------------------------------- 1 | [{ 2 | "messages":[ 3 | { 4 | "toy_system": "You are a helpful AI assistant.", 5 | "toy_input": "Give three tips for staying healthy.", 6 | "toy_output": "1.Eat a balanced diet. 2. Exercise regularly. 3. Get enough sleep." 7 | }, 8 | { 9 | "toy_input": "How to study English?", 10 | "toy_output": "1. Set clear goals. 2. Create a study plan. 3. Build vocabulary. 4. Practice speaking." 11 | } 12 | ] 13 | }, 14 | { 15 | "messages":[ 16 | { 17 | "toy_system": "You are a helpful AI assistant.", 18 | "toy_input": "How to study English?", 19 | "toy_output": "1. Set clear goals. 2. Create a study plan. 3. Build vocabulary. 4. Practice speaking." 20 | }, 21 | { 22 | "toy_input": "Give three tips for staying healthy.", 23 | "toy_output": "1.Eat a balanced diet. 2. Exercise regularly. 3. Get enough sleep." 24 | } 25 | ] 26 | }] 27 | -------------------------------------------------------------------------------- /examples/demo_data/multi_turn_1/map_fn.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | def multi_turn_1_map_fn(example): 3 | messages = example['messages'] 4 | conversation = [] 5 | for msg in messages: 6 | conversation.append({ 7 | 'system': msg['toy_system'], 8 | 'input': msg['toy_input'], 9 | 'output': msg['toy_output'] 10 | }) 11 | return {'conversation': conversation} 12 | -------------------------------------------------------------------------------- /examples/demo_data/multi_turn_2/data.json: -------------------------------------------------------------------------------- 1 | [{ 2 | "messages":[ 3 | { 4 | "role": "system", 5 | "content": "You are a helpful AI assistant." 6 | }, 7 | { 8 | "role": "user", 9 | "content": "Give three tips for staying healthy." 10 | }, 11 | { 12 | "role": "assistant", 13 | "content": "1.Eat a balanced diet. 2. Exercise regularly. 3. Get enough sleep." 14 | }, 15 | { 16 | "role": "user", 17 | "content": "How to study English?" 18 | }, 19 | { 20 | "role": "assistant", 21 | "content": "1. Set clear goals. 2. Create a study plan. 3. Build vocabulary. 4. Practice speaking." 22 | } 23 | ] 24 | }, 25 | { 26 | "messages":[ 27 | { 28 | "role": "system", 29 | "content": "You are a helpful AI assistant." 30 | }, 31 | { 32 | "role": "user", 33 | "content": "How to study English?" 34 | }, 35 | { 36 | "role": "assistant", 37 | "content": "1. Set clear goals. 2. Create a study plan. 3. Build vocabulary. 4. Practice speaking." 38 | }, 39 | { 40 | "role": "user", 41 | "content": "Give three tips for staying healthy." 42 | }, 43 | { 44 | "role": "assistant", 45 | "content": "1.Eat a balanced diet. 2. Exercise regularly. 3. Get enough sleep." 46 | } 47 | ] 48 | }] 49 | -------------------------------------------------------------------------------- /examples/demo_data/multi_turn_2/map_fn.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | def multi_turn_2_map_fn(example): 3 | messages = example['messages'] 4 | system = '' 5 | input = '' 6 | conversation = [] 7 | while messages and messages[0]['role'] == 'assistant': 8 | # Skip the first one if it is from assistant 9 | messages = messages[1:] 10 | for msg in messages: 11 | if msg['role'] == 'system': 12 | system = msg['content'] 13 | elif msg['role'] == 'user': 14 | input += msg['content'] 15 | elif msg['role'] == 'assistant': 16 | conversation.append({ 17 | 'system': system, 18 | 'input': input, 19 | 'output': msg['content'] 20 | }) 21 | system = '' 22 | input = '' 23 | else: 24 | raise NotImplementedError 25 | return {'conversation': conversation} 26 | -------------------------------------------------------------------------------- /examples/demo_data/pretrain/data.json: -------------------------------------------------------------------------------- 1 | [{ 2 | "toy_text": "I am an artificial intelligence (AI) assistant named InternLM. I was created by the Shanghai AI Laboratory and my purpose is to assist users with various tasks through natural language processing technology." 3 | }, 4 | { 5 | "toy_text": "I am an artificial intelligence programmed to assist with various types of tasks, including answering questions, providing information, and performing automated processes." 6 | }] 7 | -------------------------------------------------------------------------------- /examples/demo_data/pretrain/map_fn.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | def pretrain_map_fn(example): 3 | return { 4 | 'conversation': [{ 5 | 'input': '', 6 | 'output': example['toy_text'].strip() 7 | }] 8 | } 9 | -------------------------------------------------------------------------------- /examples/demo_data/single_turn/data.json: -------------------------------------------------------------------------------- 1 | [{ 2 | "toy_system": "You are a helpful AI assistant.", 3 | "toy_input": "Give three tips for staying healthy.", 4 | "toy_output": "1.Eat a balanced diet. 2. Exercise regularly. 3. Get enough sleep." 5 | }, 6 | { 7 | "toy_system": "You are a helpful AI assistant.", 8 | "toy_input": "How to study English?", 9 | "toy_output": "1. Set clear goals. 2. Create a study plan. 3. Build vocabulary. 4. Practice speaking." 10 | }] 11 | -------------------------------------------------------------------------------- /examples/demo_data/single_turn/map_fn.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | def single_turn_map_fn(example): 3 | return { 4 | 'conversation': [{ 5 | 'system': example['toy_system'], 6 | 'input': example['toy_input'], 7 | 'output': example['toy_output'] 8 | }] 9 | } 10 | -------------------------------------------------------------------------------- /examples/example.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/examples/example.jpg -------------------------------------------------------------------------------- /examples/huggingface_trainer/README.md: -------------------------------------------------------------------------------- 1 | # How to use XTuner in HuggingFace training pipeline 2 | 3 | ## Quick run 4 | 5 | 1. step in `examples` 6 | 7 | ```shell 8 | cd ./examples 9 | ``` 10 | 11 | 2. run training scripts 12 | 13 | ```shell 14 | # qlora-training internlm-7b with alpaca dataset 15 | python train_qlora_hf.py --model_name_or_path internlm/internlm-7b --dataset_name_or_path tatsu-lab/alpaca 16 | ``` 17 | 18 | `--model_name_or_path`: specify the model name or path to train. 19 | 20 | `--dataset_name_or_path`: specify the dataset name or path to use. 21 | 22 | ## How to customize your experiment 23 | 24 | XTuner APIs are compatible with the usage of HuggingFace's transformers. 25 | If you want to customize your experiment, you just need to pass in your hyperparameters like HuggingFace. 26 | 27 | ``` 28 | # training example 29 | python train_qlora_hf.py \ 30 | # custom training args 31 | --model_name_or_path internlm/internlm-7b \ 32 | --dataset_name_or_path tatsu-lab/alpaca \ 33 | # HuggingFace's default training args 34 | --do_train = True 35 | --per_device_train_batch_size = 1 36 | --learning_rate = 2e-5 37 | --save_strategy = 'epoch' 38 | --lr_scheduler_type = 'cosine' 39 | --logging_steps = 1 40 | ``` 41 | -------------------------------------------------------------------------------- /examples/huggingface_trainer/train_hf.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | import transformers 3 | from transformers import Trainer 4 | 5 | from xtuner.apis import DefaultTrainingArguments, build_model 6 | from xtuner.apis.datasets import alpaca_data_collator, alpaca_dataset 7 | 8 | 9 | def train(): 10 | # get DefaultTrainingArguments and to be updated with passed args 11 | parser = transformers.HfArgumentParser(DefaultTrainingArguments) 12 | training_args = parser.parse_args_into_dataclasses()[0] 13 | 14 | # init model and dataset 15 | model, tokenizer = build_model( 16 | model_name_or_path=training_args.model_name_or_path, 17 | return_tokenizer=True) 18 | train_dataset = alpaca_dataset( 19 | tokenizer=tokenizer, path=training_args.dataset_name_or_path) 20 | data_collator = alpaca_data_collator(return_hf_format=True) 21 | 22 | # build trainer 23 | trainer = Trainer( 24 | model=model, 25 | args=training_args, 26 | train_dataset=train_dataset, 27 | data_collator=data_collator) 28 | 29 | # training 30 | trainer.train() 31 | 32 | trainer.save_state() 33 | trainer.save_model(output_dir=training_args.output_dir) 34 | 35 | 36 | if __name__ == '__main__': 37 | train() 38 | -------------------------------------------------------------------------------- /examples/huggingface_trainer/train_lora_hf.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | import transformers 3 | from transformers import Trainer 4 | 5 | from xtuner.apis import DefaultTrainingArguments, build_lora_model 6 | from xtuner.apis.datasets import alpaca_data_collator, alpaca_dataset 7 | 8 | 9 | def train(): 10 | # get DefaultTrainingArguments and to be updated with passed args 11 | parser = transformers.HfArgumentParser(DefaultTrainingArguments) 12 | training_args = parser.parse_args_into_dataclasses()[0] 13 | 14 | # init model and dataset 15 | model, tokenizer = build_lora_model( 16 | model_name_or_path=training_args.model_name_or_path, 17 | return_tokenizer=True) 18 | train_dataset = alpaca_dataset( 19 | tokenizer=tokenizer, path=training_args.dataset_name_or_path) 20 | data_collator = alpaca_data_collator(return_hf_format=True) 21 | 22 | # build trainer 23 | trainer = Trainer( 24 | model=model, 25 | args=training_args, 26 | train_dataset=train_dataset, 27 | data_collator=data_collator) 28 | 29 | # training 30 | trainer.train() 31 | 32 | trainer.save_state() 33 | trainer.save_model(output_dir=training_args.output_dir) 34 | 35 | 36 | if __name__ == '__main__': 37 | train() 38 | -------------------------------------------------------------------------------- /examples/huggingface_trainer/train_qlora_hf.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | import transformers 3 | from transformers import Trainer 4 | 5 | from xtuner.apis import DefaultTrainingArguments, build_qlora_model 6 | from xtuner.apis.datasets import alpaca_data_collator, alpaca_dataset 7 | 8 | 9 | def train(): 10 | # get DefaultTrainingArguments and to be updated with passed args 11 | parser = transformers.HfArgumentParser(DefaultTrainingArguments) 12 | training_args = parser.parse_args_into_dataclasses()[0] 13 | 14 | # init model and dataset 15 | model, tokenizer = build_qlora_model( 16 | model_name_or_path=training_args.model_name_or_path, 17 | return_tokenizer=True) 18 | train_dataset = alpaca_dataset( 19 | tokenizer=tokenizer, path=training_args.dataset_name_or_path) 20 | data_collator = alpaca_data_collator(return_hf_format=True) 21 | 22 | # build trainer 23 | trainer = Trainer( 24 | model=model, 25 | args=training_args, 26 | train_dataset=train_dataset, 27 | data_collator=data_collator) 28 | 29 | # training 30 | trainer.train() 31 | 32 | trainer.save_state() 33 | trainer.save_model(output_dir=training_args.output_dir) 34 | 35 | 36 | if __name__ == '__main__': 37 | train() 38 | -------------------------------------------------------------------------------- /images/MG.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/images/MG.png -------------------------------------------------------------------------------- /images/Main-results1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/images/Main-results1.png -------------------------------------------------------------------------------- /images/More_result.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/images/More_result.png -------------------------------------------------------------------------------- /images/framework.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/images/framework.png -------------------------------------------------------------------------------- /images/teaser.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/images/teaser.png -------------------------------------------------------------------------------- /mg_llava/dataset/__init__.py: -------------------------------------------------------------------------------- 1 | from .video.batch_sampler import BatchSampler 2 | from .video.video_collate_fn import video_collate_fn 3 | from .video.video_length_grouped import VideoLengthGroupedSampler 4 | from .video.batch_sampler import VideoImageSeperateBatchSampler 5 | 6 | __all__ = ['BatchSampler', 'video_collate_fn', 'VideoLengthGroupedSampler', 'VideoImageSeperateBatchSampler'] 7 | -------------------------------------------------------------------------------- /mg_llava/dataset/__pycache__/__init__.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/mg_llava/dataset/__pycache__/__init__.cpython-310.pyc -------------------------------------------------------------------------------- /mg_llava/dataset/__pycache__/huggingface.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/mg_llava/dataset/__pycache__/huggingface.cpython-310.pyc -------------------------------------------------------------------------------- /mg_llava/dataset/__pycache__/llamavid_map_fn.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/mg_llava/dataset/__pycache__/llamavid_map_fn.cpython-310.pyc -------------------------------------------------------------------------------- /mg_llava/dataset/__pycache__/llava.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/mg_llava/dataset/__pycache__/llava.cpython-310.pyc -------------------------------------------------------------------------------- /mg_llava/dataset/__pycache__/utils.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/mg_llava/dataset/__pycache__/utils.cpython-310.pyc -------------------------------------------------------------------------------- /mg_llava/dataset/video/__pycache__/batch_sampler.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/mg_llava/dataset/video/__pycache__/batch_sampler.cpython-310.pyc -------------------------------------------------------------------------------- /mg_llava/dataset/video/__pycache__/video_collate_fn.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/mg_llava/dataset/video/__pycache__/video_collate_fn.cpython-310.pyc -------------------------------------------------------------------------------- /mg_llava/dataset/video/__pycache__/video_length_grouped.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/mg_llava/dataset/video/__pycache__/video_length_grouped.cpython-310.pyc -------------------------------------------------------------------------------- /mg_llava/evaluation/__init__.py: -------------------------------------------------------------------------------- 1 | from .mg_llava_proxy_eval_dataset import MGLLaVAProxyEvalDataset 2 | from .gqa_llava_eval_dataset import GQADataset 3 | from .vqav2_llava_eval_dataset import VQAv2Dataset 4 | from .video.video_qa_dataset import VIDEOQADataset 5 | from .video.video_mg_llava_proxy_eval_dataset import VideoObjectLLaVAProxyEvalDataset 6 | from .video.hooks import DatasetInfoHook 7 | 8 | __all__ = [ 9 | 'MGLLaVAProxyEvalDataset', 10 | 'GQADataset', 11 | 'VQAv2Dataset', 12 | 'VIDEOQADataset', 13 | 'VideoObjectLLaVAProxyEvalDataset', 14 | 'DatasetInfoHook', 15 | ] 16 | 00 -------------------------------------------------------------------------------- /mg_llava/evaluation/__pycache__/__init__.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/mg_llava/evaluation/__pycache__/__init__.cpython-310.pyc -------------------------------------------------------------------------------- /mg_llava/evaluation/__pycache__/llvid_proxy_eval_dataset.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/mg_llava/evaluation/__pycache__/llvid_proxy_eval_dataset.cpython-310.pyc -------------------------------------------------------------------------------- /mg_llava/evaluation/__pycache__/qformer_proxy_eval_dataset.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/mg_llava/evaluation/__pycache__/qformer_proxy_eval_dataset.cpython-310.pyc -------------------------------------------------------------------------------- /mg_llava/evaluation/video/hooks.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from mmengine.hooks import Hook 3 | 4 | from xtuner.registry import BUILDER 5 | from xtuner.utils import DEFAULT_IMAGE_TOKEN, IMAGE_TOKEN_INDEX 6 | 7 | 8 | def split_list(lst, value): 9 | res = [] 10 | tmp_res = [] 11 | for i in lst: 12 | if tmp_res and i == value: 13 | res.append(tmp_res) 14 | tmp_res = [] 15 | else: 16 | tmp_res.append(i) 17 | res.append(tmp_res) 18 | return res 19 | 20 | 21 | class DatasetInfoHook(Hook): 22 | 23 | def __init__(self, tokenizer, is_intern_repo_dataset=False): 24 | self.tokenizer = BUILDER.build(tokenizer) 25 | self.is_intern_repo_dataset = is_intern_repo_dataset 26 | 27 | def log(self, runner, dataset, mode='train'): 28 | runner.logger.info(f'Num {mode} samples {len(dataset)}') 29 | runner.logger.info(f'{mode} example:') 30 | # input_ids = dataset[0]['input_ids'] 31 | # if self.is_intern_repo_dataset: 32 | # input_ids = [abs(x) for x in input_ids] 33 | # # Try to split list to be compatible with IMAGE token 34 | # input_ids = split_list(input_ids, IMAGE_TOKEN_INDEX) 35 | # text = '' 36 | # for idx, ids in enumerate(input_ids): 37 | # text += self.tokenizer.decode(ids) 38 | # if idx != len(input_ids) - 1: 39 | # text += DEFAULT_IMAGE_TOKEN 40 | # runner.logger.info(text) 41 | 42 | def before_train(self, runner) -> None: 43 | do_train = runner.train_loop is not None 44 | do_eval = runner.val_loop is not None 45 | do_test = runner.test_loop is not None 46 | if do_train: 47 | train_dataset = runner.train_dataloader.dataset 48 | self.log(runner, train_dataset, mode='train') 49 | if do_eval: 50 | eval_dataset = runner.val_dataloader.dataset 51 | self.log(runner, eval_dataset, mode='eval') 52 | # if do_test: 53 | # test_dataset = runner.test_dataloader.dataset 54 | # self.log(runner, test_dataset, mode='test') 55 | 56 | def before_val(self, runner) -> None: 57 | eval_dataset = runner.val_dataloader.dataset 58 | self.log(runner, eval_dataset, mode='eval') 59 | 60 | def before_test(self, runner) -> None: 61 | test_dataset = runner.test_dataloader.dataset 62 | self.log(runner, test_dataset, mode='test') 63 | -------------------------------------------------------------------------------- /mg_llava/module/__init__.py: -------------------------------------------------------------------------------- 1 | from .fuse_model import MultiFuseObjectLLaVAModel 2 | from .box_model import BoxLLaVAModel 3 | from .mg_llava_dataset import MGLLaVADataset 4 | from .openclip_encoder import OpenCLIPVisionTower 5 | from .chat_utils import box_generator 6 | 7 | __all__ = ['MultiFuseObjectLLaVAModel', 'BoxLLaVAModel', 'MGLLaVADataset', 'OpenCLIPVisionTower', 'box_generator'] 8 | -------------------------------------------------------------------------------- /mg_llava/module/__pycache__/__init__.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/mg_llava/module/__pycache__/__init__.cpython-310.pyc -------------------------------------------------------------------------------- /mg_llava/module/__pycache__/bert_qformer.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/mg_llava/module/__pycache__/bert_qformer.cpython-310.pyc -------------------------------------------------------------------------------- /mg_llava/module/__pycache__/box_model.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/mg_llava/module/__pycache__/box_model.cpython-310.pyc -------------------------------------------------------------------------------- /mg_llava/module/__pycache__/fuse_attention_object_llava_model.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/mg_llava/module/__pycache__/fuse_attention_object_llava_model.cpython-310.pyc -------------------------------------------------------------------------------- /mg_llava/module/__pycache__/fuse_box2object_llava_model.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/mg_llava/module/__pycache__/fuse_box2object_llava_model.cpython-310.pyc -------------------------------------------------------------------------------- /mg_llava/module/__pycache__/fuse_box_object_llava_model.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/mg_llava/module/__pycache__/fuse_box_object_llava_model.cpython-310.pyc -------------------------------------------------------------------------------- /mg_llava/module/__pycache__/fuse_llamavid_dataset.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/mg_llava/module/__pycache__/fuse_llamavid_dataset.cpython-310.pyc -------------------------------------------------------------------------------- /mg_llava/module/__pycache__/fuse_llamavid_model.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/mg_llava/module/__pycache__/fuse_llamavid_model.cpython-310.pyc -------------------------------------------------------------------------------- /mg_llava/module/__pycache__/fuse_mgm_box_model.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/mg_llava/module/__pycache__/fuse_mgm_box_model.cpython-310.pyc -------------------------------------------------------------------------------- /mg_llava/module/__pycache__/fuse_mgm_object_llava_model.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/mg_llava/module/__pycache__/fuse_mgm_object_llava_model.cpython-310.pyc -------------------------------------------------------------------------------- /mg_llava/module/__pycache__/fuse_mlp_object_llava_model.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/mg_llava/module/__pycache__/fuse_mlp_object_llava_model.cpython-310.pyc -------------------------------------------------------------------------------- /mg_llava/module/__pycache__/fuse_model.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/mg_llava/module/__pycache__/fuse_model.cpython-310.pyc -------------------------------------------------------------------------------- /mg_llava/module/__pycache__/fuse_object_llava_model.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/mg_llava/module/__pycache__/fuse_object_llava_model.cpython-310.pyc -------------------------------------------------------------------------------- /mg_llava/module/__pycache__/fuse_only_model.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/mg_llava/module/__pycache__/fuse_only_model.cpython-310.pyc -------------------------------------------------------------------------------- /mg_llava/module/__pycache__/instruct_blip.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/mg_llava/module/__pycache__/instruct_blip.cpython-310.pyc -------------------------------------------------------------------------------- /mg_llava/module/__pycache__/mg_llava_dataset.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/mg_llava/module/__pycache__/mg_llava_dataset.cpython-310.pyc -------------------------------------------------------------------------------- /mg_llava/module/__pycache__/multi_fuse_object_llava_model.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/mg_llava/module/__pycache__/multi_fuse_object_llava_model.cpython-310.pyc -------------------------------------------------------------------------------- /mg_llava/module/__pycache__/openclip_encoder.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/mg_llava/module/__pycache__/openclip_encoder.cpython-310.pyc -------------------------------------------------------------------------------- /mg_llava/module/__pycache__/qformer_box_object_llava.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/mg_llava/module/__pycache__/qformer_box_object_llava.cpython-310.pyc -------------------------------------------------------------------------------- /mg_llava/module/__pycache__/qformer_object_llava.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/mg_llava/module/__pycache__/qformer_object_llava.cpython-310.pyc -------------------------------------------------------------------------------- /mg_llava/module/__pycache__/qformer_object_llava_dataset.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/mg_llava/module/__pycache__/qformer_object_llava_dataset.cpython-310.pyc -------------------------------------------------------------------------------- /mg_llava/module/__pycache__/resampler.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/mg_llava/module/__pycache__/resampler.cpython-310.pyc -------------------------------------------------------------------------------- /mg_llava/module/__pycache__/video_llava.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/mg_llava/module/__pycache__/video_llava.cpython-310.pyc -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | -r requirements/runtime.txt 2 | -r requirements/deepspeed.txt 3 | -r requirements/modelscope.txt 4 | -r requirements/mgllava.txt 5 | -------------------------------------------------------------------------------- /requirements/deepspeed.txt: -------------------------------------------------------------------------------- 1 | # Minimum 0.12.3, see https://github.com/microsoft/DeepSpeed/pull/4587 2 | deepspeed>=0.12.3 3 | mpi4py-mpich 4 | -------------------------------------------------------------------------------- /requirements/mgllava.txt: -------------------------------------------------------------------------------- 1 | pytorchvideo 2 | decord 3 | fairscale 4 | open-clip-torch 5 | openai 6 | -------------------------------------------------------------------------------- /requirements/modelscope.txt: -------------------------------------------------------------------------------- 1 | modelscope 2 | -------------------------------------------------------------------------------- /requirements/runtime.txt: -------------------------------------------------------------------------------- 1 | # Minimum 0.40.0.post4 to fix some 4-bit precision bugs 2 | bitsandbytes>=0.40.0.post4 3 | # Minimum 2.16.0 to fix some bugs, see https://github.com/huggingface/datasets/pull/6444 4 | datasets>=2.16.0 5 | einops 6 | # Minimum 0.1.2 to fix some bugs, see https://github.com/InternLM/lagent/pull/44 7 | lagent>=0.1.2 8 | # Minimum 0.10.3 to support distributed evaluation for MMBench 9 | # see https://github.com/open-mmlab/mmengine/pull/1469 10 | mmengine>=0.10.3 11 | openpyxl 12 | # Minimum 0.4.0 to support QLoRA, see https://github.com/huggingface/peft/pull/476 13 | peft>=0.4.0 14 | scikit-image 15 | scipy 16 | SentencePiece 17 | tiktoken 18 | torch==2.1.2 19 | torchvision 20 | # Minimum 4.36.0 to support `Cache` data structure used by KV Cache 21 | # Registering a causal mask in `LlamaModel` is not friendly for very large 22 | # `max_position_embeddings`. Refer to 23 | # https://github.com/huggingface/transformers/blob/v4.38.0/src/transformers/models/llama/modeling_llama.py#L921-L923 24 | transformers>=4.36.0,!=4.38.0,!=4.38.1,!=4.38.2 25 | transformers_stream_generator 26 | -------------------------------------------------------------------------------- /script/chat.sh: -------------------------------------------------------------------------------- 1 | srun -p mllm_1 \ 2 | --gres=gpu:1 \ 3 | python mg_llava/module/chat.py \ 4 | 'PATH TO MG-LLaVA-Vicuna-7B MODEL' \ 5 | --llm_name_or_path 'PATH TO Vicuna1.5-7B LLM' \ 6 | --visual_encoder_clip 'PATH TO CLIP MODEL' \ 7 | --visual_encoder_convnext 'PATH TO ConvNext MODEL' \ 8 | --ram_model 'PATH TO RAM MODEL' \ 9 | --owl_vit_model 'PATH TO OWL-VIT-2 MODEL' \ 10 | --prompt-template 'vicuna' \ 11 | --image examples/example.jpg 12 | -------------------------------------------------------------------------------- /script/test.sh: -------------------------------------------------------------------------------- 1 | JOB_NAME=$1 2 | GPUS=${GPUS:-8} 3 | GPUS_PER_NODE=${GPUS_PER_NODE:-8} 4 | CPUS_PER_TASK=${CPUS_PER_TASK:-5} 5 | SRUN_ARGS=${SRUN_ARGS:-""} 6 | PY_ARGS=${@:5} 7 | 8 | srun -p PARTITION_NAME \ 9 | --job-name=${JOB_NAME} \ 10 | --quotatype=auto \ 11 | --gres=gpu:${GPUS_PER_NODE} \ 12 | --ntasks=${GPUS} \ 13 | --ntasks-per-node=${GPUS_PER_NODE} \ 14 | --cpus-per-task=${CPUS_PER_TASK} \ 15 | --kill-on-bad-exit=1 \ 16 | python xtuner/tools/test.py \ 17 | PATH_TO_CONFIG \ 18 | --checkpoint PATH_TO_PTH \ 19 | --launcher="slurm" \ 20 | 21 | # Evaluation for Yi-1.5-34B 22 | # srun -p llmit \ 23 | # --job-name=${JOB_NAME} \ 24 | # --quotatype=auto \ 25 | # --gres=gpu:${GPUS_PER_NODE} \ 26 | # --ntasks=${GPUS} \ 27 | # --ntasks-per-node=${GPUS_PER_NODE} \ 28 | # --cpus-per-task=${CPUS_PER_TASK} \ 29 | # --kill-on-bad-exit=1 \ 30 | # python fuse_object_llava/module/test_34B/special_test.py \ 31 | # mg_llava/config/Yi-34B/fuse_more_34B_test.py \ 32 | # --work-dir work_dirs/fuse_more_yi34B_clip_L_14_336_sft_padding \ 33 | # --launcher="slurm" \ 34 | # --deepspeed deepspeed_zero3_debug -------------------------------------------------------------------------------- /script/train_llama3.sh: -------------------------------------------------------------------------------- 1 | set -e 2 | 3 | JOB_NAME=$1 4 | GPUS=${GPUS:-8} 5 | GPUS_PER_NODE=${GPUS_PER_NODE:-8} 6 | CPUS_PER_TASK=${CPUS_PER_TASK:-5} 7 | SRUN_ARGS=${SRUN_ARGS:-""} 8 | PY_ARGS=${@:5} 9 | # export MASTER_PORT=12345 10 | srun -p PARTITION_NAME \ 11 | --job-name=${JOB_NAME} \ 12 | --gres=gpu:${GPUS_PER_NODE} \ 13 | --quotatype=auto \ 14 | --ntasks=${GPUS} \ 15 | --ntasks-per-node=${GPUS_PER_NODE} \ 16 | --cpus-per-task=${CPUS_PER_TASK} \ 17 | --kill-on-bad-exit=1 \ 18 | python xtuner/tools/train.py \ 19 | mg_llava/config/llama3/fuse_more_llama3_clip_L_14_336_pretrain_padding.py \ 20 | --seed 2024 \ 21 | --launcher="slurm" \ 22 | --deepspeed deepspeed_zero2 23 | 24 | srun -p PARTITION_NAME \ 25 | --job-name=${JOB_NAME} \ 26 | --gres=gpu:${GPUS_PER_NODE} \ 27 | --ntasks=${GPUS} \ 28 | --ntasks-per-node=${GPUS_PER_NODE} \ 29 | --cpus-per-task=${CPUS_PER_TASK} \ 30 | --kill-on-bad-exit=1 \ 31 | python xtuner/tools/train.py \ 32 | mg_llava/config/llama3/fuse_more_llama3_clip_L_14_336_sft_padding.py \ 33 | --seed 2024 \ 34 | --launcher="slurm" \ 35 | --deepspeed deepspeed_zero2 36 | 37 | srun -p PARTITION_NAME \ 38 | --job-name=${JOB_NAME} \ 39 | --quotatype=auto \ 40 | --gres=gpu:${GPUS_PER_NODE} \ 41 | --ntasks=${GPUS} \ 42 | --ntasks-per-node=${GPUS_PER_NODE} \ 43 | --cpus-per-task=${CPUS_PER_TASK} \ 44 | --kill-on-bad-exit=1 \ 45 | python xtuner/tools/test.py \ 46 | mg_llava/config/llama3/fuse_more_llama3_clip_L_14_336_sft_padding.py \ 47 | --checkpoint work_dirs/fuse_more_llama3_clip_L_14_336_sft_padding/iter_43226.pth \ 48 | --launcher="slurm" \ -------------------------------------------------------------------------------- /script/train_phi3.sh: -------------------------------------------------------------------------------- 1 | set -e 2 | 3 | JOB_NAME=$1 4 | GPUS=${GPUS:-8} 5 | GPUS_PER_NODE=${GPUS_PER_NODE:-8} 6 | CPUS_PER_TASK=${CPUS_PER_TASK:-5} 7 | SRUN_ARGS=${SRUN_ARGS:-""} 8 | PY_ARGS=${@:5} 9 | # export MASTER_PORT=12345 10 | srun -p PARTITION_NAME \ 11 | --job-name=${JOB_NAME} \ 12 | --gres=gpu:${GPUS_PER_NODE} \ 13 | --quotatype=auto \ 14 | --ntasks=${GPUS} \ 15 | --ntasks-per-node=${GPUS_PER_NODE} \ 16 | --cpus-per-task=${CPUS_PER_TASK} \ 17 | --kill-on-bad-exit=1 \ 18 | python xtuner/tools/train.py \ 19 | mg_llava/config/phi3/fuse_more_phi3_clip_L_14_336_pretrain_padding.py \ 20 | --seed 2024 \ 21 | --launcher="slurm" \ 22 | --deepspeed deepspeed_zero2 23 | 24 | srun -p PARTITION_NAME \ 25 | --job-name=${JOB_NAME} \ 26 | --gres=gpu:${GPUS_PER_NODE} \ 27 | --ntasks=${GPUS} \ 28 | --ntasks-per-node=${GPUS_PER_NODE} \ 29 | --cpus-per-task=${CPUS_PER_TASK} \ 30 | --kill-on-bad-exit=1 \ 31 | python xtuner/tools/train.py \ 32 | mg_llava/config/phi3/fuse_more_phi3_clip_L_14_336_sft_padding.py \ 33 | --seed 2024 \ 34 | --launcher="slurm" \ 35 | --deepspeed deepspeed_zero2 36 | 37 | srun -p PARTITION_NAME \ 38 | --job-name=${JOB_NAME} \ 39 | --quotatype=auto \ 40 | --gres=gpu:${GPUS_PER_NODE} \ 41 | --ntasks=${GPUS} \ 42 | --ntasks-per-node=${GPUS_PER_NODE} \ 43 | --cpus-per-task=${CPUS_PER_TASK} \ 44 | --kill-on-bad-exit=1 \ 45 | python xtuner/tools/test.py \ 46 | mg_llava/config/phi3/fuse_more_phi3_clip_L_14_336_sft_padding.py \ 47 | --checkpoint work_dirs/fuse_more_phi3_clip_L_14_336_sft_padding/iter_21610.pth \ 48 | --launcher="slurm" \ -------------------------------------------------------------------------------- /script/train_pretrain.sh: -------------------------------------------------------------------------------- 1 | JOB_NAME=$1 2 | CONFIG=$2 3 | GPUS=${GPUS:-8} 4 | GPUS_PER_NODE=${GPUS_PER_NODE:-8} 5 | CPUS_PER_TASK=${CPUS_PER_TASK:-5} 6 | SRUN_ARGS=${SRUN_ARGS:-""} 7 | PY_ARGS=${@:5} 8 | export MASTER_PORT=12345 9 | srun -p PARTITION_NAME \ 10 | --job-name=${JOB_NAME} \ 11 | --gres=gpu:${GPUS_PER_NODE} \ 12 | --ntasks=${GPUS} \ 13 | --ntasks-per-node=${GPUS_PER_NODE} \ 14 | --cpus-per-task=${CPUS_PER_TASK} \ 15 | --kill-on-bad-exit=1 \ 16 | python xtuner/tools/train.py \ 17 | ${CONFIG} \ 18 | --seed 2024 \ 19 | --launcher="slurm" \ 20 | --deepspeed deepspeed_zero2 21 | -------------------------------------------------------------------------------- /script/train_sft.sh: -------------------------------------------------------------------------------- 1 | set -e 2 | 3 | JOB_NAME=$1 4 | CONFIG=$2 5 | GPUS=${GPUS:-16} 6 | GPUS_PER_NODE=${GPUS_PER_NODE:-8} 7 | CPUS_PER_TASK=${CPUS_PER_TASK:-5} 8 | SRUN_ARGS=${SRUN_ARGS:-""} 9 | PY_ARGS=${@:5} 10 | 11 | srun -p PARTITION_NAME \ 12 | --job-name=${JOB_NAME} \ 13 | --gres=gpu:${GPUS_PER_NODE} \ 14 | --ntasks=${GPUS} \ 15 | --ntasks-per-node=${GPUS_PER_NODE} \ 16 | --cpus-per-task=${CPUS_PER_TASK} \ 17 | --kill-on-bad-exit=1 \ 18 | python xtuner/tools/train.py \ 19 | ${CONFIG} \ 20 | --seed 2024 \ 21 | --launcher="slurm" \ 22 | --deepspeed deepspeed_zero2 23 | -------------------------------------------------------------------------------- /script/train_vicuna13B.sh: -------------------------------------------------------------------------------- 1 | set -e 2 | 3 | JOB_NAME=$1 4 | GPUS=${GPUS:-8} 5 | GPUS_PER_NODE=${GPUS_PER_NODE:-8} 6 | CPUS_PER_TASK=${CPUS_PER_TASK:-5} 7 | SRUN_ARGS=${SRUN_ARGS:-""} 8 | PY_ARGS=${@:5} 9 | # export MASTER_PORT=12345 10 | srun -p PARTITION_NAME \ 11 | --job-name=${JOB_NAME} \ 12 | --gres=gpu:${GPUS_PER_NODE} \ 13 | --quotatype=auto \ 14 | --ntasks=${GPUS} \ 15 | --ntasks-per-node=${GPUS_PER_NODE} \ 16 | --cpus-per-task=${CPUS_PER_TASK} \ 17 | --kill-on-bad-exit=1 \ 18 | python xtuner/tools/train.py \ 19 | mg_llava/config/vicuna/fuse_vicuna13b_clip_L_14_336_pretrain_padding.py \ 20 | --seed 2024 \ 21 | --launcher="slurm" \ 22 | --deepspeed deepspeed_zero2 23 | 24 | srun -p PARTITION_NAME \ 25 | --job-name=${JOB_NAME} \ 26 | --gres=gpu:${GPUS_PER_NODE} \ 27 | --ntasks=${GPUS} \ 28 | --ntasks-per-node=${GPUS_PER_NODE} \ 29 | --cpus-per-task=${CPUS_PER_TASK} \ 30 | --kill-on-bad-exit=1 \ 31 | python xtuner/tools/train.py \ 32 | mg_llava/config/vicuna/fuse_vicuna13b_clip_L_14_336_sft_padding.py \ 33 | --seed 2024 \ 34 | --launcher="slurm" \ 35 | --deepspeed deepspeed_zero2 36 | 37 | srun -p PARTITION_NAME \ 38 | --job-name=${JOB_NAME} \ 39 | --quotatype=auto \ 40 | --gres=gpu:${GPUS_PER_NODE} \ 41 | --ntasks=${GPUS} \ 42 | --ntasks-per-node=${GPUS_PER_NODE} \ 43 | --cpus-per-task=${CPUS_PER_TASK} \ 44 | --kill-on-bad-exit=1 \ 45 | python xtuner/tools/test.py \ 46 | mg_llava/config/vicuna/fuse_vicuna13b_clip_L_14_336_sft_padding.py \ 47 | --checkpoint work_dirs/fuse_vicuna13b_clip_L_14_336_sft_padding/iter_10805.pth \ 48 | --launcher="slurm" \ -------------------------------------------------------------------------------- /script/train_vicuna7B.sh: -------------------------------------------------------------------------------- 1 | set -e 2 | export PYTHONPATH=$PYTHONPATH:$(pwd) 3 | 4 | JOB_NAME=$1 5 | GPUS=${GPUS:-8} 6 | GPUS_PER_NODE=${GPUS_PER_NODE:-8} 7 | CPUS_PER_TASK=${CPUS_PER_TASK:-5} 8 | SRUN_ARGS=${SRUN_ARGS:-""} 9 | PY_ARGS=${@:5} 10 | # export MASTER_PORT=12345 11 | srun -p PARTITION_NAME \ 12 | --job-name=${JOB_NAME} \ 13 | --gres=gpu:${GPUS_PER_NODE} \ 14 | --quotatype=auto \ 15 | --ntasks=${GPUS} \ 16 | --ntasks-per-node=${GPUS_PER_NODE} \ 17 | --cpus-per-task=${CPUS_PER_TASK} \ 18 | --kill-on-bad-exit=1 \ 19 | python xtuner/tools/train.py \ 20 | mg_llava/config/vicuna/fuse_more_vicuna7b_clip_L_14_336_pretrain_padding.py \ 21 | --seed 2024 \ 22 | --launcher="slurm" \ 23 | --deepspeed deepspeed_zero2 24 | 25 | srun -p PARTITION_NAME \ 26 | --job-name=${JOB_NAME} \ 27 | --gres=gpu:${GPUS_PER_NODE} \ 28 | --ntasks=${GPUS} \ 29 | --ntasks-per-node=${GPUS_PER_NODE} \ 30 | --cpus-per-task=${CPUS_PER_TASK} \ 31 | --kill-on-bad-exit=1 \ 32 | python xtuner/tools/train.py \ 33 | mg_llava/config/vicuna/fuse_more_vicuna7b_clip_L_14_336_sft_padding.py \ 34 | --seed 2024 \ 35 | --launcher="slurm" \ 36 | --deepspeed deepspeed_zero2 37 | 38 | srun -p PARTITION_NAME \ 39 | --job-name=${JOB_NAME} \ 40 | --quotatype=auto \ 41 | --gres=gpu:${GPUS_PER_NODE} \ 42 | --ntasks=${GPUS} \ 43 | --ntasks-per-node=${GPUS_PER_NODE} \ 44 | --cpus-per-task=${CPUS_PER_TASK} \ 45 | --kill-on-bad-exit=1 \ 46 | python xtuner/tools/test.py \ 47 | mg_llava/config/vicuna/fuse_more_vicuna7b_clip_L_14_336_sft_padding.py \ 48 | --checkpoint work_dirs/fuse_more_vicuna7b_clip_L_14_336_sft_padding/iter_5198.pth \ 49 | --launcher="slurm" \ -------------------------------------------------------------------------------- /script/train_yi34b.sh: -------------------------------------------------------------------------------- 1 | set -e 2 | 3 | JOB_NAME=$1 4 | GPUS=${GPUS:-32} 5 | GPUS_PER_NODE=${GPUS_PER_NODE:-8} 6 | CPUS_PER_TASK=${CPUS_PER_TASK:-5} 7 | SRUN_ARGS=${SRUN_ARGS:-""} 8 | PY_ARGS=${@:5} 9 | # export MASTER_PORT=12345 10 | srun -p PARTITION_NAME \ 11 | --job-name=${JOB_NAME} \ 12 | --gres=gpu:${GPUS_PER_NODE} \ 13 | --quotatype=auto \ 14 | --ntasks=${GPUS} \ 15 | --ntasks-per-node=${GPUS_PER_NODE} \ 16 | --cpus-per-task=${CPUS_PER_TASK} \ 17 | -x SH-IDC1-10-140-0-145,SH-IDC1-10-140-0-143,SH-IDC1-10-140-0-142 \ 18 | python xtuner/tools/train.py \ 19 | mg_llava/config/Yi-34B/fuse_more_yi34B_clip_L_14_336_pretrain_padding.py \ 20 | --seed 2024 \ 21 | --launcher="slurm" \ 22 | --deepspeed deepspeed_zero3 23 | 24 | srun -p PARTITION_NAME \ 25 | --job-name=${JOB_NAME} \ 26 | --gres=gpu:${GPUS_PER_NODE} \ 27 | --ntasks=${GPUS} \ 28 | --ntasks-per-node=${GPUS_PER_NODE} \ 29 | --cpus-per-task=${CPUS_PER_TASK} \ 30 | --kill-on-bad-exit=1 \ 31 | python xtuner/tools/train.py \ 32 | mg_llava/config/Yi-34B/fuse_more_yi34B_clip_L_14_336_sft_padding.py \ 33 | --seed 2024 \ 34 | --launcher="slurm" \ 35 | --deepspeed deepspeed_zero3 36 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [isort] 2 | line_length = 79 3 | multi_line_output = 0 4 | extra_standard_library = setuptools 5 | known_first_party = xtuner 6 | known_third_party = pytest,yaml 7 | no_lines_before = STDLIB,LOCALFOLDER 8 | default_section = THIRDPARTY 9 | 10 | [yapf] 11 | BASED_ON_STYLE = pep8 12 | BLANK_LINE_BEFORE_NESTED_CLASS_OR_DEF = true 13 | SPLIT_BEFORE_EXPRESSION_AFTER_OPENING_PAREN = true 14 | 15 | [codespell] 16 | ignore-words-list = nd, ba, warmup 17 | -------------------------------------------------------------------------------- /xtuner/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | import os 3 | 4 | from mmengine.utils import digit_version 5 | 6 | from .entry_point import cli 7 | from .version import __version__, version_info 8 | 9 | HF_CEPH_HUB = os.getenv('HF_CEPH_HUB', '') 10 | HF_USE_CEPH = os.getenv('HF_USE_CEPH', 0) or HF_CEPH_HUB != '' 11 | DS_CEPH_DIR = os.getenv('DS_CEPH_DIR', None) 12 | if HF_USE_CEPH: 13 | from .utils.fileio import (patch_hf_auto_from_pretrained, 14 | patch_hf_save_pretrained) 15 | patch_hf_auto_from_pretrained(HF_CEPH_HUB) 16 | patch_hf_save_pretrained() 17 | 18 | if DS_CEPH_DIR: 19 | from .utils.fileio import patch_deepspeed_engine 20 | patch_deepspeed_engine() 21 | 22 | __all__ = [ 23 | '__version__', 'version_info', 'digit_version', 'cli', 'HF_USE_CEPH', 24 | 'DS_CEPH_DIR' 25 | ] 26 | -------------------------------------------------------------------------------- /xtuner/__pycache__/__init__.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/__pycache__/__init__.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/__pycache__/entry_point.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/__pycache__/entry_point.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/__pycache__/registry.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/__pycache__/registry.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/__pycache__/version.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/__pycache__/version.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/apis/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from .datasets import * # noqa: F401, F403 3 | from .model import * # noqa: F401, F403 4 | from .training_args import * # noqa: F401, F403 5 | -------------------------------------------------------------------------------- /xtuner/apis/datasets/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from .alpaca import (alpaca_data_collator, alpaca_dataset, 3 | alpaca_enzh_data_collator, alpaca_enzh_dataset, 4 | alpaca_zh_data_collator, alpaca_zh_dataset) 5 | from .arxiv import arxiv_data_collator, arxiv_dataset 6 | from .code_alpaca import code_alpaca_data_collator, code_alpaca_dataset 7 | from .colorist import colorist_data_collator, colorist_dataset 8 | from .lawyer import (lawyer_crime_data_collator, lawyer_crime_dataset, 9 | lawyer_data_collator, lawyer_dataset, 10 | lawyer_reference_data_collator, lawyer_reference_dataset) 11 | from .medical import medical_data_collator, medical_dataset 12 | from .moss_003_sft import (moss_003_sft_data_collator, moss_003_sft_dataset, 13 | moss_003_sft_no_plugins_data_collator, 14 | moss_003_sft_no_plugins_dataset, 15 | moss_003_sft_plugins_data_collator, 16 | moss_003_sft_plugins_dataset) 17 | from .oasst1 import oasst1_data_collator, oasst1_dataset 18 | from .open_orca import openorca_data_collator, openorca_dataset 19 | from .sql import sql_data_collator, sql_dataset 20 | from .tiny_codes import tiny_codes_data_collator, tiny_codes_dataset 21 | from .wizardlm import wizardlm_data_collator, wizardlm_dataset 22 | 23 | __all__ = [ 24 | 'alpaca_data_collator', 'alpaca_dataset', 'alpaca_enzh_data_collator', 25 | 'alpaca_enzh_dataset', 'alpaca_zh_data_collator', 'alpaca_zh_dataset', 26 | 'arxiv_data_collator', 'arxiv_dataset', 'medical_data_collator', 27 | 'medical_dataset', 'moss_003_sft_data_collator', 'moss_003_sft_dataset', 28 | 'moss_003_sft_no_plugins_data_collator', 'moss_003_sft_no_plugins_dataset', 29 | 'moss_003_sft_plugins_data_collator', 'moss_003_sft_plugins_dataset', 30 | 'oasst1_data_collator', 'oasst1_dataset', 'openorca_data_collator', 31 | 'openorca_dataset', 'lawyer_crime_dataset', 'lawyer_crime_data_collator', 32 | 'lawyer_reference_dataset', 'lawyer_reference_data_collator', 33 | 'lawyer_dataset', 'lawyer_data_collator', 'colorist_dataset', 34 | 'colorist_data_collator', 'sql_dataset', 'sql_data_collator', 35 | 'code_alpaca_dataset', 'code_alpaca_data_collator', 'tiny_codes_dataset', 36 | 'tiny_codes_data_collator', 'wizardlm_data_collator', 'wizardlm_dataset' 37 | ] 38 | -------------------------------------------------------------------------------- /xtuner/apis/datasets/arxiv.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from functools import partial 3 | 4 | from datasets import load_dataset 5 | 6 | from xtuner.dataset import process_hf_dataset 7 | from xtuner.dataset.collate_fns import default_collate_fn 8 | from xtuner.dataset.map_fns import arxiv_map_fn, template_map_fn_factory 9 | from xtuner.utils import PROMPT_TEMPLATE 10 | 11 | 12 | def arxiv_dataset(tokenizer, 13 | data_file=None, 14 | max_length=2048, 15 | prompt_template=PROMPT_TEMPLATE.default, 16 | remove_unused_columns=True, 17 | pack_to_max_length=True): 18 | template_map_fn = template_map_fn_factory(template=prompt_template) 19 | # 1. Download data from https://kaggle.com/datasets/Cornell-University/arxiv # noqa: E501 20 | # 2. Process data with `./tools/data_preprocess/arxiv.py` 21 | if data_file is None: 22 | data_file = './data/arxiv_postprocess_csAIcsCLcsCV_20200101.json' 23 | dataset_org = load_dataset(path='json', data_files=dict(train=data_file)) 24 | dataset = process_hf_dataset( 25 | dataset=dataset_org, 26 | tokenizer=tokenizer, 27 | max_length=max_length, 28 | dataset_map_fn=arxiv_map_fn, 29 | template_map_fn=template_map_fn, 30 | remove_unused_columns=remove_unused_columns, 31 | shuffle_before_pack=True, 32 | pack_to_max_length=pack_to_max_length) 33 | 34 | return dataset 35 | 36 | 37 | def arxiv_data_collator(return_hf_format=False): 38 | return partial(default_collate_fn, return_hf_format=return_hf_format) 39 | -------------------------------------------------------------------------------- /xtuner/apis/datasets/code_alpaca.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from functools import partial 3 | 4 | from datasets import load_dataset 5 | 6 | from xtuner.dataset import process_hf_dataset 7 | from xtuner.dataset.collate_fns import default_collate_fn 8 | from xtuner.dataset.map_fns import code_alpaca_map_fn, template_map_fn_factory 9 | from xtuner.utils import PROMPT_TEMPLATE 10 | 11 | 12 | def code_alpaca_dataset(tokenizer, 13 | path='HuggingFaceH4/CodeAlpaca_20K', 14 | max_length=2048, 15 | prompt_template=PROMPT_TEMPLATE.default, 16 | remove_unused_columns=True, 17 | pack_to_max_length=True): 18 | template_map_fn = template_map_fn_factory(template=prompt_template) 19 | dataset_org = load_dataset(path) 20 | dataset = process_hf_dataset( 21 | dataset=dataset_org, 22 | tokenizer=tokenizer, 23 | max_length=max_length, 24 | dataset_map_fn=code_alpaca_map_fn, 25 | template_map_fn=template_map_fn, 26 | remove_unused_columns=remove_unused_columns, 27 | shuffle_before_pack=True, 28 | pack_to_max_length=pack_to_max_length) 29 | 30 | return dataset 31 | 32 | 33 | def code_alpaca_data_collator(return_hf_format=False): 34 | return partial(default_collate_fn, return_hf_format=return_hf_format) 35 | -------------------------------------------------------------------------------- /xtuner/apis/datasets/colorist.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from functools import partial 3 | 4 | from datasets import load_dataset 5 | 6 | from xtuner.dataset import process_hf_dataset 7 | from xtuner.dataset.collate_fns import default_collate_fn 8 | from xtuner.dataset.map_fns import colors_map_fn, template_map_fn_factory 9 | from xtuner.utils import PROMPT_TEMPLATE 10 | 11 | 12 | def colorist_dataset(tokenizer, 13 | path='burkelibbey/colors', 14 | max_length=2048, 15 | prompt_template=PROMPT_TEMPLATE.default, 16 | remove_unused_columns=True, 17 | pack_to_max_length=True): 18 | template_map_fn = template_map_fn_factory(template=prompt_template) 19 | dataset_org = load_dataset(path) 20 | dataset = process_hf_dataset( 21 | dataset=dataset_org, 22 | tokenizer=tokenizer, 23 | max_length=max_length, 24 | dataset_map_fn=colors_map_fn, 25 | template_map_fn=template_map_fn, 26 | remove_unused_columns=remove_unused_columns, 27 | shuffle_before_pack=True, 28 | pack_to_max_length=pack_to_max_length) 29 | 30 | return dataset 31 | 32 | 33 | def colorist_data_collator(return_hf_format=False): 34 | return partial(default_collate_fn, return_hf_format=return_hf_format) 35 | -------------------------------------------------------------------------------- /xtuner/apis/datasets/medical.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from functools import partial 3 | 4 | from datasets import load_dataset 5 | 6 | from xtuner.dataset import process_hf_dataset 7 | from xtuner.dataset.collate_fns import default_collate_fn 8 | from xtuner.dataset.map_fns import medical_map_fn, template_map_fn_factory 9 | from xtuner.utils import PROMPT_TEMPLATE 10 | 11 | 12 | def medical_dataset(tokenizer, 13 | path='shibing624/medical', 14 | max_length=2048, 15 | prompt_template=PROMPT_TEMPLATE.default, 16 | remove_unused_columns=False, 17 | pack_to_max_length=True): 18 | template_map_fn = template_map_fn_factory(template=prompt_template) 19 | dataset_org = load_dataset(path) 20 | dataset = process_hf_dataset( 21 | dataset=dataset_org, 22 | tokenizer=tokenizer, 23 | max_length=max_length, 24 | dataset_map_fn=medical_map_fn, 25 | template_map_fn=template_map_fn, 26 | remove_unused_columns=remove_unused_columns, 27 | shuffle_before_pack=True, 28 | pack_to_max_length=pack_to_max_length) 29 | 30 | return dataset 31 | 32 | 33 | def medical_data_collator(return_hf_format=False): 34 | return partial(default_collate_fn, return_hf_format=return_hf_format) 35 | -------------------------------------------------------------------------------- /xtuner/apis/datasets/moss_003_sft.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from functools import partial 3 | 4 | from torch.utils.data import ConcatDataset 5 | 6 | from xtuner.dataset import MOSSSFTDataset 7 | from xtuner.dataset.collate_fns import default_collate_fn 8 | 9 | 10 | def moss_003_sft_dataset(tokenizer, 11 | plugins_data_file=None, 12 | no_plugins_data_file=None, 13 | bot_name=None, 14 | max_length=2048): 15 | plugins = moss_003_sft_plugins_dataset( 16 | tokenizer, 17 | data_file=plugins_data_file, 18 | bot_name=bot_name, 19 | max_length=max_length) 20 | no_plugins = moss_003_sft_no_plugins_dataset( 21 | tokenizer, 22 | data_file=no_plugins_data_file, 23 | bot_name=bot_name, 24 | max_length=max_length) 25 | dataset = ConcatDataset([plugins, no_plugins]) 26 | return dataset 27 | 28 | 29 | def moss_003_sft_data_collator(return_hf_format=False): 30 | return partial(default_collate_fn, return_hf_format=return_hf_format) 31 | 32 | 33 | def moss_003_sft_no_plugins_dataset(tokenizer, 34 | data_file=None, 35 | bot_name=None, 36 | max_length=2048): 37 | 38 | # Download data from https://huggingface.co/datasets/fnlp/moss-003-sft-data 39 | if data_file is None: 40 | data_file = './data/moss-003-sft-no-tools.jsonl' 41 | dataset = MOSSSFTDataset( 42 | data_file=data_file, 43 | bot_name=bot_name, 44 | tokenizer=tokenizer, 45 | max_length=max_length) 46 | 47 | return dataset 48 | 49 | 50 | def moss_003_sft_no_plugins_data_collator(return_hf_format=False): 51 | return partial(default_collate_fn, return_hf_format=return_hf_format) 52 | 53 | 54 | def moss_003_sft_plugins_dataset(tokenizer, 55 | data_file=None, 56 | bot_name=None, 57 | max_length=2048): 58 | 59 | # Download data from https://huggingface.co/datasets/fnlp/moss-003-sft-data 60 | if data_file is None: 61 | data_file = './data/conversations_with_tools_with_inner_instruction_no_text2image_train_all_random_meta0.5_0.1_0.01_moss_0709.jsonl' # noqa: E501 62 | dataset = MOSSSFTDataset( 63 | data_file=data_file, 64 | bot_name=bot_name, 65 | tokenizer=tokenizer, 66 | max_length=max_length) 67 | 68 | return dataset 69 | 70 | 71 | def moss_003_sft_plugins_data_collator(return_hf_format=False): 72 | return partial(default_collate_fn, return_hf_format=return_hf_format) 73 | -------------------------------------------------------------------------------- /xtuner/apis/datasets/oasst1.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from functools import partial 3 | 4 | from datasets import load_dataset 5 | 6 | from xtuner.dataset import process_hf_dataset 7 | from xtuner.dataset.collate_fns import default_collate_fn 8 | from xtuner.dataset.map_fns import oasst1_map_fn, template_map_fn_factory 9 | from xtuner.utils import PROMPT_TEMPLATE 10 | 11 | 12 | def oasst1_dataset(tokenizer, 13 | path='timdettmers/openassistant-guanaco', 14 | max_length=2048, 15 | prompt_template=PROMPT_TEMPLATE.default, 16 | remove_unused_columns=False, 17 | pack_to_max_length=True): 18 | template_map_fn = template_map_fn_factory(template=prompt_template) 19 | dataset_org = load_dataset(path) 20 | dataset = process_hf_dataset( 21 | dataset=dataset_org, 22 | tokenizer=tokenizer, 23 | max_length=max_length, 24 | dataset_map_fn=oasst1_map_fn, 25 | template_map_fn=template_map_fn, 26 | remove_unused_columns=remove_unused_columns, 27 | shuffle_before_pack=True, 28 | pack_to_max_length=pack_to_max_length) 29 | 30 | return dataset 31 | 32 | 33 | def oasst1_data_collator(return_hf_format=False): 34 | return partial(default_collate_fn, return_hf_format=return_hf_format) 35 | -------------------------------------------------------------------------------- /xtuner/apis/datasets/open_orca.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from functools import partial 3 | 4 | from datasets import load_dataset 5 | 6 | from xtuner.dataset import process_hf_dataset 7 | from xtuner.dataset.collate_fns import default_collate_fn 8 | from xtuner.dataset.map_fns import openorca_map_fn, template_map_fn_factory 9 | from xtuner.utils import PROMPT_TEMPLATE 10 | 11 | 12 | def openorca_dataset(tokenizer, 13 | path='Open-Orca/OpenOrca', 14 | max_length=2048, 15 | prompt_template=PROMPT_TEMPLATE.default, 16 | remove_unused_columns=True, 17 | pack_to_max_length=True): 18 | template_map_fn = template_map_fn_factory(template=prompt_template) 19 | dataset_org = load_dataset(path) 20 | dataset = process_hf_dataset( 21 | dataset=dataset_org, 22 | tokenizer=tokenizer, 23 | max_length=max_length, 24 | dataset_map_fn=openorca_map_fn, 25 | template_map_fn=template_map_fn, 26 | remove_unused_columns=remove_unused_columns, 27 | shuffle_before_pack=True, 28 | pack_to_max_length=pack_to_max_length) 29 | 30 | return dataset 31 | 32 | 33 | def openorca_data_collator(return_hf_format=False): 34 | return partial(default_collate_fn, return_hf_format=return_hf_format) 35 | -------------------------------------------------------------------------------- /xtuner/apis/datasets/sql.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from functools import partial 3 | 4 | from datasets import load_dataset 5 | 6 | from xtuner.dataset import process_hf_dataset 7 | from xtuner.dataset.collate_fns import default_collate_fn 8 | from xtuner.dataset.map_fns import sql_map_fn, template_map_fn_factory 9 | from xtuner.utils import PROMPT_TEMPLATE 10 | 11 | 12 | def sql_dataset(tokenizer, 13 | path='b-mc2/sql-create-context', 14 | max_length=2048, 15 | prompt_template=PROMPT_TEMPLATE.default, 16 | remove_unused_columns=True, 17 | pack_to_max_length=True): 18 | template_map_fn = template_map_fn_factory(template=prompt_template) 19 | dataset_org = load_dataset(path) 20 | dataset = process_hf_dataset( 21 | dataset=dataset_org, 22 | tokenizer=tokenizer, 23 | max_length=max_length, 24 | dataset_map_fn=sql_map_fn, 25 | template_map_fn=template_map_fn, 26 | remove_unused_columns=remove_unused_columns, 27 | shuffle_before_pack=True, 28 | pack_to_max_length=pack_to_max_length) 29 | 30 | return dataset 31 | 32 | 33 | def sql_data_collator(return_hf_format=False): 34 | return partial(default_collate_fn, return_hf_format=return_hf_format) 35 | -------------------------------------------------------------------------------- /xtuner/apis/datasets/tiny_codes.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from functools import partial 3 | 4 | from datasets import load_dataset 5 | 6 | from xtuner.dataset import process_hf_dataset 7 | from xtuner.dataset.collate_fns import default_collate_fn 8 | from xtuner.dataset.map_fns import template_map_fn_factory, tiny_codes_map_fn 9 | from xtuner.utils import PROMPT_TEMPLATE 10 | 11 | 12 | def tiny_codes_dataset(tokenizer, 13 | path='nampdn-ai/tiny-codes', 14 | max_length=2048, 15 | prompt_template=PROMPT_TEMPLATE.default, 16 | remove_unused_columns=True, 17 | pack_to_max_length=True): 18 | template_map_fn = template_map_fn_factory(template=prompt_template) 19 | dataset_org = load_dataset(path) 20 | dataset = process_hf_dataset( 21 | dataset=dataset_org, 22 | tokenizer=tokenizer, 23 | max_length=max_length, 24 | dataset_map_fn=tiny_codes_map_fn, 25 | template_map_fn=template_map_fn, 26 | remove_unused_columns=remove_unused_columns, 27 | shuffle_before_pack=True, 28 | pack_to_max_length=pack_to_max_length) 29 | 30 | return dataset 31 | 32 | 33 | def tiny_codes_data_collator(return_hf_format=False): 34 | return partial(default_collate_fn, return_hf_format=return_hf_format) 35 | -------------------------------------------------------------------------------- /xtuner/apis/datasets/wizardlm.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from functools import partial 3 | 4 | from datasets import load_dataset 5 | 6 | from xtuner.dataset import process_hf_dataset 7 | from xtuner.dataset.collate_fns import default_collate_fn 8 | from xtuner.dataset.map_fns import template_map_fn_factory, wizardlm_map_fn 9 | from xtuner.utils import PROMPT_TEMPLATE 10 | 11 | 12 | def wizardlm_dataset(tokenizer, 13 | path='WizardLM/WizardLM_evol_instruct_V2_196k', 14 | max_length=2048, 15 | prompt_template=PROMPT_TEMPLATE.default, 16 | remove_unused_columns=False, 17 | pack_to_max_length=True): 18 | template_map_fn = template_map_fn_factory(template=prompt_template) 19 | dataset_org = load_dataset(path) 20 | dataset = process_hf_dataset( 21 | dataset=dataset_org, 22 | tokenizer=tokenizer, 23 | max_length=max_length, 24 | dataset_map_fn=wizardlm_map_fn, 25 | template_map_fn=template_map_fn, 26 | remove_unused_columns=remove_unused_columns, 27 | shuffle_before_pack=True, 28 | pack_to_max_length=pack_to_max_length) 29 | 30 | return dataset 31 | 32 | 33 | def wizardlm_data_collator(return_hf_format=False): 34 | return partial(default_collate_fn, return_hf_format=return_hf_format) 35 | -------------------------------------------------------------------------------- /xtuner/apis/training_args.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from dataclasses import dataclass, field 3 | from typing import Union 4 | 5 | from transformers import TrainingArguments 6 | from transformers.trainer_utils import IntervalStrategy, SchedulerType 7 | 8 | __all__ = ['DefaultTrainingArguments'] 9 | 10 | 11 | @dataclass 12 | class DefaultTrainingArguments(TrainingArguments): 13 | # custom 14 | model_name_or_path: str = field( 15 | default=None, 16 | metadata={'help': 'model name or path.'}, 17 | ) 18 | dataset_name_or_path: str = field( 19 | default=None, 20 | metadata={'help': 'dataset name or path.'}, 21 | ) 22 | 23 | # huggingface 24 | default_output_dir = './work_dirs' 25 | default_do_train = True 26 | default_per_device_train_batch_size = 1 27 | default_learning_rate = 2e-5 28 | default_save_strategy = 'epoch' 29 | default_lr_scheduler_type = 'cosine' 30 | default_logging_steps = 5 31 | 32 | output_dir: str = field( 33 | default=default_output_dir, 34 | metadata={ 35 | 'help': ('The output directory where the model predictions and ' 36 | 'checkpoints will be written.') 37 | }) 38 | do_train: bool = field( 39 | default=default_do_train, 40 | metadata={'help': 'Whether to run training.'}) 41 | per_device_train_batch_size: int = field( 42 | default=default_per_device_train_batch_size, 43 | metadata={'help': 'Batch size per GPU/TPU core/CPU for training.'}) 44 | learning_rate: float = field( 45 | default=default_learning_rate, 46 | metadata={'help': 'The initial learning rate for AdamW.'}) 47 | save_strategy: Union[IntervalStrategy, str] = field( 48 | default=default_save_strategy, 49 | metadata={'help': 'The checkpoint save strategy to use.'}, 50 | ) 51 | lr_scheduler_type: Union[SchedulerType, str] = field( 52 | default=default_lr_scheduler_type, 53 | metadata={'help': 'The scheduler type to use.'}, 54 | ) 55 | logging_steps: float = field( 56 | default=default_logging_steps, 57 | metadata={ 58 | 'help': ('Log every X updates steps. Should be an integer or a ' 59 | 'float in range `[0,1)`. If smaller than 1, will be ' 60 | 'interpreted as ratio of total training steps.') 61 | }) 62 | -------------------------------------------------------------------------------- /xtuner/configs/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | import os 3 | 4 | 5 | def get_cfgs_name_path(): 6 | path = os.path.dirname(__file__) 7 | mapping = {} 8 | for root, dirs, files in os.walk(path): 9 | for file_ in files: 10 | if file_.endswith( 11 | ('.py', '.json') 12 | ) and not file_.startswith('.') and not file_.startswith('_'): 13 | mapping[os.path.splitext(file_)[0]] = os.path.join(root, file_) 14 | return mapping 15 | 16 | 17 | cfgs_name_path = get_cfgs_name_path() 18 | 19 | __all__ = ['cfgs_name_path'] 20 | -------------------------------------------------------------------------------- /xtuner/configs/__pycache__/__init__.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/configs/__pycache__/__init__.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/configs/cohere/README.md: -------------------------------------------------------------------------------- 1 | # Cohere 8x7B 2 | 3 | ## Install 4 | 5 | ```bash 6 | # Install the latest xtuner 7 | pip install -U 'xtuner[deepspeed]' 8 | 9 | # Cohere requires the latest version of transformers. 10 | pip install git+https://github.com/huggingface/transformers.git 11 | 12 | # Sequence parallel requires flash-attn 13 | pip install flash-attn 14 | ``` 15 | 16 | ## Full Parameter Fine-tune 17 | 18 | Full parameter fine-tune needs 64 A100-80G 19 | 20 | ### slurm 21 | 22 | Note: `$PARTITION` means the virtual partition of slurm. 23 | 24 | ```bash 25 | srun -p $PARTITION --job-name=Cohere --nodes=8 --gres=gpu:8 --ntasks-per-node=8 xtuner train cohere_100b_128k_sp32 --deepspeed deepspeed_zero3 --launcher slurm 26 | ``` 27 | 28 | ### torchrun 29 | 30 | Note: `$NODE_0_ADDR` means the ip address of the node_0 machine. 31 | 32 | ```bash 33 | # excuete on node 0 34 | NPROC_PER_NODE=8 NNODES=8 PORT=29600 ADDR=$NODE_0_ADDR NODE_RANK=0 xtuner train cohere_100b_128k_sp32 --deepspeed deepspeed_zero3 35 | 36 | # excuete on node 1 37 | NPROC_PER_NODE=8 NNODES=8 PORT=29600 ADDR=$NODE_0_ADDR NODE_RANK=1 xtuner train cohere_100b_128k_sp32 --deepspeed deepspeed_zero3 38 | ``` 39 | 40 | ### Speed 41 | 42 | 16 * A100 80G: 43 | 44 | | Model | Sequence Length | GPUs Number | Sequence Parallel World Size | Tokens per Second | TFLOPs | 45 | | :---------: | :-------------: | :---------: | :--------------------------: | :---------------: | :----: | 46 | | Cohere_100b | 128k | 64 | 32 | 97.3 | 173.4 | 47 | | Cohere_100b | 128k | 128 | 16 | 102.1 | 182.7 | 48 | | Cohere_100b | 128k | 256 | 16 | 101.3 | 181.3 | 49 | -------------------------------------------------------------------------------- /xtuner/configs/deepspeed/deepspeed_zero1.json: -------------------------------------------------------------------------------- 1 | { 2 | "gradient_accumulation_steps": "auto", 3 | "train_micro_batch_size_per_gpu": "auto", 4 | "gradient_clipping": "auto", 5 | "zero_allow_untested_optimizer": true, 6 | "zero_force_ds_cpu_optimizer": false, 7 | "zero_optimization": { 8 | "stage": 1, 9 | "overlap_comm": true 10 | }, 11 | "fp16": { 12 | "enabled": "auto", 13 | "initial_scale_power": 16 14 | }, 15 | "bf16": { 16 | "enabled": "auto" 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /xtuner/configs/deepspeed/deepspeed_zero2.json: -------------------------------------------------------------------------------- 1 | { 2 | "gradient_accumulation_steps": "auto", 3 | "train_micro_batch_size_per_gpu": "auto", 4 | "gradient_clipping": "auto", 5 | "zero_allow_untested_optimizer": true, 6 | "zero_force_ds_cpu_optimizer": false, 7 | "zero_optimization": { 8 | "stage": 2, 9 | "overlap_comm": true 10 | }, 11 | "fp16": { 12 | "enabled": "auto", 13 | "initial_scale_power": 16 14 | }, 15 | "bf16": { 16 | "enabled": "auto" 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /xtuner/configs/deepspeed/deepspeed_zero2_offload.json: -------------------------------------------------------------------------------- 1 | { 2 | "gradient_accumulation_steps": "auto", 3 | "train_micro_batch_size_per_gpu": "auto", 4 | "gradient_clipping": "auto", 5 | "zero_allow_untested_optimizer": true, 6 | "zero_force_ds_cpu_optimizer": false, 7 | "zero_optimization": { 8 | "stage": 2, 9 | "overlap_comm": true, 10 | "offload_optimizer": { 11 | "device": "cpu", 12 | "pin_memory": true 13 | } 14 | }, 15 | "fp16": { 16 | "enabled": "auto", 17 | "initial_scale_power": 16 18 | }, 19 | "bf16": { 20 | "enabled": "auto" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /xtuner/configs/deepspeed/deepspeed_zero3.json: -------------------------------------------------------------------------------- 1 | { 2 | "gradient_accumulation_steps": "auto", 3 | "train_micro_batch_size_per_gpu": "auto", 4 | "gradient_clipping": "auto", 5 | "zero_allow_untested_optimizer": true, 6 | "zero_force_ds_cpu_optimizer": false, 7 | "zero_optimization": { 8 | "stage": 3, 9 | "overlap_comm": true, 10 | "stage3_gather_16bit_weights_on_model_save": true 11 | }, 12 | "fp16": { 13 | "enabled": "auto", 14 | "initial_scale_power": 16 15 | }, 16 | "bf16": { 17 | "enabled": "auto" 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /xtuner/configs/deepspeed/deepspeed_zero3_debug.json: -------------------------------------------------------------------------------- 1 | { 2 | "gradient_accumulation_steps": "auto", 3 | "train_micro_batch_size_per_gpu": "auto", 4 | "gradient_clipping": "auto", 5 | "zero_allow_untested_optimizer": true, 6 | "zero_force_ds_cpu_optimizer": false, 7 | "zero_optimization": { 8 | "stage": 3, 9 | "overlap_comm": true, 10 | "stage3_gather_16bit_weights_on_model_save": true, 11 | "stage3_prefetch_bucket_size": 0 12 | }, 13 | "fp16": { 14 | "enabled": "auto", 15 | "initial_scale_power": 16 16 | }, 17 | "bf16": { 18 | "enabled": "auto" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /xtuner/configs/deepspeed/deepspeed_zero3_offload.json: -------------------------------------------------------------------------------- 1 | { 2 | "gradient_accumulation_steps": "auto", 3 | "train_micro_batch_size_per_gpu": "auto", 4 | "gradient_clipping": "auto", 5 | "zero_allow_untested_optimizer": true, 6 | "zero_force_ds_cpu_optimizer": false, 7 | "zero_optimization": { 8 | "stage": 3, 9 | "overlap_comm": true, 10 | "offload_optimizer": { 11 | "device": "cpu", 12 | "pin_memory": true 13 | }, 14 | "offload_param": { 15 | "device": "cpu", 16 | "pin_memory": true 17 | }, 18 | "stage3_gather_16bit_weights_on_model_save": true 19 | }, 20 | "fp16": { 21 | "enabled": "auto", 22 | "initial_scale_power": 16 23 | }, 24 | "bf16": { 25 | "enabled": "auto" 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /xtuner/configs/internlm/internlm_7b/internlm_7b_qlora_oasst1_e3_hf.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | import torch 3 | from datasets import load_dataset 4 | from peft import LoraConfig 5 | from transformers import (AutoModelForCausalLM, AutoTokenizer, 6 | BitsAndBytesConfig, Trainer, TrainingArguments) 7 | 8 | from xtuner.dataset import process_hf_dataset 9 | from xtuner.dataset.map_fns import oasst1_map_fn, template_map_fn_factory 10 | from xtuner.utils import PROMPT_TEMPLATE 11 | 12 | framework = 'huggingface' 13 | pretrained_model_name_or_path = 'internlm/internlm-7b' 14 | dataset_name_or_path = 'timdettmers/openassistant-guanaco' 15 | max_length = 2048 16 | pack_to_max_length = True 17 | prompt_template = PROMPT_TEMPLATE.default 18 | 19 | trainer = Trainer 20 | 21 | training_args = dict( 22 | type=TrainingArguments, 23 | do_train=True, 24 | learning_rate=2e-4, 25 | weight_decay=0, 26 | lr_scheduler_type='cosine', 27 | warmup_steps=100, 28 | per_device_train_batch_size=1, 29 | gradient_accumulation_steps=16, 30 | num_train_epochs=3, 31 | fp16=True, 32 | logging_steps=1, 33 | optim='paged_adamw_32bit', 34 | save_strategy='steps', 35 | save_steps=1000, 36 | save_total_limit=2, 37 | ddp_find_unused_parameters=False) 38 | 39 | tokenizer = dict( 40 | type=AutoTokenizer.from_pretrained, 41 | pretrained_model_name_or_path=pretrained_model_name_or_path, 42 | trust_remote_code=True, 43 | padding_side='right') 44 | 45 | model = dict( 46 | type=AutoModelForCausalLM.from_pretrained, 47 | pretrained_model_name_or_path=pretrained_model_name_or_path, 48 | trust_remote_code=True, 49 | torch_dtype=torch.float16, 50 | quantization_config=dict( 51 | type=BitsAndBytesConfig, 52 | load_in_4bit=True, 53 | load_in_8bit=False, 54 | llm_int8_threshold=6.0, 55 | llm_int8_has_fp16_weight=False, 56 | bnb_4bit_compute_dtype=torch.float16, 57 | bnb_4bit_use_double_quant=True, 58 | bnb_4bit_quant_type='nf4')) 59 | 60 | lora = dict( 61 | type=LoraConfig, 62 | r=64, 63 | lora_alpha=16, 64 | lora_dropout=0.1, 65 | bias='none', 66 | task_type='CAUSAL_LM') 67 | 68 | train_dataset = dict( 69 | type=process_hf_dataset, 70 | dataset=dict(type=load_dataset, path=dataset_name_or_path), 71 | tokenizer=tokenizer, 72 | max_length=max_length, 73 | dataset_map_fn=oasst1_map_fn, 74 | template_map_fn=dict( 75 | type=template_map_fn_factory, template=prompt_template), 76 | remove_unused_columns=True, 77 | shuffle_before_pack=True, 78 | pack_to_max_length=pack_to_max_length) 79 | -------------------------------------------------------------------------------- /xtuner/configs/llama/llama2_70b/llama2_70b_int8_lora_open_platypus_e1_hf.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | import torch 3 | from datasets import load_dataset 4 | from peft import LoraConfig 5 | from transformers import (AutoModelForCausalLM, AutoTokenizer, Trainer, 6 | TrainingArguments) 7 | 8 | from xtuner.dataset import process_hf_dataset 9 | from xtuner.dataset.map_fns import alpaca_map_fn, template_map_fn_factory 10 | from xtuner.utils import PROMPT_TEMPLATE 11 | 12 | framework = 'huggingface' 13 | pretrained_model_name_or_path = 'meta-llama/Llama-2-70b-hf' 14 | dataset_name_or_path = 'garage-bAInd/Open-Platypus' 15 | max_length = 2048 16 | pack_to_max_length = True 17 | prompt_template = PROMPT_TEMPLATE.llama2_chat 18 | 19 | trainer = Trainer 20 | 21 | training_args = dict( 22 | type=TrainingArguments, 23 | do_train=True, 24 | learning_rate=3e-4, 25 | weight_decay=0, 26 | lr_scheduler_type='cosine', 27 | warmup_steps=100, 28 | per_device_train_batch_size=1, 29 | gradient_accumulation_steps=16, 30 | num_train_epochs=1, 31 | fp16=True, 32 | logging_steps=1, 33 | optim='adamw_torch', 34 | save_strategy='steps', 35 | save_steps=1000, 36 | save_total_limit=2, 37 | ddp_find_unused_parameters=False) 38 | 39 | tokenizer = dict( 40 | type=AutoTokenizer.from_pretrained, 41 | pretrained_model_name_or_path=pretrained_model_name_or_path, 42 | trust_remote_code=True, 43 | padding_side='right') 44 | 45 | model = dict( 46 | type=AutoModelForCausalLM.from_pretrained, 47 | pretrained_model_name_or_path=pretrained_model_name_or_path, 48 | trust_remote_code=True, 49 | torch_dtype=torch.float16, 50 | load_in_8bit=True) 51 | 52 | lora = dict( 53 | type=LoraConfig, 54 | r=16, 55 | lora_alpha=16, 56 | lora_dropout=0.05, 57 | target_modules=['gate_proj', 'down_proj', 'up_proj'], 58 | bias='none', 59 | task_type='CAUSAL_LM') 60 | 61 | train_dataset = dict( 62 | type=process_hf_dataset, 63 | dataset=dict(type=load_dataset, path=dataset_name_or_path), 64 | tokenizer=tokenizer, 65 | max_length=max_length, 66 | dataset_map_fn=alpaca_map_fn, 67 | template_map_fn=dict( 68 | type=template_map_fn_factory, template=prompt_template), 69 | remove_unused_columns=True, 70 | shuffle_before_pack=True, 71 | pack_to_max_length=pack_to_max_length) 72 | -------------------------------------------------------------------------------- /xtuner/configs/llama/llama2_70b/llama2_70b_qlora_open_platypus_e1_hf.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | import torch 3 | from datasets import load_dataset 4 | from peft import LoraConfig 5 | from transformers import (AutoModelForCausalLM, AutoTokenizer, 6 | BitsAndBytesConfig, Trainer, TrainingArguments) 7 | 8 | from xtuner.dataset import process_hf_dataset 9 | from xtuner.dataset.map_fns import alpaca_map_fn, template_map_fn_factory 10 | from xtuner.utils import PROMPT_TEMPLATE 11 | 12 | framework = 'huggingface' 13 | pretrained_model_name_or_path = 'meta-llama/Llama-2-70b-hf' 14 | dataset_name_or_path = 'garage-bAInd/Open-Platypus' 15 | max_length = 2048 16 | pack_to_max_length = True 17 | prompt_template = PROMPT_TEMPLATE.llama2_chat 18 | 19 | trainer = Trainer 20 | 21 | training_args = dict( 22 | type=TrainingArguments, 23 | do_train=True, 24 | learning_rate=3e-4, 25 | weight_decay=0, 26 | lr_scheduler_type='cosine', 27 | warmup_steps=100, 28 | per_device_train_batch_size=1, 29 | gradient_accumulation_steps=16, 30 | num_train_epochs=1, 31 | fp16=True, 32 | logging_steps=1, 33 | optim='adamw_torch', 34 | save_strategy='steps', 35 | save_steps=1000, 36 | save_total_limit=2, 37 | ddp_find_unused_parameters=False) 38 | 39 | tokenizer = dict( 40 | type=AutoTokenizer.from_pretrained, 41 | pretrained_model_name_or_path=pretrained_model_name_or_path, 42 | trust_remote_code=True, 43 | padding_side='right') 44 | 45 | model = dict( 46 | type=AutoModelForCausalLM.from_pretrained, 47 | pretrained_model_name_or_path=pretrained_model_name_or_path, 48 | trust_remote_code=True, 49 | torch_dtype=torch.float16, 50 | quantization_config=dict( 51 | type=BitsAndBytesConfig, 52 | load_in_4bit=True, 53 | load_in_8bit=False, 54 | llm_int8_threshold=6.0, 55 | llm_int8_has_fp16_weight=False, 56 | bnb_4bit_compute_dtype=torch.float16, 57 | bnb_4bit_use_double_quant=True, 58 | bnb_4bit_quant_type='nf4')) 59 | 60 | lora = dict( 61 | type=LoraConfig, 62 | r=64, 63 | lora_alpha=16, 64 | lora_dropout=0.1, 65 | target_modules=['gate_proj', 'down_proj', 'up_proj'], 66 | bias='none', 67 | task_type='CAUSAL_LM') 68 | 69 | train_dataset = dict( 70 | type=process_hf_dataset, 71 | dataset=dict(type=load_dataset, path=dataset_name_or_path), 72 | tokenizer=tokenizer, 73 | max_length=max_length, 74 | dataset_map_fn=alpaca_map_fn, 75 | template_map_fn=dict( 76 | type=template_map_fn_factory, template=prompt_template), 77 | remove_unused_columns=True, 78 | shuffle_before_pack=True, 79 | pack_to_max_length=pack_to_max_length) 80 | -------------------------------------------------------------------------------- /xtuner/configs/llama/llama3_8b/README.md: -------------------------------------------------------------------------------- 1 | # Llama3 8B 2 | 3 | ## Install 4 | 5 | ```bash 6 | # Install the latest xtuner 7 | pip install -U 'xtuner[deepspeed]' 8 | 9 | # install the latest transformers 10 | pip install -U transformers 11 | ``` 12 | 13 | ## QLoRA Fine-tune 14 | 15 | QLoRA only need a single A100-80G 16 | 17 | ```bash 18 | xtuner train llama3_8b_instruct_qlora_alpaca_e3 19 | ``` 20 | 21 | ## Full Parameter Fine-tune 22 | 23 | Full parameter fine-tune Llama3 8B in 8k context only requires 2 * A100-80G 24 | 25 | ### torchrun 26 | 27 | ```bash 28 | NPROC_PER_NODE=${GPU_NUM} xtuner train llama3_8b_instruct_full_alpaca_e3 --deepspeed deepspeed_zero2 29 | ``` 30 | 31 | ### slurm 32 | 33 | ```bash 34 | srun ${SRUN_ARGS} xtuner train llama3_8b_instruct_full_alpaca_e3 --launcher slurm --deepspeed deepspeed_zero3 35 | ``` 36 | 37 | ### Speed 38 | 39 | | Model | Sequence Length | GPU Number | ZeRO | Sequence Parallel | Tokens per Second | TFLOPs | 40 | | :-------: | :-------------: | :--------: | :----: | :---------------: | :---------------: | :----: | 41 | | Llama3 8B | 8k | 2 | ZeRO-3 | 2 | 1037.0 | 76.8 | 42 | | Llama3 8B | 8k | 4 | ZeRO-3 | 1 | 2331.3 | 172.6 | 43 | | Llama3 8B | 8k | 8 | ZeRO-3 | 1 | 2771.2 | 205.1 | 44 | 45 | | Model | Sequence Length | GPU Number | ZeRO | Sequence Parallel | Tokens per Second | TFLOPs | 46 | | :-------: | :-------------: | :--------: | :----: | :---------------: | :---------------: | :----: | 47 | | Llama3 8B | 8k | 8 | ZeRO-3 | 1 | 2771.2 | 205.1 | 48 | | Llama3 8B | 16k | 8 | ZeRO-3 | 2 | 2320.7 | 191.7 | 49 | | Llama3 8B | 32k | 8 | ZeRO-3 | 4 | 1870.2 | 186.6 | 50 | | Llama3 8B | 64k | 8 | ZeRO-3 | 8 | 1356.4 | 182.0 | 51 | | Llama3 8B | 128k | 8 | ZeRO-3 | 8 | 875.7 | 177.7 | 52 | -------------------------------------------------------------------------------- /xtuner/configs/mixtral/README.md: -------------------------------------------------------------------------------- 1 | # Mixtral 8x7B 2 | 3 | ## Install 4 | 5 | ```bash 6 | # Install the latest xtuner 7 | pip install -U 'xtuner[deepspeed]' 8 | 9 | # Mixtral requires flash-attn 10 | pip install flash-attn 11 | 12 | # install the latest transformers 13 | pip install -U transformers 14 | ``` 15 | 16 | ## QLoRA Fine-tune 17 | 18 | QLoRA only need a single A100-80G 19 | 20 | ```bash 21 | xtuner train mixtral_8x7b_instruct_qlora_oasst1_e3 --deepspeed deepspeed_zero2 22 | ``` 23 | 24 | ## Full Parameter Fine-tune 25 | 26 | Full parameter fine-tune needs 16 A100-80G 27 | 28 | ### slurm 29 | 30 | Note: `$PARTITION` means the virtual partition of slurm. 31 | 32 | ```bash 33 | srun -p $PARTITION --job-name=mixtral --nodes=2 --gres=gpu:8 --ntasks-per-node=8 xtuner train mixtral_8x7b_instruct_full_oasst1_e3 --deepspeed deepspeed_zero3 --launcher slurm 34 | ``` 35 | 36 | ### torchrun 37 | 38 | Note: `$NODE_0_ADDR` means the ip address of the node_0 machine. 39 | 40 | ```bash 41 | # excuete on node 0 42 | NPROC_PER_NODE=8 NNODES=2 PORT=29600 ADDR=$NODE_0_ADDR NODE_RANK=0 xtuner train mixtral_8x7b_instruct_full_oasst1_e3 --deepspeed deepspeed_zero3 43 | 44 | # excuete on node 1 45 | NPROC_PER_NODE=8 NNODES=2 PORT=29600 ADDR=$NODE_0_ADDR NODE_RANK=1 xtuner train mixtral_8x7b_instruct_full_oasst1_e3 --deepspeed deepspeed_zero3 46 | ``` 47 | 48 | ### Speed 49 | 50 | 16 * A100 80G: 51 | 52 | | Model | Sequence Length | Use Varlen Attn | Sequence Parallel World Size | Tokens per Second | 53 | | :----------: | :-------------: | :-------------: | :--------------------------: | :---------------: | 54 | | mixtral_8x7b | 32k | False | 1 | 853.7 | 55 | | mixtral_8x7b | 32k | True | 1 | 910.1 | 56 | | mixtral_8x7b | 32k | False | 2 | 635.2 | 57 | | mixtral_8x7b | 32k | True | 2 | 650.9 | 58 | -------------------------------------------------------------------------------- /xtuner/dataset/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | import warnings 3 | 4 | from .concat_dataset import ConcatDataset 5 | from .huggingface import process_hf_dataset 6 | from .intern_repo import (build_packed_dataset, 7 | load_intern_repo_tokenized_dataset, 8 | load_intern_repo_untokenized_dataset) 9 | 10 | from .llava import LLaVADataset, AnyResLLaVADataset 11 | from .json_dataset import load_json_file 12 | from .modelscope import process_ms_dataset 13 | from .moss_sft import MOSSSFTDataset 14 | from .refcoco_json import (InvRefCOCOJsonDataset, RefCOCOJsonDataset, 15 | RefCOCOJsonEvalDataset) 16 | from .utils import decode_base64_to_image, expand2square, load_image 17 | from .llava_proxy_eval_dataset import LLaVAProxyEvalDataset, ChatLLaVAProxyEvalDataset 18 | from .anyres_llava_proxy_eval_dataset import AnyResLLaVAProxyEvalDataset 19 | from .mini_gemini_dataset import MiniGeminiDataset 20 | from .mini_gemini_proxy_eval_dataset import MiniGeminiProxyEvalDataset 21 | 22 | # ignore FutureWarning in hf datasets 23 | warnings.simplefilter(action='ignore', category=FutureWarning) 24 | 25 | __all__ = [ 26 | 'process_hf_dataset', 27 | 'ConcatDataset', 28 | 'MOSSSFTDataset', 29 | 'process_ms_dataset', 30 | 'LLaVADataset', 31 | 'expand2square', 32 | 'decode_base64_to_image', 33 | 'load_image', 34 | 'process_ms_dataset', 35 | 'load_intern_repo_tokenized_dataset', 36 | 'load_intern_repo_untokenized_dataset', 37 | 'build_packed_dataset', 38 | 'RefCOCOJsonDataset', 39 | 'RefCOCOJsonEvalDataset', 40 | 'InvRefCOCOJsonDataset', 41 | 'AnyResLLaVADataset', 42 | 'load_json_file', 43 | 'LLaVAProxyEvalDataset', 44 | 'ChatLLaVAProxyEvalDataset', 45 | 'AnyResLLaVAProxyEvalDataset', 46 | 'MiniGeminiDataset', 47 | 'MiniGeminiProxyEvalDataset', 48 | ] 49 | -------------------------------------------------------------------------------- /xtuner/dataset/__pycache__/__init__.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/__pycache__/__init__.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/__pycache__/anyres_llava_proxy_eval_dataset.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/__pycache__/anyres_llava_proxy_eval_dataset.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/__pycache__/concat_dataset.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/__pycache__/concat_dataset.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/__pycache__/huggingface.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/__pycache__/huggingface.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/__pycache__/intern_repo.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/__pycache__/intern_repo.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/__pycache__/json_dataset.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/__pycache__/json_dataset.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/__pycache__/llava.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/__pycache__/llava.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/__pycache__/llava_proxy_eval_dataset.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/__pycache__/llava_proxy_eval_dataset.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/__pycache__/mini_gemini_dataset.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/__pycache__/mini_gemini_dataset.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/__pycache__/mini_gemini_proxy_eval_dataset.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/__pycache__/mini_gemini_proxy_eval_dataset.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/__pycache__/modelscope.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/__pycache__/modelscope.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/__pycache__/moss_sft.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/__pycache__/moss_sft.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/__pycache__/object_llava.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/__pycache__/object_llava.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/__pycache__/refcoco_json.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/__pycache__/refcoco_json.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/__pycache__/utils.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/__pycache__/utils.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/collate_fns/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from .default_collate_fn import default_collate_fn 3 | from .mmlu_collate_fn import mmlu_collate_fn 4 | from .mm_collate_fn import mm_collate_fn 5 | 6 | __all__ = ['default_collate_fn', 'mmlu_collate_fn', 'mm_collate_fn'] 7 | -------------------------------------------------------------------------------- /xtuner/dataset/collate_fns/__pycache__/__init__.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/collate_fns/__pycache__/__init__.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/collate_fns/__pycache__/defalut_collate_fn.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/collate_fns/__pycache__/defalut_collate_fn.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/collate_fns/__pycache__/default_collate_fn.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/collate_fns/__pycache__/default_collate_fn.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/collate_fns/__pycache__/mm_collate_fn.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/collate_fns/__pycache__/mm_collate_fn.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/collate_fns/__pycache__/mmlu_collate_fn.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/collate_fns/__pycache__/mmlu_collate_fn.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/collate_fns/mmlu_collate_fn.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from typing import Dict, Sequence 3 | 4 | import torch 5 | from torch.nn.utils.rnn import pad_sequence 6 | 7 | from xtuner.utils import DEFAULT_PAD_TOKEN_INDEX, IGNORE_INDEX 8 | 9 | 10 | def mmlu_collate_fn(instances: Sequence[Dict], 11 | pad_index: int = DEFAULT_PAD_TOKEN_INDEX, 12 | return_hf_format: bool = False) -> Dict[str, torch.Tensor]: 13 | input_ids = [] 14 | labels = [] 15 | data_samples = {'labels': [], 'subjects': []} 16 | for example in instances: 17 | input_ids.append(torch.tensor(example['input_ids'])) 18 | labels.append(torch.tensor(example['labels'])) 19 | data_samples['labels'].append(example['output']) 20 | data_samples['subjects'].append(example['subject']) 21 | if len(instances) > 1: 22 | input_ids = pad_sequence( 23 | input_ids, batch_first=True, padding_value=pad_index) 24 | labels = pad_sequence( 25 | labels, batch_first=True, padding_value=IGNORE_INDEX) 26 | else: 27 | input_ids = torch.stack(input_ids) 28 | labels = torch.stack(labels) 29 | 30 | data_dict = { 31 | 'input_ids': input_ids, 32 | 'attention_mask': input_ids.ne(pad_index), 33 | 'labels': labels 34 | } 35 | 36 | if return_hf_format: 37 | return data_dict 38 | else: 39 | return {'data': data_dict, 'data_samples': data_samples} 40 | -------------------------------------------------------------------------------- /xtuner/dataset/concat_dataset.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from torch.utils.data import ConcatDataset as TorchConcatDataset 3 | 4 | from xtuner.registry import BUILDER 5 | 6 | 7 | class ConcatDataset(TorchConcatDataset): 8 | 9 | def __init__(self, datasets): 10 | datasets_instance = [] 11 | for cfg in datasets: 12 | datasets_instance.append(BUILDER.build(cfg)) 13 | super().__init__(datasets=datasets_instance) 14 | 15 | def __repr__(self): 16 | main_str = 'Dataset as a concatenation of multiple datasets. \n' 17 | main_str += ',\n'.join( 18 | [f'{repr(dataset)}' for dataset in self.datasets]) 19 | return main_str 20 | -------------------------------------------------------------------------------- /xtuner/dataset/evaluation/MMVP/gpt_eval.sh: -------------------------------------------------------------------------------- 1 | set -e 2 | answer_file=$1 3 | 4 | srun -p llmit --time 1-00:00 \ 5 | python xtuner/dataset/evaluation/MMVP/gpt_grader.py \ 6 | --answer_file $answer_file -------------------------------------------------------------------------------- /xtuner/dataset/evaluation/__init__.py: -------------------------------------------------------------------------------- 1 | from .mme_dataset import MMEDataset 2 | from .multiple_choice_dataset import MultipleChoiceDataset 3 | from .pope_dataset import POPEDataset 4 | from .hallusion_dataset import HallusionDataset 5 | from .textvqa_dataset import TextVQADataset 6 | from .gqa_dataset import GQADataset 7 | from .vqav2_dataset import VQAv2Dataset 8 | from .chartqa_dataset import ChartQADataset 9 | from .general_vqa_dataset import GeneralVQADataset 10 | from .vqa_dataset import VQADataset 11 | from .math_dataset import MathDataset 12 | from .mmvp_dataset import MMVPDataset 13 | 14 | __all__ = [ 15 | 'MMEDataset', 16 | 'MultipleChoiceDataset', 17 | 'POPEDataset', 18 | 'HallusionDataset', 19 | 'TextVQADataset', 20 | 'GQADataset', 21 | 'VQAv2Dataset', 22 | 'ChartQADataset', 23 | 'GeneralVQADataset', 24 | 'VQADataset', 25 | 'MathDataset', 26 | 'MMVPDataset', 27 | ] 28 | -------------------------------------------------------------------------------- /xtuner/dataset/evaluation/__pycache__/__init__.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/evaluation/__pycache__/__init__.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/evaluation/__pycache__/base_eval_dataset.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/evaluation/__pycache__/base_eval_dataset.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/evaluation/__pycache__/chartqa_dataset.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/evaluation/__pycache__/chartqa_dataset.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/evaluation/__pycache__/general_vqa_dataset.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/evaluation/__pycache__/general_vqa_dataset.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/evaluation/__pycache__/gqa_dataset.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/evaluation/__pycache__/gqa_dataset.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/evaluation/__pycache__/gqa_eval_utils.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/evaluation/__pycache__/gqa_eval_utils.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/evaluation/__pycache__/hallusion_dataset.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/evaluation/__pycache__/hallusion_dataset.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/evaluation/__pycache__/hallusion_llava_dataset.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/evaluation/__pycache__/hallusion_llava_dataset.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/evaluation/__pycache__/mme_dataset.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/evaluation/__pycache__/mme_dataset.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/evaluation/__pycache__/mme_llava_dataset.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/evaluation/__pycache__/mme_llava_dataset.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/evaluation/__pycache__/multiple_choice_dataset.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/evaluation/__pycache__/multiple_choice_dataset.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/evaluation/__pycache__/multiple_choice_llava_dataset.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/evaluation/__pycache__/multiple_choice_llava_dataset.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/evaluation/__pycache__/pope_dataset.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/evaluation/__pycache__/pope_dataset.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/evaluation/__pycache__/pope_llava_dataset.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/evaluation/__pycache__/pope_llava_dataset.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/evaluation/__pycache__/textvqa_dataset.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/evaluation/__pycache__/textvqa_dataset.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/evaluation/__pycache__/textvqa_llava_dataset.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/evaluation/__pycache__/textvqa_llava_dataset.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/evaluation/__pycache__/textvqa_utils.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/evaluation/__pycache__/textvqa_utils.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/evaluation/__pycache__/utils.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/evaluation/__pycache__/utils.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/evaluation/__pycache__/vqav2_dataset.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/evaluation/__pycache__/vqav2_dataset.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/evaluation/__pycache__/vqav2_utils.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/evaluation/__pycache__/vqav2_utils.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/evaluation/base_eval_dataset.py: -------------------------------------------------------------------------------- 1 | from torch.utils.data import Dataset 2 | import copy 3 | from collections.abc import Mapping 4 | from typing import Union 5 | from mmengine.config import Config 6 | import logging 7 | from mmengine.fileio import list_from_file 8 | from mmengine.logging import print_log 9 | from abc import abstractmethod 10 | 11 | 12 | class BaseEvalDataset(Dataset): 13 | 14 | METAINFO: dict = dict(name='default') 15 | 16 | def __init__(self, metainfo: Union[Mapping, Config, None] = None): 17 | self._metainfo = self._load_metainfo(copy.deepcopy(metainfo)) 18 | 19 | @classmethod 20 | def _load_metainfo(cls, 21 | metainfo: Union[Mapping, Config, None] = None) -> dict: 22 | """Collect meta information from the dictionary of meta. 23 | 24 | Args: 25 | metainfo (Mapping or Config, optional): Meta information dict. 26 | If ``metainfo`` contains existed filename, it will be 27 | parsed by ``list_from_file``. 28 | 29 | Returns: 30 | dict: Parsed meta information. 31 | """ 32 | # avoid `cls.METAINFO` being overwritten by `metainfo` 33 | cls_metainfo = copy.deepcopy(cls.METAINFO) 34 | if metainfo is None: 35 | return cls_metainfo 36 | if not isinstance(metainfo, (Mapping, Config)): 37 | raise TypeError('metainfo should be a Mapping or Config, ' 38 | f'but got {type(metainfo)}') 39 | 40 | for k, v in metainfo.items(): 41 | if isinstance(v, str): 42 | # If type of value is string, and can be loaded from 43 | # corresponding backend. it means the file name of meta file. 44 | try: 45 | cls_metainfo[k] = list_from_file(v) 46 | except (TypeError, FileNotFoundError): 47 | print_log( 48 | f'{v} is not a meta file, simply parsed as meta ' 49 | 'information', 50 | logger='current', 51 | level=logging.WARNING) 52 | cls_metainfo[k] = v 53 | else: 54 | cls_metainfo[k] = v 55 | return cls_metainfo 56 | 57 | @property 58 | def metainfo(self) -> dict: 59 | """Get meta information of dataset. 60 | 61 | Returns: 62 | dict: meta information collected from ``BaseDataset.METAINFO``, 63 | annotation file and metainfo argument during instantiation. 64 | """ 65 | return copy.deepcopy(self._metainfo) 66 | 67 | @abstractmethod 68 | def evaluate(self, results, work_dir): 69 | pass 70 | -------------------------------------------------------------------------------- /xtuner/dataset/evaluation/llava_w/gpt_eval.sh: -------------------------------------------------------------------------------- 1 | set -e 2 | 3 | PREDICTION=$1 4 | 5 | DIR=$(dirname "$PREDICTION") 6 | OUTPUTPATH="$DIR/llava_w_gpt_eval.jsonl" 7 | 8 | srun -p llmit --time 1-00:00 \ 9 | python xtuner/dataset/evaluation/llava_w/eval_gpt_review_bench.py \ 10 | --question llava-bench-in-the-wild/questions.jsonl \ 11 | --context llava-bench-in-the-wild/context.jsonl \ 12 | --rule xtuner/dataset/evaluation/llava_w/rule.json \ 13 | --answer-list \ 14 | llava-bench-in-the-wild/answers_gpt4.jsonl \ 15 | $PREDICTION \ 16 | --output $OUTPUTPATH 17 | 18 | srun -p llmit --time 1-00:00 \ 19 | python xtuner/dataset/evaluation/llava_w/summarize_gpt_review.py -f $OUTPUTPATH -------------------------------------------------------------------------------- /xtuner/dataset/evaluation/llava_w/summarize_gpt_review.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | from collections import defaultdict 4 | 5 | import numpy as np 6 | 7 | import argparse 8 | 9 | def parse_args(): 10 | parser = argparse.ArgumentParser(description='ChatGPT-based QA evaluation.') 11 | parser.add_argument('-d', '--dir', default=None) 12 | parser.add_argument('-v', '--version', default=None) 13 | parser.add_argument('-s', '--select', nargs='*', default=None) 14 | parser.add_argument('-f', '--files', nargs='*', default=[]) 15 | parser.add_argument('-i', '--ignore', nargs='*', default=[]) 16 | return parser.parse_args() 17 | 18 | 19 | if __name__ == '__main__': 20 | args = parse_args() 21 | 22 | if args.ignore is not None: 23 | args.ignore = [int(x) for x in args.ignore] 24 | 25 | if len(args.files) > 0: 26 | review_files = args.files 27 | else: 28 | review_files = [x for x in os.listdir(args.dir) if x.endswith('.jsonl') and (x.startswith('gpt4_text') or x.startswith('reviews_') or x.startswith('review_') or 'review' in args.dir)] 29 | 30 | for review_file in sorted(review_files): 31 | config = os.path.basename(review_file).replace('gpt4_text_', '').replace('.jsonl', '') 32 | if args.select is not None and any(x not in config for x in args.select): 33 | continue 34 | if '0613' in config: 35 | version = '0613' 36 | else: 37 | version = '0314' 38 | if args.version is not None and args.version != version: 39 | continue 40 | scores = defaultdict(list) 41 | print(config) 42 | with open(os.path.join(args.dir, review_file) if args.dir is not None else review_file) as f: 43 | for review_str in f: 44 | review = json.loads(review_str) 45 | if review['question_id'] in args.ignore: 46 | continue 47 | if 'category' in review: 48 | scores[review['category']].append(review['tuple']) 49 | scores['all'].append(review['tuple']) 50 | else: 51 | if 'tuple' in review: 52 | scores['all'].append(review['tuple']) 53 | else: 54 | scores['all'].append(review['score']) 55 | for k, v in sorted(scores.items()): 56 | stats = np.asarray(v).mean(0).tolist() 57 | stats = [round(x, 3) for x in stats] 58 | # print(k, stats, round(stats[1]/stats[0]*100, 1)) 59 | print(k, round(stats[1]/stats[0]*100, 1), round(stats[0] * 10, 1), round(stats[1] * 10, 1)) 60 | print('=================================') 61 | -------------------------------------------------------------------------------- /xtuner/dataset/evaluation/math_vista/__pycache__/utilities.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/evaluation/math_vista/__pycache__/utilities.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/evaluation/math_vista/gpt_eval.sh: -------------------------------------------------------------------------------- 1 | set -e 2 | 3 | input_file=$1 4 | score_file="math_vista_score.json" 5 | 6 | DIR=$(dirname "$input_file") 7 | score_file_path="$DIR/$score_file" 8 | 9 | 10 | 11 | srun -p llmit --time 1-00:00 \ 12 | python xtuner/dataset/evaluation/math_vista/extract_answer.py \ 13 | --output_file $input_file \ 14 | 15 | srun -p llmit --time 1-00:00 \ 16 | python xtuner/dataset/evaluation/math_vista/calculate_score.py \ 17 | --output_file $input_file \ 18 | --score_file $score_file_path \ 19 | --gt_file MathVista/test_mini.json -------------------------------------------------------------------------------- /xtuner/dataset/evaluation/math_vista/prompts/__pycache__/ext_ans.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/evaluation/math_vista/prompts/__pycache__/ext_ans.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/evaluation/math_vista/prompts/ext_ans.py: -------------------------------------------------------------------------------- 1 | 2 | 3 | # pids = 852, 104, 824, 506, 540 4 | 5 | demo_prompt = """ 6 | Please read the following example. Then extract the answer from the model response and type it at the end of the prompt. 7 | 8 | Hint: Please answer the question requiring an integer answer and provide the final value, e.g., 1, 2, 3, at the end. 9 | Question: Which number is missing? 10 | 11 | Model response: The number missing in the sequence is 14. 12 | 13 | Extracted answer: 14 14 | 15 | Hint: Please answer the question requiring a floating-point number with one decimal place and provide the final value, e.g., 1.2, 1.3, 1.4, at the end. 16 | Question: What is the fraction of females facing the camera? 17 | 18 | Model response: The fraction of females facing the camera is 0.6, which means that six out of ten females in the group are facing the camera. 19 | 20 | Extracted answer: 0.6 21 | 22 | Hint: Please answer the question requiring a floating-point number with two decimal places and provide the final value, e.g., 1.23, 1.34, 1.45, at the end. 23 | Question: How much money does Luca need to buy a sour apple candy and a butterscotch candy? (Unit: $) 24 | 25 | Model response: Luca needs $1.45 to buy a sour apple candy and a butterscotch candy. 26 | 27 | Extracted answer: 1.45 28 | 29 | Hint: Please answer the question requiring a Python list as an answer and provide the final list, e.g., [1, 2, 3], [1.2, 1.3, 1.4], at the end. 30 | Question: Between which two years does the line graph saw its maximum peak? 31 | 32 | Model response: The line graph saw its maximum peak between 2007 and 2008. 33 | 34 | Extracted answer: [2007, 2008] 35 | 36 | Hint: Please answer the question and provide the correct option letter, e.g., A, B, C, D, at the end. 37 | Question: What fraction of the shape is blue?\nChoices:\n(A) 3/11\n(B) 8/11\n(C) 6/11\n(D) 3/5 38 | 39 | Model response: The correct answer is (B) 8/11. 40 | 41 | Extracted answer: B 42 | """ -------------------------------------------------------------------------------- /xtuner/dataset/json_dataset.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | 4 | from datasets import Dataset, concatenate_datasets 5 | 6 | 7 | def load_json_file(data_files=None, data_dir=None, suffix=None): 8 | assert (data_files is not None) != (data_dir is not None) 9 | if data_dir is not None: 10 | data_files = os.listdir(data_dir) 11 | data_files = [os.path.join(data_dir, fn) for fn in data_files] 12 | if suffix is not None: 13 | data_files = [fp for fp in data_files if fp.endswith(suffix)] 14 | elif isinstance(data_files, str): 15 | data_files = [data_files] 16 | 17 | dataset_list = [] 18 | for fp in data_files: 19 | with open(fp, encoding='utf-8') as file: 20 | data = json.load(file) 21 | ds = Dataset.from_list(data) 22 | dataset_list.append(ds) 23 | dataset = concatenate_datasets(dataset_list) 24 | return dataset 25 | -------------------------------------------------------------------------------- /xtuner/dataset/map_fns/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from .dataset_map_fns import * # noqa: F401, F403 3 | from .template_map_fn import template_map_fn # noqa: F401 4 | from .template_map_fn import template_map_fn_factory # noqa: F401 5 | -------------------------------------------------------------------------------- /xtuner/dataset/map_fns/__pycache__/__init__.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/map_fns/__pycache__/__init__.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/map_fns/__pycache__/template_map_fn.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/map_fns/__pycache__/template_map_fn.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/map_fns/dataset_map_fns/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from .alpaca_map_fn import alpaca_map_fn 3 | from .alpaca_zh_map_fn import alpaca_zh_map_fn 4 | from .arxiv_map_fn import arxiv_map_fn 5 | from .code_alpaca_map_fn import code_alpaca_map_fn 6 | from .colors_map_fn import colors_map_fn 7 | from .crime_kg_assitant_map_fn import crime_kg_assitant_map_fn 8 | from .default_map_fn import default_map_fn 9 | from .law_reference_map_fn import law_reference_map_fn 10 | from .llava_map_fn import llava_image_only_map_fn, llava_map_fn 11 | from .medical_map_fn import medical_map_fn 12 | from .msagent_map_fn import msagent_react_map_fn 13 | from .oasst1_map_fn import oasst1_map_fn 14 | from .openai_map_fn import openai_map_fn 15 | from .openorca_map_fn import openorca_map_fn 16 | from .pretrain_map_fn import pretrain_map_fn 17 | from .sql_map_fn import sql_map_fn 18 | from .stack_exchange_map_fn import stack_exchange_map_fn 19 | from .tiny_codes_map_fn import tiny_codes_map_fn 20 | from .wizardlm_map_fn import wizardlm_map_fn 21 | 22 | DATASET_FORMAT_MAPPING = dict( 23 | alpaca=alpaca_map_fn, 24 | alpaca_zh=alpaca_zh_map_fn, 25 | arxiv=arxiv_map_fn, 26 | code_alpaca=code_alpaca_map_fn, 27 | colors=colors_map_fn, 28 | crime_kg_assitan=crime_kg_assitant_map_fn, 29 | default=default_map_fn, 30 | law_reference=law_reference_map_fn, 31 | llava_image_only=llava_image_only_map_fn, 32 | llava=llava_map_fn, 33 | medical=medical_map_fn, 34 | msagent_react=msagent_react_map_fn, 35 | oasst1=oasst1_map_fn, 36 | openai=openai_map_fn, 37 | openorca=openorca_map_fn, 38 | pretrain=pretrain_map_fn, 39 | sql=sql_map_fn, 40 | stack_exchange=stack_exchange_map_fn, 41 | tiny_codes=tiny_codes_map_fn, 42 | wizardlm=wizardlm_map_fn, 43 | ) 44 | 45 | __all__ = [ 46 | 'alpaca_map_fn', 'alpaca_zh_map_fn', 'oasst1_map_fn', 'arxiv_map_fn', 47 | 'medical_map_fn', 'openorca_map_fn', 'code_alpaca_map_fn', 48 | 'tiny_codes_map_fn', 'colors_map_fn', 'law_reference_map_fn', 49 | 'crime_kg_assitant_map_fn', 'sql_map_fn', 'openai_map_fn', 50 | 'wizardlm_map_fn', 'stack_exchange_map_fn', 'msagent_react_map_fn', 51 | 'pretrain_map_fn', 'default_map_fn', 'llava_image_only_map_fn', 52 | 'llava_map_fn', 'DATASET_FORMAT_MAPPING' 53 | ] 54 | -------------------------------------------------------------------------------- /xtuner/dataset/map_fns/dataset_map_fns/__pycache__/__init__.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/map_fns/dataset_map_fns/__pycache__/__init__.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/map_fns/dataset_map_fns/__pycache__/alpaca_map_fn.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/map_fns/dataset_map_fns/__pycache__/alpaca_map_fn.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/map_fns/dataset_map_fns/__pycache__/alpaca_zh_map_fn.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/map_fns/dataset_map_fns/__pycache__/alpaca_zh_map_fn.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/map_fns/dataset_map_fns/__pycache__/arxiv_map_fn.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/map_fns/dataset_map_fns/__pycache__/arxiv_map_fn.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/map_fns/dataset_map_fns/__pycache__/code_alpaca_map_fn.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/map_fns/dataset_map_fns/__pycache__/code_alpaca_map_fn.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/map_fns/dataset_map_fns/__pycache__/colors_map_fn.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/map_fns/dataset_map_fns/__pycache__/colors_map_fn.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/map_fns/dataset_map_fns/__pycache__/crime_kg_assitant_map_fn.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/map_fns/dataset_map_fns/__pycache__/crime_kg_assitant_map_fn.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/map_fns/dataset_map_fns/__pycache__/default_map_fn.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/map_fns/dataset_map_fns/__pycache__/default_map_fn.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/map_fns/dataset_map_fns/__pycache__/law_reference_map_fn.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/map_fns/dataset_map_fns/__pycache__/law_reference_map_fn.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/map_fns/dataset_map_fns/__pycache__/llava_map_fn.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/map_fns/dataset_map_fns/__pycache__/llava_map_fn.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/map_fns/dataset_map_fns/__pycache__/medical_map_fn.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/map_fns/dataset_map_fns/__pycache__/medical_map_fn.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/map_fns/dataset_map_fns/__pycache__/msagent_map_fn.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/map_fns/dataset_map_fns/__pycache__/msagent_map_fn.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/map_fns/dataset_map_fns/__pycache__/oasst1_map_fn.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/map_fns/dataset_map_fns/__pycache__/oasst1_map_fn.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/map_fns/dataset_map_fns/__pycache__/openai_map_fn.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/map_fns/dataset_map_fns/__pycache__/openai_map_fn.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/map_fns/dataset_map_fns/__pycache__/openorca_map_fn.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/map_fns/dataset_map_fns/__pycache__/openorca_map_fn.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/map_fns/dataset_map_fns/__pycache__/pretrain_map_fn.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/map_fns/dataset_map_fns/__pycache__/pretrain_map_fn.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/map_fns/dataset_map_fns/__pycache__/sql_map_fn.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/map_fns/dataset_map_fns/__pycache__/sql_map_fn.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/map_fns/dataset_map_fns/__pycache__/stack_exchange_map_fn.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/map_fns/dataset_map_fns/__pycache__/stack_exchange_map_fn.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/map_fns/dataset_map_fns/__pycache__/tiny_codes_map_fn.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/map_fns/dataset_map_fns/__pycache__/tiny_codes_map_fn.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/map_fns/dataset_map_fns/__pycache__/wizardlm_map_fn.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/map_fns/dataset_map_fns/__pycache__/wizardlm_map_fn.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/map_fns/dataset_map_fns/alpaca_map_fn.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | 3 | 4 | def alpaca_map_fn(example): 5 | if example.get('output') == '': 6 | return {'conversation': []} 7 | else: 8 | return { 9 | 'conversation': [{ 10 | 'input': f"{example['instruction']}\n{example['input']}", 11 | 'output': example['output'] 12 | }] 13 | } 14 | -------------------------------------------------------------------------------- /xtuner/dataset/map_fns/dataset_map_fns/alpaca_zh_map_fn.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | 3 | 4 | def alpaca_zh_map_fn(example): 5 | return { 6 | 'conversation': [{ 7 | 'input': f"{example['instruction_zh']}\n{example['input_zh']}", 8 | 'output': example['output_zh'] 9 | }] 10 | } 11 | -------------------------------------------------------------------------------- /xtuner/dataset/map_fns/dataset_map_fns/arxiv_map_fn.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from xtuner.utils import SYSTEM_TEMPLATE 3 | 4 | 5 | def arxiv_map_fn(example): 6 | return { 7 | 'conversation': [{ 8 | 'system': SYSTEM_TEMPLATE.arxiv_gentile, 9 | 'input': example['abstract'], 10 | 'output': example['title'] 11 | }] 12 | } 13 | -------------------------------------------------------------------------------- /xtuner/dataset/map_fns/dataset_map_fns/code_alpaca_map_fn.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from xtuner.utils import SYSTEM_TEMPLATE 3 | 4 | 5 | def code_alpaca_map_fn(example): 6 | return { 7 | 'conversation': [{ 8 | 'system': SYSTEM_TEMPLATE.coder, 9 | 'input': example['prompt'], 10 | 'output': example['completion'] 11 | }] 12 | } 13 | -------------------------------------------------------------------------------- /xtuner/dataset/map_fns/dataset_map_fns/colors_map_fn.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from xtuner.utils import SYSTEM_TEMPLATE 3 | 4 | 5 | def colors_map_fn(example): 6 | desc = ':'.join(example['description'].split(':')[1:]).strip() 7 | return { 8 | 'conversation': [{ 9 | 'system': SYSTEM_TEMPLATE.colorist, 10 | 'input': desc, 11 | 'output': example['color'] 12 | }] 13 | } 14 | -------------------------------------------------------------------------------- /xtuner/dataset/map_fns/dataset_map_fns/crime_kg_assitant_map_fn.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from xtuner.utils import SYSTEM_TEMPLATE 3 | 4 | 5 | def crime_kg_assitant_map_fn(example): 6 | return { 7 | 'conversation': [{ 8 | 'system': SYSTEM_TEMPLATE.lawyer, 9 | 'input': example['input'], 10 | 'output': example['output'] 11 | }] 12 | } 13 | -------------------------------------------------------------------------------- /xtuner/dataset/map_fns/dataset_map_fns/default_map_fn.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | def default_map_fn(example): 3 | return { 4 | 'conversation': [{ 5 | 'input': example['input'], 6 | 'output': example['output'] 7 | }] 8 | } 9 | -------------------------------------------------------------------------------- /xtuner/dataset/map_fns/dataset_map_fns/law_reference_map_fn.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from xtuner.utils import SYSTEM_TEMPLATE 3 | 4 | 5 | def law_reference_map_fn(example): 6 | return { 7 | 'conversation': [{ 8 | 'system': SYSTEM_TEMPLATE.lawyer, 9 | 'input': example['question'], 10 | 'output': example['answer'] 11 | }] 12 | } 13 | -------------------------------------------------------------------------------- /xtuner/dataset/map_fns/dataset_map_fns/llava_map_fn.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from xtuner.utils import DEFAULT_IMAGE_TOKEN 3 | 4 | 5 | def llava_image_only_map_fn(example): 6 | # input contains the DEFAULT_IMAGE_TOKEN only 7 | messages = example['conversations'] 8 | input = '' 9 | conversation = [] 10 | while messages and messages[0]['from'] == 'gpt': 11 | # Skip the first one if it is from gpt 12 | messages = messages[1:] 13 | for msg in messages: 14 | if msg['from'] == 'human': 15 | assert DEFAULT_IMAGE_TOKEN in msg['value'] 16 | input += DEFAULT_IMAGE_TOKEN 17 | elif msg['from'] == 'gpt': 18 | conversation.append({'input': input, 'output': msg['value']}) 19 | input = '' 20 | else: 21 | raise NotImplementedError 22 | return {'conversation': conversation} 23 | 24 | 25 | def llava_map_fn(example): 26 | messages = example['conversations'] 27 | input = '' 28 | conversation = [] 29 | while messages and messages[0]['from'] == 'gpt': 30 | # Skip the first one if it is from gpt 31 | messages = messages[1:] 32 | for msg in messages: 33 | if msg['from'] == 'human': 34 | if DEFAULT_IMAGE_TOKEN in msg['value']: 35 | msg['value'] = msg['value'].replace(DEFAULT_IMAGE_TOKEN, 36 | '').strip() 37 | msg['value'] = DEFAULT_IMAGE_TOKEN + '\n' + msg['value'] 38 | msg['value'] = msg['value'].strip() 39 | input += msg['value'] 40 | 41 | elif msg['from'] == 'gpt': 42 | conversation.append({'input': input, 'output': msg['value']}) 43 | input = '' 44 | else: 45 | raise NotImplementedError 46 | return {'conversation': conversation} 47 | -------------------------------------------------------------------------------- /xtuner/dataset/map_fns/dataset_map_fns/medical_map_fn.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from xtuner.utils import SYSTEM_TEMPLATE 3 | 4 | 5 | def medical_map_fn(example): 6 | return { 7 | 'conversation': [{ 8 | 'system': SYSTEM_TEMPLATE.medical, 9 | 'input': '{instruction}\n{input}'.format(**example), 10 | 'output': example['output'] 11 | }] 12 | } 13 | -------------------------------------------------------------------------------- /xtuner/dataset/map_fns/dataset_map_fns/oasst1_map_fn.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | def oasst1_map_fn(example): 3 | r"""Example before preprocessing: 4 | example['text'] = '### Human: Can you explain xxx' 5 | '### Assistant: Sure! xxx' 6 | '### Human: I didn't understand how xxx' 7 | '### Assistant: It has to do with a process xxx.' 8 | 9 | Example after preprocessing: 10 | example['conversation'] = [ 11 | { 12 | 'input': 'Can you explain xxx', 13 | 'output': 'Sure! xxx' 14 | }, 15 | { 16 | 'input': 'I didn't understand how xxx', 17 | 'output': 'It has to do with a process xxx.' 18 | } 19 | ] 20 | """ 21 | data = [] 22 | for sentence in example['text'].strip().split('###'): 23 | sentence = sentence.strip() 24 | if sentence[:6] == 'Human:': 25 | data.append(sentence[6:].strip()) 26 | elif sentence[:10] == 'Assistant:': 27 | data.append(sentence[10:].strip()) 28 | if len(data) % 2: 29 | # The last round of conversation solely consists of input 30 | # without any output. 31 | # Discard the input part of the last round, as this part is ignored in 32 | # the loss calculation. 33 | data.pop() 34 | conversation = [] 35 | for i in range(0, len(data), 2): 36 | single_turn_conversation = {'input': data[i], 'output': data[i + 1]} 37 | conversation.append(single_turn_conversation) 38 | return {'conversation': conversation} 39 | -------------------------------------------------------------------------------- /xtuner/dataset/map_fns/dataset_map_fns/openai_map_fn.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | def openai_map_fn(example): 3 | """ 4 | Example before preprocessing: 5 | example["messages"] = [ 6 | { "role": "system", "content": "You are an assistant that 7 | occasionally misspells words." }, 8 | { "role": "user", "content": "Tell me a story." }, 9 | { "role": "assistant", "content": "One day a student 10 | went to schoool." } 11 | ] 12 | Example after preprocessing: 13 | example["conversation"] = [ 14 | { 15 | "system": "You are an assistant that occasionally misspells 16 | words.", 17 | "input": "Tell me a story.", 18 | "output": "One day a student went to schoool." 19 | } 20 | ] 21 | """ 22 | messages = example['messages'] 23 | system = '' 24 | input = '' 25 | conversation = [] 26 | while messages and messages[0]['role'] == 'assistant': 27 | # Skip the first one if it is from assistant 28 | messages = messages[1:] 29 | for msg in messages: 30 | if msg['role'] == 'system': 31 | system = msg['content'] 32 | elif msg['role'] == 'user': 33 | input += msg['content'] 34 | elif msg['role'] == 'assistant': 35 | conversation.append({ 36 | 'system': system, 37 | 'input': input, 38 | 'output': msg['content'] 39 | }) 40 | system = '' 41 | input = '' 42 | else: 43 | raise NotImplementedError 44 | return {'conversation': conversation} 45 | -------------------------------------------------------------------------------- /xtuner/dataset/map_fns/dataset_map_fns/openorca_map_fn.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | def openorca_map_fn(example): 3 | return { 4 | 'conversation': [{ 5 | 'system': example['system_prompt'], 6 | 'input': example['question'], 7 | 'output': example['response'] 8 | }] 9 | } 10 | -------------------------------------------------------------------------------- /xtuner/dataset/map_fns/dataset_map_fns/pretrain_map_fn.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | def pretrain_map_fn(example): 3 | r"""Example before preprocessing: 4 | example['text'] = 'xxx' 5 | 6 | Example after preprocessing: 7 | example['conversation'] = [ 8 | { 9 | 'input': '', 10 | 'output': 'xxx' 11 | }, 12 | ] 13 | """ 14 | return {'conversation': [{'input': '', 'output': example['text'].strip()}]} 15 | -------------------------------------------------------------------------------- /xtuner/dataset/map_fns/dataset_map_fns/sql_map_fn.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from xtuner.utils import SYSTEM_TEMPLATE 3 | 4 | 5 | def sql_map_fn(example): 6 | return { 7 | 'conversation': [{ 8 | 'system': SYSTEM_TEMPLATE.sql, 9 | 'input': '{context}\n{question}'.format(**example), 10 | 'output': example['answer'] 11 | }] 12 | } 13 | -------------------------------------------------------------------------------- /xtuner/dataset/map_fns/dataset_map_fns/stack_exchange_map_fn.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | def stack_exchange_map_fn(example): 3 | return { 4 | 'conversation': [{ 5 | 'input': example['question'], 6 | 'output': example['response'] 7 | }] 8 | } 9 | -------------------------------------------------------------------------------- /xtuner/dataset/map_fns/dataset_map_fns/tiny_codes_map_fn.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from xtuner.utils import SYSTEM_TEMPLATE 3 | 4 | 5 | def tiny_codes_map_fn(example): 6 | return { 7 | 'conversation': [{ 8 | 'system': SYSTEM_TEMPLATE.coder, 9 | 'input': example['prompt'], 10 | 'output': example['response'] 11 | }] 12 | } 13 | -------------------------------------------------------------------------------- /xtuner/dataset/map_fns/dataset_map_fns/wizardlm_map_fn.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | def wizardlm_map_fn(example): 3 | messages = example['conversations'] 4 | input = '' 5 | conversation = [] 6 | while messages and messages[0]['from'] == 'gpt': 7 | # Skip the first one if it is from gpt 8 | messages = messages[1:] 9 | for msg in messages: 10 | if msg['from'] == 'human': 11 | input += msg['value'] 12 | elif msg['from'] == 'gpt': 13 | conversation.append({'input': input, 'output': msg['value']}) 14 | input = '' 15 | else: 16 | raise NotImplementedError 17 | return {'conversation': conversation} 18 | -------------------------------------------------------------------------------- /xtuner/dataset/map_fns/template_map_fn.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from functools import partial 3 | 4 | from mmengine.utils.misc import get_object_from_string 5 | 6 | 7 | def template_map_fn(example, template): 8 | conversation = example.get('conversation', []) 9 | for i, single_turn_conversation in enumerate(conversation): 10 | input = single_turn_conversation.get('input', '') 11 | if input is None: 12 | input = '' 13 | input_text = template.INSTRUCTION.format(input=input, round=i + 1) 14 | system = single_turn_conversation.get('system', '') 15 | if system != '' and system is not None: 16 | system = template.SYSTEM.format(system=system) 17 | input_text = system + input_text 18 | single_turn_conversation['input'] = input_text 19 | 20 | if template.get('SUFFIX', None): 21 | output_text = single_turn_conversation.get('output', '') 22 | output_text += template.SUFFIX 23 | single_turn_conversation['output'] = output_text 24 | 25 | # SUFFIX_AS_EOS is False ==> need_eos_token is True 26 | single_turn_conversation['need_eos_token'] = \ 27 | not template.get('SUFFIX_AS_EOS', False) 28 | single_turn_conversation['sep'] = template.get('SEP', '') 29 | 30 | return {'conversation': conversation} 31 | 32 | 33 | def template_map_fn_factory(template): 34 | if isinstance(template, str): # for resume 35 | template = get_object_from_string(template) 36 | return partial(template_map_fn, template=template) 37 | -------------------------------------------------------------------------------- /xtuner/dataset/modelscope.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from mmengine.config import Config, ConfigDict 3 | 4 | from xtuner.registry import BUILDER 5 | from .huggingface import process_hf_dataset 6 | 7 | 8 | def process_ms_dataset(dataset, split='train', *args, **kwargs): 9 | """Post-process the dataset loaded from the ModelScope Hub.""" 10 | 11 | if isinstance(dataset, (Config, ConfigDict)): 12 | dataset = BUILDER.build(dataset) 13 | if isinstance(dataset, dict): 14 | dataset = dataset[split] 15 | dataset = dataset.to_hf_dataset() 16 | return process_hf_dataset(dataset, *args, **kwargs) 17 | -------------------------------------------------------------------------------- /xtuner/dataset/samplers/__init__.py: -------------------------------------------------------------------------------- 1 | from .intern_repo import InternlmRepoSampler, InternRepoSampler 2 | from .length_grouped import LengthGroupedSampler 3 | 4 | __all__ = ['LengthGroupedSampler', 'InternRepoSampler', 'InternlmRepoSampler'] 5 | -------------------------------------------------------------------------------- /xtuner/dataset/samplers/__pycache__/__init__.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/samplers/__pycache__/__init__.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/samplers/__pycache__/intern_repo.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/samplers/__pycache__/intern_repo.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/dataset/samplers/__pycache__/length_grouped.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/dataset/samplers/__pycache__/length_grouped.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/engine/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from ._strategy import DeepSpeedStrategy 3 | from .hooks import (DatasetInfoHook, EvaluateChatHook, ThroughputHook, 4 | VarlenAttnArgsToMessageHubHook) 5 | from .runner import TrainLoop, ValLoop, TestLoop 6 | 7 | __all__ = [ 8 | 'EvaluateChatHook', 'DatasetInfoHook', 'ThroughputHook', 9 | 'VarlenAttnArgsToMessageHubHook', 'DeepSpeedStrategy', 'TrainLoop', 10 | 'ValLoop', 'TestLoop' 11 | ] 12 | -------------------------------------------------------------------------------- /xtuner/engine/__pycache__/__init__.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/engine/__pycache__/__init__.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/engine/_strategy/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from .deepspeed import DeepSpeedStrategy 3 | 4 | __all__ = ['DeepSpeedStrategy'] 5 | -------------------------------------------------------------------------------- /xtuner/engine/_strategy/__pycache__/__init__.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/engine/_strategy/__pycache__/__init__.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/engine/_strategy/__pycache__/deepspeed.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/engine/_strategy/__pycache__/deepspeed.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/engine/_strategy/deepspeed.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from typing import Optional 3 | 4 | from mmengine._strategy import DeepSpeedStrategy as MMEngineDeepSpeedStrategy 5 | 6 | from xtuner import DS_CEPH_DIR 7 | from xtuner.parallel.sequence import init_sequence_parallel 8 | from xtuner.utils.fileio import patch_fileio 9 | 10 | 11 | class DeepSpeedStrategy(MMEngineDeepSpeedStrategy): 12 | 13 | def __init__(self, *args, **kwargs): 14 | sequence_parallel_size = kwargs.pop('sequence_parallel_size', 1) 15 | self.sequence_parallel_size = sequence_parallel_size 16 | 17 | super().__init__(*args, **kwargs) 18 | 19 | from transformers.integrations.deepspeed import HfDeepSpeedConfig 20 | 21 | # hf_deepspeed_config has to be saved as an attribute. 22 | self.hf_deepspeed_config = HfDeepSpeedConfig(self.config) 23 | 24 | def _wrap_model(self, model): 25 | wrapper = super()._wrap_model(model) 26 | # hard code for deepspeed zero3 27 | # When utilizing Zero3, the model isn't allocated to CUDA within the 28 | # `deepspeed.initialize` process. 29 | assert hasattr(wrapper.model, 'data_preprocessor') 30 | wrapper.model.data_preprocessor.cuda() 31 | return wrapper 32 | 33 | def save_checkpoint(self, *args, **kwargs) -> None: 34 | if DS_CEPH_DIR: 35 | from os import path as osp 36 | work_dir_prefix = osp.split(self.work_dir)[0] 37 | 38 | filename = kwargs['filename'].replace(work_dir_prefix, DS_CEPH_DIR) 39 | kwargs['filename'] = filename 40 | with patch_fileio(): 41 | super().save_checkpoint(*args, **kwargs) 42 | else: 43 | super().save_checkpoint(*args, **kwargs) 44 | 45 | def load_checkpoint(self, *args, **kwargs) -> None: 46 | if DS_CEPH_DIR: 47 | 48 | with patch_fileio(): 49 | checkpoint = super().load_checkpoint(*args, **kwargs) 50 | else: 51 | checkpoint = super().load_checkpoint(*args, **kwargs) 52 | return checkpoint 53 | 54 | def resume(self, *args, **kwargs) -> None: 55 | if DS_CEPH_DIR: 56 | 57 | with patch_fileio(): 58 | checkpoint = super().resume(*args, **kwargs) 59 | else: 60 | checkpoint = super().resume(*args, **kwargs) 61 | return checkpoint 62 | 63 | def _setup_distributed( # type: ignore 64 | self, 65 | launcher: Optional[str] = None, 66 | backend: str = 'nccl', 67 | **kwargs, 68 | ): 69 | super()._setup_distributed(launcher, backend, **kwargs) 70 | init_sequence_parallel(self.sequence_parallel_size) 71 | -------------------------------------------------------------------------------- /xtuner/engine/hooks/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from .dataset_info_hook import DatasetInfoHook 3 | from .evaluate_chat_hook import EvaluateChatHook 4 | from .throughput_hook import ThroughputHook 5 | from .varlen_attn_args_to_messagehub_hook import VarlenAttnArgsToMessageHubHook 6 | 7 | __all__ = [ 8 | 'EvaluateChatHook', 'DatasetInfoHook', 'ThroughputHook', 9 | 'VarlenAttnArgsToMessageHubHook' 10 | ] 11 | -------------------------------------------------------------------------------- /xtuner/engine/hooks/__pycache__/__init__.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/engine/hooks/__pycache__/__init__.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/engine/hooks/__pycache__/dataset_info_hook.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/engine/hooks/__pycache__/dataset_info_hook.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/engine/hooks/__pycache__/evaluate_chat_hook.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/engine/hooks/__pycache__/evaluate_chat_hook.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/engine/hooks/__pycache__/throughput_hook.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/engine/hooks/__pycache__/throughput_hook.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/engine/hooks/__pycache__/varlen_attn_args_to_messagehub_hook.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/engine/hooks/__pycache__/varlen_attn_args_to_messagehub_hook.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/engine/hooks/dataset_info_hook.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from mmengine.hooks import Hook 3 | 4 | from xtuner.registry import BUILDER 5 | from xtuner.utils import DEFAULT_IMAGE_TOKEN, IMAGE_TOKEN_INDEX 6 | 7 | 8 | def split_list(lst, value): 9 | res = [] 10 | tmp_res = [] 11 | for i in lst: 12 | if i == value: 13 | res.append(tmp_res) 14 | tmp_res = [] 15 | else: 16 | tmp_res.append(i) 17 | res.append(tmp_res) 18 | return res 19 | 20 | 21 | class DatasetInfoHook(Hook): 22 | 23 | def __init__(self, tokenizer, is_intern_repo_dataset=False): 24 | self.tokenizer = BUILDER.build(tokenizer) 25 | self.is_intern_repo_dataset = is_intern_repo_dataset 26 | 27 | def log(self, runner, dataset, mode='train'): 28 | runner.logger.info(f'Num {mode} samples {len(dataset)}') 29 | runner.logger.info(f'{mode} example:') 30 | input_ids = dataset[0]['input_ids'] 31 | if self.is_intern_repo_dataset: 32 | input_ids = [abs(x) for x in input_ids] 33 | # Try to split list to be compatible with IMAGE token 34 | input_ids = split_list(input_ids, IMAGE_TOKEN_INDEX) 35 | text = '' 36 | for idx, ids in enumerate(input_ids): 37 | text += self.tokenizer.decode(ids) 38 | if idx != len(input_ids) - 1: 39 | text += DEFAULT_IMAGE_TOKEN 40 | runner.logger.info(text) 41 | 42 | def before_train(self, runner) -> None: 43 | do_train = runner.train_loop is not None 44 | do_eval = runner.val_loop is not None 45 | if do_train: 46 | train_dataset = runner.train_dataloader.dataset 47 | self.log(runner, train_dataset, mode='train') 48 | if do_eval: 49 | eval_dataset = runner.val_dataloader.dataset 50 | self.log(runner, eval_dataset, mode='eval') 51 | 52 | def before_val(self, runner) -> None: 53 | eval_dataset = runner.val_dataloader.dataset 54 | self.log(runner, eval_dataset, mode='eval') 55 | 56 | def before_test(self, runner) -> None: 57 | test_dataset = runner.test_dataloader.dataset 58 | self.log(runner, test_dataset, mode='test') 59 | -------------------------------------------------------------------------------- /xtuner/engine/hooks/varlen_attn_args_to_messagehub_hook.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from typing import Optional, Union 3 | 4 | import torch.distributed as dist 5 | from mmengine import MessageHub 6 | from mmengine.hooks import Hook 7 | 8 | DATA_BATCH = Optional[Union[dict, tuple, list]] 9 | 10 | 11 | class VarlenAttnArgsToMessageHubHook(Hook): 12 | 13 | def before_train_iter(self, 14 | runner, 15 | batch_idx: int, 16 | data_batch: dict = None) -> None: 17 | rank = dist.get_rank() 18 | message_hub = MessageHub.get_instance('varlen_attn_args') 19 | 20 | assert 'data' in data_batch.keys() 21 | data = data_batch['data'] 22 | 23 | cumulative_len = data.pop('cumulative_len') 24 | assert len(cumulative_len) == 1 25 | cumulative_len = cumulative_len[0].cuda() 26 | message_hub.update_info(f'cumulative_len_rank_{rank}', cumulative_len) 27 | 28 | max_seqlen = data.pop('max_seqlen') 29 | message_hub.update_info(f'max_seqlen_rank_{rank}', max_seqlen) 30 | 31 | def after_train_iter(self, 32 | runner, 33 | batch_idx: int, 34 | data_batch: DATA_BATCH = None, 35 | outputs: Optional[dict] = None) -> None: 36 | rank = dist.get_rank() 37 | message_hub = MessageHub.get_instance('varlen_attn_args') 38 | message_hub.update_info(f'cumulative_len_rank_{rank}', None) 39 | message_hub.update_info(f'max_seqlen_rank_{rank}', None) 40 | -------------------------------------------------------------------------------- /xtuner/engine/runner/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from .loops import TrainLoop, ValLoop, TestLoop, TestLoop1, TestLoop2 3 | 4 | __all__ = ['TrainLoop', 'ValLoop', 'TestLoop', 'TestLoop1', 'TestLoop2'] 5 | -------------------------------------------------------------------------------- /xtuner/engine/runner/__pycache__/__init__.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/engine/runner/__pycache__/__init__.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/engine/runner/__pycache__/loops.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/engine/runner/__pycache__/loops.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/evaluation/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from .metrics import MMLUMetric 3 | 4 | __all__ = ['MMLUMetric'] 5 | -------------------------------------------------------------------------------- /xtuner/evaluation/metrics/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from .mmlu_metric import MMLUMetric 3 | 4 | __all__ = ['MMLUMetric'] 5 | -------------------------------------------------------------------------------- /xtuner/model/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from .llava import LLaVAModel 3 | from .sft import SupervisedFinetune 4 | from .anyres_llava import AnyResLLaVAModel 5 | from .mini_gemini import MiniGeminiModel 6 | 7 | __all__ = ['SupervisedFinetune', 'LLaVAModel', 'AnyResLLaVAModel', 'MiniGeminiModel'] 8 | -------------------------------------------------------------------------------- /xtuner/model/__pycache__/__init__.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/model/__pycache__/__init__.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/model/__pycache__/anyres_llava.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/model/__pycache__/anyres_llava.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/model/__pycache__/llava.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/model/__pycache__/llava.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/model/__pycache__/mini_gemini.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/model/__pycache__/mini_gemini.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/model/__pycache__/object_llava.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/model/__pycache__/object_llava.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/model/__pycache__/sft.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/model/__pycache__/sft.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/model/__pycache__/utils.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/model/__pycache__/utils.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/model/modules/__init__.py: -------------------------------------------------------------------------------- 1 | from .dispatch import dispatch_modules 2 | from .projector import ProjectorConfig, ProjectorModel 3 | 4 | __all__ = ['dispatch_modules', 'ProjectorConfig', 'ProjectorModel'] 5 | -------------------------------------------------------------------------------- /xtuner/model/modules/__pycache__/__init__.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/model/modules/__pycache__/__init__.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/model/modules/dispatch/__pycache__/__init__.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/model/modules/dispatch/__pycache__/__init__.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/model/modules/dispatch/__pycache__/attention.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/model/modules/dispatch/__pycache__/attention.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/model/modules/dispatch/__pycache__/baichuan.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/model/modules/dispatch/__pycache__/baichuan.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/model/modules/dispatch/__pycache__/llama.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/model/modules/dispatch/__pycache__/llama.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/model/modules/dispatch/__pycache__/phi3.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/model/modules/dispatch/__pycache__/phi3.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/model/modules/dispatch/__pycache__/utils.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/model/modules/dispatch/__pycache__/utils.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/model/modules/dispatch/__pycache__/yi.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/model/modules/dispatch/__pycache__/yi.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/model/modules/dispatch/triton_kernels/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from .layer_norm import layer_norm_forward 3 | from .rms_norm import rms_norm_forward 4 | from .rotary import apply_rotary_emb 5 | 6 | __all__ = ['rms_norm_forward', 'layer_norm_forward', 'apply_rotary_emb'] 7 | -------------------------------------------------------------------------------- /xtuner/model/modules/dispatch/triton_kernels/__pycache__/__init__.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/model/modules/dispatch/triton_kernels/__pycache__/__init__.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/model/modules/dispatch/triton_kernels/__pycache__/layer_norm.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/model/modules/dispatch/triton_kernels/__pycache__/layer_norm.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/model/modules/dispatch/triton_kernels/__pycache__/rms_norm.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/model/modules/dispatch/triton_kernels/__pycache__/rms_norm.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/model/modules/dispatch/triton_kernels/__pycache__/rotary.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/model/modules/dispatch/triton_kernels/__pycache__/rotary.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/model/modules/dispatch/triton_kernels/layer_norm.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | import torch 3 | import torch.nn.functional as F 4 | 5 | 6 | def layer_norm_forward(self, hidden_states): 7 | input_dtype = hidden_states.dtype 8 | hidden_states = hidden_states.to(torch.float32) 9 | hidden_states = F.layer_norm( 10 | hidden_states, (hidden_states.shape[-1], ), eps=self.variance_epsilon) 11 | hidden_states = self.weight.to(torch.float32) * hidden_states 12 | return hidden_states.to(input_dtype) 13 | -------------------------------------------------------------------------------- /xtuner/model/modules/dispatch/utils.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn.functional as F 3 | 4 | try: 5 | from flash_attn.bert_padding import index_first_axis, unpad_input 6 | except ImportError: 7 | pass 8 | 9 | 10 | def _get_unpad_data(attention_mask): 11 | seqlens_in_batch = attention_mask.sum(dim=-1, dtype=torch.int32) 12 | indices = torch.nonzero(attention_mask.flatten(), as_tuple=False).flatten() 13 | max_seqlen_in_batch = seqlens_in_batch.max().item() 14 | cu_seqlens = F.pad( 15 | torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.torch.int32), (1, 0)) 16 | return ( 17 | indices, 18 | cu_seqlens, 19 | max_seqlen_in_batch, 20 | ) 21 | 22 | 23 | def upad_qkv(query_layer, key_layer, value_layer, attention_mask, 24 | query_length): 25 | indices_k, cu_seqlens_k, max_seqlen_in_batch_k = _get_unpad_data( 26 | attention_mask) 27 | batch_size, kv_seq_len, num_key_value_heads, head_dim = key_layer.shape 28 | 29 | key_layer = index_first_axis( 30 | key_layer.reshape(batch_size * kv_seq_len, num_key_value_heads, 31 | head_dim), indices_k) 32 | value_layer = index_first_axis( 33 | value_layer.reshape(batch_size * kv_seq_len, num_key_value_heads, 34 | head_dim), indices_k) 35 | if query_length == kv_seq_len: 36 | # Different from the origin version as sequence parallel change 37 | # the number of attention heads. 38 | query_layer = index_first_axis( 39 | query_layer.reshape(batch_size * kv_seq_len, -1, head_dim), 40 | indices_k) 41 | cu_seqlens_q = cu_seqlens_k 42 | max_seqlen_in_batch_q = max_seqlen_in_batch_k 43 | indices_q = indices_k 44 | elif query_length == 1: 45 | max_seqlen_in_batch_q = 1 46 | cu_seqlens_q = torch.arange( 47 | batch_size + 1, dtype=torch.int32, device=query_layer.device 48 | ) # There is a memcpy here, that is very bad. 49 | indices_q = cu_seqlens_q[:-1] 50 | query_layer = query_layer.squeeze(1) 51 | else: 52 | # The -q_len: slice assumes left padding. 53 | attention_mask = attention_mask[:, -query_length:] 54 | query_layer, indices_q, cu_seqlens_q, max_seqlen_in_batch_q = \ 55 | unpad_input(query_layer, attention_mask) 56 | 57 | return ( 58 | query_layer, 59 | key_layer, 60 | value_layer, 61 | indices_q, 62 | (cu_seqlens_q, cu_seqlens_k), 63 | (max_seqlen_in_batch_q, max_seqlen_in_batch_k), 64 | ) 65 | -------------------------------------------------------------------------------- /xtuner/model/modules/projector/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from transformers import AutoConfig, AutoModel 3 | 4 | from .configuration_projector import ProjectorConfig 5 | from .modeling_projector import ProjectorModel 6 | 7 | AutoConfig.register('projector', ProjectorConfig) 8 | AutoModel.register(ProjectorConfig, ProjectorModel) 9 | 10 | __all__ = ['ProjectorConfig', 'ProjectorModel'] 11 | -------------------------------------------------------------------------------- /xtuner/model/modules/projector/__pycache__/__init__.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/model/modules/projector/__pycache__/__init__.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/model/modules/projector/__pycache__/configuration_projector.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/model/modules/projector/__pycache__/configuration_projector.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/model/modules/projector/__pycache__/modeling_projector.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/model/modules/projector/__pycache__/modeling_projector.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/model/modules/projector/configuration_projector.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from transformers import PretrainedConfig 3 | 4 | 5 | class ProjectorConfig(PretrainedConfig): 6 | model_type = 'projector' 7 | _auto_class = 'AutoConfig' 8 | 9 | def __init__( 10 | self, 11 | visual_hidden_size=4096, 12 | llm_hidden_size=4096, 13 | depth=2, 14 | hidden_act='gelu', 15 | bias=True, 16 | **kwargs, 17 | ): 18 | self.visual_hidden_size = visual_hidden_size 19 | self.llm_hidden_size = llm_hidden_size 20 | self.depth = depth 21 | self.hidden_act = hidden_act 22 | self.bias = bias 23 | super().__init__(**kwargs) 24 | -------------------------------------------------------------------------------- /xtuner/model/modules/projector/modeling_projector.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | import torch 3 | import torch.nn as nn 4 | from transformers import PreTrainedModel 5 | from transformers.activations import ACT2FN 6 | 7 | from .configuration_projector import ProjectorConfig 8 | 9 | 10 | class ProjectorModel(PreTrainedModel): 11 | _auto_class = 'AutoModel' 12 | config_class = ProjectorConfig 13 | base_model_prefix = 'model' 14 | supports_gradient_checkpointing = True 15 | 16 | def __init__(self, config: ProjectorConfig) -> None: 17 | super().__init__(config) 18 | self.gradient_checkpointing = False 19 | 20 | modules = [ 21 | nn.Linear( 22 | config.visual_hidden_size, 23 | config.llm_hidden_size, 24 | bias=config.bias) 25 | ] 26 | for _ in range(1, config.depth): 27 | modules.append(ACT2FN[config.hidden_act]) 28 | modules.append( 29 | nn.Linear( 30 | config.llm_hidden_size, 31 | config.llm_hidden_size, 32 | bias=config.bias)) 33 | self.model = nn.Sequential(*modules) 34 | 35 | def enable_input_require_grads(self): 36 | 37 | def make_inputs_require_grad(module, input, output): 38 | output.requires_grad_(True) 39 | 40 | self.model.register_forward_hook(make_inputs_require_grad) 41 | 42 | def _set_gradient_checkpointing(self, module, value=False): 43 | if isinstance(module, ProjectorModel): 44 | module.gradient_checkpointing = value 45 | 46 | def forward(self, x): 47 | if self.gradient_checkpointing and self.training: 48 | layer_outputs = torch.utils.checkpoint.checkpoint(self.model, x) 49 | else: 50 | layer_outputs = self.model(x) 51 | return layer_outputs 52 | -------------------------------------------------------------------------------- /xtuner/parallel/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from .sequence import * # noqa: F401, F403 3 | -------------------------------------------------------------------------------- /xtuner/parallel/__pycache__/__init__.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/parallel/__pycache__/__init__.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/parallel/sequence/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from mmengine.dist import init_dist 3 | 4 | from .attention import (post_process_for_sequence_parallel_attn, 5 | pre_process_for_sequence_parallel_attn, 6 | sequence_parallel_wrapper) 7 | from .comm import (all_to_all, gather_for_sequence_parallel, 8 | gather_forward_split_backward, split_for_sequence_parallel, 9 | split_forward_gather_backward) 10 | from .data_collate import pad_for_sequence_parallel 11 | from .reduce_loss import reduce_sequence_parallel_loss 12 | from .sampler import SequenceParallelSampler 13 | from .setup_distributed import (get_data_parallel_group, 14 | get_data_parallel_rank, 15 | get_data_parallel_world_size, 16 | get_sequence_parallel_group, 17 | get_sequence_parallel_rank, 18 | get_sequence_parallel_world_size, 19 | init_sequence_parallel) 20 | 21 | __all__ = [ 22 | 'sequence_parallel_wrapper', 'pre_process_for_sequence_parallel_attn', 23 | 'post_process_for_sequence_parallel_attn', 'pad_for_sequence_parallel', 24 | 'split_for_sequence_parallel', 'SequenceParallelSampler', 25 | 'init_sequence_parallel', 'get_sequence_parallel_group', 26 | 'get_sequence_parallel_world_size', 'get_sequence_parallel_rank', 27 | 'get_data_parallel_group', 'get_data_parallel_world_size', 28 | 'get_data_parallel_rank', 'reduce_sequence_parallel_loss', 'init_dist', 29 | 'all_to_all', 'gather_for_sequence_parallel', 30 | 'split_forward_gather_backward', 'gather_forward_split_backward' 31 | ] 32 | -------------------------------------------------------------------------------- /xtuner/parallel/sequence/__pycache__/__init__.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/parallel/sequence/__pycache__/__init__.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/parallel/sequence/__pycache__/attention.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/parallel/sequence/__pycache__/attention.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/parallel/sequence/__pycache__/comm.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/parallel/sequence/__pycache__/comm.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/parallel/sequence/__pycache__/data_collate.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/parallel/sequence/__pycache__/data_collate.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/parallel/sequence/__pycache__/reduce_loss.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/parallel/sequence/__pycache__/reduce_loss.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/parallel/sequence/__pycache__/sampler.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/parallel/sequence/__pycache__/sampler.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/parallel/sequence/__pycache__/setup_distributed.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/parallel/sequence/__pycache__/setup_distributed.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/parallel/sequence/data_collate.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | import torch 3 | 4 | from xtuner.utils import DEFAULT_PAD_TOKEN_INDEX, IGNORE_INDEX 5 | from .setup_distributed import get_sequence_parallel_world_size 6 | 7 | 8 | def pad_for_sequence_parallel(tokens, 9 | labels=None, 10 | position_ids=None, 11 | attention_mask=None, 12 | tokens_pad_index=DEFAULT_PAD_TOKEN_INDEX, 13 | labels_pad_index=IGNORE_INDEX, 14 | position_ids_pad_index=0, 15 | attention_mask_pad_index=0): 16 | if labels is not None: 17 | assert tokens.shape == labels.shape 18 | if position_ids is not None: 19 | assert tokens.shape == position_ids.shape 20 | if attention_mask is not None: 21 | assert tokens.shape == attention_mask.shape 22 | 23 | bs, seq_len = tokens.shape 24 | seq_parallel_world_size = get_sequence_parallel_world_size() 25 | if seq_len % seq_parallel_world_size == 0: 26 | return tokens, labels, position_ids, attention_mask 27 | 28 | pad_num = seq_parallel_world_size - (seq_len % seq_parallel_world_size) 29 | pad = torch.full((bs, pad_num), 30 | tokens_pad_index, 31 | dtype=tokens.dtype, 32 | device=tokens.device) 33 | tokens = torch.cat([tokens, pad], dim=1) 34 | 35 | if labels is not None: 36 | pad = torch.full((bs, pad_num), 37 | labels_pad_index, 38 | dtype=labels.dtype, 39 | device=labels.device) 40 | labels = torch.cat([labels, pad], dim=1) 41 | 42 | if position_ids is not None: 43 | pad = torch.full((bs, pad_num), 44 | position_ids_pad_index, 45 | dtype=position_ids.dtype, 46 | device=position_ids.device) 47 | position_ids = torch.cat([position_ids, pad], dim=1) 48 | 49 | if attention_mask is not None: 50 | pad = torch.full((bs, pad_num), 51 | attention_mask_pad_index, 52 | dtype=attention_mask.dtype, 53 | device=attention_mask.device) 54 | attention_mask = torch.cat([attention_mask, pad], dim=1) 55 | 56 | return tokens, labels, position_ids, attention_mask 57 | -------------------------------------------------------------------------------- /xtuner/parallel/sequence/reduce_loss.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.distributed as dist 3 | 4 | from .setup_distributed import get_sequence_parallel_group 5 | 6 | 7 | def reduce_sequence_parallel_loss(mean_loss, num_tokens_for_loss): 8 | sequence_parallel_group = get_sequence_parallel_group() 9 | if num_tokens_for_loss == 0: 10 | # convert nan to 0 just for logging 11 | mean_loss = torch.nan_to_num(mean_loss) 12 | loss_sum = mean_loss * num_tokens_for_loss 13 | dist.all_reduce(loss_sum, group=sequence_parallel_group) 14 | dist.all_reduce(num_tokens_for_loss, group=sequence_parallel_group) 15 | 16 | loss = loss_sum / num_tokens_for_loss 17 | return loss 18 | -------------------------------------------------------------------------------- /xtuner/parallel/sequence/sampler.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | import math 3 | from typing import Optional, Sized 4 | 5 | from mmengine.dataset import DefaultSampler 6 | from mmengine.dist import sync_random_seed 7 | 8 | from .setup_distributed import (get_data_parallel_rank, 9 | get_data_parallel_world_size) 10 | 11 | 12 | class SequenceParallelSampler(DefaultSampler): 13 | 14 | def __init__(self, 15 | dataset: Sized, 16 | shuffle: bool = True, 17 | seed: Optional[int] = None, 18 | round_up: bool = True) -> None: 19 | rank = get_data_parallel_rank() 20 | world_size = get_data_parallel_world_size() 21 | self.rank = rank 22 | self.world_size = world_size 23 | 24 | self.dataset = dataset 25 | self.shuffle = shuffle 26 | if seed is None: 27 | seed = sync_random_seed() 28 | self.seed = seed 29 | self.epoch = 0 30 | self.round_up = round_up 31 | 32 | if self.round_up: 33 | self.num_samples = math.ceil(len(self.dataset) / world_size) 34 | self.total_size = self.num_samples * self.world_size 35 | else: 36 | self.num_samples = math.ceil( 37 | (len(self.dataset) - rank) / world_size) 38 | self.total_size = len(self.dataset) 39 | -------------------------------------------------------------------------------- /xtuner/registry.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from mmengine.registry import Registry 3 | 4 | __all__ = ['BUILDER', 'MAP_FUNC'] 5 | 6 | BUILDER = Registry('builder') 7 | MAP_FUNC = Registry('map_fn') 8 | -------------------------------------------------------------------------------- /xtuner/tools/__pycache__/utils.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/tools/__pycache__/utils.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/tools/copy_cfg.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | import argparse 3 | import os.path as osp 4 | import shutil 5 | 6 | from mmengine.utils import mkdir_or_exist 7 | 8 | from xtuner.configs import cfgs_name_path 9 | 10 | 11 | def parse_args(): 12 | parser = argparse.ArgumentParser() 13 | parser.add_argument('config_name', help='config name') 14 | parser.add_argument('save_dir', help='save directory for copied config') 15 | args = parser.parse_args() 16 | return args 17 | 18 | 19 | def add_copy_suffix(string): 20 | file_name, ext = osp.splitext(string) 21 | return f'{file_name}_copy{ext}' 22 | 23 | 24 | def main(): 25 | args = parse_args() 26 | mkdir_or_exist(args.save_dir) 27 | config_path = cfgs_name_path[args.config_name] 28 | save_path = osp.join(args.save_dir, 29 | add_copy_suffix(osp.basename(config_path))) 30 | shutil.copyfile(config_path, save_path) 31 | print(f'Copy to {save_path}') 32 | 33 | 34 | if __name__ == '__main__': 35 | main() 36 | -------------------------------------------------------------------------------- /xtuner/tools/data_preprocess/arxiv.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | import argparse 3 | import json 4 | from datetime import datetime 5 | 6 | 7 | def parse_args(): 8 | parser = argparse.ArgumentParser() 9 | parser.add_argument('src_file', help='source file path') 10 | parser.add_argument('dst_file', help='destination file path') 11 | parser.add_argument( 12 | '--categories', 13 | nargs='+', 14 | default=['cs.AI', 'cs.CL', 'cs.CV'], 15 | help='target categories') 16 | parser.add_argument( 17 | '--start-date', 18 | default='2020-01-01', 19 | help='start date (format: YYYY-MM-DD)') 20 | 21 | args = parser.parse_args() 22 | return args 23 | 24 | 25 | def has_intersection(list1, list2): 26 | set1 = set(list1) 27 | set2 = set(list2) 28 | return len(set1.intersection(set2)) > 0 29 | 30 | 31 | def read_json_file(file_path): 32 | data = [] 33 | with open(file_path) as file: 34 | for line in file: 35 | try: 36 | json_data = json.loads(line) 37 | data.append(json_data) 38 | except json.JSONDecodeError: 39 | print(f'Failed to parse line: {line}') 40 | return data 41 | 42 | 43 | def main(): 44 | args = parse_args() 45 | json_data = read_json_file(args.src_file) 46 | from_time = datetime.strptime(args.start_date, '%Y-%m-%d') 47 | filtered_data = [ 48 | item for item in json_data 49 | if has_intersection(args.categories, item['categories'].split()) 50 | and datetime.strptime(item['update_date'], '%Y-%m-%d') >= from_time 51 | ] 52 | 53 | with open(args.dst_file, 'w') as file: 54 | json.dump(filtered_data, file) 55 | 56 | print(f'Save to {args.dst_file}\n{len(filtered_data)} items') 57 | 58 | 59 | if __name__ == '__main__': 60 | main() 61 | -------------------------------------------------------------------------------- /xtuner/tools/data_preprocess/convert_refcoco.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | import argparse 3 | import json 4 | 5 | from xtuner.dataset.refcoco_json import RefCOCOJsonDataset 6 | 7 | 8 | def parse_args(): 9 | parser = argparse.ArgumentParser() 10 | parser.add_argument( 11 | '--ann-path', 12 | default='data/refcoco_annotations', 13 | help='Refcoco annotation path', 14 | ) 15 | parser.add_argument( 16 | '--image-path', 17 | default='data/llava_data/llava_images/coco/train2017', 18 | help='COCO image path', 19 | ) 20 | parser.add_argument( 21 | '--save-path', default='./', help='The folder to save converted data') 22 | args = parser.parse_args() 23 | return args 24 | 25 | 26 | if __name__ == '__main__': 27 | args = parse_args() 28 | 29 | data_info = [ 30 | ('refcoco', 'unc'), 31 | ('refcoco+', 'unc'), 32 | ('refcocog', 'umd'), 33 | ] 34 | all_data = [] 35 | for dataset, split in data_info: 36 | data = RefCOCOJsonDataset.get_data_json( 37 | ann_path=args.ann_path, 38 | image_path=args.image_path, 39 | dataset=dataset, 40 | splitBy=split, 41 | )[0] 42 | all_data.extend(data) 43 | save_path = args.save_path + '/train.json' 44 | with open(save_path, 'w') as f: 45 | print(f'save to {save_path} with {len(all_data)} items.') 46 | print(all_data[0]) 47 | json.dump(all_data, f, indent=4) 48 | -------------------------------------------------------------------------------- /xtuner/tools/get_data_order.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | import argparse 3 | import os 4 | 5 | 6 | def parse_args(): 7 | parser = argparse.ArgumentParser() 8 | parser.add_argument('--data-folder', help='Data folder') 9 | parser.add_argument('--save-folder', help='The folder to save data order.') 10 | parser.add_argument( 11 | '--file-type', 12 | default='.bin', 13 | help='We want to get the order of the file in this type.') 14 | args = parser.parse_args() 15 | return args 16 | 17 | 18 | def save_data_order(data_folder, save_folder, file_type='.bin'): 19 | assert os.path.exists(data_folder), f'{data_folder} does not exist.' 20 | triples = list(os.walk(data_folder, followlinks=True)) 21 | data_order = [] 22 | for root, dirs, files in triples: 23 | dirs.sort() 24 | print(f'Reading {root}...') 25 | for fn in sorted(files): 26 | if fn.endswith(file_type): 27 | fp = os.path.join(root, fn) 28 | # Using relative paths so that you can get the same result 29 | # on different clusters 30 | fp = fp.replace(data_folder, '')[1:] 31 | data_order.append(fp) 32 | 33 | save_path = os.path.join(save_folder, 'data_order.txt') 34 | with open(save_path, 'w') as f: 35 | for fp in data_order: 36 | f.write(fp + '\n') 37 | 38 | 39 | if __name__ == '__main__': 40 | args = parse_args() 41 | save_data_order(args.data_folder, args.save_folder, args.file_type) 42 | -------------------------------------------------------------------------------- /xtuner/tools/list_cfg.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | import argparse 3 | 4 | from xtuner.configs import cfgs_name_path 5 | 6 | 7 | def parse_args(): 8 | parser = argparse.ArgumentParser() 9 | parser.add_argument( 10 | '-p', '--pattern', default=None, help='Pattern for fuzzy matching') 11 | args = parser.parse_args() 12 | return args 13 | 14 | 15 | def main(pattern=None): 16 | args = parse_args() 17 | configs_names = sorted(list(cfgs_name_path.keys())) 18 | print('==========================CONFIGS===========================') 19 | if args.pattern is not None: 20 | print(f'PATTERN: {args.pattern}') 21 | print('-------------------------------') 22 | for name in configs_names: 23 | if args.pattern is None or args.pattern.lower() in name.lower(): 24 | print(name) 25 | print('=============================================================') 26 | 27 | 28 | if __name__ == '__main__': 29 | main() 30 | -------------------------------------------------------------------------------- /xtuner/tools/list_dataset_format.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from xtuner.dataset.map_fns import DATASET_FORMAT_MAPPING 3 | 4 | 5 | def main(): 6 | dataset_format = DATASET_FORMAT_MAPPING.keys() 7 | print('======================DATASET_FORMAT======================') 8 | for format in dataset_format: 9 | print(format) 10 | print('==========================================================') 11 | 12 | 13 | if __name__ == '__main__': 14 | main() 15 | -------------------------------------------------------------------------------- /xtuner/tools/log_dataset.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | import argparse 3 | 4 | from mmengine.config import Config 5 | 6 | from xtuner.registry import BUILDER 7 | 8 | 9 | def parse_args(): 10 | parser = argparse.ArgumentParser(description='Log processed dataset.') 11 | parser.add_argument('config', help='config file name or path.') 12 | # chose which kind of dataset style to show 13 | parser.add_argument( 14 | '--show', 15 | default='text', 16 | choices=['text', 'masked_text', 'input_ids', 'labels', 'all'], 17 | help='which kind of dataset style to show') 18 | args = parser.parse_args() 19 | return args 20 | 21 | 22 | def main(): 23 | args = parse_args() 24 | 25 | cfg = Config.fromfile(args.config) 26 | 27 | tokenizer = BUILDER.build(cfg.tokenizer) 28 | if cfg.get('framework', 'mmengine').lower() == 'huggingface': 29 | train_dataset = BUILDER.build(cfg.train_dataset) 30 | else: 31 | train_dataset = BUILDER.build(cfg.train_dataloader.dataset) 32 | 33 | if args.show == 'text' or args.show == 'all': 34 | print('#' * 20 + ' text ' + '#' * 20) 35 | print(tokenizer.decode(train_dataset[0]['input_ids'])) 36 | if args.show == 'masked_text' or args.show == 'all': 37 | print('#' * 20 + ' text(masked) ' + '#' * 20) 38 | masked_text = ' '.join( 39 | ['[-100]' for i in train_dataset[0]['labels'] if i == -100]) 40 | unmasked_text = tokenizer.decode( 41 | [i for i in train_dataset[0]['labels'] if i != -100]) 42 | print(masked_text + ' ' + unmasked_text) 43 | if args.show == 'input_ids' or args.show == 'all': 44 | print('#' * 20 + ' input_ids ' + '#' * 20) 45 | print(train_dataset[0]['input_ids']) 46 | if args.show == 'labels' or args.show == 'all': 47 | print('#' * 20 + ' labels ' + '#' * 20) 48 | print(train_dataset[0]['labels']) 49 | 50 | 51 | if __name__ == '__main__': 52 | main() 53 | -------------------------------------------------------------------------------- /xtuner/tools/model_converters/merge.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | import argparse 3 | 4 | import torch 5 | from peft import PeftModel 6 | from transformers import (AutoModelForCausalLM, AutoTokenizer, 7 | CLIPImageProcessor, CLIPVisionModel) 8 | 9 | from xtuner.model.utils import LoadWoInit 10 | 11 | 12 | def parse_args(): 13 | parser = argparse.ArgumentParser( 14 | description='Merge a HuggingFace adapter to base model') 15 | parser.add_argument('model_name_or_path', help='model name or path') 16 | parser.add_argument('adapter_name_or_path', help='adapter name or path') 17 | parser.add_argument( 18 | 'save_dir', help='the directory to save the merged model') 19 | parser.add_argument( 20 | '--max-shard-size', 21 | type=str, 22 | default='2GB', 23 | help='Only applicable for LLM. The maximum size for ' 24 | 'each sharded checkpoint.') 25 | parser.add_argument( 26 | '--is-clip', 27 | action='store_true', 28 | help='Indicate if the model is a clip model') 29 | parser.add_argument( 30 | '--device', 31 | default='cuda', 32 | choices=('cuda', 'cpu', 'auto'), 33 | help='Indicate the device') 34 | 35 | args = parser.parse_args() 36 | return args 37 | 38 | 39 | def main(): 40 | args = parse_args() 41 | if args.is_clip: 42 | with LoadWoInit(): 43 | model = CLIPVisionModel.from_pretrained( 44 | args.model_name_or_path, device_map=args.device) 45 | processor = CLIPImageProcessor.from_pretrained(args.model_name_or_path) 46 | else: 47 | with LoadWoInit(): 48 | model = AutoModelForCausalLM.from_pretrained( 49 | args.model_name_or_path, 50 | torch_dtype=torch.float16, 51 | low_cpu_mem_usage=True, 52 | device_map=args.device, 53 | trust_remote_code=True) 54 | processor = AutoTokenizer.from_pretrained( 55 | args.model_name_or_path, trust_remote_code=True) 56 | model_unmerged = PeftModel.from_pretrained( 57 | model, 58 | args.adapter_name_or_path, 59 | device_map=args.device, 60 | is_trainable=False, 61 | trust_remote_code=True) 62 | model_merged = model_unmerged.merge_and_unload() 63 | print(f'Saving to {args.save_dir}...') 64 | model_merged.save_pretrained( 65 | args.save_dir, 66 | safe_serialization=False, 67 | max_shard_size=args.max_shard_size) 68 | processor.save_pretrained(args.save_dir) 69 | print('All done!') 70 | 71 | 72 | if __name__ == '__main__': 73 | main() 74 | -------------------------------------------------------------------------------- /xtuner/tools/model_converters/split.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | import argparse 3 | import copy 4 | import json 5 | import os 6 | import os.path as osp 7 | import shutil 8 | 9 | import torch 10 | from mmengine.utils import mkdir_or_exist 11 | 12 | 13 | def parse_args(): 14 | parser = argparse.ArgumentParser( 15 | description='Split a HuggingFace model to the smallest sharded one') 16 | parser.add_argument('src_dir', help='the directory of the model') 17 | parser.add_argument('dst_dir', help='the directory to save the new model') 18 | args = parser.parse_args() 19 | return args 20 | 21 | 22 | def main(): 23 | args = parse_args() 24 | mkdir_or_exist(args.dst_dir) 25 | 26 | all_files = os.listdir(args.src_dir) 27 | for name in all_files: 28 | if not name.startswith(('pytorch_model', '.')): 29 | src_path = osp.join(args.src_dir, name) 30 | dst_path = osp.join(args.dst_dir, name) 31 | shutil.copy(src_path, dst_path) 32 | 33 | with open(osp.join(args.src_dir, 'pytorch_model.bin.index.json')) as f: 34 | index = json.load(f) 35 | 36 | n_shard = len(index['weight_map']) 37 | new_index = copy.deepcopy(index) 38 | new_index['weight_map'] = {} 39 | cnt = 1 40 | 41 | checkpoints = set(index['weight_map'].values()) 42 | for ckpt in checkpoints: 43 | state_dict = torch.load( 44 | osp.join(args.src_dir, ckpt), map_location='cuda') 45 | keys = sorted(list(state_dict.keys())) 46 | for k in keys: 47 | new_state_dict_name = 'pytorch_model-{:05d}-of-{:05d}.bin'.format( 48 | cnt, n_shard) 49 | new_index['weight_map'][k] = new_state_dict_name 50 | new_state_dict = {k: state_dict[k]} 51 | torch.save(new_state_dict, 52 | osp.join(args.dst_dir, new_state_dict_name)) 53 | cnt += 1 54 | del state_dict 55 | torch.cuda.empty_cache() 56 | with open(osp.join(args.dst_dir, 'pytorch_model.bin.index.json'), 57 | 'w') as f: 58 | json.dump(new_index, f) 59 | assert new_index['weight_map'].keys() == index['weight_map'].keys( 60 | ), 'Mismatch on `weight_map`!' 61 | 62 | 63 | if __name__ == '__main__': 64 | main() 65 | -------------------------------------------------------------------------------- /xtuner/tools/plugins/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from .api import plugins_api 3 | 4 | __all__ = ['plugins_api'] 5 | -------------------------------------------------------------------------------- /xtuner/tools/plugins/api.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | import re 3 | 4 | 5 | def plugins_api(input_str, 6 | calculate_open=True, 7 | solve_open=True, 8 | search_open=True): 9 | 10 | pattern = r'(Solve|solve|Solver|solver|Calculate|calculate|Calculator|calculator|Search)\("([^"]*)"\)' # noqa: E501 11 | 12 | matches = re.findall(pattern, input_str) 13 | 14 | converted_str = '<|Results|>:\n' 15 | 16 | for i in range(len(matches)): 17 | if matches[i][0] in [ 18 | 'Calculate', 'calculate' 19 | 'Calculator', 'calculator' 20 | ]: 21 | if calculate_open: 22 | from .calculate import Calculate 23 | result = Calculate(matches[i][1]) 24 | else: 25 | result = None 26 | converted_str += f"Calculate(\"{matches[i][1]}\") => {result}\n" 27 | elif matches[i][0] in ['Solve', 'solve', 'Solver', 'solver']: 28 | if solve_open: 29 | from .solve import Solve 30 | result = Solve(matches[i][1]) 31 | else: 32 | result = None 33 | converted_str += f"Solve(\"{matches[i][1]}\") =>\n{result}\n" 34 | elif matches[i][0] == 'Search': 35 | if search_open: 36 | from .search import Search 37 | result = Search(matches[i][1]) 38 | else: 39 | result = None 40 | converted_str += f"Search(\"{matches[i][1]}\") =>\n{result}" 41 | 42 | converted_str += '\n' 43 | return converted_str 44 | -------------------------------------------------------------------------------- /xtuner/tools/plugins/calculate.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from math import * # noqa: F401, F403 3 | 4 | 5 | def Calculate(expression): 6 | res = '' 7 | for exp in expression.split(';'): 8 | try: 9 | res += '{:.2f};'.format(eval(exp.replace('^', '**'))) 10 | except Exception: 11 | res += 'No result.' 12 | if res[-1] == ';': 13 | res = res[:-1] 14 | return res 15 | -------------------------------------------------------------------------------- /xtuner/tools/plugins/search.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | import os 3 | import sys 4 | 5 | import requests 6 | 7 | try: 8 | SERPER_API_KEY = os.environ['SERPER_API_KEY'] 9 | except Exception: 10 | print('Please obtain the `SERPER_API_KEY` from https://serper.dev and ' 11 | 'set it using `export SERPER_API_KEY=xxx`.') 12 | sys.exit(1) 13 | 14 | 15 | def parse_results(results, k=10): 16 | snippets = [] 17 | 18 | for result in results['organic'][:k]: 19 | if 'snippet' in result: 20 | snippets.append(result['snippet']) 21 | for attribute, value in result.get('attributes', {}).items(): 22 | snippets.append(f'{attribute}: {value}.') 23 | return snippets 24 | 25 | 26 | def search(api_key, search_term, **kwargs): 27 | headers = { 28 | 'X-API-KEY': api_key, 29 | 'Content-Type': 'application/json', 30 | } 31 | params = { 32 | 'q': search_term, 33 | **{key: value 34 | for key, value in kwargs.items() if value is not None}, 35 | } 36 | try: 37 | response = requests.post( 38 | 'https://google.serper.dev/search', 39 | headers=headers, 40 | params=params, 41 | timeout=5) 42 | except Exception as e: 43 | return -1, str(e) 44 | return response.status_code, response.json() 45 | 46 | 47 | def Search(q, k=10): 48 | status_code, response = search(SERPER_API_KEY, q) 49 | if status_code != 200: 50 | ret = 'None\n' 51 | else: 52 | text = parse_results(response, k=k) 53 | ret = '' 54 | for idx, res in enumerate(text): 55 | ret += f"<|{idx+1}|>: '{res}'\n" 56 | return ret 57 | -------------------------------------------------------------------------------- /xtuner/tools/plugins/solve.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | import math 3 | import re 4 | from math import * # noqa: F401, F403 5 | 6 | from sympy import Eq, solve, symbols 7 | 8 | from .calculate import Calculate 9 | 10 | 11 | def Solve(equations_str): 12 | try: 13 | equations_str = equations_str.replace(' ', '') 14 | equations_ori = re.split(r'[,;]+', equations_str) 15 | equations_str = equations_str.replace('^', '**') 16 | equations_str = re.sub(r'(\(.*\))([a-zA-Z])', r'\1 * \2', 17 | equations_str) 18 | equations_str = re.sub(r'(\d+)([a-zA-Z])', r'\1 * \2', equations_str) 19 | equations_str = equations_str.replace('pi', str(math.pi)) 20 | equations = re.split(r'[,;]+', equations_str) 21 | vars_list = list(set(re.findall(r'[a-zA-Z]+', equations_str))) 22 | vars = {var: symbols(var) for var in vars_list} 23 | 24 | output = '' 25 | eqs = [] 26 | for eq in equations: 27 | if '=' in eq: 28 | left, right = eq.split('=') 29 | eqs.append( 30 | Eq( 31 | eval(left.strip(), {}, vars), 32 | eval(right.strip(), {}, vars))) 33 | solutions = solve(eqs, vars, dict=True) 34 | 35 | vars_values = {var: [] for var in vars_list} 36 | if isinstance(solutions, list): 37 | for idx, solution in enumerate(solutions): 38 | for var, sol in solution.items(): 39 | output += f'{var}_{idx} = {sol}\n' 40 | vars_values[str(var)].append(sol) 41 | else: 42 | for var, sol in solutions.items(): 43 | output += f'{var} = {sol}\n' 44 | vars_values[str(var)].append(sol) 45 | for eq, eq_o in zip(equations, equations_ori): 46 | if '=' not in eq: 47 | for var in vars_list: 48 | need_note = True if len(vars_values[var]) > 1 else False 49 | for idx, value in enumerate(vars_values[var]): 50 | eq_to_calc = eq.replace(var, str(value)) 51 | calc_result = Calculate(eq_to_calc) 52 | if need_note: 53 | eq_name = eq_o.replace(var, f'{var}_{idx}') 54 | else: 55 | eq_name = eq_o 56 | if calc_result != 'No results.': 57 | output += f'{eq_name} = {calc_result}\n' 58 | 59 | return output.strip() 60 | except Exception: 61 | return 'No result.' 62 | -------------------------------------------------------------------------------- /xtuner/tools/process_untokenized_llava_data.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | import argparse 3 | import warnings 4 | 5 | from mmengine import Config 6 | 7 | from xtuner.registry import BUILDER 8 | 9 | # ignore FutureWarning in hf datasets 10 | warnings.simplefilter(action='ignore', category=FutureWarning) 11 | 12 | 13 | def parse_args(): 14 | parser = argparse.ArgumentParser() 15 | parser.add_argument('config', help='config file name or path.') 16 | parser.add_argument('--save-folder', help='The folder to save data order.') 17 | args = parser.parse_args() 18 | return args 19 | 20 | 21 | def build_llava_dataset(config): 22 | dataset = BUILDER.build(config.train_dataloader.dataset) 23 | return dataset 24 | 25 | 26 | if __name__ == '__main__': 27 | args = parse_args() 28 | cfg = Config.fromfile(args.config) 29 | 30 | llava_dataset = build_llava_dataset(cfg) 31 | text_data = llava_dataset.text_data 32 | 33 | text_data.save_to_disk(args.save_folder) 34 | -------------------------------------------------------------------------------- /xtuner/utils/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from .constants import (DEFAULT_IMAGE_TOKEN, DEFAULT_PAD_TOKEN_INDEX, 3 | IGNORE_INDEX, IMAGE_TOKEN_INDEX) 4 | from .stop_criteria import StopWordStoppingCriteria 5 | from .templates import PROMPT_TEMPLATE, SYSTEM_TEMPLATE 6 | 7 | __all__ = [ 8 | 'IGNORE_INDEX', 'DEFAULT_PAD_TOKEN_INDEX', 'PROMPT_TEMPLATE', 9 | 'DEFAULT_IMAGE_TOKEN', 'SYSTEM_TEMPLATE', 'StopWordStoppingCriteria', 10 | 'IMAGE_TOKEN_INDEX' 11 | ] 12 | -------------------------------------------------------------------------------- /xtuner/utils/__pycache__/__init__.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/utils/__pycache__/__init__.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/utils/__pycache__/constants.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/utils/__pycache__/constants.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/utils/__pycache__/fileio.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/utils/__pycache__/fileio.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/utils/__pycache__/stop_criteria.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/utils/__pycache__/stop_criteria.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/utils/__pycache__/templates.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PhoenixZ810/MG-LLaVA/3237152d4697416baa739a8b4eaa544d4ea3499c/xtuner/utils/__pycache__/templates.cpython-310.pyc -------------------------------------------------------------------------------- /xtuner/utils/constants.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | IGNORE_INDEX = -100 3 | DEFAULT_PAD_TOKEN_INDEX = 0 4 | IMAGE_TOKEN_INDEX = -200 5 | DEFAULT_IMAGE_TOKEN = '' 6 | -------------------------------------------------------------------------------- /xtuner/utils/stop_criteria.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | from transformers import StoppingCriteria 3 | 4 | 5 | class StopWordStoppingCriteria(StoppingCriteria): 6 | """StopWord stopping criteria.""" 7 | 8 | def __init__(self, tokenizer, stop_word): 9 | self.tokenizer = tokenizer 10 | self.stop_word = stop_word 11 | self.length = len(self.stop_word) 12 | 13 | def __call__(self, input_ids, *args, **kwargs) -> bool: 14 | cur_text = self.tokenizer.decode(input_ids[0]) 15 | cur_text = cur_text.replace('\r', '').replace('\n', '') 16 | return cur_text[-self.length:] == self.stop_word 17 | -------------------------------------------------------------------------------- /xtuner/version.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) OpenMMLab. All rights reserved. 2 | __version__ = '0.1.18' 3 | short_version = __version__ 4 | 5 | 6 | def parse_version_info(version_str): 7 | """Parse a version string into a tuple. 8 | 9 | Args: 10 | version_str (str): The version string. 11 | Returns: 12 | tuple[int or str]: The version info, e.g., "1.3.0" is parsed into 13 | (1, 3, 0), and "2.0.0rc1" is parsed into (2, 0, 0, 'rc1'). 14 | """ 15 | version_info = [] 16 | for x in version_str.split('.'): 17 | if x.isdigit(): 18 | version_info.append(int(x)) 19 | elif x.find('rc') != -1: 20 | patch_version = x.split('rc') 21 | version_info.append(int(patch_version[0])) 22 | version_info.append(f'rc{patch_version[1]}') 23 | return tuple(version_info) 24 | 25 | 26 | version_info = parse_version_info(__version__) 27 | --------------------------------------------------------------------------------