├── README.md ├── eval ├── refcoco │ ├── evaluate_grounding.py │ └── evaluate_grounding_modelsplit.py ├── seed │ ├── calculation.py │ ├── evaluate_seed.py │ └── evaluate_seed_modelsplit.py └── vqa │ ├── analyze_small_large_model_results.py │ ├── convert_gqa_for_eval.py │ ├── evaluate_vqa.py │ ├── evaluate_vqa_modelsplit.py │ ├── infographicsvqa_eval.py │ └── textvqa_eval.py ├── internvl ├── conversation.py ├── dist_utils.py ├── model │ ├── __init__.py │ ├── internlm2 │ │ ├── configuration_internlm2.py │ │ ├── modeling_internlm2.py │ │ ├── tokenization_internlm2.py │ │ └── tokenization_internlm2_fast.py │ ├── internvl_chat │ │ ├── __init__.py │ │ ├── configuration_intern_vit.py │ │ ├── configuration_internvl_chat.py │ │ ├── flash_attention.py │ │ ├── modeling_intern_vit.py │ │ └── modeling_internvl_chat.py │ ├── llama │ │ ├── __init__.py │ │ ├── configuration_llama.py │ │ ├── convert_llama_weights_to_hf.py │ │ ├── modeling_flax_llama.py │ │ ├── modeling_llama.py │ │ ├── tokenization_llama.py │ │ └── tokenization_llama_fast.py │ ├── phi3 │ │ ├── configuration_phi3.py │ │ └── modeling_phi3.py │ └── qwen2 │ │ ├── __init__.py │ │ ├── configuration_qwen2.py │ │ ├── modeling_qwen2.py │ │ ├── tokenization_qwen2.py │ │ └── tokenization_qwen2_fast.py ├── patch │ ├── __init__.py │ ├── llama2_flash_attn_monkey_patch.py │ ├── llama_flash_attn_monkey_patch.py │ ├── llama_rmsnorm_monkey_patch.py │ ├── pad_data_collator.py │ └── train_sampler_patch.py └── train │ ├── __init__.py │ ├── constants.py │ ├── dataset.py │ ├── internvl_chat_finetune.py │ ├── internvl_chat_pretrain.py │ └── trainer_monkey_patch.py ├── logo.png ├── misc.py ├── run.py ├── textvqa2B-26B.sh └── tools ├── convert_parquet.py ├── convert_to_int8.py ├── extract_mlp.py ├── extract_video_frames.py ├── extract_vit.py ├── json2jsonl.py ├── jsonl2jsonl.py ├── replace_llm.py └── resize_pos_embed.py /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NUS-HPC-AI-Lab/SGL/HEAD/README.md -------------------------------------------------------------------------------- /eval/refcoco/evaluate_grounding.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NUS-HPC-AI-Lab/SGL/HEAD/eval/refcoco/evaluate_grounding.py -------------------------------------------------------------------------------- /eval/refcoco/evaluate_grounding_modelsplit.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NUS-HPC-AI-Lab/SGL/HEAD/eval/refcoco/evaluate_grounding_modelsplit.py -------------------------------------------------------------------------------- /eval/seed/calculation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NUS-HPC-AI-Lab/SGL/HEAD/eval/seed/calculation.py -------------------------------------------------------------------------------- /eval/seed/evaluate_seed.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NUS-HPC-AI-Lab/SGL/HEAD/eval/seed/evaluate_seed.py -------------------------------------------------------------------------------- /eval/seed/evaluate_seed_modelsplit.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NUS-HPC-AI-Lab/SGL/HEAD/eval/seed/evaluate_seed_modelsplit.py -------------------------------------------------------------------------------- /eval/vqa/analyze_small_large_model_results.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NUS-HPC-AI-Lab/SGL/HEAD/eval/vqa/analyze_small_large_model_results.py -------------------------------------------------------------------------------- /eval/vqa/convert_gqa_for_eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NUS-HPC-AI-Lab/SGL/HEAD/eval/vqa/convert_gqa_for_eval.py -------------------------------------------------------------------------------- /eval/vqa/evaluate_vqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NUS-HPC-AI-Lab/SGL/HEAD/eval/vqa/evaluate_vqa.py -------------------------------------------------------------------------------- /eval/vqa/evaluate_vqa_modelsplit.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NUS-HPC-AI-Lab/SGL/HEAD/eval/vqa/evaluate_vqa_modelsplit.py -------------------------------------------------------------------------------- /eval/vqa/infographicsvqa_eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NUS-HPC-AI-Lab/SGL/HEAD/eval/vqa/infographicsvqa_eval.py -------------------------------------------------------------------------------- /eval/vqa/textvqa_eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NUS-HPC-AI-Lab/SGL/HEAD/eval/vqa/textvqa_eval.py -------------------------------------------------------------------------------- /internvl/conversation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NUS-HPC-AI-Lab/SGL/HEAD/internvl/conversation.py -------------------------------------------------------------------------------- /internvl/dist_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NUS-HPC-AI-Lab/SGL/HEAD/internvl/dist_utils.py -------------------------------------------------------------------------------- /internvl/model/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /internvl/model/internlm2/configuration_internlm2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NUS-HPC-AI-Lab/SGL/HEAD/internvl/model/internlm2/configuration_internlm2.py -------------------------------------------------------------------------------- /internvl/model/internlm2/modeling_internlm2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NUS-HPC-AI-Lab/SGL/HEAD/internvl/model/internlm2/modeling_internlm2.py -------------------------------------------------------------------------------- /internvl/model/internlm2/tokenization_internlm2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NUS-HPC-AI-Lab/SGL/HEAD/internvl/model/internlm2/tokenization_internlm2.py -------------------------------------------------------------------------------- /internvl/model/internlm2/tokenization_internlm2_fast.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NUS-HPC-AI-Lab/SGL/HEAD/internvl/model/internlm2/tokenization_internlm2_fast.py -------------------------------------------------------------------------------- /internvl/model/internvl_chat/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NUS-HPC-AI-Lab/SGL/HEAD/internvl/model/internvl_chat/__init__.py -------------------------------------------------------------------------------- /internvl/model/internvl_chat/configuration_intern_vit.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NUS-HPC-AI-Lab/SGL/HEAD/internvl/model/internvl_chat/configuration_intern_vit.py -------------------------------------------------------------------------------- /internvl/model/internvl_chat/configuration_internvl_chat.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NUS-HPC-AI-Lab/SGL/HEAD/internvl/model/internvl_chat/configuration_internvl_chat.py -------------------------------------------------------------------------------- /internvl/model/internvl_chat/flash_attention.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NUS-HPC-AI-Lab/SGL/HEAD/internvl/model/internvl_chat/flash_attention.py -------------------------------------------------------------------------------- /internvl/model/internvl_chat/modeling_intern_vit.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NUS-HPC-AI-Lab/SGL/HEAD/internvl/model/internvl_chat/modeling_intern_vit.py -------------------------------------------------------------------------------- /internvl/model/internvl_chat/modeling_internvl_chat.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NUS-HPC-AI-Lab/SGL/HEAD/internvl/model/internvl_chat/modeling_internvl_chat.py -------------------------------------------------------------------------------- /internvl/model/llama/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NUS-HPC-AI-Lab/SGL/HEAD/internvl/model/llama/__init__.py -------------------------------------------------------------------------------- /internvl/model/llama/configuration_llama.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NUS-HPC-AI-Lab/SGL/HEAD/internvl/model/llama/configuration_llama.py -------------------------------------------------------------------------------- /internvl/model/llama/convert_llama_weights_to_hf.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NUS-HPC-AI-Lab/SGL/HEAD/internvl/model/llama/convert_llama_weights_to_hf.py -------------------------------------------------------------------------------- /internvl/model/llama/modeling_flax_llama.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NUS-HPC-AI-Lab/SGL/HEAD/internvl/model/llama/modeling_flax_llama.py -------------------------------------------------------------------------------- /internvl/model/llama/modeling_llama.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NUS-HPC-AI-Lab/SGL/HEAD/internvl/model/llama/modeling_llama.py -------------------------------------------------------------------------------- /internvl/model/llama/tokenization_llama.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NUS-HPC-AI-Lab/SGL/HEAD/internvl/model/llama/tokenization_llama.py -------------------------------------------------------------------------------- /internvl/model/llama/tokenization_llama_fast.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NUS-HPC-AI-Lab/SGL/HEAD/internvl/model/llama/tokenization_llama_fast.py -------------------------------------------------------------------------------- /internvl/model/phi3/configuration_phi3.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NUS-HPC-AI-Lab/SGL/HEAD/internvl/model/phi3/configuration_phi3.py -------------------------------------------------------------------------------- /internvl/model/phi3/modeling_phi3.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NUS-HPC-AI-Lab/SGL/HEAD/internvl/model/phi3/modeling_phi3.py -------------------------------------------------------------------------------- /internvl/model/qwen2/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NUS-HPC-AI-Lab/SGL/HEAD/internvl/model/qwen2/__init__.py -------------------------------------------------------------------------------- /internvl/model/qwen2/configuration_qwen2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NUS-HPC-AI-Lab/SGL/HEAD/internvl/model/qwen2/configuration_qwen2.py -------------------------------------------------------------------------------- /internvl/model/qwen2/modeling_qwen2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NUS-HPC-AI-Lab/SGL/HEAD/internvl/model/qwen2/modeling_qwen2.py -------------------------------------------------------------------------------- /internvl/model/qwen2/tokenization_qwen2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NUS-HPC-AI-Lab/SGL/HEAD/internvl/model/qwen2/tokenization_qwen2.py -------------------------------------------------------------------------------- /internvl/model/qwen2/tokenization_qwen2_fast.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NUS-HPC-AI-Lab/SGL/HEAD/internvl/model/qwen2/tokenization_qwen2_fast.py -------------------------------------------------------------------------------- /internvl/patch/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NUS-HPC-AI-Lab/SGL/HEAD/internvl/patch/__init__.py -------------------------------------------------------------------------------- /internvl/patch/llama2_flash_attn_monkey_patch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NUS-HPC-AI-Lab/SGL/HEAD/internvl/patch/llama2_flash_attn_monkey_patch.py -------------------------------------------------------------------------------- /internvl/patch/llama_flash_attn_monkey_patch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NUS-HPC-AI-Lab/SGL/HEAD/internvl/patch/llama_flash_attn_monkey_patch.py -------------------------------------------------------------------------------- /internvl/patch/llama_rmsnorm_monkey_patch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NUS-HPC-AI-Lab/SGL/HEAD/internvl/patch/llama_rmsnorm_monkey_patch.py -------------------------------------------------------------------------------- /internvl/patch/pad_data_collator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NUS-HPC-AI-Lab/SGL/HEAD/internvl/patch/pad_data_collator.py -------------------------------------------------------------------------------- /internvl/patch/train_sampler_patch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NUS-HPC-AI-Lab/SGL/HEAD/internvl/patch/train_sampler_patch.py -------------------------------------------------------------------------------- /internvl/train/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /internvl/train/constants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NUS-HPC-AI-Lab/SGL/HEAD/internvl/train/constants.py -------------------------------------------------------------------------------- /internvl/train/dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NUS-HPC-AI-Lab/SGL/HEAD/internvl/train/dataset.py -------------------------------------------------------------------------------- /internvl/train/internvl_chat_finetune.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NUS-HPC-AI-Lab/SGL/HEAD/internvl/train/internvl_chat_finetune.py -------------------------------------------------------------------------------- /internvl/train/internvl_chat_pretrain.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NUS-HPC-AI-Lab/SGL/HEAD/internvl/train/internvl_chat_pretrain.py -------------------------------------------------------------------------------- /internvl/train/trainer_monkey_patch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NUS-HPC-AI-Lab/SGL/HEAD/internvl/train/trainer_monkey_patch.py -------------------------------------------------------------------------------- /logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NUS-HPC-AI-Lab/SGL/HEAD/logo.png -------------------------------------------------------------------------------- /misc.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NUS-HPC-AI-Lab/SGL/HEAD/misc.py -------------------------------------------------------------------------------- /run.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NUS-HPC-AI-Lab/SGL/HEAD/run.py -------------------------------------------------------------------------------- /textvqa2B-26B.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NUS-HPC-AI-Lab/SGL/HEAD/textvqa2B-26B.sh -------------------------------------------------------------------------------- /tools/convert_parquet.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NUS-HPC-AI-Lab/SGL/HEAD/tools/convert_parquet.py -------------------------------------------------------------------------------- /tools/convert_to_int8.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NUS-HPC-AI-Lab/SGL/HEAD/tools/convert_to_int8.py -------------------------------------------------------------------------------- /tools/extract_mlp.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NUS-HPC-AI-Lab/SGL/HEAD/tools/extract_mlp.py -------------------------------------------------------------------------------- /tools/extract_video_frames.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NUS-HPC-AI-Lab/SGL/HEAD/tools/extract_video_frames.py -------------------------------------------------------------------------------- /tools/extract_vit.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NUS-HPC-AI-Lab/SGL/HEAD/tools/extract_vit.py -------------------------------------------------------------------------------- /tools/json2jsonl.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NUS-HPC-AI-Lab/SGL/HEAD/tools/json2jsonl.py -------------------------------------------------------------------------------- /tools/jsonl2jsonl.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NUS-HPC-AI-Lab/SGL/HEAD/tools/jsonl2jsonl.py -------------------------------------------------------------------------------- /tools/replace_llm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NUS-HPC-AI-Lab/SGL/HEAD/tools/replace_llm.py -------------------------------------------------------------------------------- /tools/resize_pos_embed.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NUS-HPC-AI-Lab/SGL/HEAD/tools/resize_pos_embed.py --------------------------------------------------------------------------------