├── .gitignore ├── LICENSE ├── README.md ├── assets ├── magic.mp4 └── old_man.png ├── environment.yaml ├── eval ├── eval_houreval.py ├── eval_hourvideo.py ├── eval_longvideobench.py ├── eval_lvbench.py ├── eval_mlvu.py ├── eval_mvbench.py ├── eval_nextqa.py ├── eval_videomme.py ├── utils_hourvideo.py ├── utils_longvideobench.py ├── utils_lvbench.py ├── utils_mlvu.py ├── utils_mvbench.py ├── utils_nextqa.py └── utils_videomme.py ├── hf_mtask_trainer ├── __init__.py ├── mixins.py ├── state.py ├── trainer.py └── types.py ├── models ├── __init__.py ├── conversation.py └── vamba_mamba2 │ ├── configuration_qwen2_vl.py │ ├── image_processing_qwen2_vl.py │ ├── mamba2_block.py │ ├── mamba2_mixer.py │ ├── modeling_qwen2_vl.py │ └── processing_qwen2_vl.py ├── scripts ├── pretrain_vamba.sh └── sft_vamba.sh ├── tools ├── chat_utils.py └── vamba_chat.py └── train ├── __init__.py ├── accelerate_configs ├── accelerate_config_ddp.yaml ├── accelerate_config_fsdp.yaml ├── accelerate_config_zero1.yaml ├── accelerate_config_zero2.yaml ├── accelerate_config_zero2_offload.yaml ├── accelerate_config_zero3++.yaml ├── accelerate_config_zero3.yaml ├── accelerate_config_zero3_offload++.yaml ├── accelerate_config_zero3_offload.yaml ├── accelerate_config_zero3_param_offload.yaml ├── accelerate_config_zero3_slurm.yaml └── ds_zero3_config.json ├── conversation.py ├── data.py ├── data_configs ├── data_cc12m_pixelprose_packed.yaml └── data_llava_ov_video_packed.yaml ├── data_utils.py ├── train_utils.py ├── train_vamba.py └── zero_configs ├── zero2.json ├── zero3++.json ├── zero3.json ├── zero3_offload++.json └── zero3_offload.json /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TIGER-AI-Lab/Vamba/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TIGER-AI-Lab/Vamba/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TIGER-AI-Lab/Vamba/HEAD/README.md -------------------------------------------------------------------------------- /assets/magic.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TIGER-AI-Lab/Vamba/HEAD/assets/magic.mp4 -------------------------------------------------------------------------------- /assets/old_man.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TIGER-AI-Lab/Vamba/HEAD/assets/old_man.png -------------------------------------------------------------------------------- /environment.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TIGER-AI-Lab/Vamba/HEAD/environment.yaml -------------------------------------------------------------------------------- /eval/eval_houreval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TIGER-AI-Lab/Vamba/HEAD/eval/eval_houreval.py -------------------------------------------------------------------------------- /eval/eval_hourvideo.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TIGER-AI-Lab/Vamba/HEAD/eval/eval_hourvideo.py -------------------------------------------------------------------------------- /eval/eval_longvideobench.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TIGER-AI-Lab/Vamba/HEAD/eval/eval_longvideobench.py -------------------------------------------------------------------------------- /eval/eval_lvbench.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TIGER-AI-Lab/Vamba/HEAD/eval/eval_lvbench.py -------------------------------------------------------------------------------- /eval/eval_mlvu.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TIGER-AI-Lab/Vamba/HEAD/eval/eval_mlvu.py -------------------------------------------------------------------------------- /eval/eval_mvbench.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TIGER-AI-Lab/Vamba/HEAD/eval/eval_mvbench.py -------------------------------------------------------------------------------- /eval/eval_nextqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TIGER-AI-Lab/Vamba/HEAD/eval/eval_nextqa.py -------------------------------------------------------------------------------- /eval/eval_videomme.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TIGER-AI-Lab/Vamba/HEAD/eval/eval_videomme.py -------------------------------------------------------------------------------- /eval/utils_hourvideo.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TIGER-AI-Lab/Vamba/HEAD/eval/utils_hourvideo.py -------------------------------------------------------------------------------- /eval/utils_longvideobench.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TIGER-AI-Lab/Vamba/HEAD/eval/utils_longvideobench.py -------------------------------------------------------------------------------- /eval/utils_lvbench.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TIGER-AI-Lab/Vamba/HEAD/eval/utils_lvbench.py -------------------------------------------------------------------------------- /eval/utils_mlvu.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TIGER-AI-Lab/Vamba/HEAD/eval/utils_mlvu.py -------------------------------------------------------------------------------- /eval/utils_mvbench.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TIGER-AI-Lab/Vamba/HEAD/eval/utils_mvbench.py -------------------------------------------------------------------------------- /eval/utils_nextqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TIGER-AI-Lab/Vamba/HEAD/eval/utils_nextqa.py -------------------------------------------------------------------------------- /eval/utils_videomme.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TIGER-AI-Lab/Vamba/HEAD/eval/utils_videomme.py -------------------------------------------------------------------------------- /hf_mtask_trainer/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TIGER-AI-Lab/Vamba/HEAD/hf_mtask_trainer/__init__.py -------------------------------------------------------------------------------- /hf_mtask_trainer/mixins.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TIGER-AI-Lab/Vamba/HEAD/hf_mtask_trainer/mixins.py -------------------------------------------------------------------------------- /hf_mtask_trainer/state.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TIGER-AI-Lab/Vamba/HEAD/hf_mtask_trainer/state.py -------------------------------------------------------------------------------- /hf_mtask_trainer/trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TIGER-AI-Lab/Vamba/HEAD/hf_mtask_trainer/trainer.py -------------------------------------------------------------------------------- /hf_mtask_trainer/types.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TIGER-AI-Lab/Vamba/HEAD/hf_mtask_trainer/types.py -------------------------------------------------------------------------------- /models/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /models/conversation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TIGER-AI-Lab/Vamba/HEAD/models/conversation.py -------------------------------------------------------------------------------- /models/vamba_mamba2/configuration_qwen2_vl.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TIGER-AI-Lab/Vamba/HEAD/models/vamba_mamba2/configuration_qwen2_vl.py -------------------------------------------------------------------------------- /models/vamba_mamba2/image_processing_qwen2_vl.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TIGER-AI-Lab/Vamba/HEAD/models/vamba_mamba2/image_processing_qwen2_vl.py -------------------------------------------------------------------------------- /models/vamba_mamba2/mamba2_block.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TIGER-AI-Lab/Vamba/HEAD/models/vamba_mamba2/mamba2_block.py -------------------------------------------------------------------------------- /models/vamba_mamba2/mamba2_mixer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TIGER-AI-Lab/Vamba/HEAD/models/vamba_mamba2/mamba2_mixer.py -------------------------------------------------------------------------------- /models/vamba_mamba2/modeling_qwen2_vl.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TIGER-AI-Lab/Vamba/HEAD/models/vamba_mamba2/modeling_qwen2_vl.py -------------------------------------------------------------------------------- /models/vamba_mamba2/processing_qwen2_vl.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TIGER-AI-Lab/Vamba/HEAD/models/vamba_mamba2/processing_qwen2_vl.py -------------------------------------------------------------------------------- /scripts/pretrain_vamba.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TIGER-AI-Lab/Vamba/HEAD/scripts/pretrain_vamba.sh -------------------------------------------------------------------------------- /scripts/sft_vamba.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TIGER-AI-Lab/Vamba/HEAD/scripts/sft_vamba.sh -------------------------------------------------------------------------------- /tools/chat_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TIGER-AI-Lab/Vamba/HEAD/tools/chat_utils.py -------------------------------------------------------------------------------- /tools/vamba_chat.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TIGER-AI-Lab/Vamba/HEAD/tools/vamba_chat.py -------------------------------------------------------------------------------- /train/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /train/accelerate_configs/accelerate_config_ddp.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TIGER-AI-Lab/Vamba/HEAD/train/accelerate_configs/accelerate_config_ddp.yaml -------------------------------------------------------------------------------- /train/accelerate_configs/accelerate_config_fsdp.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TIGER-AI-Lab/Vamba/HEAD/train/accelerate_configs/accelerate_config_fsdp.yaml -------------------------------------------------------------------------------- /train/accelerate_configs/accelerate_config_zero1.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TIGER-AI-Lab/Vamba/HEAD/train/accelerate_configs/accelerate_config_zero1.yaml -------------------------------------------------------------------------------- /train/accelerate_configs/accelerate_config_zero2.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TIGER-AI-Lab/Vamba/HEAD/train/accelerate_configs/accelerate_config_zero2.yaml -------------------------------------------------------------------------------- /train/accelerate_configs/accelerate_config_zero2_offload.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TIGER-AI-Lab/Vamba/HEAD/train/accelerate_configs/accelerate_config_zero2_offload.yaml -------------------------------------------------------------------------------- /train/accelerate_configs/accelerate_config_zero3++.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TIGER-AI-Lab/Vamba/HEAD/train/accelerate_configs/accelerate_config_zero3++.yaml -------------------------------------------------------------------------------- /train/accelerate_configs/accelerate_config_zero3.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TIGER-AI-Lab/Vamba/HEAD/train/accelerate_configs/accelerate_config_zero3.yaml -------------------------------------------------------------------------------- /train/accelerate_configs/accelerate_config_zero3_offload++.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TIGER-AI-Lab/Vamba/HEAD/train/accelerate_configs/accelerate_config_zero3_offload++.yaml -------------------------------------------------------------------------------- /train/accelerate_configs/accelerate_config_zero3_offload.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TIGER-AI-Lab/Vamba/HEAD/train/accelerate_configs/accelerate_config_zero3_offload.yaml -------------------------------------------------------------------------------- /train/accelerate_configs/accelerate_config_zero3_param_offload.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TIGER-AI-Lab/Vamba/HEAD/train/accelerate_configs/accelerate_config_zero3_param_offload.yaml -------------------------------------------------------------------------------- /train/accelerate_configs/accelerate_config_zero3_slurm.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TIGER-AI-Lab/Vamba/HEAD/train/accelerate_configs/accelerate_config_zero3_slurm.yaml -------------------------------------------------------------------------------- /train/accelerate_configs/ds_zero3_config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TIGER-AI-Lab/Vamba/HEAD/train/accelerate_configs/ds_zero3_config.json -------------------------------------------------------------------------------- /train/conversation.py: -------------------------------------------------------------------------------- 1 | from models.conversation import * -------------------------------------------------------------------------------- /train/data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TIGER-AI-Lab/Vamba/HEAD/train/data.py -------------------------------------------------------------------------------- /train/data_configs/data_cc12m_pixelprose_packed.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TIGER-AI-Lab/Vamba/HEAD/train/data_configs/data_cc12m_pixelprose_packed.yaml -------------------------------------------------------------------------------- /train/data_configs/data_llava_ov_video_packed.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TIGER-AI-Lab/Vamba/HEAD/train/data_configs/data_llava_ov_video_packed.yaml -------------------------------------------------------------------------------- /train/data_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TIGER-AI-Lab/Vamba/HEAD/train/data_utils.py -------------------------------------------------------------------------------- /train/train_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TIGER-AI-Lab/Vamba/HEAD/train/train_utils.py -------------------------------------------------------------------------------- /train/train_vamba.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TIGER-AI-Lab/Vamba/HEAD/train/train_vamba.py -------------------------------------------------------------------------------- /train/zero_configs/zero2.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TIGER-AI-Lab/Vamba/HEAD/train/zero_configs/zero2.json -------------------------------------------------------------------------------- /train/zero_configs/zero3++.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TIGER-AI-Lab/Vamba/HEAD/train/zero_configs/zero3++.json -------------------------------------------------------------------------------- /train/zero_configs/zero3.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TIGER-AI-Lab/Vamba/HEAD/train/zero_configs/zero3.json -------------------------------------------------------------------------------- /train/zero_configs/zero3_offload++.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TIGER-AI-Lab/Vamba/HEAD/train/zero_configs/zero3_offload++.json -------------------------------------------------------------------------------- /train/zero_configs/zero3_offload.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/TIGER-AI-Lab/Vamba/HEAD/train/zero_configs/zero3_offload.json --------------------------------------------------------------------------------