├── .gitignore ├── LICENSE ├── README.md ├── assets └── architecture.png ├── eval.py ├── pyproject.toml ├── scripts ├── eval │ ├── lvbench.sh │ ├── mlvu.sh │ ├── mmvu.sh │ ├── mvbench.sh │ └── videomme.sh ├── train │ ├── gemma │ │ ├── finetune_gemma_video.sh │ │ ├── pretrain_gemma_video.sh │ │ └── train_gemma_video.sh │ ├── phi │ │ ├── finetune_video.sh │ │ ├── pretrain_video.sh │ │ └── train_phi_video.sh │ └── qwen2 │ │ ├── finetune_qwen2_video.sh │ │ ├── pretrain_qwen2_video.sh │ │ └── train_qwen2_base_video.sh ├── zero2.json └── zero3.json └── tinyllava ├── __init__.py ├── data ├── __init__.py ├── dataset.py ├── image_preprocess.py ├── template │ ├── __init__.py │ ├── base.py │ ├── formatter.py │ ├── gemma_template.py │ ├── phi_template.py │ ├── pretrain_template.py │ └── qwen2_base_template.py ├── text_preprocess.py └── video_preprocess.py ├── eval ├── __init__.py ├── eval_lvbench.py ├── eval_mlvu.py ├── eval_mmvu.py ├── eval_mvbench.py ├── eval_videomme.py ├── lvbenchdataset.py └── run_tiny_llava.py ├── model ├── __init__.py ├── configuration_tinyllava.py ├── connector │ ├── __init__.py │ ├── base.py │ ├── groupresampler.py │ └── naiveresampler.py ├── llm │ ├── __init__.py │ ├── gemma.py │ ├── phi.py │ └── qwen2.py ├── load_model.py ├── modeling_tinyllava.py └── vision_tower │ ├── __init__.py │ ├── base.py │ ├── clip.py │ ├── dinov2.py │ └── siglip.py ├── serve ├── __init__.py ├── app.py ├── cli.py └── examples │ ├── extreme_ironing.jpg │ └── waterview.jpg ├── train ├── __init__.py ├── custom_finetune.py ├── tinyllava_trainer.py └── train.py ├── training_recipe ├── __init__.py ├── base.py └── common_recipe.py └── utils ├── __init__.py ├── arguments.py ├── constants.py ├── data_utils.py ├── eval_utils.py ├── import_module.py ├── logging.py ├── message.py └── train_utils.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/README.md -------------------------------------------------------------------------------- /assets/architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/assets/architecture.png -------------------------------------------------------------------------------- /eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/eval.py -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/pyproject.toml -------------------------------------------------------------------------------- /scripts/eval/lvbench.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/scripts/eval/lvbench.sh -------------------------------------------------------------------------------- /scripts/eval/mlvu.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/scripts/eval/mlvu.sh -------------------------------------------------------------------------------- /scripts/eval/mmvu.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/scripts/eval/mmvu.sh -------------------------------------------------------------------------------- /scripts/eval/mvbench.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/scripts/eval/mvbench.sh -------------------------------------------------------------------------------- /scripts/eval/videomme.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/scripts/eval/videomme.sh -------------------------------------------------------------------------------- /scripts/train/gemma/finetune_gemma_video.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/scripts/train/gemma/finetune_gemma_video.sh -------------------------------------------------------------------------------- /scripts/train/gemma/pretrain_gemma_video.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/scripts/train/gemma/pretrain_gemma_video.sh -------------------------------------------------------------------------------- /scripts/train/gemma/train_gemma_video.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/scripts/train/gemma/train_gemma_video.sh -------------------------------------------------------------------------------- /scripts/train/phi/finetune_video.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/scripts/train/phi/finetune_video.sh -------------------------------------------------------------------------------- /scripts/train/phi/pretrain_video.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/scripts/train/phi/pretrain_video.sh -------------------------------------------------------------------------------- /scripts/train/phi/train_phi_video.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/scripts/train/phi/train_phi_video.sh -------------------------------------------------------------------------------- /scripts/train/qwen2/finetune_qwen2_video.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/scripts/train/qwen2/finetune_qwen2_video.sh -------------------------------------------------------------------------------- /scripts/train/qwen2/pretrain_qwen2_video.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/scripts/train/qwen2/pretrain_qwen2_video.sh -------------------------------------------------------------------------------- /scripts/train/qwen2/train_qwen2_base_video.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/scripts/train/qwen2/train_qwen2_base_video.sh -------------------------------------------------------------------------------- /scripts/zero2.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/scripts/zero2.json -------------------------------------------------------------------------------- /scripts/zero3.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/scripts/zero3.json -------------------------------------------------------------------------------- /tinyllava/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tinyllava/data/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/tinyllava/data/__init__.py -------------------------------------------------------------------------------- /tinyllava/data/dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/tinyllava/data/dataset.py -------------------------------------------------------------------------------- /tinyllava/data/image_preprocess.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/tinyllava/data/image_preprocess.py -------------------------------------------------------------------------------- /tinyllava/data/template/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/tinyllava/data/template/__init__.py -------------------------------------------------------------------------------- /tinyllava/data/template/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/tinyllava/data/template/base.py -------------------------------------------------------------------------------- /tinyllava/data/template/formatter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/tinyllava/data/template/formatter.py -------------------------------------------------------------------------------- /tinyllava/data/template/gemma_template.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/tinyllava/data/template/gemma_template.py -------------------------------------------------------------------------------- /tinyllava/data/template/phi_template.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/tinyllava/data/template/phi_template.py -------------------------------------------------------------------------------- /tinyllava/data/template/pretrain_template.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/tinyllava/data/template/pretrain_template.py -------------------------------------------------------------------------------- /tinyllava/data/template/qwen2_base_template.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/tinyllava/data/template/qwen2_base_template.py -------------------------------------------------------------------------------- /tinyllava/data/text_preprocess.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/tinyllava/data/text_preprocess.py -------------------------------------------------------------------------------- /tinyllava/data/video_preprocess.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/tinyllava/data/video_preprocess.py -------------------------------------------------------------------------------- /tinyllava/eval/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tinyllava/eval/eval_lvbench.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/tinyllava/eval/eval_lvbench.py -------------------------------------------------------------------------------- /tinyllava/eval/eval_mlvu.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/tinyllava/eval/eval_mlvu.py -------------------------------------------------------------------------------- /tinyllava/eval/eval_mmvu.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/tinyllava/eval/eval_mmvu.py -------------------------------------------------------------------------------- /tinyllava/eval/eval_mvbench.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/tinyllava/eval/eval_mvbench.py -------------------------------------------------------------------------------- /tinyllava/eval/eval_videomme.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/tinyllava/eval/eval_videomme.py -------------------------------------------------------------------------------- /tinyllava/eval/lvbenchdataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/tinyllava/eval/lvbenchdataset.py -------------------------------------------------------------------------------- /tinyllava/eval/run_tiny_llava.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/tinyllava/eval/run_tiny_llava.py -------------------------------------------------------------------------------- /tinyllava/model/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/tinyllava/model/__init__.py -------------------------------------------------------------------------------- /tinyllava/model/configuration_tinyllava.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/tinyllava/model/configuration_tinyllava.py -------------------------------------------------------------------------------- /tinyllava/model/connector/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/tinyllava/model/connector/__init__.py -------------------------------------------------------------------------------- /tinyllava/model/connector/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/tinyllava/model/connector/base.py -------------------------------------------------------------------------------- /tinyllava/model/connector/groupresampler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/tinyllava/model/connector/groupresampler.py -------------------------------------------------------------------------------- /tinyllava/model/connector/naiveresampler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/tinyllava/model/connector/naiveresampler.py -------------------------------------------------------------------------------- /tinyllava/model/llm/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/tinyllava/model/llm/__init__.py -------------------------------------------------------------------------------- /tinyllava/model/llm/gemma.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/tinyllava/model/llm/gemma.py -------------------------------------------------------------------------------- /tinyllava/model/llm/phi.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/tinyllava/model/llm/phi.py -------------------------------------------------------------------------------- /tinyllava/model/llm/qwen2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/tinyllava/model/llm/qwen2.py -------------------------------------------------------------------------------- /tinyllava/model/load_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/tinyllava/model/load_model.py -------------------------------------------------------------------------------- /tinyllava/model/modeling_tinyllava.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/tinyllava/model/modeling_tinyllava.py -------------------------------------------------------------------------------- /tinyllava/model/vision_tower/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/tinyllava/model/vision_tower/__init__.py -------------------------------------------------------------------------------- /tinyllava/model/vision_tower/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/tinyllava/model/vision_tower/base.py -------------------------------------------------------------------------------- /tinyllava/model/vision_tower/clip.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/tinyllava/model/vision_tower/clip.py -------------------------------------------------------------------------------- /tinyllava/model/vision_tower/dinov2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/tinyllava/model/vision_tower/dinov2.py -------------------------------------------------------------------------------- /tinyllava/model/vision_tower/siglip.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/tinyllava/model/vision_tower/siglip.py -------------------------------------------------------------------------------- /tinyllava/serve/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tinyllava/serve/app.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/tinyllava/serve/app.py -------------------------------------------------------------------------------- /tinyllava/serve/cli.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/tinyllava/serve/cli.py -------------------------------------------------------------------------------- /tinyllava/serve/examples/extreme_ironing.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/tinyllava/serve/examples/extreme_ironing.jpg -------------------------------------------------------------------------------- /tinyllava/serve/examples/waterview.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/tinyllava/serve/examples/waterview.jpg -------------------------------------------------------------------------------- /tinyllava/train/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/tinyllava/train/__init__.py -------------------------------------------------------------------------------- /tinyllava/train/custom_finetune.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/tinyllava/train/custom_finetune.py -------------------------------------------------------------------------------- /tinyllava/train/tinyllava_trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/tinyllava/train/tinyllava_trainer.py -------------------------------------------------------------------------------- /tinyllava/train/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/tinyllava/train/train.py -------------------------------------------------------------------------------- /tinyllava/training_recipe/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/tinyllava/training_recipe/__init__.py -------------------------------------------------------------------------------- /tinyllava/training_recipe/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/tinyllava/training_recipe/base.py -------------------------------------------------------------------------------- /tinyllava/training_recipe/common_recipe.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/tinyllava/training_recipe/common_recipe.py -------------------------------------------------------------------------------- /tinyllava/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/tinyllava/utils/__init__.py -------------------------------------------------------------------------------- /tinyllava/utils/arguments.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/tinyllava/utils/arguments.py -------------------------------------------------------------------------------- /tinyllava/utils/constants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/tinyllava/utils/constants.py -------------------------------------------------------------------------------- /tinyllava/utils/data_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/tinyllava/utils/data_utils.py -------------------------------------------------------------------------------- /tinyllava/utils/eval_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/tinyllava/utils/eval_utils.py -------------------------------------------------------------------------------- /tinyllava/utils/import_module.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/tinyllava/utils/import_module.py -------------------------------------------------------------------------------- /tinyllava/utils/logging.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/tinyllava/utils/logging.py -------------------------------------------------------------------------------- /tinyllava/utils/message.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/tinyllava/utils/message.py -------------------------------------------------------------------------------- /tinyllava/utils/train_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ZhangXJ199/TinyLLaVA-Video/HEAD/tinyllava/utils/train_utils.py --------------------------------------------------------------------------------