├── .gitignore ├── README.md ├── app.py ├── captioner ├── README.md ├── SimHei.ttf ├── app.py ├── fast_captioner_lmdeploy.py ├── slide_captioner_lmdeploy.py └── videos_to_describe.json ├── examples ├── C_1_0.mp4 ├── sample_demo_1.mp4 └── yoga.mp4 ├── keyframe_extraction_example.py ├── llava ├── __init__.py ├── constants.py ├── conversation.py ├── eval │ ├── evaluate_benchmark_1_correctness.py │ ├── evaluate_benchmark_2_detailed_orientation.py │ ├── evaluate_benchmark_3_context.py │ ├── evaluate_benchmark_4_temporal.py │ ├── evaluate_benchmark_5_consistency.py │ ├── model_vqa_loader.py │ ├── model_vqa_tempcompass.py │ ├── run_llava.py │ └── video │ │ ├── eval_mvbench.py │ │ ├── eval_vbench.py │ │ ├── general_utils.py │ │ ├── mvbench_utils.py │ │ └── vbench_utils.py ├── mm_utils.py ├── model │ ├── __init__.py │ ├── apply_delta.py │ ├── builder.py │ ├── consolidate.py │ ├── language_model │ │ ├── llava_llama.py │ │ ├── llava_mistral.py │ │ └── llava_mpt.py │ ├── llava_arch.py │ ├── make_delta.py │ ├── multimodal_encoder │ │ ├── builder.py │ │ ├── clip_encoder.py │ │ └── siglip_encoder.py │ ├── multimodal_projector │ │ └── builder.py │ └── utils.py ├── serve │ └── gradio_utils.py ├── train │ ├── llava_trainer.py │ ├── train.py │ └── train_mem.py ├── utils.py └── video_utils.py ├── pyproject.toml ├── run.py └── scripts ├── eval └── 8b │ ├── mvbench.sh │ ├── tempcompass.sh │ └── vbench.sh └── train └── 8b ├── lora_finetune_153k.sh └── lora_full_finetune_181k_add-share28k.sh /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShareGPT4Omni/ShareGPT4Video/HEAD/.gitignore -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShareGPT4Omni/ShareGPT4Video/HEAD/README.md -------------------------------------------------------------------------------- /app.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShareGPT4Omni/ShareGPT4Video/HEAD/app.py -------------------------------------------------------------------------------- /captioner/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShareGPT4Omni/ShareGPT4Video/HEAD/captioner/README.md -------------------------------------------------------------------------------- /captioner/SimHei.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShareGPT4Omni/ShareGPT4Video/HEAD/captioner/SimHei.ttf -------------------------------------------------------------------------------- /captioner/app.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShareGPT4Omni/ShareGPT4Video/HEAD/captioner/app.py -------------------------------------------------------------------------------- /captioner/fast_captioner_lmdeploy.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShareGPT4Omni/ShareGPT4Video/HEAD/captioner/fast_captioner_lmdeploy.py -------------------------------------------------------------------------------- /captioner/slide_captioner_lmdeploy.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShareGPT4Omni/ShareGPT4Video/HEAD/captioner/slide_captioner_lmdeploy.py -------------------------------------------------------------------------------- /captioner/videos_to_describe.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShareGPT4Omni/ShareGPT4Video/HEAD/captioner/videos_to_describe.json -------------------------------------------------------------------------------- /examples/C_1_0.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShareGPT4Omni/ShareGPT4Video/HEAD/examples/C_1_0.mp4 -------------------------------------------------------------------------------- /examples/sample_demo_1.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShareGPT4Omni/ShareGPT4Video/HEAD/examples/sample_demo_1.mp4 -------------------------------------------------------------------------------- /examples/yoga.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShareGPT4Omni/ShareGPT4Video/HEAD/examples/yoga.mp4 -------------------------------------------------------------------------------- /keyframe_extraction_example.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShareGPT4Omni/ShareGPT4Video/HEAD/keyframe_extraction_example.py -------------------------------------------------------------------------------- /llava/__init__.py: -------------------------------------------------------------------------------- 1 | # from .model import LlavaLlamaForCausalLM 2 | -------------------------------------------------------------------------------- /llava/constants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShareGPT4Omni/ShareGPT4Video/HEAD/llava/constants.py -------------------------------------------------------------------------------- /llava/conversation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShareGPT4Omni/ShareGPT4Video/HEAD/llava/conversation.py -------------------------------------------------------------------------------- /llava/eval/evaluate_benchmark_1_correctness.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShareGPT4Omni/ShareGPT4Video/HEAD/llava/eval/evaluate_benchmark_1_correctness.py -------------------------------------------------------------------------------- /llava/eval/evaluate_benchmark_2_detailed_orientation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShareGPT4Omni/ShareGPT4Video/HEAD/llava/eval/evaluate_benchmark_2_detailed_orientation.py -------------------------------------------------------------------------------- /llava/eval/evaluate_benchmark_3_context.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShareGPT4Omni/ShareGPT4Video/HEAD/llava/eval/evaluate_benchmark_3_context.py -------------------------------------------------------------------------------- /llava/eval/evaluate_benchmark_4_temporal.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShareGPT4Omni/ShareGPT4Video/HEAD/llava/eval/evaluate_benchmark_4_temporal.py -------------------------------------------------------------------------------- /llava/eval/evaluate_benchmark_5_consistency.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShareGPT4Omni/ShareGPT4Video/HEAD/llava/eval/evaluate_benchmark_5_consistency.py -------------------------------------------------------------------------------- /llava/eval/model_vqa_loader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShareGPT4Omni/ShareGPT4Video/HEAD/llava/eval/model_vqa_loader.py -------------------------------------------------------------------------------- /llava/eval/model_vqa_tempcompass.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShareGPT4Omni/ShareGPT4Video/HEAD/llava/eval/model_vqa_tempcompass.py -------------------------------------------------------------------------------- /llava/eval/run_llava.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShareGPT4Omni/ShareGPT4Video/HEAD/llava/eval/run_llava.py -------------------------------------------------------------------------------- /llava/eval/video/eval_mvbench.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShareGPT4Omni/ShareGPT4Video/HEAD/llava/eval/video/eval_mvbench.py -------------------------------------------------------------------------------- /llava/eval/video/eval_vbench.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShareGPT4Omni/ShareGPT4Video/HEAD/llava/eval/video/eval_vbench.py -------------------------------------------------------------------------------- /llava/eval/video/general_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShareGPT4Omni/ShareGPT4Video/HEAD/llava/eval/video/general_utils.py -------------------------------------------------------------------------------- /llava/eval/video/mvbench_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShareGPT4Omni/ShareGPT4Video/HEAD/llava/eval/video/mvbench_utils.py -------------------------------------------------------------------------------- /llava/eval/video/vbench_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShareGPT4Omni/ShareGPT4Video/HEAD/llava/eval/video/vbench_utils.py -------------------------------------------------------------------------------- /llava/mm_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShareGPT4Omni/ShareGPT4Video/HEAD/llava/mm_utils.py -------------------------------------------------------------------------------- /llava/model/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShareGPT4Omni/ShareGPT4Video/HEAD/llava/model/__init__.py -------------------------------------------------------------------------------- /llava/model/apply_delta.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShareGPT4Omni/ShareGPT4Video/HEAD/llava/model/apply_delta.py -------------------------------------------------------------------------------- /llava/model/builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShareGPT4Omni/ShareGPT4Video/HEAD/llava/model/builder.py -------------------------------------------------------------------------------- /llava/model/consolidate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShareGPT4Omni/ShareGPT4Video/HEAD/llava/model/consolidate.py -------------------------------------------------------------------------------- /llava/model/language_model/llava_llama.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShareGPT4Omni/ShareGPT4Video/HEAD/llava/model/language_model/llava_llama.py -------------------------------------------------------------------------------- /llava/model/language_model/llava_mistral.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShareGPT4Omni/ShareGPT4Video/HEAD/llava/model/language_model/llava_mistral.py -------------------------------------------------------------------------------- /llava/model/language_model/llava_mpt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShareGPT4Omni/ShareGPT4Video/HEAD/llava/model/language_model/llava_mpt.py -------------------------------------------------------------------------------- /llava/model/llava_arch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShareGPT4Omni/ShareGPT4Video/HEAD/llava/model/llava_arch.py -------------------------------------------------------------------------------- /llava/model/make_delta.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShareGPT4Omni/ShareGPT4Video/HEAD/llava/model/make_delta.py -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShareGPT4Omni/ShareGPT4Video/HEAD/llava/model/multimodal_encoder/builder.py -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/clip_encoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShareGPT4Omni/ShareGPT4Video/HEAD/llava/model/multimodal_encoder/clip_encoder.py -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/siglip_encoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShareGPT4Omni/ShareGPT4Video/HEAD/llava/model/multimodal_encoder/siglip_encoder.py -------------------------------------------------------------------------------- /llava/model/multimodal_projector/builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShareGPT4Omni/ShareGPT4Video/HEAD/llava/model/multimodal_projector/builder.py -------------------------------------------------------------------------------- /llava/model/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShareGPT4Omni/ShareGPT4Video/HEAD/llava/model/utils.py -------------------------------------------------------------------------------- /llava/serve/gradio_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShareGPT4Omni/ShareGPT4Video/HEAD/llava/serve/gradio_utils.py -------------------------------------------------------------------------------- /llava/train/llava_trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShareGPT4Omni/ShareGPT4Video/HEAD/llava/train/llava_trainer.py -------------------------------------------------------------------------------- /llava/train/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShareGPT4Omni/ShareGPT4Video/HEAD/llava/train/train.py -------------------------------------------------------------------------------- /llava/train/train_mem.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShareGPT4Omni/ShareGPT4Video/HEAD/llava/train/train_mem.py -------------------------------------------------------------------------------- /llava/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShareGPT4Omni/ShareGPT4Video/HEAD/llava/utils.py -------------------------------------------------------------------------------- /llava/video_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShareGPT4Omni/ShareGPT4Video/HEAD/llava/video_utils.py -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShareGPT4Omni/ShareGPT4Video/HEAD/pyproject.toml -------------------------------------------------------------------------------- /run.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShareGPT4Omni/ShareGPT4Video/HEAD/run.py -------------------------------------------------------------------------------- /scripts/eval/8b/mvbench.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShareGPT4Omni/ShareGPT4Video/HEAD/scripts/eval/8b/mvbench.sh -------------------------------------------------------------------------------- /scripts/eval/8b/tempcompass.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShareGPT4Omni/ShareGPT4Video/HEAD/scripts/eval/8b/tempcompass.sh -------------------------------------------------------------------------------- /scripts/eval/8b/vbench.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShareGPT4Omni/ShareGPT4Video/HEAD/scripts/eval/8b/vbench.sh -------------------------------------------------------------------------------- /scripts/train/8b/lora_finetune_153k.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShareGPT4Omni/ShareGPT4Video/HEAD/scripts/train/8b/lora_finetune_153k.sh -------------------------------------------------------------------------------- /scripts/train/8b/lora_full_finetune_181k_add-share28k.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ShareGPT4Omni/ShareGPT4Video/HEAD/scripts/train/8b/lora_full_finetune_181k_add-share28k.sh --------------------------------------------------------------------------------