├── .gitignore ├── DATA.md ├── README.md ├── assert ├── data.png ├── logo.png ├── module.png ├── performance.png ├── teaser.jpg └── zeroshot.png ├── dataset ├── __init__.py ├── base_dataset.py ├── it_dataset.py ├── utils.py └── video_utils.py ├── docs └── PoolLLaVA_Report.pdf ├── example ├── 1917.mov ├── 1917.mp4 ├── bear.jpg ├── cooking.mp4 ├── dog.png ├── jesse_dance.mp4 ├── working.mp4 └── yoga.mp4 ├── models ├── __init__.py └── pllava │ ├── __init__.py │ ├── configuration_pllava.py │ ├── convert_pllava_weights_to_hf.py │ ├── modeling_pllava.py │ └── processing_pllava.py ├── python_scripts └── hf.py ├── requirements.no_torch.txt ├── requirements.torch.txt ├── requirements.txt ├── scripts ├── accel_config_deepspeed_zero2.yaml ├── accel_config_deepspeed_zero3_offload.yaml ├── accel_config_deepspeed_zero3_offload_multinode.yaml ├── accel_config_deepspeed_zero3_offload_multinode_1.yaml ├── accel_config_deepspeed_zero3_offload_multinode_2.yaml ├── accel_config_deepspeed_zero3_offload_singlegpu.yaml ├── accel_config_multigpu.yaml ├── accel_config_multinode.yaml ├── accel_config_singlegpu.yaml ├── demo.sh ├── eval.sh ├── eval_yiprompt.sh ├── gallery.sh ├── train_pllava.sh ├── train_pllava_13b.sh ├── train_pllava_34b.sh └── train_pllava_7b.sh ├── tasks ├── eval │ ├── demo │ │ ├── __init__.py │ │ ├── pllava_demo.py │ │ ├── show_compare.py │ │ └── show_gallery.py │ ├── eval_utils.py │ ├── model_utils.py │ ├── mvbench │ │ ├── __init__.py │ │ └── pllava_eval_mvbench.py │ ├── recaption │ │ ├── __init__.py │ │ ├── pllava_recaption.py │ │ └── show_recaption.py │ ├── vcgbench │ │ ├── __init__.py │ │ ├── pllava_eval_vcgbench.py │ │ └── show_vcg.py │ └── videoqabench │ │ ├── __init__.py │ │ └── pllava_eval_videoqabench.py ├── shared_utils.py └── train │ ├── config_pllava_nframe.py │ ├── config_pllava_nframe_yiprompt.py │ ├── instruction_data.py │ └── train_pllava_nframe_accel.py └── utils ├── basic_utils.py ├── config.py ├── config_utils.py ├── distributed.py ├── easydict.py ├── logger.py ├── optimizer.py └── scheduler.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/.gitignore -------------------------------------------------------------------------------- /DATA.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/DATA.md -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/README.md -------------------------------------------------------------------------------- /assert/data.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/assert/data.png -------------------------------------------------------------------------------- /assert/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/assert/logo.png -------------------------------------------------------------------------------- /assert/module.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/assert/module.png -------------------------------------------------------------------------------- /assert/performance.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/assert/performance.png -------------------------------------------------------------------------------- /assert/teaser.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/assert/teaser.jpg -------------------------------------------------------------------------------- /assert/zeroshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/assert/zeroshot.png -------------------------------------------------------------------------------- /dataset/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/dataset/__init__.py -------------------------------------------------------------------------------- /dataset/base_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/dataset/base_dataset.py -------------------------------------------------------------------------------- /dataset/it_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/dataset/it_dataset.py -------------------------------------------------------------------------------- /dataset/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/dataset/utils.py -------------------------------------------------------------------------------- /dataset/video_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/dataset/video_utils.py -------------------------------------------------------------------------------- /docs/PoolLLaVA_Report.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/docs/PoolLLaVA_Report.pdf -------------------------------------------------------------------------------- /example/1917.mov: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/example/1917.mov -------------------------------------------------------------------------------- /example/1917.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/example/1917.mp4 -------------------------------------------------------------------------------- /example/bear.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/example/bear.jpg -------------------------------------------------------------------------------- /example/cooking.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/example/cooking.mp4 -------------------------------------------------------------------------------- /example/dog.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/example/dog.png -------------------------------------------------------------------------------- /example/jesse_dance.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/example/jesse_dance.mp4 -------------------------------------------------------------------------------- /example/working.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/example/working.mp4 -------------------------------------------------------------------------------- /example/yoga.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/example/yoga.mp4 -------------------------------------------------------------------------------- /models/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /models/pllava/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/models/pllava/__init__.py -------------------------------------------------------------------------------- /models/pllava/configuration_pllava.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/models/pllava/configuration_pllava.py -------------------------------------------------------------------------------- /models/pllava/convert_pllava_weights_to_hf.py: -------------------------------------------------------------------------------- 1 | # Not yet -------------------------------------------------------------------------------- /models/pllava/modeling_pllava.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/models/pllava/modeling_pllava.py -------------------------------------------------------------------------------- /models/pllava/processing_pllava.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/models/pllava/processing_pllava.py -------------------------------------------------------------------------------- /python_scripts/hf.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/python_scripts/hf.py -------------------------------------------------------------------------------- /requirements.no_torch.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/requirements.no_torch.txt -------------------------------------------------------------------------------- /requirements.torch.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/requirements.torch.txt -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/requirements.txt -------------------------------------------------------------------------------- /scripts/accel_config_deepspeed_zero2.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/scripts/accel_config_deepspeed_zero2.yaml -------------------------------------------------------------------------------- /scripts/accel_config_deepspeed_zero3_offload.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/scripts/accel_config_deepspeed_zero3_offload.yaml -------------------------------------------------------------------------------- /scripts/accel_config_deepspeed_zero3_offload_multinode.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/scripts/accel_config_deepspeed_zero3_offload_multinode.yaml -------------------------------------------------------------------------------- /scripts/accel_config_deepspeed_zero3_offload_multinode_1.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/scripts/accel_config_deepspeed_zero3_offload_multinode_1.yaml -------------------------------------------------------------------------------- /scripts/accel_config_deepspeed_zero3_offload_multinode_2.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/scripts/accel_config_deepspeed_zero3_offload_multinode_2.yaml -------------------------------------------------------------------------------- /scripts/accel_config_deepspeed_zero3_offload_singlegpu.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/scripts/accel_config_deepspeed_zero3_offload_singlegpu.yaml -------------------------------------------------------------------------------- /scripts/accel_config_multigpu.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/scripts/accel_config_multigpu.yaml -------------------------------------------------------------------------------- /scripts/accel_config_multinode.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/scripts/accel_config_multinode.yaml -------------------------------------------------------------------------------- /scripts/accel_config_singlegpu.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/scripts/accel_config_singlegpu.yaml -------------------------------------------------------------------------------- /scripts/demo.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/scripts/demo.sh -------------------------------------------------------------------------------- /scripts/eval.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/scripts/eval.sh -------------------------------------------------------------------------------- /scripts/eval_yiprompt.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/scripts/eval_yiprompt.sh -------------------------------------------------------------------------------- /scripts/gallery.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/scripts/gallery.sh -------------------------------------------------------------------------------- /scripts/train_pllava.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/scripts/train_pllava.sh -------------------------------------------------------------------------------- /scripts/train_pllava_13b.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/scripts/train_pllava_13b.sh -------------------------------------------------------------------------------- /scripts/train_pllava_34b.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/scripts/train_pllava_34b.sh -------------------------------------------------------------------------------- /scripts/train_pllava_7b.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/scripts/train_pllava_7b.sh -------------------------------------------------------------------------------- /tasks/eval/demo/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/tasks/eval/demo/__init__.py -------------------------------------------------------------------------------- /tasks/eval/demo/pllava_demo.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/tasks/eval/demo/pllava_demo.py -------------------------------------------------------------------------------- /tasks/eval/demo/show_compare.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/tasks/eval/demo/show_compare.py -------------------------------------------------------------------------------- /tasks/eval/demo/show_gallery.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/tasks/eval/demo/show_gallery.py -------------------------------------------------------------------------------- /tasks/eval/eval_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/tasks/eval/eval_utils.py -------------------------------------------------------------------------------- /tasks/eval/model_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/tasks/eval/model_utils.py -------------------------------------------------------------------------------- /tasks/eval/mvbench/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/tasks/eval/mvbench/__init__.py -------------------------------------------------------------------------------- /tasks/eval/mvbench/pllava_eval_mvbench.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/tasks/eval/mvbench/pllava_eval_mvbench.py -------------------------------------------------------------------------------- /tasks/eval/recaption/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/tasks/eval/recaption/__init__.py -------------------------------------------------------------------------------- /tasks/eval/recaption/pllava_recaption.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/tasks/eval/recaption/pllava_recaption.py -------------------------------------------------------------------------------- /tasks/eval/recaption/show_recaption.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/tasks/eval/recaption/show_recaption.py -------------------------------------------------------------------------------- /tasks/eval/vcgbench/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/tasks/eval/vcgbench/__init__.py -------------------------------------------------------------------------------- /tasks/eval/vcgbench/pllava_eval_vcgbench.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/tasks/eval/vcgbench/pllava_eval_vcgbench.py -------------------------------------------------------------------------------- /tasks/eval/vcgbench/show_vcg.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/tasks/eval/vcgbench/show_vcg.py -------------------------------------------------------------------------------- /tasks/eval/videoqabench/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/tasks/eval/videoqabench/__init__.py -------------------------------------------------------------------------------- /tasks/eval/videoqabench/pllava_eval_videoqabench.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/tasks/eval/videoqabench/pllava_eval_videoqabench.py -------------------------------------------------------------------------------- /tasks/shared_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/tasks/shared_utils.py -------------------------------------------------------------------------------- /tasks/train/config_pllava_nframe.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/tasks/train/config_pllava_nframe.py -------------------------------------------------------------------------------- /tasks/train/config_pllava_nframe_yiprompt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/tasks/train/config_pllava_nframe_yiprompt.py -------------------------------------------------------------------------------- /tasks/train/instruction_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/tasks/train/instruction_data.py -------------------------------------------------------------------------------- /tasks/train/train_pllava_nframe_accel.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/tasks/train/train_pllava_nframe_accel.py -------------------------------------------------------------------------------- /utils/basic_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/utils/basic_utils.py -------------------------------------------------------------------------------- /utils/config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/utils/config.py -------------------------------------------------------------------------------- /utils/config_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/utils/config_utils.py -------------------------------------------------------------------------------- /utils/distributed.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/utils/distributed.py -------------------------------------------------------------------------------- /utils/easydict.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/utils/easydict.py -------------------------------------------------------------------------------- /utils/logger.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/utils/logger.py -------------------------------------------------------------------------------- /utils/optimizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/utils/optimizer.py -------------------------------------------------------------------------------- /utils/scheduler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/magic-research/PLLaVA/HEAD/utils/scheduler.py --------------------------------------------------------------------------------