├── .gitignore ├── README.md ├── common.py ├── configs ├── __pycache__ │ └── instruction_data.cpython-311.pyc ├── config_lvchat.json ├── config_videochat2.json ├── data.py ├── instruction_data.py └── model.py ├── data ├── dataset ├── __init__.py ├── __pycache__ │ ├── __init__.cpython-311.pyc │ ├── base_dataset.cpython-311.pyc │ ├── dataloader.cpython-311.pyc │ ├── it_dataset.cpython-311.pyc │ ├── pt_dataset.cpython-311.pyc │ ├── utils.cpython-311.pyc │ ├── video_transforms.cpython-311.pyc │ └── video_utils.cpython-311.pyc ├── base_dataset.py ├── dataloader.py ├── it_dataset.py ├── pt_dataset.py ├── utils.py ├── video_transforms.py └── video_utils.py ├── download_street_scnene.sh ├── eval_tacos.py ├── inference.py ├── models ├── __init__.py ├── __pycache__ │ ├── __init__.cpython-311.pyc │ └── videochat2_it_long.cpython-311.pyc ├── bert │ ├── __init__.py │ ├── builder.py │ ├── tokenization_bert.py │ └── xbert.py ├── blip2 │ ├── Qformer.py │ ├── __init__.py │ ├── blip2.py │ ├── builder.py │ ├── modeling_llama.py │ ├── modeling_llama_mem.py │ ├── utils.py │ └── vit.py ├── criterions.py ├── utils.py ├── videochat2_it.py ├── videochat2_it_long.py ├── videochat2_pt.py └── videochat2_qformer.py ├── mv_eval.py ├── mvbench.py ├── output └── 7b_stage4 │ ├── config.json │ └── train.log ├── requirements.txt ├── run_7b_stage4.sh ├── run_egoschema.sh ├── run_mvbench.sh ├── run_tacos.sh ├── scripts └── config_7b_stage4.py ├── streetscene.py ├── tasks ├── __pycache__ │ └── shared_utils.cpython-311.pyc ├── retrieval_utils.py ├── shared_utils.py ├── shared_utils_qformer.py ├── train_it.py ├── train_it_long.py ├── train_pt.py └── train_qformer.py ├── train_it_long.py ├── utils ├── __pycache__ │ ├── basic_utils.cpython-311.pyc │ ├── config.cpython-311.pyc │ ├── config_utils.cpython-311.pyc │ ├── distributed.cpython-311.pyc │ ├── easydict.cpython-311.pyc │ ├── logger.cpython-311.pyc │ ├── optimizer.cpython-311.pyc │ └── scheduler.cpython-311.pyc ├── basic_utils.py ├── config.py ├── config_utils.py ├── distributed.py ├── easydict.py ├── logger.py ├── optimizer.py └── scheduler.py ├── video_models └── videochat2_process.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/.gitignore -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/README.md -------------------------------------------------------------------------------- /common.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/common.py -------------------------------------------------------------------------------- /configs/__pycache__/instruction_data.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/configs/__pycache__/instruction_data.cpython-311.pyc -------------------------------------------------------------------------------- /configs/config_lvchat.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/configs/config_lvchat.json -------------------------------------------------------------------------------- /configs/config_videochat2.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/configs/config_videochat2.json -------------------------------------------------------------------------------- /configs/data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/configs/data.py -------------------------------------------------------------------------------- /configs/instruction_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/configs/instruction_data.py -------------------------------------------------------------------------------- /configs/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/configs/model.py -------------------------------------------------------------------------------- /data: -------------------------------------------------------------------------------- 1 | ../VideoChat2/data -------------------------------------------------------------------------------- /dataset/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/dataset/__init__.py -------------------------------------------------------------------------------- /dataset/__pycache__/__init__.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/dataset/__pycache__/__init__.cpython-311.pyc -------------------------------------------------------------------------------- /dataset/__pycache__/base_dataset.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/dataset/__pycache__/base_dataset.cpython-311.pyc -------------------------------------------------------------------------------- /dataset/__pycache__/dataloader.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/dataset/__pycache__/dataloader.cpython-311.pyc -------------------------------------------------------------------------------- /dataset/__pycache__/it_dataset.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/dataset/__pycache__/it_dataset.cpython-311.pyc -------------------------------------------------------------------------------- /dataset/__pycache__/pt_dataset.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/dataset/__pycache__/pt_dataset.cpython-311.pyc -------------------------------------------------------------------------------- /dataset/__pycache__/utils.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/dataset/__pycache__/utils.cpython-311.pyc -------------------------------------------------------------------------------- /dataset/__pycache__/video_transforms.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/dataset/__pycache__/video_transforms.cpython-311.pyc -------------------------------------------------------------------------------- /dataset/__pycache__/video_utils.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/dataset/__pycache__/video_utils.cpython-311.pyc -------------------------------------------------------------------------------- /dataset/base_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/dataset/base_dataset.py -------------------------------------------------------------------------------- /dataset/dataloader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/dataset/dataloader.py -------------------------------------------------------------------------------- /dataset/it_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/dataset/it_dataset.py -------------------------------------------------------------------------------- /dataset/pt_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/dataset/pt_dataset.py -------------------------------------------------------------------------------- /dataset/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/dataset/utils.py -------------------------------------------------------------------------------- /dataset/video_transforms.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/dataset/video_transforms.py -------------------------------------------------------------------------------- /dataset/video_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/dataset/video_utils.py -------------------------------------------------------------------------------- /download_street_scnene.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/download_street_scnene.sh -------------------------------------------------------------------------------- /eval_tacos.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/eval_tacos.py -------------------------------------------------------------------------------- /inference.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/inference.py -------------------------------------------------------------------------------- /models/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /models/__pycache__/__init__.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/models/__pycache__/__init__.cpython-311.pyc -------------------------------------------------------------------------------- /models/__pycache__/videochat2_it_long.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/models/__pycache__/videochat2_it_long.cpython-311.pyc -------------------------------------------------------------------------------- /models/bert/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /models/bert/builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/models/bert/builder.py -------------------------------------------------------------------------------- /models/bert/tokenization_bert.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/models/bert/tokenization_bert.py -------------------------------------------------------------------------------- /models/bert/xbert.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/models/bert/xbert.py -------------------------------------------------------------------------------- /models/blip2/Qformer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/models/blip2/Qformer.py -------------------------------------------------------------------------------- /models/blip2/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /models/blip2/blip2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/models/blip2/blip2.py -------------------------------------------------------------------------------- /models/blip2/builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/models/blip2/builder.py -------------------------------------------------------------------------------- /models/blip2/modeling_llama.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/models/blip2/modeling_llama.py -------------------------------------------------------------------------------- /models/blip2/modeling_llama_mem.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/models/blip2/modeling_llama_mem.py -------------------------------------------------------------------------------- /models/blip2/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/models/blip2/utils.py -------------------------------------------------------------------------------- /models/blip2/vit.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/models/blip2/vit.py -------------------------------------------------------------------------------- /models/criterions.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/models/criterions.py -------------------------------------------------------------------------------- /models/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/models/utils.py -------------------------------------------------------------------------------- /models/videochat2_it.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/models/videochat2_it.py -------------------------------------------------------------------------------- /models/videochat2_it_long.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/models/videochat2_it_long.py -------------------------------------------------------------------------------- /models/videochat2_pt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/models/videochat2_pt.py -------------------------------------------------------------------------------- /models/videochat2_qformer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/models/videochat2_qformer.py -------------------------------------------------------------------------------- /mv_eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/mv_eval.py -------------------------------------------------------------------------------- /mvbench.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/mvbench.py -------------------------------------------------------------------------------- /output/7b_stage4/config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/output/7b_stage4/config.json -------------------------------------------------------------------------------- /output/7b_stage4/train.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/output/7b_stage4/train.log -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/requirements.txt -------------------------------------------------------------------------------- /run_7b_stage4.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/run_7b_stage4.sh -------------------------------------------------------------------------------- /run_egoschema.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/run_egoschema.sh -------------------------------------------------------------------------------- /run_mvbench.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/run_mvbench.sh -------------------------------------------------------------------------------- /run_tacos.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/run_tacos.sh -------------------------------------------------------------------------------- /scripts/config_7b_stage4.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/scripts/config_7b_stage4.py -------------------------------------------------------------------------------- /streetscene.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/streetscene.py -------------------------------------------------------------------------------- /tasks/__pycache__/shared_utils.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/tasks/__pycache__/shared_utils.cpython-311.pyc -------------------------------------------------------------------------------- /tasks/retrieval_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/tasks/retrieval_utils.py -------------------------------------------------------------------------------- /tasks/shared_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/tasks/shared_utils.py -------------------------------------------------------------------------------- /tasks/shared_utils_qformer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/tasks/shared_utils_qformer.py -------------------------------------------------------------------------------- /tasks/train_it.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/tasks/train_it.py -------------------------------------------------------------------------------- /tasks/train_it_long.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/tasks/train_it_long.py -------------------------------------------------------------------------------- /tasks/train_pt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/tasks/train_pt.py -------------------------------------------------------------------------------- /tasks/train_qformer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/tasks/train_qformer.py -------------------------------------------------------------------------------- /train_it_long.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/train_it_long.py -------------------------------------------------------------------------------- /utils/__pycache__/basic_utils.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/utils/__pycache__/basic_utils.cpython-311.pyc -------------------------------------------------------------------------------- /utils/__pycache__/config.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/utils/__pycache__/config.cpython-311.pyc -------------------------------------------------------------------------------- /utils/__pycache__/config_utils.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/utils/__pycache__/config_utils.cpython-311.pyc -------------------------------------------------------------------------------- /utils/__pycache__/distributed.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/utils/__pycache__/distributed.cpython-311.pyc -------------------------------------------------------------------------------- /utils/__pycache__/easydict.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/utils/__pycache__/easydict.cpython-311.pyc -------------------------------------------------------------------------------- /utils/__pycache__/logger.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/utils/__pycache__/logger.cpython-311.pyc -------------------------------------------------------------------------------- /utils/__pycache__/optimizer.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/utils/__pycache__/optimizer.cpython-311.pyc -------------------------------------------------------------------------------- /utils/__pycache__/scheduler.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/utils/__pycache__/scheduler.cpython-311.pyc -------------------------------------------------------------------------------- /utils/basic_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/utils/basic_utils.py -------------------------------------------------------------------------------- /utils/config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/utils/config.py -------------------------------------------------------------------------------- /utils/config_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/utils/config_utils.py -------------------------------------------------------------------------------- /utils/distributed.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/utils/distributed.py -------------------------------------------------------------------------------- /utils/easydict.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/utils/easydict.py -------------------------------------------------------------------------------- /utils/logger.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/utils/logger.py -------------------------------------------------------------------------------- /utils/optimizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/utils/optimizer.py -------------------------------------------------------------------------------- /utils/scheduler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/utils/scheduler.py -------------------------------------------------------------------------------- /video_models: -------------------------------------------------------------------------------- 1 | ../VideoChat2/video_models/ -------------------------------------------------------------------------------- /videochat2_process.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyu-ustc/LVChat/HEAD/videochat2_process.py --------------------------------------------------------------------------------