├── .gitignore ├── LICENSE ├── LICENSE_Lavis.md ├── LICENSE_Minigpt4.md ├── LICENSE_timechat.md ├── README.md ├── cat_and_chicken.mp4 ├── docs └── DATA.md ├── environment-npu.yaml ├── eval.sh ├── eval_configs ├── blip2-image-text-matching.yaml ├── videollama-cross-96-interpolation.yaml ├── videollama-diverse-96.yaml ├── videollama-entropy-96.yaml ├── videollama-slot-96-interpolation-no-real-time.yaml ├── videollama-slot-96-interpolation-no-time-token.yaml ├── videollama-slot-96-interpolation-second-lora.yaml ├── videollama-slot-96-interpolation.yaml ├── videollama-slot-96.yaml └── videollama-slot-96_text_input.yaml ├── evaluate.py ├── example.py ├── figures └── vtg-lm-overview.png ├── fusion_result.json ├── ge_check_op.json ├── gradio_demo.py ├── install_requirements-npu.sh ├── install_requirements-v100.sh ├── metrics ├── README.md ├── dvc │ ├── SODA │ │ ├── LICENSE │ │ ├── README.md │ │ ├── dataset.py │ │ ├── nlpeval │ │ │ ├── bert_f_score.py │ │ │ ├── bert_r_score.py │ │ │ └── mover.py │ │ ├── requirements.txt │ │ ├── soda.py │ │ └── utils.py │ ├── __init__.py │ ├── eval_dvc.py │ ├── eval_dvc_anet.py │ ├── eval_soda.py │ ├── example_gt_file.json │ ├── example_pred_file.json │ └── metrics │ │ ├── README.md │ │ ├── cider.py │ │ ├── cider_scorer.py │ │ ├── data │ │ └── paraphrase-en.gz │ │ ├── eval_soda.py │ │ ├── meteor-1.5.jar │ │ ├── meteor.py │ │ ├── ptbtokenizer.py │ │ └── stanford-corenlp-3.4.1.jar ├── tvg │ ├── eval_tvg.py │ ├── eval_tvg.sh │ ├── example_gt_file.json │ └── example_pred_file.json └── vhd │ ├── eval_highlights.sh │ ├── eval_vhd.py │ ├── example_gt_file.json │ ├── example_pred_file.json │ ├── metrics.json │ └── utils.py ├── prompts ├── dvc.txt ├── mr.txt └── vhd.txt ├── requirements-v100.txt ├── train.py ├── train_configs ├── videochat2 │ └── pretrain-codebook-sampler.yaml └── videollama │ ├── fintune │ ├── activitynet-slot-sample-fmt-96.yaml │ ├── charades-slot-sample-fmt-96.yaml │ ├── qvhighlight-slot-sample-fmt-96.yaml │ └── youcook-slot-sample-fmt-96.yaml │ └── pretrain-slot-sample-fmt-96.yaml ├── utils ├── asr │ ├── asr.py │ ├── asr_data_process.py │ ├── clean_asr.py │ └── extract_audio.py ├── compress_video_data.py ├── construct_it_data │ ├── merge.py │ ├── process_action_data.py │ ├── process_dvc_data.py │ ├── process_grounding_data.py │ ├── process_highlight_data.py │ ├── process_tsg_data.py │ └── prompts │ │ ├── action_locate_prompts.json │ │ ├── dvc_prompts.json │ │ ├── highlight_prompts.json │ │ ├── tsp_prompts.json │ │ ├── vdedit_prompts.txt │ │ ├── video_grounding_prompts.json │ │ └── video_summarize_prompts.json ├── format_dvc.py ├── format_tvg.py ├── format_vhd.py ├── get_coco_format.py ├── parse_annotation_dvc.py ├── parse_annotation_tvg.py ├── parse_annotation_vhd.py ├── parse_annotation_vs.py ├── process_hirest.py └── process_valley.py └── vtgllm ├── __init__.py ├── common ├── __init__.py ├── config.py ├── dist_utils.py ├── gradcam.py ├── logger.py ├── losses.py ├── optims.py ├── registry.py └── utils.py ├── configs ├── datasets │ ├── cc_sbu │ │ ├── align.yaml │ │ └── defaults.yaml │ ├── instruct │ │ ├── charades_instruct.yaml │ │ ├── llava_instruct.yaml │ │ ├── qvhighlights_instruct.yaml │ │ ├── time_instruct.yaml │ │ ├── valley72k_instruct.yaml │ │ ├── webvid_instruct.yaml │ │ └── youcook2_instruct.yaml │ ├── laion │ │ └── defaults.yaml │ └── webvid │ │ └── defaults.yaml ├── default.yaml └── models │ ├── blip2_pretrain.yaml │ ├── minigpt4.yaml │ └── vtgllm.yaml ├── conversation ├── __init__.py ├── conversation_video.py └── conversation_video_batch.py ├── datasets ├── __init__.py ├── builders │ ├── __init__.py │ ├── base_dataset_builder.py │ ├── image_text_pair_builder.py │ ├── instruct_builder.py │ └── video_caption_builder.py ├── data_utils.py └── datasets │ ├── __init__.py │ ├── base_dataset.py │ ├── caption_datasets.py │ ├── cc_sbu_dataset.py │ ├── dataloader_utils.py │ ├── laion_dataset.py │ ├── llava_instruct_dataset.py │ ├── video_instruct_dataset.py │ └── webvid_datasets.py ├── models ├── Qformer.py ├── __init__.py ├── base_model.py ├── blip2.py ├── blip2_image_text_matching.py ├── blip2_outputs.py ├── blip2_qformer.py ├── eva_vit.py ├── modeling_llama.py ├── vit.py └── vtgllm.py ├── processors ├── __init__.py ├── base_processor.py ├── blip_processors.py ├── functional_video.py ├── randaugment.py ├── transforms_video.py └── video_processor.py ├── runners ├── __init__.py ├── runner_base.py └── test.py └── tasks ├── __init__.py ├── base_task.py ├── image_text_pretrain.py └── video_text_pretrain.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/LICENSE -------------------------------------------------------------------------------- /LICENSE_Lavis.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/LICENSE_Lavis.md -------------------------------------------------------------------------------- /LICENSE_Minigpt4.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/LICENSE_Minigpt4.md -------------------------------------------------------------------------------- /LICENSE_timechat.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/LICENSE_timechat.md -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/README.md -------------------------------------------------------------------------------- /cat_and_chicken.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/cat_and_chicken.mp4 -------------------------------------------------------------------------------- /docs/DATA.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/docs/DATA.md -------------------------------------------------------------------------------- /environment-npu.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/environment-npu.yaml -------------------------------------------------------------------------------- /eval.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/eval.sh -------------------------------------------------------------------------------- /eval_configs/blip2-image-text-matching.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/eval_configs/blip2-image-text-matching.yaml -------------------------------------------------------------------------------- /eval_configs/videollama-cross-96-interpolation.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/eval_configs/videollama-cross-96-interpolation.yaml -------------------------------------------------------------------------------- /eval_configs/videollama-diverse-96.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/eval_configs/videollama-diverse-96.yaml -------------------------------------------------------------------------------- /eval_configs/videollama-entropy-96.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/eval_configs/videollama-entropy-96.yaml -------------------------------------------------------------------------------- /eval_configs/videollama-slot-96-interpolation-no-real-time.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/eval_configs/videollama-slot-96-interpolation-no-real-time.yaml -------------------------------------------------------------------------------- /eval_configs/videollama-slot-96-interpolation-no-time-token.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/eval_configs/videollama-slot-96-interpolation-no-time-token.yaml -------------------------------------------------------------------------------- /eval_configs/videollama-slot-96-interpolation-second-lora.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/eval_configs/videollama-slot-96-interpolation-second-lora.yaml -------------------------------------------------------------------------------- /eval_configs/videollama-slot-96-interpolation.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/eval_configs/videollama-slot-96-interpolation.yaml -------------------------------------------------------------------------------- /eval_configs/videollama-slot-96.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/eval_configs/videollama-slot-96.yaml -------------------------------------------------------------------------------- /eval_configs/videollama-slot-96_text_input.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/eval_configs/videollama-slot-96_text_input.yaml -------------------------------------------------------------------------------- /evaluate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/evaluate.py -------------------------------------------------------------------------------- /example.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/example.py -------------------------------------------------------------------------------- /figures/vtg-lm-overview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/figures/vtg-lm-overview.png -------------------------------------------------------------------------------- /fusion_result.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/fusion_result.json -------------------------------------------------------------------------------- /ge_check_op.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/ge_check_op.json -------------------------------------------------------------------------------- /gradio_demo.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/gradio_demo.py -------------------------------------------------------------------------------- /install_requirements-npu.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/install_requirements-npu.sh -------------------------------------------------------------------------------- /install_requirements-v100.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/install_requirements-v100.sh -------------------------------------------------------------------------------- /metrics/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/metrics/README.md -------------------------------------------------------------------------------- /metrics/dvc/SODA/LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/metrics/dvc/SODA/LICENSE -------------------------------------------------------------------------------- /metrics/dvc/SODA/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/metrics/dvc/SODA/README.md -------------------------------------------------------------------------------- /metrics/dvc/SODA/dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/metrics/dvc/SODA/dataset.py -------------------------------------------------------------------------------- /metrics/dvc/SODA/nlpeval/bert_f_score.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/metrics/dvc/SODA/nlpeval/bert_f_score.py -------------------------------------------------------------------------------- /metrics/dvc/SODA/nlpeval/bert_r_score.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/metrics/dvc/SODA/nlpeval/bert_r_score.py -------------------------------------------------------------------------------- /metrics/dvc/SODA/nlpeval/mover.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/metrics/dvc/SODA/nlpeval/mover.py -------------------------------------------------------------------------------- /metrics/dvc/SODA/requirements.txt: -------------------------------------------------------------------------------- 1 | numpy==1.19.1 2 | tqdm==4.48.2 3 | -------------------------------------------------------------------------------- /metrics/dvc/SODA/soda.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/metrics/dvc/SODA/soda.py -------------------------------------------------------------------------------- /metrics/dvc/SODA/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/metrics/dvc/SODA/utils.py -------------------------------------------------------------------------------- /metrics/dvc/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/metrics/dvc/__init__.py -------------------------------------------------------------------------------- /metrics/dvc/eval_dvc.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/metrics/dvc/eval_dvc.py -------------------------------------------------------------------------------- /metrics/dvc/eval_dvc_anet.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/metrics/dvc/eval_dvc_anet.py -------------------------------------------------------------------------------- /metrics/dvc/eval_soda.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/metrics/dvc/eval_soda.py -------------------------------------------------------------------------------- /metrics/dvc/example_gt_file.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/metrics/dvc/example_gt_file.json -------------------------------------------------------------------------------- /metrics/dvc/example_pred_file.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/metrics/dvc/example_pred_file.json -------------------------------------------------------------------------------- /metrics/dvc/metrics/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/metrics/dvc/metrics/README.md -------------------------------------------------------------------------------- /metrics/dvc/metrics/cider.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/metrics/dvc/metrics/cider.py -------------------------------------------------------------------------------- /metrics/dvc/metrics/cider_scorer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/metrics/dvc/metrics/cider_scorer.py -------------------------------------------------------------------------------- /metrics/dvc/metrics/data/paraphrase-en.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/metrics/dvc/metrics/data/paraphrase-en.gz -------------------------------------------------------------------------------- /metrics/dvc/metrics/eval_soda.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/metrics/dvc/metrics/eval_soda.py -------------------------------------------------------------------------------- /metrics/dvc/metrics/meteor-1.5.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/metrics/dvc/metrics/meteor-1.5.jar -------------------------------------------------------------------------------- /metrics/dvc/metrics/meteor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/metrics/dvc/metrics/meteor.py -------------------------------------------------------------------------------- /metrics/dvc/metrics/ptbtokenizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/metrics/dvc/metrics/ptbtokenizer.py -------------------------------------------------------------------------------- /metrics/dvc/metrics/stanford-corenlp-3.4.1.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/metrics/dvc/metrics/stanford-corenlp-3.4.1.jar -------------------------------------------------------------------------------- /metrics/tvg/eval_tvg.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/metrics/tvg/eval_tvg.py -------------------------------------------------------------------------------- /metrics/tvg/eval_tvg.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/metrics/tvg/eval_tvg.sh -------------------------------------------------------------------------------- /metrics/tvg/example_gt_file.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/metrics/tvg/example_gt_file.json -------------------------------------------------------------------------------- /metrics/tvg/example_pred_file.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/metrics/tvg/example_pred_file.json -------------------------------------------------------------------------------- /metrics/vhd/eval_highlights.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/metrics/vhd/eval_highlights.sh -------------------------------------------------------------------------------- /metrics/vhd/eval_vhd.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/metrics/vhd/eval_vhd.py -------------------------------------------------------------------------------- /metrics/vhd/example_gt_file.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/metrics/vhd/example_gt_file.json -------------------------------------------------------------------------------- /metrics/vhd/example_pred_file.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/metrics/vhd/example_pred_file.json -------------------------------------------------------------------------------- /metrics/vhd/metrics.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/metrics/vhd/metrics.json -------------------------------------------------------------------------------- /metrics/vhd/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/metrics/vhd/utils.py -------------------------------------------------------------------------------- /prompts/dvc.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/prompts/dvc.txt -------------------------------------------------------------------------------- /prompts/mr.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/prompts/mr.txt -------------------------------------------------------------------------------- /prompts/vhd.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/prompts/vhd.txt -------------------------------------------------------------------------------- /requirements-v100.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/requirements-v100.txt -------------------------------------------------------------------------------- /train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/train.py -------------------------------------------------------------------------------- /train_configs/videochat2/pretrain-codebook-sampler.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/train_configs/videochat2/pretrain-codebook-sampler.yaml -------------------------------------------------------------------------------- /train_configs/videollama/fintune/activitynet-slot-sample-fmt-96.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/train_configs/videollama/fintune/activitynet-slot-sample-fmt-96.yaml -------------------------------------------------------------------------------- /train_configs/videollama/fintune/charades-slot-sample-fmt-96.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/train_configs/videollama/fintune/charades-slot-sample-fmt-96.yaml -------------------------------------------------------------------------------- /train_configs/videollama/fintune/qvhighlight-slot-sample-fmt-96.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/train_configs/videollama/fintune/qvhighlight-slot-sample-fmt-96.yaml -------------------------------------------------------------------------------- /train_configs/videollama/fintune/youcook-slot-sample-fmt-96.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/train_configs/videollama/fintune/youcook-slot-sample-fmt-96.yaml -------------------------------------------------------------------------------- /train_configs/videollama/pretrain-slot-sample-fmt-96.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/train_configs/videollama/pretrain-slot-sample-fmt-96.yaml -------------------------------------------------------------------------------- /utils/asr/asr.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/utils/asr/asr.py -------------------------------------------------------------------------------- /utils/asr/asr_data_process.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/utils/asr/asr_data_process.py -------------------------------------------------------------------------------- /utils/asr/clean_asr.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/utils/asr/clean_asr.py -------------------------------------------------------------------------------- /utils/asr/extract_audio.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/utils/asr/extract_audio.py -------------------------------------------------------------------------------- /utils/compress_video_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/utils/compress_video_data.py -------------------------------------------------------------------------------- /utils/construct_it_data/merge.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/utils/construct_it_data/merge.py -------------------------------------------------------------------------------- /utils/construct_it_data/process_action_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/utils/construct_it_data/process_action_data.py -------------------------------------------------------------------------------- /utils/construct_it_data/process_dvc_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/utils/construct_it_data/process_dvc_data.py -------------------------------------------------------------------------------- /utils/construct_it_data/process_grounding_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/utils/construct_it_data/process_grounding_data.py -------------------------------------------------------------------------------- /utils/construct_it_data/process_highlight_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/utils/construct_it_data/process_highlight_data.py -------------------------------------------------------------------------------- /utils/construct_it_data/process_tsg_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/utils/construct_it_data/process_tsg_data.py -------------------------------------------------------------------------------- /utils/construct_it_data/prompts/action_locate_prompts.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/utils/construct_it_data/prompts/action_locate_prompts.json -------------------------------------------------------------------------------- /utils/construct_it_data/prompts/dvc_prompts.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/utils/construct_it_data/prompts/dvc_prompts.json -------------------------------------------------------------------------------- /utils/construct_it_data/prompts/highlight_prompts.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/utils/construct_it_data/prompts/highlight_prompts.json -------------------------------------------------------------------------------- /utils/construct_it_data/prompts/tsp_prompts.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/utils/construct_it_data/prompts/tsp_prompts.json -------------------------------------------------------------------------------- /utils/construct_it_data/prompts/vdedit_prompts.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/utils/construct_it_data/prompts/vdedit_prompts.txt -------------------------------------------------------------------------------- /utils/construct_it_data/prompts/video_grounding_prompts.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/utils/construct_it_data/prompts/video_grounding_prompts.json -------------------------------------------------------------------------------- /utils/construct_it_data/prompts/video_summarize_prompts.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/utils/construct_it_data/prompts/video_summarize_prompts.json -------------------------------------------------------------------------------- /utils/format_dvc.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/utils/format_dvc.py -------------------------------------------------------------------------------- /utils/format_tvg.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/utils/format_tvg.py -------------------------------------------------------------------------------- /utils/format_vhd.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/utils/format_vhd.py -------------------------------------------------------------------------------- /utils/get_coco_format.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/utils/get_coco_format.py -------------------------------------------------------------------------------- /utils/parse_annotation_dvc.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/utils/parse_annotation_dvc.py -------------------------------------------------------------------------------- /utils/parse_annotation_tvg.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/utils/parse_annotation_tvg.py -------------------------------------------------------------------------------- /utils/parse_annotation_vhd.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/utils/parse_annotation_vhd.py -------------------------------------------------------------------------------- /utils/parse_annotation_vs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/utils/parse_annotation_vs.py -------------------------------------------------------------------------------- /utils/process_hirest.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/utils/process_hirest.py -------------------------------------------------------------------------------- /utils/process_valley.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/utils/process_valley.py -------------------------------------------------------------------------------- /vtgllm/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/__init__.py -------------------------------------------------------------------------------- /vtgllm/common/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /vtgllm/common/config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/common/config.py -------------------------------------------------------------------------------- /vtgllm/common/dist_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/common/dist_utils.py -------------------------------------------------------------------------------- /vtgllm/common/gradcam.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/common/gradcam.py -------------------------------------------------------------------------------- /vtgllm/common/logger.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/common/logger.py -------------------------------------------------------------------------------- /vtgllm/common/losses.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/common/losses.py -------------------------------------------------------------------------------- /vtgllm/common/optims.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/common/optims.py -------------------------------------------------------------------------------- /vtgllm/common/registry.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/common/registry.py -------------------------------------------------------------------------------- /vtgllm/common/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/common/utils.py -------------------------------------------------------------------------------- /vtgllm/configs/datasets/cc_sbu/align.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/configs/datasets/cc_sbu/align.yaml -------------------------------------------------------------------------------- /vtgllm/configs/datasets/cc_sbu/defaults.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/configs/datasets/cc_sbu/defaults.yaml -------------------------------------------------------------------------------- /vtgllm/configs/datasets/instruct/charades_instruct.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/configs/datasets/instruct/charades_instruct.yaml -------------------------------------------------------------------------------- /vtgllm/configs/datasets/instruct/llava_instruct.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/configs/datasets/instruct/llava_instruct.yaml -------------------------------------------------------------------------------- /vtgllm/configs/datasets/instruct/qvhighlights_instruct.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/configs/datasets/instruct/qvhighlights_instruct.yaml -------------------------------------------------------------------------------- /vtgllm/configs/datasets/instruct/time_instruct.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/configs/datasets/instruct/time_instruct.yaml -------------------------------------------------------------------------------- /vtgllm/configs/datasets/instruct/valley72k_instruct.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/configs/datasets/instruct/valley72k_instruct.yaml -------------------------------------------------------------------------------- /vtgllm/configs/datasets/instruct/webvid_instruct.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/configs/datasets/instruct/webvid_instruct.yaml -------------------------------------------------------------------------------- /vtgllm/configs/datasets/instruct/youcook2_instruct.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/configs/datasets/instruct/youcook2_instruct.yaml -------------------------------------------------------------------------------- /vtgllm/configs/datasets/laion/defaults.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/configs/datasets/laion/defaults.yaml -------------------------------------------------------------------------------- /vtgllm/configs/datasets/webvid/defaults.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/configs/datasets/webvid/defaults.yaml -------------------------------------------------------------------------------- /vtgllm/configs/default.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/configs/default.yaml -------------------------------------------------------------------------------- /vtgllm/configs/models/blip2_pretrain.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/configs/models/blip2_pretrain.yaml -------------------------------------------------------------------------------- /vtgllm/configs/models/minigpt4.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/configs/models/minigpt4.yaml -------------------------------------------------------------------------------- /vtgllm/configs/models/vtgllm.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/configs/models/vtgllm.yaml -------------------------------------------------------------------------------- /vtgllm/conversation/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /vtgllm/conversation/conversation_video.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/conversation/conversation_video.py -------------------------------------------------------------------------------- /vtgllm/conversation/conversation_video_batch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/conversation/conversation_video_batch.py -------------------------------------------------------------------------------- /vtgllm/datasets/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /vtgllm/datasets/builders/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/datasets/builders/__init__.py -------------------------------------------------------------------------------- /vtgllm/datasets/builders/base_dataset_builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/datasets/builders/base_dataset_builder.py -------------------------------------------------------------------------------- /vtgllm/datasets/builders/image_text_pair_builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/datasets/builders/image_text_pair_builder.py -------------------------------------------------------------------------------- /vtgllm/datasets/builders/instruct_builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/datasets/builders/instruct_builder.py -------------------------------------------------------------------------------- /vtgllm/datasets/builders/video_caption_builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/datasets/builders/video_caption_builder.py -------------------------------------------------------------------------------- /vtgllm/datasets/data_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/datasets/data_utils.py -------------------------------------------------------------------------------- /vtgllm/datasets/datasets/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /vtgllm/datasets/datasets/base_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/datasets/datasets/base_dataset.py -------------------------------------------------------------------------------- /vtgllm/datasets/datasets/caption_datasets.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/datasets/datasets/caption_datasets.py -------------------------------------------------------------------------------- /vtgllm/datasets/datasets/cc_sbu_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/datasets/datasets/cc_sbu_dataset.py -------------------------------------------------------------------------------- /vtgllm/datasets/datasets/dataloader_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/datasets/datasets/dataloader_utils.py -------------------------------------------------------------------------------- /vtgllm/datasets/datasets/laion_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/datasets/datasets/laion_dataset.py -------------------------------------------------------------------------------- /vtgllm/datasets/datasets/llava_instruct_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/datasets/datasets/llava_instruct_dataset.py -------------------------------------------------------------------------------- /vtgllm/datasets/datasets/video_instruct_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/datasets/datasets/video_instruct_dataset.py -------------------------------------------------------------------------------- /vtgllm/datasets/datasets/webvid_datasets.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/datasets/datasets/webvid_datasets.py -------------------------------------------------------------------------------- /vtgllm/models/Qformer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/models/Qformer.py -------------------------------------------------------------------------------- /vtgllm/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/models/__init__.py -------------------------------------------------------------------------------- /vtgllm/models/base_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/models/base_model.py -------------------------------------------------------------------------------- /vtgllm/models/blip2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/models/blip2.py -------------------------------------------------------------------------------- /vtgllm/models/blip2_image_text_matching.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/models/blip2_image_text_matching.py -------------------------------------------------------------------------------- /vtgllm/models/blip2_outputs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/models/blip2_outputs.py -------------------------------------------------------------------------------- /vtgllm/models/blip2_qformer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/models/blip2_qformer.py -------------------------------------------------------------------------------- /vtgllm/models/eva_vit.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/models/eva_vit.py -------------------------------------------------------------------------------- /vtgllm/models/modeling_llama.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/models/modeling_llama.py -------------------------------------------------------------------------------- /vtgllm/models/vit.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/models/vit.py -------------------------------------------------------------------------------- /vtgllm/models/vtgllm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/models/vtgllm.py -------------------------------------------------------------------------------- /vtgllm/processors/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/processors/__init__.py -------------------------------------------------------------------------------- /vtgllm/processors/base_processor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/processors/base_processor.py -------------------------------------------------------------------------------- /vtgllm/processors/blip_processors.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/processors/blip_processors.py -------------------------------------------------------------------------------- /vtgllm/processors/functional_video.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/processors/functional_video.py -------------------------------------------------------------------------------- /vtgllm/processors/randaugment.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/processors/randaugment.py -------------------------------------------------------------------------------- /vtgllm/processors/transforms_video.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/processors/transforms_video.py -------------------------------------------------------------------------------- /vtgllm/processors/video_processor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/processors/video_processor.py -------------------------------------------------------------------------------- /vtgllm/runners/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/runners/__init__.py -------------------------------------------------------------------------------- /vtgllm/runners/runner_base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/runners/runner_base.py -------------------------------------------------------------------------------- /vtgllm/runners/test.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /vtgllm/tasks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/tasks/__init__.py -------------------------------------------------------------------------------- /vtgllm/tasks/base_task.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/tasks/base_task.py -------------------------------------------------------------------------------- /vtgllm/tasks/image_text_pretrain.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/tasks/image_text_pretrain.py -------------------------------------------------------------------------------- /vtgllm/tasks/video_text_pretrain.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/VTG-LLM/HEAD/vtgllm/tasks/video_text_pretrain.py --------------------------------------------------------------------------------