├── .gitignore ├── README.md ├── args.py ├── base ├── __init__.py ├── base_data_loader.py ├── base_dataset.py ├── base_model.py └── base_trainer.py ├── configs ├── ft │ ├── didemo_o2t-select.json │ ├── lsmdc_mc-select.json │ ├── lsmdc_o2t-select.json │ ├── msrvtt_mc-select.json │ ├── msrvtt_o2t-select.json │ ├── msrvtt_qa-select.json │ ├── msvd_o2t-select.json │ ├── msvd_qa-select.json │ └── tgif_qa-frame-select.json └── pt │ └── o2t-cl-local-select-loss-cc.json ├── data_loader ├── ConceptualCaptions_dataset.py ├── DiDeMo_dataset.py ├── LSMDC_dataset.py ├── MSRVTT_dataset.py ├── MSVD_dataset.py ├── TGIF_dataset.py ├── WebVid_dataset.py ├── data_loader.py └── transforms.py ├── figs └── framework.png ├── logger ├── __init__.py ├── logger.py ├── logger_config.json └── visualization.py ├── meta_data ├── DiDeMo_test.tsv ├── DiDeMo_train.tsv ├── DiDeMo_val.tsv ├── MSVD_test.tsv ├── MSVD_train.tsv ├── MSVD_val.tsv ├── cc3m_validation_success_full.tsv ├── frameqa_test.jsonl ├── frameqa_train.jsonl ├── frameqa_trainval_ans2label.json ├── frameqa_val.jsonl ├── msrvtt_mc_test.jsonl ├── msrvtt_qa_test.jsonl ├── msrvtt_qa_train.jsonl ├── msrvtt_qa_val.jsonl ├── msrvtt_train_ans2label.json ├── msvd_answer_set.txt ├── msvd_test_qa_encode.json ├── msvd_train_qa_encode.json ├── msvd_val_qa_encode.json ├── msvd_youtube_mapping.txt └── webvid_validation_success_full.tsv ├── model ├── __init__.py ├── loss.py ├── metric.py ├── model.py ├── object_transformer.py └── video_qa_mdoel.py ├── parse_config_dist_multi.py ├── requirements.txt ├── train_dist_multi.py ├── train_dist_multi_mc.py ├── train_dist_multi_qa.py ├── trainer ├── __init__.py └── trainer_dist.py └── utils ├── __init__.py ├── html.py ├── util.py └── visualizer.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/.gitignore -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/README.md -------------------------------------------------------------------------------- /args.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/args.py -------------------------------------------------------------------------------- /base/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/base/__init__.py -------------------------------------------------------------------------------- /base/base_data_loader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/base/base_data_loader.py -------------------------------------------------------------------------------- /base/base_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/base/base_dataset.py -------------------------------------------------------------------------------- /base/base_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/base/base_model.py -------------------------------------------------------------------------------- /base/base_trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/base/base_trainer.py -------------------------------------------------------------------------------- /configs/ft/didemo_o2t-select.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/configs/ft/didemo_o2t-select.json -------------------------------------------------------------------------------- /configs/ft/lsmdc_mc-select.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/configs/ft/lsmdc_mc-select.json -------------------------------------------------------------------------------- /configs/ft/lsmdc_o2t-select.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/configs/ft/lsmdc_o2t-select.json -------------------------------------------------------------------------------- /configs/ft/msrvtt_mc-select.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/configs/ft/msrvtt_mc-select.json -------------------------------------------------------------------------------- /configs/ft/msrvtt_o2t-select.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/configs/ft/msrvtt_o2t-select.json -------------------------------------------------------------------------------- /configs/ft/msrvtt_qa-select.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/configs/ft/msrvtt_qa-select.json -------------------------------------------------------------------------------- /configs/ft/msvd_o2t-select.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/configs/ft/msvd_o2t-select.json -------------------------------------------------------------------------------- /configs/ft/msvd_qa-select.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/configs/ft/msvd_qa-select.json -------------------------------------------------------------------------------- /configs/ft/tgif_qa-frame-select.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/configs/ft/tgif_qa-frame-select.json -------------------------------------------------------------------------------- /configs/pt/o2t-cl-local-select-loss-cc.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/configs/pt/o2t-cl-local-select-loss-cc.json -------------------------------------------------------------------------------- /data_loader/ConceptualCaptions_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/data_loader/ConceptualCaptions_dataset.py -------------------------------------------------------------------------------- /data_loader/DiDeMo_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/data_loader/DiDeMo_dataset.py -------------------------------------------------------------------------------- /data_loader/LSMDC_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/data_loader/LSMDC_dataset.py -------------------------------------------------------------------------------- /data_loader/MSRVTT_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/data_loader/MSRVTT_dataset.py -------------------------------------------------------------------------------- /data_loader/MSVD_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/data_loader/MSVD_dataset.py -------------------------------------------------------------------------------- /data_loader/TGIF_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/data_loader/TGIF_dataset.py -------------------------------------------------------------------------------- /data_loader/WebVid_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/data_loader/WebVid_dataset.py -------------------------------------------------------------------------------- /data_loader/data_loader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/data_loader/data_loader.py -------------------------------------------------------------------------------- /data_loader/transforms.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/data_loader/transforms.py -------------------------------------------------------------------------------- /figs/framework.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/figs/framework.png -------------------------------------------------------------------------------- /logger/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/logger/__init__.py -------------------------------------------------------------------------------- /logger/logger.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/logger/logger.py -------------------------------------------------------------------------------- /logger/logger_config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/logger/logger_config.json -------------------------------------------------------------------------------- /logger/visualization.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/logger/visualization.py -------------------------------------------------------------------------------- /meta_data/DiDeMo_test.tsv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/meta_data/DiDeMo_test.tsv -------------------------------------------------------------------------------- /meta_data/DiDeMo_train.tsv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/meta_data/DiDeMo_train.tsv -------------------------------------------------------------------------------- /meta_data/DiDeMo_val.tsv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/meta_data/DiDeMo_val.tsv -------------------------------------------------------------------------------- /meta_data/MSVD_test.tsv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/meta_data/MSVD_test.tsv -------------------------------------------------------------------------------- /meta_data/MSVD_train.tsv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/meta_data/MSVD_train.tsv -------------------------------------------------------------------------------- /meta_data/MSVD_val.tsv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/meta_data/MSVD_val.tsv -------------------------------------------------------------------------------- /meta_data/cc3m_validation_success_full.tsv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/meta_data/cc3m_validation_success_full.tsv -------------------------------------------------------------------------------- /meta_data/frameqa_test.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/meta_data/frameqa_test.jsonl -------------------------------------------------------------------------------- /meta_data/frameqa_train.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/meta_data/frameqa_train.jsonl -------------------------------------------------------------------------------- /meta_data/frameqa_trainval_ans2label.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/meta_data/frameqa_trainval_ans2label.json -------------------------------------------------------------------------------- /meta_data/frameqa_val.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/meta_data/frameqa_val.jsonl -------------------------------------------------------------------------------- /meta_data/msrvtt_mc_test.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/meta_data/msrvtt_mc_test.jsonl -------------------------------------------------------------------------------- /meta_data/msrvtt_qa_test.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/meta_data/msrvtt_qa_test.jsonl -------------------------------------------------------------------------------- /meta_data/msrvtt_qa_train.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/meta_data/msrvtt_qa_train.jsonl -------------------------------------------------------------------------------- /meta_data/msrvtt_qa_val.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/meta_data/msrvtt_qa_val.jsonl -------------------------------------------------------------------------------- /meta_data/msrvtt_train_ans2label.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/meta_data/msrvtt_train_ans2label.json -------------------------------------------------------------------------------- /meta_data/msvd_answer_set.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/meta_data/msvd_answer_set.txt -------------------------------------------------------------------------------- /meta_data/msvd_test_qa_encode.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/meta_data/msvd_test_qa_encode.json -------------------------------------------------------------------------------- /meta_data/msvd_train_qa_encode.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/meta_data/msvd_train_qa_encode.json -------------------------------------------------------------------------------- /meta_data/msvd_val_qa_encode.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/meta_data/msvd_val_qa_encode.json -------------------------------------------------------------------------------- /meta_data/msvd_youtube_mapping.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/meta_data/msvd_youtube_mapping.txt -------------------------------------------------------------------------------- /meta_data/webvid_validation_success_full.tsv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/meta_data/webvid_validation_success_full.tsv -------------------------------------------------------------------------------- /model/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /model/loss.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/model/loss.py -------------------------------------------------------------------------------- /model/metric.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/model/metric.py -------------------------------------------------------------------------------- /model/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/model/model.py -------------------------------------------------------------------------------- /model/object_transformer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/model/object_transformer.py -------------------------------------------------------------------------------- /model/video_qa_mdoel.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/model/video_qa_mdoel.py -------------------------------------------------------------------------------- /parse_config_dist_multi.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/parse_config_dist_multi.py -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/requirements.txt -------------------------------------------------------------------------------- /train_dist_multi.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/train_dist_multi.py -------------------------------------------------------------------------------- /train_dist_multi_mc.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/train_dist_multi_mc.py -------------------------------------------------------------------------------- /train_dist_multi_qa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/train_dist_multi_qa.py -------------------------------------------------------------------------------- /trainer/__init__.py: -------------------------------------------------------------------------------- 1 | from .trainer_dist import * 2 | -------------------------------------------------------------------------------- /trainer/trainer_dist.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/trainer/trainer_dist.py -------------------------------------------------------------------------------- /utils/__init__.py: -------------------------------------------------------------------------------- 1 | from .util import * -------------------------------------------------------------------------------- /utils/html.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/utils/html.py -------------------------------------------------------------------------------- /utils/util.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/utils/util.py -------------------------------------------------------------------------------- /utils/visualizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/showlab/DemoVLP/HEAD/utils/visualizer.py --------------------------------------------------------------------------------