├── README.md ├── caption_mplug.py ├── caption_mplug_scst.py ├── configs ├── caption_mplug_base.yaml ├── caption_mplug_base_scst.yaml ├── caption_mplug_large.yaml ├── caption_mplug_large_scst.yaml ├── config_bert.json ├── config_bert_stride3.json ├── ds_config.json ├── grounding_mplug_large.yaml ├── pretrain_mplug_base.yaml ├── pretrain_mplug_large.yaml ├── retrieval_coco_mplug_large.yaml ├── retrieval_flickr30k_mplug_large.yaml ├── retrieval_msrvtt_mplug_large.yaml ├── videocap_vatex_mplug_large.yaml ├── videoqa_msrvtt_mplug_base.yaml ├── vqa_mplug_base.yaml └── vqa_mplug_large.yaml ├── dataset ├── __init__.py ├── caption_dataset.py ├── grounding_dataset.py ├── nlvr_dataset.py ├── randaugment.py ├── rouge.py ├── utils.py ├── ve_dataset.py ├── vg_transforms.py ├── video_dataset.py ├── videoqa_dataset.py └── vqa_dataset.py ├── grounding_mplug.py ├── models ├── __init__.py ├── clip │ ├── __init__.py │ ├── bpe_simple_vocab_16e6.txt.gz │ ├── clip.py │ ├── model.py │ └── simple_tokenizer.py ├── model_caption_mplug.py ├── model_caption_mplug_vatex.py ├── model_grounding_mplug.py ├── model_retrieval_mplug.py ├── model_vqa_mplug.py ├── modeling_mplug.py ├── predictor.py ├── tokenization_bert.py ├── visual_transformers.py └── vit.py ├── mplug_framework.png ├── optim ├── __init__.py ├── adafactor.py ├── adahessian.py ├── adamp.py ├── adamw.py ├── lookahead.py ├── nadam.py ├── novograd.py ├── nvnovograd.py ├── optim_factory.py ├── radam.py ├── rmsprop_tf.py └── sgdp.py ├── requirements.txt ├── retrieval_img_mplug.py ├── retrieval_vid_mplug.py ├── scheduler ├── __init__.py ├── cosine_lr.py ├── plateau_lr.py ├── scheduler.py ├── scheduler_factory.py ├── step_lr.py └── tanh_lr.py ├── scripts ├── caption_mplug_base.sh ├── caption_mplug_base_scst.sh ├── caption_mplug_large.sh ├── caption_mplug_large_scst.sh ├── grounding_mplug_large.sh ├── retrieval_coco_mplug_large.sh ├── retrieval_flickr30k_mplug_large.sh ├── retrieval_msrvtt_mplug_large_zs.sh ├── videocap_vatex_mplug_large.sh ├── videoqa_msrvtt_mplug_base.sh ├── vqa_mplug_base.sh └── vqa_mplug_large.sh ├── utils.py ├── vgTools └── utils │ ├── box_utils.py │ ├── eval_utils.py │ ├── loss_utils.py │ ├── misc.py │ ├── transforms.py │ └── word_utils.py ├── videocap_mplug.py ├── videoqa_mplug.py ├── vqaTools ├── __init__.py ├── vqa.py └── vqaEval.py └── vqa_mplug.py /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/README.md -------------------------------------------------------------------------------- /caption_mplug.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/caption_mplug.py -------------------------------------------------------------------------------- /caption_mplug_scst.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/caption_mplug_scst.py -------------------------------------------------------------------------------- /configs/caption_mplug_base.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/configs/caption_mplug_base.yaml -------------------------------------------------------------------------------- /configs/caption_mplug_base_scst.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/configs/caption_mplug_base_scst.yaml -------------------------------------------------------------------------------- /configs/caption_mplug_large.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/configs/caption_mplug_large.yaml -------------------------------------------------------------------------------- /configs/caption_mplug_large_scst.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/configs/caption_mplug_large_scst.yaml -------------------------------------------------------------------------------- /configs/config_bert.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/configs/config_bert.json -------------------------------------------------------------------------------- /configs/config_bert_stride3.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/configs/config_bert_stride3.json -------------------------------------------------------------------------------- /configs/ds_config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/configs/ds_config.json -------------------------------------------------------------------------------- /configs/grounding_mplug_large.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/configs/grounding_mplug_large.yaml -------------------------------------------------------------------------------- /configs/pretrain_mplug_base.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/configs/pretrain_mplug_base.yaml -------------------------------------------------------------------------------- /configs/pretrain_mplug_large.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/configs/pretrain_mplug_large.yaml -------------------------------------------------------------------------------- /configs/retrieval_coco_mplug_large.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/configs/retrieval_coco_mplug_large.yaml -------------------------------------------------------------------------------- /configs/retrieval_flickr30k_mplug_large.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/configs/retrieval_flickr30k_mplug_large.yaml -------------------------------------------------------------------------------- /configs/retrieval_msrvtt_mplug_large.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/configs/retrieval_msrvtt_mplug_large.yaml -------------------------------------------------------------------------------- /configs/videocap_vatex_mplug_large.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/configs/videocap_vatex_mplug_large.yaml -------------------------------------------------------------------------------- /configs/videoqa_msrvtt_mplug_base.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/configs/videoqa_msrvtt_mplug_base.yaml -------------------------------------------------------------------------------- /configs/vqa_mplug_base.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/configs/vqa_mplug_base.yaml -------------------------------------------------------------------------------- /configs/vqa_mplug_large.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/configs/vqa_mplug_large.yaml -------------------------------------------------------------------------------- /dataset/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/dataset/__init__.py -------------------------------------------------------------------------------- /dataset/caption_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/dataset/caption_dataset.py -------------------------------------------------------------------------------- /dataset/grounding_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/dataset/grounding_dataset.py -------------------------------------------------------------------------------- /dataset/nlvr_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/dataset/nlvr_dataset.py -------------------------------------------------------------------------------- /dataset/randaugment.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/dataset/randaugment.py -------------------------------------------------------------------------------- /dataset/rouge.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/dataset/rouge.py -------------------------------------------------------------------------------- /dataset/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/dataset/utils.py -------------------------------------------------------------------------------- /dataset/ve_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/dataset/ve_dataset.py -------------------------------------------------------------------------------- /dataset/vg_transforms.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/dataset/vg_transforms.py -------------------------------------------------------------------------------- /dataset/video_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/dataset/video_dataset.py -------------------------------------------------------------------------------- /dataset/videoqa_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/dataset/videoqa_dataset.py -------------------------------------------------------------------------------- /dataset/vqa_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/dataset/vqa_dataset.py -------------------------------------------------------------------------------- /grounding_mplug.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/grounding_mplug.py -------------------------------------------------------------------------------- /models/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /models/clip/__init__.py: -------------------------------------------------------------------------------- 1 | from .clip import * 2 | -------------------------------------------------------------------------------- /models/clip/bpe_simple_vocab_16e6.txt.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/models/clip/bpe_simple_vocab_16e6.txt.gz -------------------------------------------------------------------------------- /models/clip/clip.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/models/clip/clip.py -------------------------------------------------------------------------------- /models/clip/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/models/clip/model.py -------------------------------------------------------------------------------- /models/clip/simple_tokenizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/models/clip/simple_tokenizer.py -------------------------------------------------------------------------------- /models/model_caption_mplug.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/models/model_caption_mplug.py -------------------------------------------------------------------------------- /models/model_caption_mplug_vatex.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/models/model_caption_mplug_vatex.py -------------------------------------------------------------------------------- /models/model_grounding_mplug.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/models/model_grounding_mplug.py -------------------------------------------------------------------------------- /models/model_retrieval_mplug.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/models/model_retrieval_mplug.py -------------------------------------------------------------------------------- /models/model_vqa_mplug.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/models/model_vqa_mplug.py -------------------------------------------------------------------------------- /models/modeling_mplug.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/models/modeling_mplug.py -------------------------------------------------------------------------------- /models/predictor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/models/predictor.py -------------------------------------------------------------------------------- /models/tokenization_bert.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/models/tokenization_bert.py -------------------------------------------------------------------------------- /models/visual_transformers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/models/visual_transformers.py -------------------------------------------------------------------------------- /models/vit.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/models/vit.py -------------------------------------------------------------------------------- /mplug_framework.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/mplug_framework.png -------------------------------------------------------------------------------- /optim/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/optim/__init__.py -------------------------------------------------------------------------------- /optim/adafactor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/optim/adafactor.py -------------------------------------------------------------------------------- /optim/adahessian.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/optim/adahessian.py -------------------------------------------------------------------------------- /optim/adamp.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/optim/adamp.py -------------------------------------------------------------------------------- /optim/adamw.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/optim/adamw.py -------------------------------------------------------------------------------- /optim/lookahead.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/optim/lookahead.py -------------------------------------------------------------------------------- /optim/nadam.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/optim/nadam.py -------------------------------------------------------------------------------- /optim/novograd.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/optim/novograd.py -------------------------------------------------------------------------------- /optim/nvnovograd.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/optim/nvnovograd.py -------------------------------------------------------------------------------- /optim/optim_factory.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/optim/optim_factory.py -------------------------------------------------------------------------------- /optim/radam.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/optim/radam.py -------------------------------------------------------------------------------- /optim/rmsprop_tf.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/optim/rmsprop_tf.py -------------------------------------------------------------------------------- /optim/sgdp.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/optim/sgdp.py -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/requirements.txt -------------------------------------------------------------------------------- /retrieval_img_mplug.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/retrieval_img_mplug.py -------------------------------------------------------------------------------- /retrieval_vid_mplug.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/retrieval_vid_mplug.py -------------------------------------------------------------------------------- /scheduler/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/scheduler/__init__.py -------------------------------------------------------------------------------- /scheduler/cosine_lr.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/scheduler/cosine_lr.py -------------------------------------------------------------------------------- /scheduler/plateau_lr.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/scheduler/plateau_lr.py -------------------------------------------------------------------------------- /scheduler/scheduler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/scheduler/scheduler.py -------------------------------------------------------------------------------- /scheduler/scheduler_factory.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/scheduler/scheduler_factory.py -------------------------------------------------------------------------------- /scheduler/step_lr.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/scheduler/step_lr.py -------------------------------------------------------------------------------- /scheduler/tanh_lr.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/scheduler/tanh_lr.py -------------------------------------------------------------------------------- /scripts/caption_mplug_base.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/scripts/caption_mplug_base.sh -------------------------------------------------------------------------------- /scripts/caption_mplug_base_scst.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/scripts/caption_mplug_base_scst.sh -------------------------------------------------------------------------------- /scripts/caption_mplug_large.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/scripts/caption_mplug_large.sh -------------------------------------------------------------------------------- /scripts/caption_mplug_large_scst.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/scripts/caption_mplug_large_scst.sh -------------------------------------------------------------------------------- /scripts/grounding_mplug_large.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/scripts/grounding_mplug_large.sh -------------------------------------------------------------------------------- /scripts/retrieval_coco_mplug_large.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/scripts/retrieval_coco_mplug_large.sh -------------------------------------------------------------------------------- /scripts/retrieval_flickr30k_mplug_large.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/scripts/retrieval_flickr30k_mplug_large.sh -------------------------------------------------------------------------------- /scripts/retrieval_msrvtt_mplug_large_zs.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/scripts/retrieval_msrvtt_mplug_large_zs.sh -------------------------------------------------------------------------------- /scripts/videocap_vatex_mplug_large.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/scripts/videocap_vatex_mplug_large.sh -------------------------------------------------------------------------------- /scripts/videoqa_msrvtt_mplug_base.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/scripts/videoqa_msrvtt_mplug_base.sh -------------------------------------------------------------------------------- /scripts/vqa_mplug_base.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/scripts/vqa_mplug_base.sh -------------------------------------------------------------------------------- /scripts/vqa_mplug_large.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/scripts/vqa_mplug_large.sh -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/utils.py -------------------------------------------------------------------------------- /vgTools/utils/box_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/vgTools/utils/box_utils.py -------------------------------------------------------------------------------- /vgTools/utils/eval_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/vgTools/utils/eval_utils.py -------------------------------------------------------------------------------- /vgTools/utils/loss_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/vgTools/utils/loss_utils.py -------------------------------------------------------------------------------- /vgTools/utils/misc.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/vgTools/utils/misc.py -------------------------------------------------------------------------------- /vgTools/utils/transforms.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/vgTools/utils/transforms.py -------------------------------------------------------------------------------- /vgTools/utils/word_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/vgTools/utils/word_utils.py -------------------------------------------------------------------------------- /videocap_mplug.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/videocap_mplug.py -------------------------------------------------------------------------------- /videoqa_mplug.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/videoqa_mplug.py -------------------------------------------------------------------------------- /vqaTools/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'aagrawal' 2 | -------------------------------------------------------------------------------- /vqaTools/vqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/vqaTools/vqa.py -------------------------------------------------------------------------------- /vqaTools/vqaEval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/vqaTools/vqaEval.py -------------------------------------------------------------------------------- /vqa_mplug.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-PLUG/mPLUG/HEAD/vqa_mplug.py --------------------------------------------------------------------------------