├── README.md ├── config ├── bert-base-uncased_weight_name.json ├── bert-large-uncased_weight_name.json ├── bert_base_2layer_2conect.json ├── bert_base_4layer_4conect.json ├── bert_base_6layer_6conect.json ├── bert_base_8layer_8conect.json ├── bert_base_baseline.json ├── bert_large_2layer_2conect.json ├── bert_large_4layer_4conect.json ├── bert_large_6layer_6conect.json └── bert_large_baseline.json ├── data └── README.md ├── eval_retrieval.py ├── eval_tasks.py ├── fig ├── vilbert.png └── vilbert_trim.png ├── requirements.txt ├── script ├── conceptual_caption_preprocess_sequential_train.py ├── conceptual_caption_preprocess_sequential_train_segment.py ├── conceptual_caption_preprocess_sequential_val.py ├── convert_lmdb_VCR.py ├── convert_lmdb_VCR_gt.py ├── convert_lmdb_flickr30k.py ├── convert_lmdb_refcoco+.py ├── convert_lmdb_refcoco+_gt.py ├── convert_test_lmdb.py └── convert_trainval_lmdb.py ├── tools ├── DownloadConcptualCaption │ ├── LICENSE │ ├── README.md │ ├── download_data.py │ └── downloaded_report.ipynb ├── __init__.py └── refer │ ├── LICENSE │ ├── Makefile │ ├── README.md │ ├── __init__.py │ ├── data │ └── README.md │ ├── evaluation │ ├── __init__.py │ ├── bleu │ │ ├── LICENSE │ │ ├── __init__.py │ │ ├── bleu.py │ │ └── bleu_scorer.py │ ├── cider │ │ ├── __init__.py │ │ ├── cider.py │ │ └── cider_scorer.py │ ├── meteor │ │ ├── __init__.py │ │ ├── data │ │ │ └── paraphrase-en.gz │ │ ├── meteor-1.5.jar │ │ └── meteor.py │ ├── readme.txt │ ├── refEvaluation.py │ ├── rouge │ │ ├── __init__.py │ │ └── rouge.py │ └── tokenizer │ │ ├── __init__.py │ │ ├── ptbtokenizer.py │ │ └── stanford-corenlp-3.4.1.jar │ ├── external │ ├── .gitignore │ ├── README.md │ ├── __init__.py │ ├── _mask.cpython-36m-x86_64-linux-gnu.so │ ├── _mask.cpython-37m-x86_64-linux-gnu.so │ ├── _mask.pyx │ ├── mask.py │ ├── maskApi.c │ └── maskApi.h │ ├── pyEvalDemo.ipynb │ ├── pyReferDemo.ipynb │ ├── refer.py │ ├── setup.py │ └── test │ ├── sample_expressions_testA.json │ └── sample_expressions_testB.json ├── train_baseline.py ├── train_concap.py ├── train_tasks.py ├── vilbert ├── __init__.py ├── basebert.py ├── datasets │ ├── __init__.py │ ├── _image_features_reader.py │ ├── concept_cap_dataset.py │ ├── refer_expression_dataset.py │ ├── retreival_dataset.py │ ├── vcr_dataset.py │ └── vqa_dataset.py ├── optimization.py ├── task_utils.py ├── utils.py └── vilbert.py └── vlbert_tasks.yml /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/README.md -------------------------------------------------------------------------------- /config/bert-base-uncased_weight_name.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/config/bert-base-uncased_weight_name.json -------------------------------------------------------------------------------- /config/bert-large-uncased_weight_name.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/config/bert-large-uncased_weight_name.json -------------------------------------------------------------------------------- /config/bert_base_2layer_2conect.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/config/bert_base_2layer_2conect.json -------------------------------------------------------------------------------- /config/bert_base_4layer_4conect.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/config/bert_base_4layer_4conect.json -------------------------------------------------------------------------------- /config/bert_base_6layer_6conect.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/config/bert_base_6layer_6conect.json -------------------------------------------------------------------------------- /config/bert_base_8layer_8conect.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/config/bert_base_8layer_8conect.json -------------------------------------------------------------------------------- /config/bert_base_baseline.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/config/bert_base_baseline.json -------------------------------------------------------------------------------- /config/bert_large_2layer_2conect.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/config/bert_large_2layer_2conect.json -------------------------------------------------------------------------------- /config/bert_large_4layer_4conect.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/config/bert_large_4layer_4conect.json -------------------------------------------------------------------------------- /config/bert_large_6layer_6conect.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/config/bert_large_6layer_6conect.json -------------------------------------------------------------------------------- /config/bert_large_baseline.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/config/bert_large_baseline.json -------------------------------------------------------------------------------- /data/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/data/README.md -------------------------------------------------------------------------------- /eval_retrieval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/eval_retrieval.py -------------------------------------------------------------------------------- /eval_tasks.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/eval_tasks.py -------------------------------------------------------------------------------- /fig/vilbert.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/fig/vilbert.png -------------------------------------------------------------------------------- /fig/vilbert_trim.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/fig/vilbert_trim.png -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/requirements.txt -------------------------------------------------------------------------------- /script/conceptual_caption_preprocess_sequential_train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/script/conceptual_caption_preprocess_sequential_train.py -------------------------------------------------------------------------------- /script/conceptual_caption_preprocess_sequential_train_segment.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/script/conceptual_caption_preprocess_sequential_train_segment.py -------------------------------------------------------------------------------- /script/conceptual_caption_preprocess_sequential_val.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/script/conceptual_caption_preprocess_sequential_val.py -------------------------------------------------------------------------------- /script/convert_lmdb_VCR.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/script/convert_lmdb_VCR.py -------------------------------------------------------------------------------- /script/convert_lmdb_VCR_gt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/script/convert_lmdb_VCR_gt.py -------------------------------------------------------------------------------- /script/convert_lmdb_flickr30k.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/script/convert_lmdb_flickr30k.py -------------------------------------------------------------------------------- /script/convert_lmdb_refcoco+.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/script/convert_lmdb_refcoco+.py -------------------------------------------------------------------------------- /script/convert_lmdb_refcoco+_gt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/script/convert_lmdb_refcoco+_gt.py -------------------------------------------------------------------------------- /script/convert_test_lmdb.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/script/convert_test_lmdb.py -------------------------------------------------------------------------------- /script/convert_trainval_lmdb.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/script/convert_trainval_lmdb.py -------------------------------------------------------------------------------- /tools/DownloadConcptualCaption/LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/tools/DownloadConcptualCaption/LICENSE -------------------------------------------------------------------------------- /tools/DownloadConcptualCaption/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/tools/DownloadConcptualCaption/README.md -------------------------------------------------------------------------------- /tools/DownloadConcptualCaption/download_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/tools/DownloadConcptualCaption/download_data.py -------------------------------------------------------------------------------- /tools/DownloadConcptualCaption/downloaded_report.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/tools/DownloadConcptualCaption/downloaded_report.ipynb -------------------------------------------------------------------------------- /tools/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'tylin' 2 | -------------------------------------------------------------------------------- /tools/refer/LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/tools/refer/LICENSE -------------------------------------------------------------------------------- /tools/refer/Makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/tools/refer/Makefile -------------------------------------------------------------------------------- /tools/refer/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/tools/refer/README.md -------------------------------------------------------------------------------- /tools/refer/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'tylin' 2 | -------------------------------------------------------------------------------- /tools/refer/data/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/tools/refer/data/README.md -------------------------------------------------------------------------------- /tools/refer/evaluation/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'licheng' 2 | 3 | 4 | -------------------------------------------------------------------------------- /tools/refer/evaluation/bleu/LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/tools/refer/evaluation/bleu/LICENSE -------------------------------------------------------------------------------- /tools/refer/evaluation/bleu/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'tylin' 2 | -------------------------------------------------------------------------------- /tools/refer/evaluation/bleu/bleu.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/tools/refer/evaluation/bleu/bleu.py -------------------------------------------------------------------------------- /tools/refer/evaluation/bleu/bleu_scorer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/tools/refer/evaluation/bleu/bleu_scorer.py -------------------------------------------------------------------------------- /tools/refer/evaluation/cider/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'tylin' 2 | -------------------------------------------------------------------------------- /tools/refer/evaluation/cider/cider.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/tools/refer/evaluation/cider/cider.py -------------------------------------------------------------------------------- /tools/refer/evaluation/cider/cider_scorer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/tools/refer/evaluation/cider/cider_scorer.py -------------------------------------------------------------------------------- /tools/refer/evaluation/meteor/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'tylin' 2 | -------------------------------------------------------------------------------- /tools/refer/evaluation/meteor/data/paraphrase-en.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/tools/refer/evaluation/meteor/data/paraphrase-en.gz -------------------------------------------------------------------------------- /tools/refer/evaluation/meteor/meteor-1.5.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/tools/refer/evaluation/meteor/meteor-1.5.jar -------------------------------------------------------------------------------- /tools/refer/evaluation/meteor/meteor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/tools/refer/evaluation/meteor/meteor.py -------------------------------------------------------------------------------- /tools/refer/evaluation/readme.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/tools/refer/evaluation/readme.txt -------------------------------------------------------------------------------- /tools/refer/evaluation/refEvaluation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/tools/refer/evaluation/refEvaluation.py -------------------------------------------------------------------------------- /tools/refer/evaluation/rouge/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'vrama91' 2 | -------------------------------------------------------------------------------- /tools/refer/evaluation/rouge/rouge.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/tools/refer/evaluation/rouge/rouge.py -------------------------------------------------------------------------------- /tools/refer/evaluation/tokenizer/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'hfang' 2 | -------------------------------------------------------------------------------- /tools/refer/evaluation/tokenizer/ptbtokenizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/tools/refer/evaluation/tokenizer/ptbtokenizer.py -------------------------------------------------------------------------------- /tools/refer/evaluation/tokenizer/stanford-corenlp-3.4.1.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/tools/refer/evaluation/tokenizer/stanford-corenlp-3.4.1.jar -------------------------------------------------------------------------------- /tools/refer/external/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/tools/refer/external/.gitignore -------------------------------------------------------------------------------- /tools/refer/external/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/tools/refer/external/README.md -------------------------------------------------------------------------------- /tools/refer/external/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'tylin' 2 | -------------------------------------------------------------------------------- /tools/refer/external/_mask.cpython-36m-x86_64-linux-gnu.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/tools/refer/external/_mask.cpython-36m-x86_64-linux-gnu.so -------------------------------------------------------------------------------- /tools/refer/external/_mask.cpython-37m-x86_64-linux-gnu.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/tools/refer/external/_mask.cpython-37m-x86_64-linux-gnu.so -------------------------------------------------------------------------------- /tools/refer/external/_mask.pyx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/tools/refer/external/_mask.pyx -------------------------------------------------------------------------------- /tools/refer/external/mask.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/tools/refer/external/mask.py -------------------------------------------------------------------------------- /tools/refer/external/maskApi.c: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/tools/refer/external/maskApi.c -------------------------------------------------------------------------------- /tools/refer/external/maskApi.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/tools/refer/external/maskApi.h -------------------------------------------------------------------------------- /tools/refer/pyEvalDemo.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/tools/refer/pyEvalDemo.ipynb -------------------------------------------------------------------------------- /tools/refer/pyReferDemo.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/tools/refer/pyReferDemo.ipynb -------------------------------------------------------------------------------- /tools/refer/refer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/tools/refer/refer.py -------------------------------------------------------------------------------- /tools/refer/setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/tools/refer/setup.py -------------------------------------------------------------------------------- /tools/refer/test/sample_expressions_testA.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/tools/refer/test/sample_expressions_testA.json -------------------------------------------------------------------------------- /tools/refer/test/sample_expressions_testB.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/tools/refer/test/sample_expressions_testB.json -------------------------------------------------------------------------------- /train_baseline.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/train_baseline.py -------------------------------------------------------------------------------- /train_concap.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/train_concap.py -------------------------------------------------------------------------------- /train_tasks.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/train_tasks.py -------------------------------------------------------------------------------- /vilbert/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /vilbert/basebert.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/vilbert/basebert.py -------------------------------------------------------------------------------- /vilbert/datasets/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/vilbert/datasets/__init__.py -------------------------------------------------------------------------------- /vilbert/datasets/_image_features_reader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/vilbert/datasets/_image_features_reader.py -------------------------------------------------------------------------------- /vilbert/datasets/concept_cap_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/vilbert/datasets/concept_cap_dataset.py -------------------------------------------------------------------------------- /vilbert/datasets/refer_expression_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/vilbert/datasets/refer_expression_dataset.py -------------------------------------------------------------------------------- /vilbert/datasets/retreival_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/vilbert/datasets/retreival_dataset.py -------------------------------------------------------------------------------- /vilbert/datasets/vcr_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/vilbert/datasets/vcr_dataset.py -------------------------------------------------------------------------------- /vilbert/datasets/vqa_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/vilbert/datasets/vqa_dataset.py -------------------------------------------------------------------------------- /vilbert/optimization.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/vilbert/optimization.py -------------------------------------------------------------------------------- /vilbert/task_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/vilbert/task_utils.py -------------------------------------------------------------------------------- /vilbert/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/vilbert/utils.py -------------------------------------------------------------------------------- /vilbert/vilbert.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/vilbert/vilbert.py -------------------------------------------------------------------------------- /vlbert_tasks.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jiasenlu/vilbert_beta/HEAD/vlbert_tasks.yml --------------------------------------------------------------------------------