├── README.md ├── ctx ├── .gitignore ├── README.md ├── dataset.py ├── encode_captions.py ├── encode_images.py ├── images │ └── txt_ctx.png └── retrieve_captions.py ├── environment.yml ├── images └── model.jpg └── m2 ├── .gitignore ├── README.md ├── data ├── __init__.py ├── dataset.py ├── example.py ├── field.py ├── tsv_file.py ├── utils.py └── vocab.py ├── evaluation ├── __init__.py ├── bleu │ ├── __init__.py │ ├── bleu.py │ └── bleu_scorer.py ├── cider │ ├── __init__.py │ ├── cider.py │ └── cider_scorer.py ├── meteor │ ├── __init__.py │ ├── data │ │ └── paraphrase-en.gz │ ├── meteor-1.5.jar │ └── meteor.py ├── rouge │ ├── __init__.py │ └── rouge.py ├── spice │ ├── __init__.py │ ├── get_stanford_models.py │ ├── lib │ │ ├── Meteor-1.5.jar │ │ ├── SceneGraphParser-1.0.jar │ │ ├── ejml-0.23.jar │ │ ├── fst-2.47.jar │ │ ├── guava-19.0.jar │ │ ├── hamcrest-core-1.3.jar │ │ ├── jackson-core-2.5.3.jar │ │ ├── javassist-3.19.0-GA.jar │ │ ├── json-simple-1.1.1.jar │ │ ├── junit-4.12.jar │ │ ├── lmdbjni-0.4.6.jar │ │ ├── lmdbjni-linux64-0.4.6.jar │ │ ├── lmdbjni-osx64-0.4.6.jar │ │ ├── lmdbjni-win64-0.4.6.jar │ │ ├── objenesis-2.4.jar │ │ ├── slf4j-api-1.7.12.jar │ │ └── slf4j-simple-1.7.21.jar │ ├── spice-1.0.jar │ └── spice.py ├── stanford-corenlp-3.4.1.jar └── tokenizer.py ├── models ├── __init__.py ├── beam_search │ ├── __init__.py │ └── beam_search.py ├── captioning_model.py ├── containers.py └── transformer │ ├── __init__.py │ ├── attention.py │ ├── decoders.py │ ├── encoders.py │ ├── projector.py │ ├── transformer.py │ └── utils.py ├── requirements.txt ├── train.py ├── utils ├── __init__.py ├── typing.py └── utils.py └── vocab └── vocab_coco.pkl /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/README.md -------------------------------------------------------------------------------- /ctx/.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__ 2 | datasets 3 | outputs 4 | *.bak -------------------------------------------------------------------------------- /ctx/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/ctx/README.md -------------------------------------------------------------------------------- /ctx/dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/ctx/dataset.py -------------------------------------------------------------------------------- /ctx/encode_captions.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/ctx/encode_captions.py -------------------------------------------------------------------------------- /ctx/encode_images.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/ctx/encode_images.py -------------------------------------------------------------------------------- /ctx/images/txt_ctx.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/ctx/images/txt_ctx.png -------------------------------------------------------------------------------- /ctx/retrieve_captions.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/ctx/retrieve_captions.py -------------------------------------------------------------------------------- /environment.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/environment.yml -------------------------------------------------------------------------------- /images/model.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/images/model.jpg -------------------------------------------------------------------------------- /m2/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/m2/.gitignore -------------------------------------------------------------------------------- /m2/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/m2/README.md -------------------------------------------------------------------------------- /m2/data/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/m2/data/__init__.py -------------------------------------------------------------------------------- /m2/data/dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/m2/data/dataset.py -------------------------------------------------------------------------------- /m2/data/example.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/m2/data/example.py -------------------------------------------------------------------------------- /m2/data/field.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/m2/data/field.py -------------------------------------------------------------------------------- /m2/data/tsv_file.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/m2/data/tsv_file.py -------------------------------------------------------------------------------- /m2/data/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/m2/data/utils.py -------------------------------------------------------------------------------- /m2/data/vocab.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/m2/data/vocab.py -------------------------------------------------------------------------------- /m2/evaluation/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/m2/evaluation/__init__.py -------------------------------------------------------------------------------- /m2/evaluation/bleu/__init__.py: -------------------------------------------------------------------------------- 1 | from .bleu import Bleu -------------------------------------------------------------------------------- /m2/evaluation/bleu/bleu.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/m2/evaluation/bleu/bleu.py -------------------------------------------------------------------------------- /m2/evaluation/bleu/bleu_scorer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/m2/evaluation/bleu/bleu_scorer.py -------------------------------------------------------------------------------- /m2/evaluation/cider/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/m2/evaluation/cider/__init__.py -------------------------------------------------------------------------------- /m2/evaluation/cider/cider.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/m2/evaluation/cider/cider.py -------------------------------------------------------------------------------- /m2/evaluation/cider/cider_scorer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/m2/evaluation/cider/cider_scorer.py -------------------------------------------------------------------------------- /m2/evaluation/meteor/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/m2/evaluation/meteor/__init__.py -------------------------------------------------------------------------------- /m2/evaluation/meteor/data/paraphrase-en.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/m2/evaluation/meteor/data/paraphrase-en.gz -------------------------------------------------------------------------------- /m2/evaluation/meteor/meteor-1.5.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/m2/evaluation/meteor/meteor-1.5.jar -------------------------------------------------------------------------------- /m2/evaluation/meteor/meteor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/m2/evaluation/meteor/meteor.py -------------------------------------------------------------------------------- /m2/evaluation/rouge/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/m2/evaluation/rouge/__init__.py -------------------------------------------------------------------------------- /m2/evaluation/rouge/rouge.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/m2/evaluation/rouge/rouge.py -------------------------------------------------------------------------------- /m2/evaluation/spice/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /m2/evaluation/spice/get_stanford_models.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/m2/evaluation/spice/get_stanford_models.py -------------------------------------------------------------------------------- /m2/evaluation/spice/lib/Meteor-1.5.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/m2/evaluation/spice/lib/Meteor-1.5.jar -------------------------------------------------------------------------------- /m2/evaluation/spice/lib/SceneGraphParser-1.0.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/m2/evaluation/spice/lib/SceneGraphParser-1.0.jar -------------------------------------------------------------------------------- /m2/evaluation/spice/lib/ejml-0.23.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/m2/evaluation/spice/lib/ejml-0.23.jar -------------------------------------------------------------------------------- /m2/evaluation/spice/lib/fst-2.47.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/m2/evaluation/spice/lib/fst-2.47.jar -------------------------------------------------------------------------------- /m2/evaluation/spice/lib/guava-19.0.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/m2/evaluation/spice/lib/guava-19.0.jar -------------------------------------------------------------------------------- /m2/evaluation/spice/lib/hamcrest-core-1.3.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/m2/evaluation/spice/lib/hamcrest-core-1.3.jar -------------------------------------------------------------------------------- /m2/evaluation/spice/lib/jackson-core-2.5.3.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/m2/evaluation/spice/lib/jackson-core-2.5.3.jar -------------------------------------------------------------------------------- /m2/evaluation/spice/lib/javassist-3.19.0-GA.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/m2/evaluation/spice/lib/javassist-3.19.0-GA.jar -------------------------------------------------------------------------------- /m2/evaluation/spice/lib/json-simple-1.1.1.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/m2/evaluation/spice/lib/json-simple-1.1.1.jar -------------------------------------------------------------------------------- /m2/evaluation/spice/lib/junit-4.12.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/m2/evaluation/spice/lib/junit-4.12.jar -------------------------------------------------------------------------------- /m2/evaluation/spice/lib/lmdbjni-0.4.6.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/m2/evaluation/spice/lib/lmdbjni-0.4.6.jar -------------------------------------------------------------------------------- /m2/evaluation/spice/lib/lmdbjni-linux64-0.4.6.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/m2/evaluation/spice/lib/lmdbjni-linux64-0.4.6.jar -------------------------------------------------------------------------------- /m2/evaluation/spice/lib/lmdbjni-osx64-0.4.6.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/m2/evaluation/spice/lib/lmdbjni-osx64-0.4.6.jar -------------------------------------------------------------------------------- /m2/evaluation/spice/lib/lmdbjni-win64-0.4.6.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/m2/evaluation/spice/lib/lmdbjni-win64-0.4.6.jar -------------------------------------------------------------------------------- /m2/evaluation/spice/lib/objenesis-2.4.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/m2/evaluation/spice/lib/objenesis-2.4.jar -------------------------------------------------------------------------------- /m2/evaluation/spice/lib/slf4j-api-1.7.12.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/m2/evaluation/spice/lib/slf4j-api-1.7.12.jar -------------------------------------------------------------------------------- /m2/evaluation/spice/lib/slf4j-simple-1.7.21.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/m2/evaluation/spice/lib/slf4j-simple-1.7.21.jar -------------------------------------------------------------------------------- /m2/evaluation/spice/spice-1.0.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/m2/evaluation/spice/spice-1.0.jar -------------------------------------------------------------------------------- /m2/evaluation/spice/spice.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/m2/evaluation/spice/spice.py -------------------------------------------------------------------------------- /m2/evaluation/stanford-corenlp-3.4.1.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/m2/evaluation/stanford-corenlp-3.4.1.jar -------------------------------------------------------------------------------- /m2/evaluation/tokenizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/m2/evaluation/tokenizer.py -------------------------------------------------------------------------------- /m2/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/m2/models/__init__.py -------------------------------------------------------------------------------- /m2/models/beam_search/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/m2/models/beam_search/__init__.py -------------------------------------------------------------------------------- /m2/models/beam_search/beam_search.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/m2/models/beam_search/beam_search.py -------------------------------------------------------------------------------- /m2/models/captioning_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/m2/models/captioning_model.py -------------------------------------------------------------------------------- /m2/models/containers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/m2/models/containers.py -------------------------------------------------------------------------------- /m2/models/transformer/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/m2/models/transformer/__init__.py -------------------------------------------------------------------------------- /m2/models/transformer/attention.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/m2/models/transformer/attention.py -------------------------------------------------------------------------------- /m2/models/transformer/decoders.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/m2/models/transformer/decoders.py -------------------------------------------------------------------------------- /m2/models/transformer/encoders.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/m2/models/transformer/encoders.py -------------------------------------------------------------------------------- /m2/models/transformer/projector.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/m2/models/transformer/projector.py -------------------------------------------------------------------------------- /m2/models/transformer/transformer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/m2/models/transformer/transformer.py -------------------------------------------------------------------------------- /m2/models/transformer/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/m2/models/transformer/utils.py -------------------------------------------------------------------------------- /m2/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/m2/requirements.txt -------------------------------------------------------------------------------- /m2/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/m2/train.py -------------------------------------------------------------------------------- /m2/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/m2/utils/__init__.py -------------------------------------------------------------------------------- /m2/utils/typing.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/m2/utils/typing.py -------------------------------------------------------------------------------- /m2/utils/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/m2/utils/utils.py -------------------------------------------------------------------------------- /m2/vocab/vocab_coco.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GT-RIPL/Xmodal-Ctx/HEAD/m2/vocab/vocab_coco.pkl --------------------------------------------------------------------------------