├── .gitignore ├── LICENSE ├── README.md ├── config ├── minigpt4.yaml └── prerained_minigpt4_7b.pth ├── constants.py ├── dataloader.py ├── datasets ├── CC3M │ ├── cc3m_val.tsv │ ├── preprocess_images.py │ └── val │ │ ├── 000007775.jpg │ │ ├── 000007775.txt │ │ ├── 000009256.jpg │ │ └── 000009256.txt ├── MMDialog │ └── test │ │ ├── -5632896413199993329.jpg │ │ ├── -5763052819421198282.jpg │ │ ├── 2682085160072908899.jpg │ │ ├── 3433276376275926389.jpg │ │ └── test_conversations.txt └── VIST │ ├── val_cleaned.json │ ├── val_image_mapping.json │ └── val_images │ ├── 000000000.jpg │ ├── 000000003.jpg │ ├── 000000004.jpg │ ├── 000000005.jpg │ └── 000000007.jpg ├── examples ├── 000000005.jpg ├── 000000007.jpg └── playground.py ├── fid_real_feaures.pt ├── figs ├── structure.png └── teaser.png ├── metric.py ├── minigpt4 ├── __init__.py ├── common │ ├── __init__.py │ ├── config.py │ ├── dist_utils.py │ ├── gradcam.py │ ├── logger.py │ ├── optims.py │ ├── registry.py │ └── utils.py ├── configs │ ├── datasets │ │ ├── cc_sbu │ │ │ ├── align.yaml │ │ │ └── defaults.yaml │ │ └── laion │ │ │ └── defaults.yaml │ ├── default.yaml │ └── models │ │ └── minigpt4.yaml ├── conversation │ ├── __init__.py │ └── conversation.py ├── datasets │ ├── __init__.py │ ├── builders │ │ ├── __init__.py │ │ ├── base_dataset_builder.py │ │ └── image_text_pair_builder.py │ ├── data_utils.py │ └── datasets │ │ ├── __init__.py │ │ ├── base_dataset.py │ │ ├── caption_datasets.py │ │ ├── cc_sbu_dataset.py │ │ ├── dataloader_utils.py │ │ └── laion_dataset.py ├── models │ ├── Qformer.py │ ├── __init__.py │ ├── base_model.py │ ├── blip2.py │ ├── blip2_outputs.py │ ├── eva_vit.py │ ├── generation_helper.py │ ├── mini_gpt4.py │ ├── mini_gpt5.py │ └── modeling_llama.py ├── processors │ ├── __init__.py │ ├── base_processor.py │ ├── blip_processors.py │ └── randaugment.py ├── runners │ ├── __init__.py │ └── runner_base.py └── tasks │ ├── __init__.py │ ├── base_task.py │ └── image_text_pretrain.py ├── model.py ├── requirements.txt ├── train_eval.py └── utils.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/README.md -------------------------------------------------------------------------------- /config/minigpt4.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/config/minigpt4.yaml -------------------------------------------------------------------------------- /config/prerained_minigpt4_7b.pth: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/config/prerained_minigpt4_7b.pth -------------------------------------------------------------------------------- /constants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/constants.py -------------------------------------------------------------------------------- /dataloader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/dataloader.py -------------------------------------------------------------------------------- /datasets/CC3M/cc3m_val.tsv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/datasets/CC3M/cc3m_val.tsv -------------------------------------------------------------------------------- /datasets/CC3M/preprocess_images.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/datasets/CC3M/preprocess_images.py -------------------------------------------------------------------------------- /datasets/CC3M/val/000007775.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/datasets/CC3M/val/000007775.jpg -------------------------------------------------------------------------------- /datasets/CC3M/val/000007775.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/datasets/CC3M/val/000007775.txt -------------------------------------------------------------------------------- /datasets/CC3M/val/000009256.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/datasets/CC3M/val/000009256.jpg -------------------------------------------------------------------------------- /datasets/CC3M/val/000009256.txt: -------------------------------------------------------------------------------- 1 | shot of the full tree . -------------------------------------------------------------------------------- /datasets/MMDialog/test/-5632896413199993329.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/datasets/MMDialog/test/-5632896413199993329.jpg -------------------------------------------------------------------------------- /datasets/MMDialog/test/-5763052819421198282.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/datasets/MMDialog/test/-5763052819421198282.jpg -------------------------------------------------------------------------------- /datasets/MMDialog/test/2682085160072908899.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/datasets/MMDialog/test/2682085160072908899.jpg -------------------------------------------------------------------------------- /datasets/MMDialog/test/3433276376275926389.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/datasets/MMDialog/test/3433276376275926389.jpg -------------------------------------------------------------------------------- /datasets/MMDialog/test/test_conversations.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/datasets/MMDialog/test/test_conversations.txt -------------------------------------------------------------------------------- /datasets/VIST/val_cleaned.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/datasets/VIST/val_cleaned.json -------------------------------------------------------------------------------- /datasets/VIST/val_image_mapping.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/datasets/VIST/val_image_mapping.json -------------------------------------------------------------------------------- /datasets/VIST/val_images/000000000.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/datasets/VIST/val_images/000000000.jpg -------------------------------------------------------------------------------- /datasets/VIST/val_images/000000003.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/datasets/VIST/val_images/000000003.jpg -------------------------------------------------------------------------------- /datasets/VIST/val_images/000000004.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/datasets/VIST/val_images/000000004.jpg -------------------------------------------------------------------------------- /datasets/VIST/val_images/000000005.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/datasets/VIST/val_images/000000005.jpg -------------------------------------------------------------------------------- /datasets/VIST/val_images/000000007.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/datasets/VIST/val_images/000000007.jpg -------------------------------------------------------------------------------- /examples/000000005.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/examples/000000005.jpg -------------------------------------------------------------------------------- /examples/000000007.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/examples/000000007.jpg -------------------------------------------------------------------------------- /examples/playground.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/examples/playground.py -------------------------------------------------------------------------------- /fid_real_feaures.pt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/fid_real_feaures.pt -------------------------------------------------------------------------------- /figs/structure.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/figs/structure.png -------------------------------------------------------------------------------- /figs/teaser.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/figs/teaser.png -------------------------------------------------------------------------------- /metric.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/metric.py -------------------------------------------------------------------------------- /minigpt4/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/minigpt4/__init__.py -------------------------------------------------------------------------------- /minigpt4/common/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /minigpt4/common/config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/minigpt4/common/config.py -------------------------------------------------------------------------------- /minigpt4/common/dist_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/minigpt4/common/dist_utils.py -------------------------------------------------------------------------------- /minigpt4/common/gradcam.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/minigpt4/common/gradcam.py -------------------------------------------------------------------------------- /minigpt4/common/logger.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/minigpt4/common/logger.py -------------------------------------------------------------------------------- /minigpt4/common/optims.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/minigpt4/common/optims.py -------------------------------------------------------------------------------- /minigpt4/common/registry.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/minigpt4/common/registry.py -------------------------------------------------------------------------------- /minigpt4/common/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/minigpt4/common/utils.py -------------------------------------------------------------------------------- /minigpt4/configs/datasets/cc_sbu/align.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/minigpt4/configs/datasets/cc_sbu/align.yaml -------------------------------------------------------------------------------- /minigpt4/configs/datasets/cc_sbu/defaults.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/minigpt4/configs/datasets/cc_sbu/defaults.yaml -------------------------------------------------------------------------------- /minigpt4/configs/datasets/laion/defaults.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/minigpt4/configs/datasets/laion/defaults.yaml -------------------------------------------------------------------------------- /minigpt4/configs/default.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/minigpt4/configs/default.yaml -------------------------------------------------------------------------------- /minigpt4/configs/models/minigpt4.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/minigpt4/configs/models/minigpt4.yaml -------------------------------------------------------------------------------- /minigpt4/conversation/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /minigpt4/conversation/conversation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/minigpt4/conversation/conversation.py -------------------------------------------------------------------------------- /minigpt4/datasets/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /minigpt4/datasets/builders/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/minigpt4/datasets/builders/__init__.py -------------------------------------------------------------------------------- /minigpt4/datasets/builders/base_dataset_builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/minigpt4/datasets/builders/base_dataset_builder.py -------------------------------------------------------------------------------- /minigpt4/datasets/builders/image_text_pair_builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/minigpt4/datasets/builders/image_text_pair_builder.py -------------------------------------------------------------------------------- /minigpt4/datasets/data_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/minigpt4/datasets/data_utils.py -------------------------------------------------------------------------------- /minigpt4/datasets/datasets/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /minigpt4/datasets/datasets/base_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/minigpt4/datasets/datasets/base_dataset.py -------------------------------------------------------------------------------- /minigpt4/datasets/datasets/caption_datasets.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/minigpt4/datasets/datasets/caption_datasets.py -------------------------------------------------------------------------------- /minigpt4/datasets/datasets/cc_sbu_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/minigpt4/datasets/datasets/cc_sbu_dataset.py -------------------------------------------------------------------------------- /minigpt4/datasets/datasets/dataloader_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/minigpt4/datasets/datasets/dataloader_utils.py -------------------------------------------------------------------------------- /minigpt4/datasets/datasets/laion_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/minigpt4/datasets/datasets/laion_dataset.py -------------------------------------------------------------------------------- /minigpt4/models/Qformer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/minigpt4/models/Qformer.py -------------------------------------------------------------------------------- /minigpt4/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/minigpt4/models/__init__.py -------------------------------------------------------------------------------- /minigpt4/models/base_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/minigpt4/models/base_model.py -------------------------------------------------------------------------------- /minigpt4/models/blip2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/minigpt4/models/blip2.py -------------------------------------------------------------------------------- /minigpt4/models/blip2_outputs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/minigpt4/models/blip2_outputs.py -------------------------------------------------------------------------------- /minigpt4/models/eva_vit.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/minigpt4/models/eva_vit.py -------------------------------------------------------------------------------- /minigpt4/models/generation_helper.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/minigpt4/models/generation_helper.py -------------------------------------------------------------------------------- /minigpt4/models/mini_gpt4.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/minigpt4/models/mini_gpt4.py -------------------------------------------------------------------------------- /minigpt4/models/mini_gpt5.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/minigpt4/models/mini_gpt5.py -------------------------------------------------------------------------------- /minigpt4/models/modeling_llama.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/minigpt4/models/modeling_llama.py -------------------------------------------------------------------------------- /minigpt4/processors/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/minigpt4/processors/__init__.py -------------------------------------------------------------------------------- /minigpt4/processors/base_processor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/minigpt4/processors/base_processor.py -------------------------------------------------------------------------------- /minigpt4/processors/blip_processors.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/minigpt4/processors/blip_processors.py -------------------------------------------------------------------------------- /minigpt4/processors/randaugment.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/minigpt4/processors/randaugment.py -------------------------------------------------------------------------------- /minigpt4/runners/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/minigpt4/runners/__init__.py -------------------------------------------------------------------------------- /minigpt4/runners/runner_base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/minigpt4/runners/runner_base.py -------------------------------------------------------------------------------- /minigpt4/tasks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/minigpt4/tasks/__init__.py -------------------------------------------------------------------------------- /minigpt4/tasks/base_task.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/minigpt4/tasks/base_task.py -------------------------------------------------------------------------------- /minigpt4/tasks/image_text_pretrain.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/minigpt4/tasks/image_text_pretrain.py -------------------------------------------------------------------------------- /model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/model.py -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/requirements.txt -------------------------------------------------------------------------------- /train_eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/train_eval.py -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/eric-ai-lab/MiniGPT-5/HEAD/utils.py --------------------------------------------------------------------------------