├── .gitignore ├── LICENSE ├── README.md ├── code ├── dataset │ ├── README_1_STAGE.md │ ├── README_2_STAGE.md │ ├── convert_cc_sbu.py │ ├── convert_laion.py │ ├── download_cc_sbu.sh │ └── download_laion.sh ├── demo_stage1.py ├── demo_stage1.sh ├── demo_stage2.py ├── demo_stage2.sh ├── eval_configs │ ├── seechat_eval_stage1.yaml │ └── seechat_eval_stage2.yaml ├── prompts │ └── alignment.txt ├── requirements.txt ├── seechat │ ├── __init__.py │ ├── common │ │ ├── __init__.py │ │ ├── config.py │ │ ├── dist_utils.py │ │ ├── gradcam.py │ │ ├── logger.py │ │ ├── optims.py │ │ ├── registry.py │ │ └── utils.py │ ├── configs │ │ ├── datasets │ │ │ ├── cc_sbu │ │ │ │ ├── align.yaml │ │ │ │ └── defaults.yaml │ │ │ ├── laion │ │ │ │ └── defaults.yaml │ │ │ └── r2d2 │ │ │ │ └── defaults.yaml │ │ ├── default.yaml │ │ └── models │ │ │ ├── seechat.yaml │ │ │ └── seechat_glm.yaml │ ├── conversation │ │ ├── __init__.py │ │ └── conversation.py │ ├── datasets │ │ ├── __init__.py │ │ ├── builders │ │ │ ├── __init__.py │ │ │ ├── base_dataset_builder.py │ │ │ └── image_text_pair_builder.py │ │ ├── data_utils.py │ │ └── datasets │ │ │ ├── __init__.py │ │ │ ├── base_dataset.py │ │ │ ├── caption_datasets.py │ │ │ ├── cc_sbu_dataset.py │ │ │ ├── dataloader_utils.py │ │ │ ├── laion_dataset.py │ │ │ └── r2d2_dataset.py │ ├── models │ │ ├── Qformer.py │ │ ├── __init__.py │ │ ├── base_model.py │ │ ├── blip2.py │ │ ├── blip2_outputs.py │ │ ├── eva_vit.py │ │ ├── modeling_llama.py │ │ └── see_chat.py │ ├── processors │ │ ├── __init__.py │ │ ├── base_processor.py │ │ ├── blip_processors.py │ │ └── randaugment.py │ ├── runners │ │ ├── __init__.py │ │ └── runner_base.py │ └── tasks │ │ ├── __init__.py │ │ ├── base_task.py │ │ └── image_text_pretrain.py ├── tools.py └── tools_box │ └── img2dataset_tools │ ├── __init__.py │ ├── architecture.md │ ├── blurrer.py │ ├── distributor.py │ ├── downloader.py │ ├── logger.py │ ├── main.py │ ├── reader.py │ ├── resizer.py │ └── writer.py ├── doc └── img │ ├── MLLM model structure.png │ ├── SEEChat-demo1.png │ ├── captionwinrate.png │ └── round_dialog.png ├── environment.yml └── models └── chatglm-6b ├── LICENSE.txt ├── MODEL_LICENSE.txt ├── README.md ├── config.json ├── configuration_chatglm.py ├── modeling_chatglm.py ├── pytorch_model.bin.index.json ├── quantization.py ├── raw_modeling_chatglm.py ├── test_modeling_chatglm.py ├── tokenization_chatglm.py └── tokenizer_config.json /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/README.md -------------------------------------------------------------------------------- /code/dataset/README_1_STAGE.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/dataset/README_1_STAGE.md -------------------------------------------------------------------------------- /code/dataset/README_2_STAGE.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/dataset/README_2_STAGE.md -------------------------------------------------------------------------------- /code/dataset/convert_cc_sbu.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/dataset/convert_cc_sbu.py -------------------------------------------------------------------------------- /code/dataset/convert_laion.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/dataset/convert_laion.py -------------------------------------------------------------------------------- /code/dataset/download_cc_sbu.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/dataset/download_cc_sbu.sh -------------------------------------------------------------------------------- /code/dataset/download_laion.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/dataset/download_laion.sh -------------------------------------------------------------------------------- /code/demo_stage1.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/demo_stage1.py -------------------------------------------------------------------------------- /code/demo_stage1.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/demo_stage1.sh -------------------------------------------------------------------------------- /code/demo_stage2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/demo_stage2.py -------------------------------------------------------------------------------- /code/demo_stage2.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/demo_stage2.sh -------------------------------------------------------------------------------- /code/eval_configs/seechat_eval_stage1.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/eval_configs/seechat_eval_stage1.yaml -------------------------------------------------------------------------------- /code/eval_configs/seechat_eval_stage2.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/eval_configs/seechat_eval_stage2.yaml -------------------------------------------------------------------------------- /code/prompts/alignment.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/prompts/alignment.txt -------------------------------------------------------------------------------- /code/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/requirements.txt -------------------------------------------------------------------------------- /code/seechat/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/seechat/__init__.py -------------------------------------------------------------------------------- /code/seechat/common/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /code/seechat/common/config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/seechat/common/config.py -------------------------------------------------------------------------------- /code/seechat/common/dist_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/seechat/common/dist_utils.py -------------------------------------------------------------------------------- /code/seechat/common/gradcam.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/seechat/common/gradcam.py -------------------------------------------------------------------------------- /code/seechat/common/logger.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/seechat/common/logger.py -------------------------------------------------------------------------------- /code/seechat/common/optims.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/seechat/common/optims.py -------------------------------------------------------------------------------- /code/seechat/common/registry.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/seechat/common/registry.py -------------------------------------------------------------------------------- /code/seechat/common/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/seechat/common/utils.py -------------------------------------------------------------------------------- /code/seechat/configs/datasets/cc_sbu/align.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/seechat/configs/datasets/cc_sbu/align.yaml -------------------------------------------------------------------------------- /code/seechat/configs/datasets/cc_sbu/defaults.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/seechat/configs/datasets/cc_sbu/defaults.yaml -------------------------------------------------------------------------------- /code/seechat/configs/datasets/laion/defaults.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/seechat/configs/datasets/laion/defaults.yaml -------------------------------------------------------------------------------- /code/seechat/configs/datasets/r2d2/defaults.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/seechat/configs/datasets/r2d2/defaults.yaml -------------------------------------------------------------------------------- /code/seechat/configs/default.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/seechat/configs/default.yaml -------------------------------------------------------------------------------- /code/seechat/configs/models/seechat.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/seechat/configs/models/seechat.yaml -------------------------------------------------------------------------------- /code/seechat/configs/models/seechat_glm.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/seechat/configs/models/seechat_glm.yaml -------------------------------------------------------------------------------- /code/seechat/conversation/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /code/seechat/conversation/conversation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/seechat/conversation/conversation.py -------------------------------------------------------------------------------- /code/seechat/datasets/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /code/seechat/datasets/builders/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/seechat/datasets/builders/__init__.py -------------------------------------------------------------------------------- /code/seechat/datasets/builders/base_dataset_builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/seechat/datasets/builders/base_dataset_builder.py -------------------------------------------------------------------------------- /code/seechat/datasets/builders/image_text_pair_builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/seechat/datasets/builders/image_text_pair_builder.py -------------------------------------------------------------------------------- /code/seechat/datasets/data_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/seechat/datasets/data_utils.py -------------------------------------------------------------------------------- /code/seechat/datasets/datasets/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /code/seechat/datasets/datasets/base_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/seechat/datasets/datasets/base_dataset.py -------------------------------------------------------------------------------- /code/seechat/datasets/datasets/caption_datasets.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/seechat/datasets/datasets/caption_datasets.py -------------------------------------------------------------------------------- /code/seechat/datasets/datasets/cc_sbu_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/seechat/datasets/datasets/cc_sbu_dataset.py -------------------------------------------------------------------------------- /code/seechat/datasets/datasets/dataloader_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/seechat/datasets/datasets/dataloader_utils.py -------------------------------------------------------------------------------- /code/seechat/datasets/datasets/laion_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/seechat/datasets/datasets/laion_dataset.py -------------------------------------------------------------------------------- /code/seechat/datasets/datasets/r2d2_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/seechat/datasets/datasets/r2d2_dataset.py -------------------------------------------------------------------------------- /code/seechat/models/Qformer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/seechat/models/Qformer.py -------------------------------------------------------------------------------- /code/seechat/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/seechat/models/__init__.py -------------------------------------------------------------------------------- /code/seechat/models/base_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/seechat/models/base_model.py -------------------------------------------------------------------------------- /code/seechat/models/blip2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/seechat/models/blip2.py -------------------------------------------------------------------------------- /code/seechat/models/blip2_outputs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/seechat/models/blip2_outputs.py -------------------------------------------------------------------------------- /code/seechat/models/eva_vit.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/seechat/models/eva_vit.py -------------------------------------------------------------------------------- /code/seechat/models/modeling_llama.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/seechat/models/modeling_llama.py -------------------------------------------------------------------------------- /code/seechat/models/see_chat.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/seechat/models/see_chat.py -------------------------------------------------------------------------------- /code/seechat/processors/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/seechat/processors/__init__.py -------------------------------------------------------------------------------- /code/seechat/processors/base_processor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/seechat/processors/base_processor.py -------------------------------------------------------------------------------- /code/seechat/processors/blip_processors.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/seechat/processors/blip_processors.py -------------------------------------------------------------------------------- /code/seechat/processors/randaugment.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/seechat/processors/randaugment.py -------------------------------------------------------------------------------- /code/seechat/runners/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/seechat/runners/__init__.py -------------------------------------------------------------------------------- /code/seechat/runners/runner_base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/seechat/runners/runner_base.py -------------------------------------------------------------------------------- /code/seechat/tasks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/seechat/tasks/__init__.py -------------------------------------------------------------------------------- /code/seechat/tasks/base_task.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/seechat/tasks/base_task.py -------------------------------------------------------------------------------- /code/seechat/tasks/image_text_pretrain.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/seechat/tasks/image_text_pretrain.py -------------------------------------------------------------------------------- /code/tools.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/tools.py -------------------------------------------------------------------------------- /code/tools_box/img2dataset_tools/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/tools_box/img2dataset_tools/__init__.py -------------------------------------------------------------------------------- /code/tools_box/img2dataset_tools/architecture.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/tools_box/img2dataset_tools/architecture.md -------------------------------------------------------------------------------- /code/tools_box/img2dataset_tools/blurrer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/tools_box/img2dataset_tools/blurrer.py -------------------------------------------------------------------------------- /code/tools_box/img2dataset_tools/distributor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/tools_box/img2dataset_tools/distributor.py -------------------------------------------------------------------------------- /code/tools_box/img2dataset_tools/downloader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/tools_box/img2dataset_tools/downloader.py -------------------------------------------------------------------------------- /code/tools_box/img2dataset_tools/logger.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/tools_box/img2dataset_tools/logger.py -------------------------------------------------------------------------------- /code/tools_box/img2dataset_tools/main.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/tools_box/img2dataset_tools/main.py -------------------------------------------------------------------------------- /code/tools_box/img2dataset_tools/reader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/tools_box/img2dataset_tools/reader.py -------------------------------------------------------------------------------- /code/tools_box/img2dataset_tools/resizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/tools_box/img2dataset_tools/resizer.py -------------------------------------------------------------------------------- /code/tools_box/img2dataset_tools/writer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/code/tools_box/img2dataset_tools/writer.py -------------------------------------------------------------------------------- /doc/img/MLLM model structure.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/doc/img/MLLM model structure.png -------------------------------------------------------------------------------- /doc/img/SEEChat-demo1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/doc/img/SEEChat-demo1.png -------------------------------------------------------------------------------- /doc/img/captionwinrate.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/doc/img/captionwinrate.png -------------------------------------------------------------------------------- /doc/img/round_dialog.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/doc/img/round_dialog.png -------------------------------------------------------------------------------- /environment.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/environment.yml -------------------------------------------------------------------------------- /models/chatglm-6b/LICENSE.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/models/chatglm-6b/LICENSE.txt -------------------------------------------------------------------------------- /models/chatglm-6b/MODEL_LICENSE.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/models/chatglm-6b/MODEL_LICENSE.txt -------------------------------------------------------------------------------- /models/chatglm-6b/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/models/chatglm-6b/README.md -------------------------------------------------------------------------------- /models/chatglm-6b/config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/models/chatglm-6b/config.json -------------------------------------------------------------------------------- /models/chatglm-6b/configuration_chatglm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/models/chatglm-6b/configuration_chatglm.py -------------------------------------------------------------------------------- /models/chatglm-6b/modeling_chatglm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/models/chatglm-6b/modeling_chatglm.py -------------------------------------------------------------------------------- /models/chatglm-6b/pytorch_model.bin.index.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/models/chatglm-6b/pytorch_model.bin.index.json -------------------------------------------------------------------------------- /models/chatglm-6b/quantization.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/models/chatglm-6b/quantization.py -------------------------------------------------------------------------------- /models/chatglm-6b/raw_modeling_chatglm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/models/chatglm-6b/raw_modeling_chatglm.py -------------------------------------------------------------------------------- /models/chatglm-6b/test_modeling_chatglm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/models/chatglm-6b/test_modeling_chatglm.py -------------------------------------------------------------------------------- /models/chatglm-6b/tokenization_chatglm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/models/chatglm-6b/tokenization_chatglm.py -------------------------------------------------------------------------------- /models/chatglm-6b/tokenizer_config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/360CVGroup/SEEChat/HEAD/models/chatglm-6b/tokenizer_config.json --------------------------------------------------------------------------------