├── .deepspeed_env ├── .gitignore ├── LICENSE ├── MODEL_LICENSE ├── README.md ├── assets ├── app_case.jpg ├── cases.png ├── data_framework.png ├── eg_comdata.png ├── eg_commath20.png ├── eg_commath3.png ├── eg_commath326.png ├── eg_comtest.png ├── logo.jpg ├── modeling_framework.png └── web_demo-min.png ├── cogcom ├── data │ ├── README.md │ ├── ann_visual.py │ ├── convert_to_wds.py │ ├── gen_steps_absurd.py │ ├── gen_steps_txt.py │ ├── nohup.out │ ├── prepare │ │ ├── process_gqa.py │ │ ├── process_okvqa.py │ │ ├── process_stvqa.py │ │ ├── process_tdiuc.py │ │ ├── process_textvqa.py │ │ └── process_vqav2.py │ ├── pro_steps_absurd.py │ ├── pro_steps_txt.py │ ├── run_convert_wds.sh │ ├── run_prepare.sh │ ├── run_visual_ann.sh │ ├── tools │ │ ├── __init__.py │ │ ├── gpt4.py │ │ ├── gpt4v.py │ │ └── groundingdino.py │ └── utils │ │ ├── __init__.py │ │ ├── template_util.py │ │ ├── templates_v2.json │ │ └── visualize.ipynb ├── demo │ ├── cli_demo_hf.py │ ├── cli_demo_sat.py │ └── web_demo.py ├── finetune.py ├── finetune.sh ├── models │ ├── __init__.py │ ├── cogcom_model.py │ ├── com_memory.py │ ├── eva2.py │ └── mixin.py ├── test_config_bf16_zero1off.json └── utils │ ├── __init__.py │ ├── chat.py │ ├── com_dataset.py │ ├── com_utils.py │ ├── grounding_parser.py │ ├── instr_dataset.py │ ├── language.py │ ├── template.py │ └── vision.py └── requirements.txt /.deepspeed_env: -------------------------------------------------------------------------------- 1 | SAT_HOME=~/.sat_models 2 | LOCAL_WORLD_SIZE=8 -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zai-org/CogCoM/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zai-org/CogCoM/HEAD/LICENSE -------------------------------------------------------------------------------- /MODEL_LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zai-org/CogCoM/HEAD/MODEL_LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zai-org/CogCoM/HEAD/README.md -------------------------------------------------------------------------------- /assets/app_case.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zai-org/CogCoM/HEAD/assets/app_case.jpg -------------------------------------------------------------------------------- /assets/cases.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zai-org/CogCoM/HEAD/assets/cases.png -------------------------------------------------------------------------------- /assets/data_framework.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zai-org/CogCoM/HEAD/assets/data_framework.png -------------------------------------------------------------------------------- /assets/eg_comdata.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zai-org/CogCoM/HEAD/assets/eg_comdata.png -------------------------------------------------------------------------------- /assets/eg_commath20.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zai-org/CogCoM/HEAD/assets/eg_commath20.png -------------------------------------------------------------------------------- /assets/eg_commath3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zai-org/CogCoM/HEAD/assets/eg_commath3.png -------------------------------------------------------------------------------- /assets/eg_commath326.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zai-org/CogCoM/HEAD/assets/eg_commath326.png -------------------------------------------------------------------------------- /assets/eg_comtest.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zai-org/CogCoM/HEAD/assets/eg_comtest.png -------------------------------------------------------------------------------- /assets/logo.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zai-org/CogCoM/HEAD/assets/logo.jpg -------------------------------------------------------------------------------- /assets/modeling_framework.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zai-org/CogCoM/HEAD/assets/modeling_framework.png -------------------------------------------------------------------------------- /assets/web_demo-min.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zai-org/CogCoM/HEAD/assets/web_demo-min.png -------------------------------------------------------------------------------- /cogcom/data/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zai-org/CogCoM/HEAD/cogcom/data/README.md -------------------------------------------------------------------------------- /cogcom/data/ann_visual.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zai-org/CogCoM/HEAD/cogcom/data/ann_visual.py -------------------------------------------------------------------------------- /cogcom/data/convert_to_wds.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zai-org/CogCoM/HEAD/cogcom/data/convert_to_wds.py -------------------------------------------------------------------------------- /cogcom/data/gen_steps_absurd.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zai-org/CogCoM/HEAD/cogcom/data/gen_steps_absurd.py -------------------------------------------------------------------------------- /cogcom/data/gen_steps_txt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zai-org/CogCoM/HEAD/cogcom/data/gen_steps_txt.py -------------------------------------------------------------------------------- /cogcom/data/nohup.out: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zai-org/CogCoM/HEAD/cogcom/data/nohup.out -------------------------------------------------------------------------------- /cogcom/data/prepare/process_gqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zai-org/CogCoM/HEAD/cogcom/data/prepare/process_gqa.py -------------------------------------------------------------------------------- /cogcom/data/prepare/process_okvqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zai-org/CogCoM/HEAD/cogcom/data/prepare/process_okvqa.py -------------------------------------------------------------------------------- /cogcom/data/prepare/process_stvqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zai-org/CogCoM/HEAD/cogcom/data/prepare/process_stvqa.py -------------------------------------------------------------------------------- /cogcom/data/prepare/process_tdiuc.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zai-org/CogCoM/HEAD/cogcom/data/prepare/process_tdiuc.py -------------------------------------------------------------------------------- /cogcom/data/prepare/process_textvqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zai-org/CogCoM/HEAD/cogcom/data/prepare/process_textvqa.py -------------------------------------------------------------------------------- /cogcom/data/prepare/process_vqav2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zai-org/CogCoM/HEAD/cogcom/data/prepare/process_vqav2.py -------------------------------------------------------------------------------- /cogcom/data/pro_steps_absurd.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zai-org/CogCoM/HEAD/cogcom/data/pro_steps_absurd.py -------------------------------------------------------------------------------- /cogcom/data/pro_steps_txt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zai-org/CogCoM/HEAD/cogcom/data/pro_steps_txt.py -------------------------------------------------------------------------------- /cogcom/data/run_convert_wds.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zai-org/CogCoM/HEAD/cogcom/data/run_convert_wds.sh -------------------------------------------------------------------------------- /cogcom/data/run_prepare.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zai-org/CogCoM/HEAD/cogcom/data/run_prepare.sh -------------------------------------------------------------------------------- /cogcom/data/run_visual_ann.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zai-org/CogCoM/HEAD/cogcom/data/run_visual_ann.sh -------------------------------------------------------------------------------- /cogcom/data/tools/__init__.py: -------------------------------------------------------------------------------- 1 | # init -------------------------------------------------------------------------------- /cogcom/data/tools/gpt4.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zai-org/CogCoM/HEAD/cogcom/data/tools/gpt4.py -------------------------------------------------------------------------------- /cogcom/data/tools/gpt4v.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zai-org/CogCoM/HEAD/cogcom/data/tools/gpt4v.py -------------------------------------------------------------------------------- /cogcom/data/tools/groundingdino.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zai-org/CogCoM/HEAD/cogcom/data/tools/groundingdino.py -------------------------------------------------------------------------------- /cogcom/data/utils/__init__.py: -------------------------------------------------------------------------------- 1 | # init -------------------------------------------------------------------------------- /cogcom/data/utils/template_util.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zai-org/CogCoM/HEAD/cogcom/data/utils/template_util.py -------------------------------------------------------------------------------- /cogcom/data/utils/templates_v2.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zai-org/CogCoM/HEAD/cogcom/data/utils/templates_v2.json -------------------------------------------------------------------------------- /cogcom/data/utils/visualize.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zai-org/CogCoM/HEAD/cogcom/data/utils/visualize.ipynb -------------------------------------------------------------------------------- /cogcom/demo/cli_demo_hf.py: -------------------------------------------------------------------------------- 1 | # TO-DO -------------------------------------------------------------------------------- /cogcom/demo/cli_demo_sat.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zai-org/CogCoM/HEAD/cogcom/demo/cli_demo_sat.py -------------------------------------------------------------------------------- /cogcom/demo/web_demo.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zai-org/CogCoM/HEAD/cogcom/demo/web_demo.py -------------------------------------------------------------------------------- /cogcom/finetune.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zai-org/CogCoM/HEAD/cogcom/finetune.py -------------------------------------------------------------------------------- /cogcom/finetune.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zai-org/CogCoM/HEAD/cogcom/finetune.sh -------------------------------------------------------------------------------- /cogcom/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zai-org/CogCoM/HEAD/cogcom/models/__init__.py -------------------------------------------------------------------------------- /cogcom/models/cogcom_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zai-org/CogCoM/HEAD/cogcom/models/cogcom_model.py -------------------------------------------------------------------------------- /cogcom/models/com_memory.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zai-org/CogCoM/HEAD/cogcom/models/com_memory.py -------------------------------------------------------------------------------- /cogcom/models/eva2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zai-org/CogCoM/HEAD/cogcom/models/eva2.py -------------------------------------------------------------------------------- /cogcom/models/mixin.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zai-org/CogCoM/HEAD/cogcom/models/mixin.py -------------------------------------------------------------------------------- /cogcom/test_config_bf16_zero1off.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zai-org/CogCoM/HEAD/cogcom/test_config_bf16_zero1off.json -------------------------------------------------------------------------------- /cogcom/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zai-org/CogCoM/HEAD/cogcom/utils/__init__.py -------------------------------------------------------------------------------- /cogcom/utils/chat.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zai-org/CogCoM/HEAD/cogcom/utils/chat.py -------------------------------------------------------------------------------- /cogcom/utils/com_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zai-org/CogCoM/HEAD/cogcom/utils/com_dataset.py -------------------------------------------------------------------------------- /cogcom/utils/com_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zai-org/CogCoM/HEAD/cogcom/utils/com_utils.py -------------------------------------------------------------------------------- /cogcom/utils/grounding_parser.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zai-org/CogCoM/HEAD/cogcom/utils/grounding_parser.py -------------------------------------------------------------------------------- /cogcom/utils/instr_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zai-org/CogCoM/HEAD/cogcom/utils/instr_dataset.py -------------------------------------------------------------------------------- /cogcom/utils/language.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zai-org/CogCoM/HEAD/cogcom/utils/language.py -------------------------------------------------------------------------------- /cogcom/utils/template.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zai-org/CogCoM/HEAD/cogcom/utils/template.py -------------------------------------------------------------------------------- /cogcom/utils/vision.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zai-org/CogCoM/HEAD/cogcom/utils/vision.py -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zai-org/CogCoM/HEAD/requirements.txt --------------------------------------------------------------------------------