├── Figures ├── GUI_overview.png ├── Gui_icon.png └── radar.jpg ├── GUI_Vid ├── configs │ ├── config.json │ ├── config_bert.json │ ├── data.py │ ├── instruction_data.py │ └── model.py ├── dataset │ ├── __init__.py │ ├── __pycache__ │ │ ├── __init__.cpython-39.pyc │ │ ├── base_dataset.cpython-39.pyc │ │ ├── dataloader.cpython-39.pyc │ │ ├── it_dataset.cpython-39.pyc │ │ ├── pt_dataset.cpython-39.pyc │ │ ├── utils.cpython-39.pyc │ │ ├── video_transforms.cpython-39.pyc │ │ └── video_utils.cpython-39.pyc │ ├── base_dataset.py │ ├── dataloader.py │ ├── it_dataset.py │ ├── pt_dataset.py │ ├── utils.py │ ├── video_transforms.py │ └── video_utils.py ├── demo_local.py ├── models │ ├── __init__.py │ ├── __pycache__ │ │ ├── __init__.cpython-311.pyc │ │ ├── __init__.cpython-39.pyc │ │ ├── videochat2_it.cpython-311.pyc │ │ └── videochat2_it.cpython-39.pyc │ ├── bert │ │ ├── __init__.py │ │ ├── builder.py │ │ ├── tokenization_bert.py │ │ └── xbert.py │ ├── blip2 │ │ ├── Qformer.py │ │ ├── __init__.py │ │ ├── __pycache__ │ │ │ ├── Qformer.cpython-39.pyc │ │ │ ├── __init__.cpython-39.pyc │ │ │ ├── blip2.cpython-39.pyc │ │ │ ├── modeling_llama.cpython-39.pyc │ │ │ └── vit.cpython-39.pyc │ │ ├── blip2.py │ │ ├── builder.py │ │ ├── modeling_llama.py │ │ ├── modeling_llama_mem.py │ │ ├── utils.py │ │ └── vit.py │ ├── criterions.py │ ├── utils.py │ ├── videochat2_it.py │ ├── videochat2_pt.py │ └── videochat2_qformer.py ├── requirements.txt ├── scripts │ ├── config_7b_stage1.py │ ├── config_7b_stage2.py │ ├── config_7b_stage3.py │ ├── katna.py │ ├── run_7b_stage1.sh │ ├── run_7b_stage2.sh │ └── run_7b_stage3.sh └── utils │ ├── __pycache__ │ ├── config.cpython-311.pyc │ ├── config.cpython-39.pyc │ ├── distributed.cpython-39.pyc │ ├── easydict.cpython-311.pyc │ └── easydict.cpython-39.pyc │ ├── basic_utils.py │ ├── config.py │ ├── config_utils.py │ ├── distributed.py │ ├── easydict.py │ ├── logger.py │ ├── optimizer.py │ └── scheduler.py └── Readme.md /Figures/GUI_overview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/Figures/GUI_overview.png -------------------------------------------------------------------------------- /Figures/Gui_icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/Figures/Gui_icon.png -------------------------------------------------------------------------------- /Figures/radar.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/Figures/radar.jpg -------------------------------------------------------------------------------- /GUI_Vid/configs/config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/configs/config.json -------------------------------------------------------------------------------- /GUI_Vid/configs/config_bert.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/configs/config_bert.json -------------------------------------------------------------------------------- /GUI_Vid/configs/data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/configs/data.py -------------------------------------------------------------------------------- /GUI_Vid/configs/instruction_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/configs/instruction_data.py -------------------------------------------------------------------------------- /GUI_Vid/configs/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/configs/model.py -------------------------------------------------------------------------------- /GUI_Vid/dataset/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/dataset/__init__.py -------------------------------------------------------------------------------- /GUI_Vid/dataset/__pycache__/__init__.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/dataset/__pycache__/__init__.cpython-39.pyc -------------------------------------------------------------------------------- /GUI_Vid/dataset/__pycache__/base_dataset.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/dataset/__pycache__/base_dataset.cpython-39.pyc -------------------------------------------------------------------------------- /GUI_Vid/dataset/__pycache__/dataloader.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/dataset/__pycache__/dataloader.cpython-39.pyc -------------------------------------------------------------------------------- /GUI_Vid/dataset/__pycache__/it_dataset.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/dataset/__pycache__/it_dataset.cpython-39.pyc -------------------------------------------------------------------------------- /GUI_Vid/dataset/__pycache__/pt_dataset.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/dataset/__pycache__/pt_dataset.cpython-39.pyc -------------------------------------------------------------------------------- /GUI_Vid/dataset/__pycache__/utils.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/dataset/__pycache__/utils.cpython-39.pyc -------------------------------------------------------------------------------- /GUI_Vid/dataset/__pycache__/video_transforms.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/dataset/__pycache__/video_transforms.cpython-39.pyc -------------------------------------------------------------------------------- /GUI_Vid/dataset/__pycache__/video_utils.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/dataset/__pycache__/video_utils.cpython-39.pyc -------------------------------------------------------------------------------- /GUI_Vid/dataset/base_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/dataset/base_dataset.py -------------------------------------------------------------------------------- /GUI_Vid/dataset/dataloader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/dataset/dataloader.py -------------------------------------------------------------------------------- /GUI_Vid/dataset/it_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/dataset/it_dataset.py -------------------------------------------------------------------------------- /GUI_Vid/dataset/pt_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/dataset/pt_dataset.py -------------------------------------------------------------------------------- /GUI_Vid/dataset/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/dataset/utils.py -------------------------------------------------------------------------------- /GUI_Vid/dataset/video_transforms.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/dataset/video_transforms.py -------------------------------------------------------------------------------- /GUI_Vid/dataset/video_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/dataset/video_utils.py -------------------------------------------------------------------------------- /GUI_Vid/demo_local.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/demo_local.py -------------------------------------------------------------------------------- /GUI_Vid/models/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /GUI_Vid/models/__pycache__/__init__.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/models/__pycache__/__init__.cpython-311.pyc -------------------------------------------------------------------------------- /GUI_Vid/models/__pycache__/__init__.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/models/__pycache__/__init__.cpython-39.pyc -------------------------------------------------------------------------------- /GUI_Vid/models/__pycache__/videochat2_it.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/models/__pycache__/videochat2_it.cpython-311.pyc -------------------------------------------------------------------------------- /GUI_Vid/models/__pycache__/videochat2_it.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/models/__pycache__/videochat2_it.cpython-39.pyc -------------------------------------------------------------------------------- /GUI_Vid/models/bert/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /GUI_Vid/models/bert/builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/models/bert/builder.py -------------------------------------------------------------------------------- /GUI_Vid/models/bert/tokenization_bert.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/models/bert/tokenization_bert.py -------------------------------------------------------------------------------- /GUI_Vid/models/bert/xbert.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/models/bert/xbert.py -------------------------------------------------------------------------------- /GUI_Vid/models/blip2/Qformer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/models/blip2/Qformer.py -------------------------------------------------------------------------------- /GUI_Vid/models/blip2/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /GUI_Vid/models/blip2/__pycache__/Qformer.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/models/blip2/__pycache__/Qformer.cpython-39.pyc -------------------------------------------------------------------------------- /GUI_Vid/models/blip2/__pycache__/__init__.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/models/blip2/__pycache__/__init__.cpython-39.pyc -------------------------------------------------------------------------------- /GUI_Vid/models/blip2/__pycache__/blip2.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/models/blip2/__pycache__/blip2.cpython-39.pyc -------------------------------------------------------------------------------- /GUI_Vid/models/blip2/__pycache__/modeling_llama.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/models/blip2/__pycache__/modeling_llama.cpython-39.pyc -------------------------------------------------------------------------------- /GUI_Vid/models/blip2/__pycache__/vit.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/models/blip2/__pycache__/vit.cpython-39.pyc -------------------------------------------------------------------------------- /GUI_Vid/models/blip2/blip2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/models/blip2/blip2.py -------------------------------------------------------------------------------- /GUI_Vid/models/blip2/builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/models/blip2/builder.py -------------------------------------------------------------------------------- /GUI_Vid/models/blip2/modeling_llama.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/models/blip2/modeling_llama.py -------------------------------------------------------------------------------- /GUI_Vid/models/blip2/modeling_llama_mem.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/models/blip2/modeling_llama_mem.py -------------------------------------------------------------------------------- /GUI_Vid/models/blip2/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/models/blip2/utils.py -------------------------------------------------------------------------------- /GUI_Vid/models/blip2/vit.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/models/blip2/vit.py -------------------------------------------------------------------------------- /GUI_Vid/models/criterions.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/models/criterions.py -------------------------------------------------------------------------------- /GUI_Vid/models/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/models/utils.py -------------------------------------------------------------------------------- /GUI_Vid/models/videochat2_it.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/models/videochat2_it.py -------------------------------------------------------------------------------- /GUI_Vid/models/videochat2_pt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/models/videochat2_pt.py -------------------------------------------------------------------------------- /GUI_Vid/models/videochat2_qformer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/models/videochat2_qformer.py -------------------------------------------------------------------------------- /GUI_Vid/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/requirements.txt -------------------------------------------------------------------------------- /GUI_Vid/scripts/config_7b_stage1.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/scripts/config_7b_stage1.py -------------------------------------------------------------------------------- /GUI_Vid/scripts/config_7b_stage2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/scripts/config_7b_stage2.py -------------------------------------------------------------------------------- /GUI_Vid/scripts/config_7b_stage3.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/scripts/config_7b_stage3.py -------------------------------------------------------------------------------- /GUI_Vid/scripts/katna.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/scripts/katna.py -------------------------------------------------------------------------------- /GUI_Vid/scripts/run_7b_stage1.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/scripts/run_7b_stage1.sh -------------------------------------------------------------------------------- /GUI_Vid/scripts/run_7b_stage2.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/scripts/run_7b_stage2.sh -------------------------------------------------------------------------------- /GUI_Vid/scripts/run_7b_stage3.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/scripts/run_7b_stage3.sh -------------------------------------------------------------------------------- /GUI_Vid/utils/__pycache__/config.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/utils/__pycache__/config.cpython-311.pyc -------------------------------------------------------------------------------- /GUI_Vid/utils/__pycache__/config.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/utils/__pycache__/config.cpython-39.pyc -------------------------------------------------------------------------------- /GUI_Vid/utils/__pycache__/distributed.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/utils/__pycache__/distributed.cpython-39.pyc -------------------------------------------------------------------------------- /GUI_Vid/utils/__pycache__/easydict.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/utils/__pycache__/easydict.cpython-311.pyc -------------------------------------------------------------------------------- /GUI_Vid/utils/__pycache__/easydict.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/utils/__pycache__/easydict.cpython-39.pyc -------------------------------------------------------------------------------- /GUI_Vid/utils/basic_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/utils/basic_utils.py -------------------------------------------------------------------------------- /GUI_Vid/utils/config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/utils/config.py -------------------------------------------------------------------------------- /GUI_Vid/utils/config_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/utils/config_utils.py -------------------------------------------------------------------------------- /GUI_Vid/utils/distributed.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/utils/distributed.py -------------------------------------------------------------------------------- /GUI_Vid/utils/easydict.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/utils/easydict.py -------------------------------------------------------------------------------- /GUI_Vid/utils/logger.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/utils/logger.py -------------------------------------------------------------------------------- /GUI_Vid/utils/optimizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/utils/optimizer.py -------------------------------------------------------------------------------- /GUI_Vid/utils/scheduler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/GUI_Vid/utils/scheduler.py -------------------------------------------------------------------------------- /Readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dongping-Chen/GUI-World/HEAD/Readme.md --------------------------------------------------------------------------------