├── .gitignore ├── README.md ├── assets ├── dataset.jpg ├── example1.jpg ├── example4.jpg ├── hawk.png └── performance.jpg ├── configs ├── config.json ├── dataset_utils.py └── instruction_data.py ├── data_preparing ├── anetc.py ├── charades.py ├── check_grounding_results.ipynb ├── internvid.py ├── nextgqa.py └── videoqa.py ├── dataset ├── __init__.py ├── base_dataset.py ├── dataloader.py ├── it_dataset.py ├── pt_dataset.py ├── utils.py ├── video_transforms.py └── video_utils.py ├── demo.ipynb ├── internvid_g ├── README.md ├── code │ ├── caption_clips.py │ ├── clip_sim.py │ ├── download_videos.py │ └── ground_data_construction.py └── scripts │ ├── extract_of.sh │ ├── run.sh │ ├── run_1121 │ ├── blip2.sh │ ├── check_videos.sh │ ├── clip_filter.sh │ ├── config_7b.json │ ├── ground_data_construction.sh │ ├── llama2.sh │ ├── text2tag.sh │ └── videochat.sh │ └── run_llama2.sh ├── models ├── __init__.py ├── bert │ ├── __init__.py │ ├── builder.py │ ├── tokenization_bert.py │ └── xbert.py ├── blip2 │ ├── Qformer.py │ ├── __init__.py │ ├── blip2.py │ ├── builder.py │ ├── modeling_llama.py │ ├── modeling_llama_mem.py │ ├── utils.py │ └── vit.py ├── criterions.py ├── hawkeye_it.py └── utils.py ├── requirements.txt ├── scripts ├── test │ ├── recursive_grounding.sh │ └── videoqa.sh └── train │ ├── anetc.py │ ├── anetc.sh │ ├── charades_sta.py │ ├── charades_sta.sh │ ├── config_7b_stage3.py │ └── run_7b_stage3.sh ├── tasks ├── shared_utils.py ├── test.py ├── test_recursive_grounding.py └── train_it.py └── utils ├── basic_utils.py ├── config.py ├── config_utils.py ├── distributed.py ├── easydict.py ├── logger.py ├── optimizer.py └── scheduler.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/.gitignore -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/README.md -------------------------------------------------------------------------------- /assets/dataset.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/assets/dataset.jpg -------------------------------------------------------------------------------- /assets/example1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/assets/example1.jpg -------------------------------------------------------------------------------- /assets/example4.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/assets/example4.jpg -------------------------------------------------------------------------------- /assets/hawk.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/assets/hawk.png -------------------------------------------------------------------------------- /assets/performance.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/assets/performance.jpg -------------------------------------------------------------------------------- /configs/config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/configs/config.json -------------------------------------------------------------------------------- /configs/dataset_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/configs/dataset_utils.py -------------------------------------------------------------------------------- /configs/instruction_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/configs/instruction_data.py -------------------------------------------------------------------------------- /data_preparing/anetc.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/data_preparing/anetc.py -------------------------------------------------------------------------------- /data_preparing/charades.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/data_preparing/charades.py -------------------------------------------------------------------------------- /data_preparing/check_grounding_results.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/data_preparing/check_grounding_results.ipynb -------------------------------------------------------------------------------- /data_preparing/internvid.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/data_preparing/internvid.py -------------------------------------------------------------------------------- /data_preparing/nextgqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/data_preparing/nextgqa.py -------------------------------------------------------------------------------- /data_preparing/videoqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/data_preparing/videoqa.py -------------------------------------------------------------------------------- /dataset/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/dataset/__init__.py -------------------------------------------------------------------------------- /dataset/base_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/dataset/base_dataset.py -------------------------------------------------------------------------------- /dataset/dataloader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/dataset/dataloader.py -------------------------------------------------------------------------------- /dataset/it_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/dataset/it_dataset.py -------------------------------------------------------------------------------- /dataset/pt_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/dataset/pt_dataset.py -------------------------------------------------------------------------------- /dataset/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/dataset/utils.py -------------------------------------------------------------------------------- /dataset/video_transforms.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/dataset/video_transforms.py -------------------------------------------------------------------------------- /dataset/video_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/dataset/video_utils.py -------------------------------------------------------------------------------- /demo.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/demo.ipynb -------------------------------------------------------------------------------- /internvid_g/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/internvid_g/README.md -------------------------------------------------------------------------------- /internvid_g/code/caption_clips.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/internvid_g/code/caption_clips.py -------------------------------------------------------------------------------- /internvid_g/code/clip_sim.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/internvid_g/code/clip_sim.py -------------------------------------------------------------------------------- /internvid_g/code/download_videos.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/internvid_g/code/download_videos.py -------------------------------------------------------------------------------- /internvid_g/code/ground_data_construction.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/internvid_g/code/ground_data_construction.py -------------------------------------------------------------------------------- /internvid_g/scripts/extract_of.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/internvid_g/scripts/extract_of.sh -------------------------------------------------------------------------------- /internvid_g/scripts/run.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/internvid_g/scripts/run.sh -------------------------------------------------------------------------------- /internvid_g/scripts/run_1121/blip2.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/internvid_g/scripts/run_1121/blip2.sh -------------------------------------------------------------------------------- /internvid_g/scripts/run_1121/check_videos.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/internvid_g/scripts/run_1121/check_videos.sh -------------------------------------------------------------------------------- /internvid_g/scripts/run_1121/clip_filter.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/internvid_g/scripts/run_1121/clip_filter.sh -------------------------------------------------------------------------------- /internvid_g/scripts/run_1121/config_7b.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/internvid_g/scripts/run_1121/config_7b.json -------------------------------------------------------------------------------- /internvid_g/scripts/run_1121/ground_data_construction.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/internvid_g/scripts/run_1121/ground_data_construction.sh -------------------------------------------------------------------------------- /internvid_g/scripts/run_1121/llama2.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/internvid_g/scripts/run_1121/llama2.sh -------------------------------------------------------------------------------- /internvid_g/scripts/run_1121/text2tag.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/internvid_g/scripts/run_1121/text2tag.sh -------------------------------------------------------------------------------- /internvid_g/scripts/run_1121/videochat.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/internvid_g/scripts/run_1121/videochat.sh -------------------------------------------------------------------------------- /internvid_g/scripts/run_llama2.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/internvid_g/scripts/run_llama2.sh -------------------------------------------------------------------------------- /models/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /models/bert/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /models/bert/builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/models/bert/builder.py -------------------------------------------------------------------------------- /models/bert/tokenization_bert.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/models/bert/tokenization_bert.py -------------------------------------------------------------------------------- /models/bert/xbert.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/models/bert/xbert.py -------------------------------------------------------------------------------- /models/blip2/Qformer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/models/blip2/Qformer.py -------------------------------------------------------------------------------- /models/blip2/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /models/blip2/blip2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/models/blip2/blip2.py -------------------------------------------------------------------------------- /models/blip2/builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/models/blip2/builder.py -------------------------------------------------------------------------------- /models/blip2/modeling_llama.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/models/blip2/modeling_llama.py -------------------------------------------------------------------------------- /models/blip2/modeling_llama_mem.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/models/blip2/modeling_llama_mem.py -------------------------------------------------------------------------------- /models/blip2/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/models/blip2/utils.py -------------------------------------------------------------------------------- /models/blip2/vit.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/models/blip2/vit.py -------------------------------------------------------------------------------- /models/criterions.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/models/criterions.py -------------------------------------------------------------------------------- /models/hawkeye_it.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/models/hawkeye_it.py -------------------------------------------------------------------------------- /models/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/models/utils.py -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/requirements.txt -------------------------------------------------------------------------------- /scripts/test/recursive_grounding.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/scripts/test/recursive_grounding.sh -------------------------------------------------------------------------------- /scripts/test/videoqa.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/scripts/test/videoqa.sh -------------------------------------------------------------------------------- /scripts/train/anetc.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/scripts/train/anetc.py -------------------------------------------------------------------------------- /scripts/train/anetc.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/scripts/train/anetc.sh -------------------------------------------------------------------------------- /scripts/train/charades_sta.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/scripts/train/charades_sta.py -------------------------------------------------------------------------------- /scripts/train/charades_sta.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/scripts/train/charades_sta.sh -------------------------------------------------------------------------------- /scripts/train/config_7b_stage3.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/scripts/train/config_7b_stage3.py -------------------------------------------------------------------------------- /scripts/train/run_7b_stage3.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/scripts/train/run_7b_stage3.sh -------------------------------------------------------------------------------- /tasks/shared_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/tasks/shared_utils.py -------------------------------------------------------------------------------- /tasks/test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/tasks/test.py -------------------------------------------------------------------------------- /tasks/test_recursive_grounding.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/tasks/test_recursive_grounding.py -------------------------------------------------------------------------------- /tasks/train_it.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/tasks/train_it.py -------------------------------------------------------------------------------- /utils/basic_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/utils/basic_utils.py -------------------------------------------------------------------------------- /utils/config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/utils/config.py -------------------------------------------------------------------------------- /utils/config_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/utils/config_utils.py -------------------------------------------------------------------------------- /utils/distributed.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/utils/distributed.py -------------------------------------------------------------------------------- /utils/easydict.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/utils/easydict.py -------------------------------------------------------------------------------- /utils/logger.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/utils/logger.py -------------------------------------------------------------------------------- /utils/optimizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/utils/optimizer.py -------------------------------------------------------------------------------- /utils/scheduler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yellow-binary-tree/HawkEye/HEAD/utils/scheduler.py --------------------------------------------------------------------------------