├── .gitmodules ├── LICENSE ├── README.md ├── base_models ├── __init__.py ├── inspyrenet │ ├── InSPyReNet.py │ ├── __init__.py │ ├── backbones │ │ ├── Res2Net_v1b.py │ │ └── SwinTransformer.py │ ├── modules │ │ ├── attention_module.py │ │ ├── context_module.py │ │ ├── decoder_module.py │ │ └── layers.py │ ├── optim │ │ ├── __init__.py │ │ ├── losses.py │ │ └── scheduler.py │ └── saliency_transforms.py ├── tcl │ ├── __init__.py │ ├── tcl_config_bert.json │ ├── tcl_model_pretrain.py │ ├── tcl_tokenization_bert.py │ ├── tcl_vit.py │ └── tcl_xbert.py └── xvlm │ ├── config_bert.json │ ├── swin_transformer.py │ ├── vit.py │ ├── xbert.py │ └── xvlm.py ├── configs ├── __init__.py ├── base_config.yaml ├── covr_config.yaml ├── gqa_config.yaml └── nextqa_config.yaml ├── data ├── queries_gqa.csv ├── queries_sample.csv └── queries_vsr.csv ├── datasets ├── __init__.py └── dataset.py ├── download_models.sh ├── image_patch.py ├── main_batch.py ├── prompts ├── chatapi_covr_nonrecursive_noclass.prompt ├── chatapi_covr_recursive_noclass.prompt ├── chatapi_gqa_nonrecursive.prompt ├── chatapi_gqa_recursivenew_noclass.prompt ├── chatapi_gqa_retrieval_samefile.prompt ├── chatapi_nextqa_nonrecursive.prompt ├── chatapi_nextqa_recursive.prompt ├── choices │ ├── fixed_code │ │ ├── blip2.prompt │ │ ├── imagesblip2.prompt │ │ └── videoblip2.prompt │ ├── gqa_prompts.json │ ├── gqa_prompts_dict_nonrecursive.json │ ├── gqa_prompts_dict_nonrecursive_new.json │ ├── gqa_prompts_dict_nonrecursive_new2.json │ ├── gqa_prompts_dict_recursive.json │ ├── gqa_prompts_dict_recursive_new.json │ └── gqa_prompts_dict_recursive_new2.json └── gpt3 │ └── gpt3_qa.txt ├── requirements.txt ├── run.sh ├── setup.sh ├── setup_env.sh ├── teaser.jpg ├── test_result_acc.py ├── utils.py ├── video_segment.py ├── vision_models.py └── vision_processes.py /.gitmodules: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/.gitmodules -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/README.md -------------------------------------------------------------------------------- /base_models/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /base_models/inspyrenet/InSPyReNet.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/base_models/inspyrenet/InSPyReNet.py -------------------------------------------------------------------------------- /base_models/inspyrenet/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /base_models/inspyrenet/backbones/Res2Net_v1b.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/base_models/inspyrenet/backbones/Res2Net_v1b.py -------------------------------------------------------------------------------- /base_models/inspyrenet/backbones/SwinTransformer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/base_models/inspyrenet/backbones/SwinTransformer.py -------------------------------------------------------------------------------- /base_models/inspyrenet/modules/attention_module.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/base_models/inspyrenet/modules/attention_module.py -------------------------------------------------------------------------------- /base_models/inspyrenet/modules/context_module.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/base_models/inspyrenet/modules/context_module.py -------------------------------------------------------------------------------- /base_models/inspyrenet/modules/decoder_module.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/base_models/inspyrenet/modules/decoder_module.py -------------------------------------------------------------------------------- /base_models/inspyrenet/modules/layers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/base_models/inspyrenet/modules/layers.py -------------------------------------------------------------------------------- /base_models/inspyrenet/optim/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/base_models/inspyrenet/optim/__init__.py -------------------------------------------------------------------------------- /base_models/inspyrenet/optim/losses.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/base_models/inspyrenet/optim/losses.py -------------------------------------------------------------------------------- /base_models/inspyrenet/optim/scheduler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/base_models/inspyrenet/optim/scheduler.py -------------------------------------------------------------------------------- /base_models/inspyrenet/saliency_transforms.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/base_models/inspyrenet/saliency_transforms.py -------------------------------------------------------------------------------- /base_models/tcl/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /base_models/tcl/tcl_config_bert.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/base_models/tcl/tcl_config_bert.json -------------------------------------------------------------------------------- /base_models/tcl/tcl_model_pretrain.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/base_models/tcl/tcl_model_pretrain.py -------------------------------------------------------------------------------- /base_models/tcl/tcl_tokenization_bert.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/base_models/tcl/tcl_tokenization_bert.py -------------------------------------------------------------------------------- /base_models/tcl/tcl_vit.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/base_models/tcl/tcl_vit.py -------------------------------------------------------------------------------- /base_models/tcl/tcl_xbert.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/base_models/tcl/tcl_xbert.py -------------------------------------------------------------------------------- /base_models/xvlm/config_bert.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/base_models/xvlm/config_bert.json -------------------------------------------------------------------------------- /base_models/xvlm/swin_transformer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/base_models/xvlm/swin_transformer.py -------------------------------------------------------------------------------- /base_models/xvlm/vit.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/base_models/xvlm/vit.py -------------------------------------------------------------------------------- /base_models/xvlm/xbert.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/base_models/xvlm/xbert.py -------------------------------------------------------------------------------- /base_models/xvlm/xvlm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/base_models/xvlm/xvlm.py -------------------------------------------------------------------------------- /configs/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/configs/__init__.py -------------------------------------------------------------------------------- /configs/base_config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/configs/base_config.yaml -------------------------------------------------------------------------------- /configs/covr_config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/configs/covr_config.yaml -------------------------------------------------------------------------------- /configs/gqa_config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/configs/gqa_config.yaml -------------------------------------------------------------------------------- /configs/nextqa_config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/configs/nextqa_config.yaml -------------------------------------------------------------------------------- /data/queries_gqa.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/data/queries_gqa.csv -------------------------------------------------------------------------------- /data/queries_sample.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/data/queries_sample.csv -------------------------------------------------------------------------------- /data/queries_vsr.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/data/queries_vsr.csv -------------------------------------------------------------------------------- /datasets/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/datasets/__init__.py -------------------------------------------------------------------------------- /datasets/dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/datasets/dataset.py -------------------------------------------------------------------------------- /download_models.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/download_models.sh -------------------------------------------------------------------------------- /image_patch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/image_patch.py -------------------------------------------------------------------------------- /main_batch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/main_batch.py -------------------------------------------------------------------------------- /prompts/chatapi_covr_nonrecursive_noclass.prompt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/prompts/chatapi_covr_nonrecursive_noclass.prompt -------------------------------------------------------------------------------- /prompts/chatapi_covr_recursive_noclass.prompt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/prompts/chatapi_covr_recursive_noclass.prompt -------------------------------------------------------------------------------- /prompts/chatapi_gqa_nonrecursive.prompt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/prompts/chatapi_gqa_nonrecursive.prompt -------------------------------------------------------------------------------- /prompts/chatapi_gqa_recursivenew_noclass.prompt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/prompts/chatapi_gqa_recursivenew_noclass.prompt -------------------------------------------------------------------------------- /prompts/chatapi_gqa_retrieval_samefile.prompt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/prompts/chatapi_gqa_retrieval_samefile.prompt -------------------------------------------------------------------------------- /prompts/chatapi_nextqa_nonrecursive.prompt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/prompts/chatapi_nextqa_nonrecursive.prompt -------------------------------------------------------------------------------- /prompts/chatapi_nextqa_recursive.prompt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/prompts/chatapi_nextqa_recursive.prompt -------------------------------------------------------------------------------- /prompts/choices/fixed_code/blip2.prompt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/prompts/choices/fixed_code/blip2.prompt -------------------------------------------------------------------------------- /prompts/choices/fixed_code/imagesblip2.prompt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/prompts/choices/fixed_code/imagesblip2.prompt -------------------------------------------------------------------------------- /prompts/choices/fixed_code/videoblip2.prompt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/prompts/choices/fixed_code/videoblip2.prompt -------------------------------------------------------------------------------- /prompts/choices/gqa_prompts.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/prompts/choices/gqa_prompts.json -------------------------------------------------------------------------------- /prompts/choices/gqa_prompts_dict_nonrecursive.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/prompts/choices/gqa_prompts_dict_nonrecursive.json -------------------------------------------------------------------------------- /prompts/choices/gqa_prompts_dict_nonrecursive_new.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/prompts/choices/gqa_prompts_dict_nonrecursive_new.json -------------------------------------------------------------------------------- /prompts/choices/gqa_prompts_dict_nonrecursive_new2.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/prompts/choices/gqa_prompts_dict_nonrecursive_new2.json -------------------------------------------------------------------------------- /prompts/choices/gqa_prompts_dict_recursive.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/prompts/choices/gqa_prompts_dict_recursive.json -------------------------------------------------------------------------------- /prompts/choices/gqa_prompts_dict_recursive_new.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/prompts/choices/gqa_prompts_dict_recursive_new.json -------------------------------------------------------------------------------- /prompts/choices/gqa_prompts_dict_recursive_new2.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/prompts/choices/gqa_prompts_dict_recursive_new2.json -------------------------------------------------------------------------------- /prompts/gpt3/gpt3_qa.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/prompts/gpt3/gpt3_qa.txt -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/requirements.txt -------------------------------------------------------------------------------- /run.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/run.sh -------------------------------------------------------------------------------- /setup.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/setup.sh -------------------------------------------------------------------------------- /setup_env.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/setup_env.sh -------------------------------------------------------------------------------- /teaser.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/teaser.jpg -------------------------------------------------------------------------------- /test_result_acc.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/test_result_acc.py -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/utils.py -------------------------------------------------------------------------------- /video_segment.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/video_segment.py -------------------------------------------------------------------------------- /vision_models.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/vision_models.py -------------------------------------------------------------------------------- /vision_processes.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/para-lost/RVP/HEAD/vision_processes.py --------------------------------------------------------------------------------