├── .gitignore ├── .gitmodules ├── LICENSE ├── README.md ├── base_models ├── __init__.py ├── inspyrenet │ ├── InSPyReNet.py │ ├── __init__.py │ ├── backbones │ │ ├── Res2Net_v1b.py │ │ └── SwinTransformer.py │ ├── modules │ │ ├── attention_module.py │ │ ├── context_module.py │ │ ├── decoder_module.py │ │ └── layers.py │ ├── optim │ │ ├── __init__.py │ │ ├── losses.py │ │ └── scheduler.py │ └── saliency_transforms.py ├── tcl │ ├── __init__.py │ ├── tcl_config_bert.json │ ├── tcl_model_pretrain.py │ ├── tcl_tokenization_bert.py │ ├── tcl_vit.py │ └── tcl_xbert.py └── xvlm │ ├── config_bert.json │ ├── swin_transformer.py │ ├── vit.py │ ├── xbert.py │ └── xvlm.py ├── configs ├── __init__.py ├── base_config.yaml ├── benchmarks │ ├── gqa.yaml │ ├── nextqa.yaml │ ├── okvqa.yaml │ └── refcoco.yaml ├── config_codellama.yaml └── my_config.yaml ├── data └── queries.csv ├── datasets ├── __init__.py ├── gqa.py ├── my_dataset.py ├── nextqa.py ├── okvqa.py └── refcoco.py ├── download_models.sh ├── image_patch.py ├── main_batch.py ├── main_simple.ipynb ├── main_simple_lib.py ├── prompts ├── api.prompt ├── benchmarks │ ├── gqa.prompt │ ├── nextqa.prompt │ ├── okvqa.prompt │ └── refcoco.prompt ├── chatapi.prompt ├── fixed_code │ ├── blip2.prompt │ ├── blip2_video.prompt │ └── glip.prompt └── gpt3 │ ├── gpt3_process_guess.txt │ ├── gpt3_qa.txt │ └── video_question.txt ├── requirements.txt ├── setup.sh ├── setup_env.sh ├── teaser.gif ├── useful_lists ├── possible_options.json └── random_negatives.txt ├── utils.py ├── video_segment.py ├── vision_models.py └── vision_processes.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/.gitignore -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/.gitmodules -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/README.md -------------------------------------------------------------------------------- /base_models/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /base_models/inspyrenet/InSPyReNet.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/base_models/inspyrenet/InSPyReNet.py -------------------------------------------------------------------------------- /base_models/inspyrenet/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /base_models/inspyrenet/backbones/Res2Net_v1b.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/base_models/inspyrenet/backbones/Res2Net_v1b.py -------------------------------------------------------------------------------- /base_models/inspyrenet/backbones/SwinTransformer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/base_models/inspyrenet/backbones/SwinTransformer.py -------------------------------------------------------------------------------- /base_models/inspyrenet/modules/attention_module.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/base_models/inspyrenet/modules/attention_module.py -------------------------------------------------------------------------------- /base_models/inspyrenet/modules/context_module.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/base_models/inspyrenet/modules/context_module.py -------------------------------------------------------------------------------- /base_models/inspyrenet/modules/decoder_module.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/base_models/inspyrenet/modules/decoder_module.py -------------------------------------------------------------------------------- /base_models/inspyrenet/modules/layers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/base_models/inspyrenet/modules/layers.py -------------------------------------------------------------------------------- /base_models/inspyrenet/optim/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/base_models/inspyrenet/optim/__init__.py -------------------------------------------------------------------------------- /base_models/inspyrenet/optim/losses.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/base_models/inspyrenet/optim/losses.py -------------------------------------------------------------------------------- /base_models/inspyrenet/optim/scheduler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/base_models/inspyrenet/optim/scheduler.py -------------------------------------------------------------------------------- /base_models/inspyrenet/saliency_transforms.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/base_models/inspyrenet/saliency_transforms.py -------------------------------------------------------------------------------- /base_models/tcl/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /base_models/tcl/tcl_config_bert.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/base_models/tcl/tcl_config_bert.json -------------------------------------------------------------------------------- /base_models/tcl/tcl_model_pretrain.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/base_models/tcl/tcl_model_pretrain.py -------------------------------------------------------------------------------- /base_models/tcl/tcl_tokenization_bert.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/base_models/tcl/tcl_tokenization_bert.py -------------------------------------------------------------------------------- /base_models/tcl/tcl_vit.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/base_models/tcl/tcl_vit.py -------------------------------------------------------------------------------- /base_models/tcl/tcl_xbert.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/base_models/tcl/tcl_xbert.py -------------------------------------------------------------------------------- /base_models/xvlm/config_bert.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/base_models/xvlm/config_bert.json -------------------------------------------------------------------------------- /base_models/xvlm/swin_transformer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/base_models/xvlm/swin_transformer.py -------------------------------------------------------------------------------- /base_models/xvlm/vit.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/base_models/xvlm/vit.py -------------------------------------------------------------------------------- /base_models/xvlm/xbert.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/base_models/xvlm/xbert.py -------------------------------------------------------------------------------- /base_models/xvlm/xvlm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/base_models/xvlm/xvlm.py -------------------------------------------------------------------------------- /configs/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/configs/__init__.py -------------------------------------------------------------------------------- /configs/base_config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/configs/base_config.yaml -------------------------------------------------------------------------------- /configs/benchmarks/gqa.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/configs/benchmarks/gqa.yaml -------------------------------------------------------------------------------- /configs/benchmarks/nextqa.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/configs/benchmarks/nextqa.yaml -------------------------------------------------------------------------------- /configs/benchmarks/okvqa.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/configs/benchmarks/okvqa.yaml -------------------------------------------------------------------------------- /configs/benchmarks/refcoco.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/configs/benchmarks/refcoco.yaml -------------------------------------------------------------------------------- /configs/config_codellama.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/configs/config_codellama.yaml -------------------------------------------------------------------------------- /configs/my_config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/configs/my_config.yaml -------------------------------------------------------------------------------- /data/queries.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/data/queries.csv -------------------------------------------------------------------------------- /datasets/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/datasets/__init__.py -------------------------------------------------------------------------------- /datasets/gqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/datasets/gqa.py -------------------------------------------------------------------------------- /datasets/my_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/datasets/my_dataset.py -------------------------------------------------------------------------------- /datasets/nextqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/datasets/nextqa.py -------------------------------------------------------------------------------- /datasets/okvqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/datasets/okvqa.py -------------------------------------------------------------------------------- /datasets/refcoco.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/datasets/refcoco.py -------------------------------------------------------------------------------- /download_models.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/download_models.sh -------------------------------------------------------------------------------- /image_patch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/image_patch.py -------------------------------------------------------------------------------- /main_batch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/main_batch.py -------------------------------------------------------------------------------- /main_simple.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/main_simple.ipynb -------------------------------------------------------------------------------- /main_simple_lib.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/main_simple_lib.py -------------------------------------------------------------------------------- /prompts/api.prompt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/prompts/api.prompt -------------------------------------------------------------------------------- /prompts/benchmarks/gqa.prompt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/prompts/benchmarks/gqa.prompt -------------------------------------------------------------------------------- /prompts/benchmarks/nextqa.prompt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/prompts/benchmarks/nextqa.prompt -------------------------------------------------------------------------------- /prompts/benchmarks/okvqa.prompt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/prompts/benchmarks/okvqa.prompt -------------------------------------------------------------------------------- /prompts/benchmarks/refcoco.prompt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/prompts/benchmarks/refcoco.prompt -------------------------------------------------------------------------------- /prompts/chatapi.prompt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/prompts/chatapi.prompt -------------------------------------------------------------------------------- /prompts/fixed_code/blip2.prompt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/prompts/fixed_code/blip2.prompt -------------------------------------------------------------------------------- /prompts/fixed_code/blip2_video.prompt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/prompts/fixed_code/blip2_video.prompt -------------------------------------------------------------------------------- /prompts/fixed_code/glip.prompt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/prompts/fixed_code/glip.prompt -------------------------------------------------------------------------------- /prompts/gpt3/gpt3_process_guess.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/prompts/gpt3/gpt3_process_guess.txt -------------------------------------------------------------------------------- /prompts/gpt3/gpt3_qa.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/prompts/gpt3/gpt3_qa.txt -------------------------------------------------------------------------------- /prompts/gpt3/video_question.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/prompts/gpt3/video_question.txt -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/requirements.txt -------------------------------------------------------------------------------- /setup.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/setup.sh -------------------------------------------------------------------------------- /setup_env.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/setup_env.sh -------------------------------------------------------------------------------- /teaser.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/teaser.gif -------------------------------------------------------------------------------- /useful_lists/possible_options.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/useful_lists/possible_options.json -------------------------------------------------------------------------------- /useful_lists/random_negatives.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/useful_lists/random_negatives.txt -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/utils.py -------------------------------------------------------------------------------- /video_segment.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/video_segment.py -------------------------------------------------------------------------------- /vision_models.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/vision_models.py -------------------------------------------------------------------------------- /vision_processes.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cvlab-columbia/viper/HEAD/vision_processes.py --------------------------------------------------------------------------------