├── .gitignore ├── EVA_clip ├── CLIP.png ├── README.md ├── __init__.py ├── benchmark.md ├── bpe_simple_vocab_16e6.txt.gz ├── clip.py ├── eva_clip.py ├── eva_model.py ├── model.py ├── model_configs │ └── EVA_CLIP_g_14.json ├── simple_tokenizer.py └── vit_model.py ├── LICENSE ├── README.md ├── args.py ├── assets ├── joint_model.png └── main_figure_compressed.png ├── clip4caption ├── .gitignore ├── README.md ├── dataloaders │ ├── dataloader_hodini_feats.py │ ├── dataloader_msrvtt_feats.py │ ├── dataloader_msrvtt_raw.py │ ├── dataloader_msvd_feats.py │ ├── dataloader_msvd_raw.py │ └── rawvideo_util.py ├── feature_extractor │ ├── README.md │ ├── clip_feature_extractor.py │ ├── modules │ │ ├── .gitkeep │ │ ├── cross-base │ │ │ └── cross_config.json │ │ ├── file_utils.py │ │ ├── modeling.py │ │ ├── module_clip.py │ │ ├── module_cross.py │ │ ├── optimization.py │ │ ├── until_config.py │ │ └── until_module.py │ ├── pretrained_clip4clip │ │ ├── msrvtt │ │ │ └── .gitkeep │ │ └── msvd │ │ │ └── .gitkeep │ ├── util.py │ └── utility │ │ ├── dataset.py │ │ ├── util.py │ │ └── vocabulary.py ├── modules │ ├── beam.py │ ├── decoder-base │ │ └── decoder_config.json │ ├── file_utils.py │ ├── modeling.py │ ├── module_bert.py │ ├── module_decoder.py │ ├── module_visual.py │ ├── optimization.py │ ├── tokenization.py │ ├── until_config.py │ ├── until_module.py │ └── visual-base │ │ └── visual_config.json └── train.py ├── custom_video_pipeline.ipynb ├── data ├── evaluation │ ├── categories.json │ └── formatted_moment_evaluation_gt.json ├── splits │ ├── all_data_test.json │ ├── all_data_test_negative_samples.json │ ├── all_data_train.json │ └── all_data_val.json └── val_testing │ ├── README.md │ ├── all_data_test.json │ └── formated_moment_evaluation_gt.json ├── dist_utils.py ├── evaluate.py ├── extraction ├── README.md ├── video_features │ ├── README.md │ ├── check_feature_size.py │ ├── download_videos.py │ ├── extract_features.py │ └── extract_frames.py └── whisper_ASR │ ├── README.md │ ├── extract_ASR.py │ ├── extract_ASR_embedding.py │ └── extract_audio.py ├── hirest_dataset.py ├── inference_video_retrieval.py ├── modeling.py ├── requirements.txt ├── run.py ├── scripts ├── run.sh ├── run_end_to_end.sh ├── run_val.sh ├── score.sh └── score_val.sh ├── trainer_base.py ├── utils.py ├── val_inference_and_evaluation.ipynb └── video_retrieval_inference_and_evaluation.ipynb /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/.gitignore -------------------------------------------------------------------------------- /EVA_clip/CLIP.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/EVA_clip/CLIP.png -------------------------------------------------------------------------------- /EVA_clip/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/EVA_clip/README.md -------------------------------------------------------------------------------- /EVA_clip/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/EVA_clip/__init__.py -------------------------------------------------------------------------------- /EVA_clip/benchmark.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/EVA_clip/benchmark.md -------------------------------------------------------------------------------- /EVA_clip/bpe_simple_vocab_16e6.txt.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/EVA_clip/bpe_simple_vocab_16e6.txt.gz -------------------------------------------------------------------------------- /EVA_clip/clip.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/EVA_clip/clip.py -------------------------------------------------------------------------------- /EVA_clip/eva_clip.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/EVA_clip/eva_clip.py -------------------------------------------------------------------------------- /EVA_clip/eva_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/EVA_clip/eva_model.py -------------------------------------------------------------------------------- /EVA_clip/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/EVA_clip/model.py -------------------------------------------------------------------------------- /EVA_clip/model_configs/EVA_CLIP_g_14.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/EVA_clip/model_configs/EVA_CLIP_g_14.json -------------------------------------------------------------------------------- /EVA_clip/simple_tokenizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/EVA_clip/simple_tokenizer.py -------------------------------------------------------------------------------- /EVA_clip/vit_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/EVA_clip/vit_model.py -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/README.md -------------------------------------------------------------------------------- /args.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/args.py -------------------------------------------------------------------------------- /assets/joint_model.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/assets/joint_model.png -------------------------------------------------------------------------------- /assets/main_figure_compressed.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/assets/main_figure_compressed.png -------------------------------------------------------------------------------- /clip4caption/.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/clip4caption/.gitignore -------------------------------------------------------------------------------- /clip4caption/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/clip4caption/README.md -------------------------------------------------------------------------------- /clip4caption/dataloaders/dataloader_hodini_feats.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/clip4caption/dataloaders/dataloader_hodini_feats.py -------------------------------------------------------------------------------- /clip4caption/dataloaders/dataloader_msrvtt_feats.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/clip4caption/dataloaders/dataloader_msrvtt_feats.py -------------------------------------------------------------------------------- /clip4caption/dataloaders/dataloader_msrvtt_raw.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/clip4caption/dataloaders/dataloader_msrvtt_raw.py -------------------------------------------------------------------------------- /clip4caption/dataloaders/dataloader_msvd_feats.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/clip4caption/dataloaders/dataloader_msvd_feats.py -------------------------------------------------------------------------------- /clip4caption/dataloaders/dataloader_msvd_raw.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/clip4caption/dataloaders/dataloader_msvd_raw.py -------------------------------------------------------------------------------- /clip4caption/dataloaders/rawvideo_util.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/clip4caption/dataloaders/rawvideo_util.py -------------------------------------------------------------------------------- /clip4caption/feature_extractor/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/clip4caption/feature_extractor/README.md -------------------------------------------------------------------------------- /clip4caption/feature_extractor/clip_feature_extractor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/clip4caption/feature_extractor/clip_feature_extractor.py -------------------------------------------------------------------------------- /clip4caption/feature_extractor/modules/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /clip4caption/feature_extractor/modules/cross-base/cross_config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/clip4caption/feature_extractor/modules/cross-base/cross_config.json -------------------------------------------------------------------------------- /clip4caption/feature_extractor/modules/file_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/clip4caption/feature_extractor/modules/file_utils.py -------------------------------------------------------------------------------- /clip4caption/feature_extractor/modules/modeling.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/clip4caption/feature_extractor/modules/modeling.py -------------------------------------------------------------------------------- /clip4caption/feature_extractor/modules/module_clip.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/clip4caption/feature_extractor/modules/module_clip.py -------------------------------------------------------------------------------- /clip4caption/feature_extractor/modules/module_cross.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/clip4caption/feature_extractor/modules/module_cross.py -------------------------------------------------------------------------------- /clip4caption/feature_extractor/modules/optimization.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/clip4caption/feature_extractor/modules/optimization.py -------------------------------------------------------------------------------- /clip4caption/feature_extractor/modules/until_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/clip4caption/feature_extractor/modules/until_config.py -------------------------------------------------------------------------------- /clip4caption/feature_extractor/modules/until_module.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/clip4caption/feature_extractor/modules/until_module.py -------------------------------------------------------------------------------- /clip4caption/feature_extractor/pretrained_clip4clip/msrvtt/.gitkeep: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /clip4caption/feature_extractor/pretrained_clip4clip/msvd/.gitkeep: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /clip4caption/feature_extractor/util.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/clip4caption/feature_extractor/util.py -------------------------------------------------------------------------------- /clip4caption/feature_extractor/utility/dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/clip4caption/feature_extractor/utility/dataset.py -------------------------------------------------------------------------------- /clip4caption/feature_extractor/utility/util.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/clip4caption/feature_extractor/utility/util.py -------------------------------------------------------------------------------- /clip4caption/feature_extractor/utility/vocabulary.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/clip4caption/feature_extractor/utility/vocabulary.py -------------------------------------------------------------------------------- /clip4caption/modules/beam.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/clip4caption/modules/beam.py -------------------------------------------------------------------------------- /clip4caption/modules/decoder-base/decoder_config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/clip4caption/modules/decoder-base/decoder_config.json -------------------------------------------------------------------------------- /clip4caption/modules/file_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/clip4caption/modules/file_utils.py -------------------------------------------------------------------------------- /clip4caption/modules/modeling.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/clip4caption/modules/modeling.py -------------------------------------------------------------------------------- /clip4caption/modules/module_bert.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/clip4caption/modules/module_bert.py -------------------------------------------------------------------------------- /clip4caption/modules/module_decoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/clip4caption/modules/module_decoder.py -------------------------------------------------------------------------------- /clip4caption/modules/module_visual.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/clip4caption/modules/module_visual.py -------------------------------------------------------------------------------- /clip4caption/modules/optimization.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/clip4caption/modules/optimization.py -------------------------------------------------------------------------------- /clip4caption/modules/tokenization.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/clip4caption/modules/tokenization.py -------------------------------------------------------------------------------- /clip4caption/modules/until_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/clip4caption/modules/until_config.py -------------------------------------------------------------------------------- /clip4caption/modules/until_module.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/clip4caption/modules/until_module.py -------------------------------------------------------------------------------- /clip4caption/modules/visual-base/visual_config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/clip4caption/modules/visual-base/visual_config.json -------------------------------------------------------------------------------- /clip4caption/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/clip4caption/train.py -------------------------------------------------------------------------------- /custom_video_pipeline.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/custom_video_pipeline.ipynb -------------------------------------------------------------------------------- /data/evaluation/categories.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/data/evaluation/categories.json -------------------------------------------------------------------------------- /data/evaluation/formatted_moment_evaluation_gt.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/data/evaluation/formatted_moment_evaluation_gt.json -------------------------------------------------------------------------------- /data/splits/all_data_test.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/data/splits/all_data_test.json -------------------------------------------------------------------------------- /data/splits/all_data_test_negative_samples.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/data/splits/all_data_test_negative_samples.json -------------------------------------------------------------------------------- /data/splits/all_data_train.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/data/splits/all_data_train.json -------------------------------------------------------------------------------- /data/splits/all_data_val.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/data/splits/all_data_val.json -------------------------------------------------------------------------------- /data/val_testing/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/data/val_testing/README.md -------------------------------------------------------------------------------- /data/val_testing/all_data_test.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/data/val_testing/all_data_test.json -------------------------------------------------------------------------------- /data/val_testing/formated_moment_evaluation_gt.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/data/val_testing/formated_moment_evaluation_gt.json -------------------------------------------------------------------------------- /dist_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/dist_utils.py -------------------------------------------------------------------------------- /evaluate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/evaluate.py -------------------------------------------------------------------------------- /extraction/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/extraction/README.md -------------------------------------------------------------------------------- /extraction/video_features/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/extraction/video_features/README.md -------------------------------------------------------------------------------- /extraction/video_features/check_feature_size.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/extraction/video_features/check_feature_size.py -------------------------------------------------------------------------------- /extraction/video_features/download_videos.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/extraction/video_features/download_videos.py -------------------------------------------------------------------------------- /extraction/video_features/extract_features.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/extraction/video_features/extract_features.py -------------------------------------------------------------------------------- /extraction/video_features/extract_frames.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/extraction/video_features/extract_frames.py -------------------------------------------------------------------------------- /extraction/whisper_ASR/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/extraction/whisper_ASR/README.md -------------------------------------------------------------------------------- /extraction/whisper_ASR/extract_ASR.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/extraction/whisper_ASR/extract_ASR.py -------------------------------------------------------------------------------- /extraction/whisper_ASR/extract_ASR_embedding.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/extraction/whisper_ASR/extract_ASR_embedding.py -------------------------------------------------------------------------------- /extraction/whisper_ASR/extract_audio.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/extraction/whisper_ASR/extract_audio.py -------------------------------------------------------------------------------- /hirest_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/hirest_dataset.py -------------------------------------------------------------------------------- /inference_video_retrieval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/inference_video_retrieval.py -------------------------------------------------------------------------------- /modeling.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/modeling.py -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/requirements.txt -------------------------------------------------------------------------------- /run.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/run.py -------------------------------------------------------------------------------- /scripts/run.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/scripts/run.sh -------------------------------------------------------------------------------- /scripts/run_end_to_end.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/scripts/run_end_to_end.sh -------------------------------------------------------------------------------- /scripts/run_val.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/scripts/run_val.sh -------------------------------------------------------------------------------- /scripts/score.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/scripts/score.sh -------------------------------------------------------------------------------- /scripts/score_val.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/scripts/score_val.sh -------------------------------------------------------------------------------- /trainer_base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/trainer_base.py -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/utils.py -------------------------------------------------------------------------------- /val_inference_and_evaluation.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/val_inference_and_evaluation.ipynb -------------------------------------------------------------------------------- /video_retrieval_inference_and_evaluation.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/j-min/HiREST/HEAD/video_retrieval_inference_and_evaluation.ipynb --------------------------------------------------------------------------------