├── DATASET.md ├── LICENSE ├── data └── how2 │ ├── how2_s3d_train.lst │ └── how2_s3d_val.lst ├── docs ├── method.jpg └── observation.jpg ├── endtask.md ├── eval_zeroshot_align.py ├── locallaunch.py ├── mmpt ├── __init__.py ├── datasets │ ├── __init__.py │ ├── fairseqmmdataset.py │ └── mmdataset.py ├── evaluators │ ├── __init__.py │ ├── evaluator.py │ ├── metric.py │ └── predictor.py ├── losses │ ├── __init__.py │ ├── fairseqmmloss.py │ ├── loss.py │ └── nce.py ├── models │ ├── __init__.py │ ├── fairseqmmmodel.py │ ├── mmfusion.py │ ├── mmfusionnlg.py │ └── transformermodel.py ├── modules │ ├── __init__.py │ ├── mm.py │ ├── retri.py │ └── vectorpool.py ├── processors │ ├── __init__.py │ ├── dedupprocessor.py │ ├── dsprocessor.py │ ├── how2processor.py │ ├── how2retriprocessor.py │ ├── models │ │ └── s3dg.py │ └── processor.py ├── tasks │ ├── __init__.py │ ├── fairseqmmtask.py │ ├── milncetask.py │ ├── retritask.py │ ├── task.py │ └── vlmtask.py └── utils │ ├── __init__.py │ ├── load_config.py │ ├── metrics.py │ └── shardedtensor.py ├── mmpt_cli ├── localjob.py └── predict.py ├── pretraining.md ├── projects └── retri │ └── norton │ ├── coin_ft.yaml │ ├── how2_pretrain.yaml │ ├── test_coin_ft.yaml │ ├── test_coin_zs.yaml │ ├── test_how2align_zs.yaml │ ├── test_vtt_ft.yaml │ ├── test_vtt_zs.yaml │ ├── test_vttqa_ft.yaml │ ├── test_vttqa_zs.yaml │ ├── test_youcook_fullvideo_bg_zs.yaml │ ├── test_youcook_fullvideo_zs.yaml │ ├── test_youcook_zs.yaml │ ├── vtt_ft.yaml │ └── vttqa_ft.yaml ├── readme.md ├── run.sh ├── scripts ├── text_token_extractor │ ├── configs │ │ └── bert-base-uncased.yaml │ └── pretokenization.py └── video_feature_extractor │ ├── extract.py │ ├── how2 │ └── s3d.sh │ ├── model.py │ ├── pathbuilder.py │ ├── preprocessing.py │ ├── random_sequence_shuffler.py │ ├── shard_feature.py │ └── videoreader.py └── train.py /DATASET.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/DATASET.md -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/LICENSE -------------------------------------------------------------------------------- /data/how2/how2_s3d_train.lst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/data/how2/how2_s3d_train.lst -------------------------------------------------------------------------------- /data/how2/how2_s3d_val.lst: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/data/how2/how2_s3d_val.lst -------------------------------------------------------------------------------- /docs/method.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/docs/method.jpg -------------------------------------------------------------------------------- /docs/observation.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/docs/observation.jpg -------------------------------------------------------------------------------- /endtask.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/endtask.md -------------------------------------------------------------------------------- /eval_zeroshot_align.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/eval_zeroshot_align.py -------------------------------------------------------------------------------- /locallaunch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/locallaunch.py -------------------------------------------------------------------------------- /mmpt/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/mmpt/__init__.py -------------------------------------------------------------------------------- /mmpt/datasets/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/mmpt/datasets/__init__.py -------------------------------------------------------------------------------- /mmpt/datasets/fairseqmmdataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/mmpt/datasets/fairseqmmdataset.py -------------------------------------------------------------------------------- /mmpt/datasets/mmdataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/mmpt/datasets/mmdataset.py -------------------------------------------------------------------------------- /mmpt/evaluators/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/mmpt/evaluators/__init__.py -------------------------------------------------------------------------------- /mmpt/evaluators/evaluator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/mmpt/evaluators/evaluator.py -------------------------------------------------------------------------------- /mmpt/evaluators/metric.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/mmpt/evaluators/metric.py -------------------------------------------------------------------------------- /mmpt/evaluators/predictor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/mmpt/evaluators/predictor.py -------------------------------------------------------------------------------- /mmpt/losses/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/mmpt/losses/__init__.py -------------------------------------------------------------------------------- /mmpt/losses/fairseqmmloss.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/mmpt/losses/fairseqmmloss.py -------------------------------------------------------------------------------- /mmpt/losses/loss.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/mmpt/losses/loss.py -------------------------------------------------------------------------------- /mmpt/losses/nce.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/mmpt/losses/nce.py -------------------------------------------------------------------------------- /mmpt/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/mmpt/models/__init__.py -------------------------------------------------------------------------------- /mmpt/models/fairseqmmmodel.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/mmpt/models/fairseqmmmodel.py -------------------------------------------------------------------------------- /mmpt/models/mmfusion.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/mmpt/models/mmfusion.py -------------------------------------------------------------------------------- /mmpt/models/mmfusionnlg.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/mmpt/models/mmfusionnlg.py -------------------------------------------------------------------------------- /mmpt/models/transformermodel.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/mmpt/models/transformermodel.py -------------------------------------------------------------------------------- /mmpt/modules/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/mmpt/modules/__init__.py -------------------------------------------------------------------------------- /mmpt/modules/mm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/mmpt/modules/mm.py -------------------------------------------------------------------------------- /mmpt/modules/retri.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/mmpt/modules/retri.py -------------------------------------------------------------------------------- /mmpt/modules/vectorpool.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/mmpt/modules/vectorpool.py -------------------------------------------------------------------------------- /mmpt/processors/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/mmpt/processors/__init__.py -------------------------------------------------------------------------------- /mmpt/processors/dedupprocessor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/mmpt/processors/dedupprocessor.py -------------------------------------------------------------------------------- /mmpt/processors/dsprocessor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/mmpt/processors/dsprocessor.py -------------------------------------------------------------------------------- /mmpt/processors/how2processor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/mmpt/processors/how2processor.py -------------------------------------------------------------------------------- /mmpt/processors/how2retriprocessor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/mmpt/processors/how2retriprocessor.py -------------------------------------------------------------------------------- /mmpt/processors/models/s3dg.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/mmpt/processors/models/s3dg.py -------------------------------------------------------------------------------- /mmpt/processors/processor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/mmpt/processors/processor.py -------------------------------------------------------------------------------- /mmpt/tasks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/mmpt/tasks/__init__.py -------------------------------------------------------------------------------- /mmpt/tasks/fairseqmmtask.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/mmpt/tasks/fairseqmmtask.py -------------------------------------------------------------------------------- /mmpt/tasks/milncetask.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/mmpt/tasks/milncetask.py -------------------------------------------------------------------------------- /mmpt/tasks/retritask.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/mmpt/tasks/retritask.py -------------------------------------------------------------------------------- /mmpt/tasks/task.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/mmpt/tasks/task.py -------------------------------------------------------------------------------- /mmpt/tasks/vlmtask.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/mmpt/tasks/vlmtask.py -------------------------------------------------------------------------------- /mmpt/utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/mmpt/utils/__init__.py -------------------------------------------------------------------------------- /mmpt/utils/load_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/mmpt/utils/load_config.py -------------------------------------------------------------------------------- /mmpt/utils/metrics.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/mmpt/utils/metrics.py -------------------------------------------------------------------------------- /mmpt/utils/shardedtensor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/mmpt/utils/shardedtensor.py -------------------------------------------------------------------------------- /mmpt_cli/localjob.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/mmpt_cli/localjob.py -------------------------------------------------------------------------------- /mmpt_cli/predict.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/mmpt_cli/predict.py -------------------------------------------------------------------------------- /pretraining.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/pretraining.md -------------------------------------------------------------------------------- /projects/retri/norton/coin_ft.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/projects/retri/norton/coin_ft.yaml -------------------------------------------------------------------------------- /projects/retri/norton/how2_pretrain.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/projects/retri/norton/how2_pretrain.yaml -------------------------------------------------------------------------------- /projects/retri/norton/test_coin_ft.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/projects/retri/norton/test_coin_ft.yaml -------------------------------------------------------------------------------- /projects/retri/norton/test_coin_zs.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/projects/retri/norton/test_coin_zs.yaml -------------------------------------------------------------------------------- /projects/retri/norton/test_how2align_zs.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/projects/retri/norton/test_how2align_zs.yaml -------------------------------------------------------------------------------- /projects/retri/norton/test_vtt_ft.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/projects/retri/norton/test_vtt_ft.yaml -------------------------------------------------------------------------------- /projects/retri/norton/test_vtt_zs.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/projects/retri/norton/test_vtt_zs.yaml -------------------------------------------------------------------------------- /projects/retri/norton/test_vttqa_ft.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/projects/retri/norton/test_vttqa_ft.yaml -------------------------------------------------------------------------------- /projects/retri/norton/test_vttqa_zs.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/projects/retri/norton/test_vttqa_zs.yaml -------------------------------------------------------------------------------- /projects/retri/norton/test_youcook_fullvideo_bg_zs.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/projects/retri/norton/test_youcook_fullvideo_bg_zs.yaml -------------------------------------------------------------------------------- /projects/retri/norton/test_youcook_fullvideo_zs.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/projects/retri/norton/test_youcook_fullvideo_zs.yaml -------------------------------------------------------------------------------- /projects/retri/norton/test_youcook_zs.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/projects/retri/norton/test_youcook_zs.yaml -------------------------------------------------------------------------------- /projects/retri/norton/vtt_ft.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/projects/retri/norton/vtt_ft.yaml -------------------------------------------------------------------------------- /projects/retri/norton/vttqa_ft.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/projects/retri/norton/vttqa_ft.yaml -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/readme.md -------------------------------------------------------------------------------- /run.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/run.sh -------------------------------------------------------------------------------- /scripts/text_token_extractor/configs/bert-base-uncased.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/scripts/text_token_extractor/configs/bert-base-uncased.yaml -------------------------------------------------------------------------------- /scripts/text_token_extractor/pretokenization.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/scripts/text_token_extractor/pretokenization.py -------------------------------------------------------------------------------- /scripts/video_feature_extractor/extract.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/scripts/video_feature_extractor/extract.py -------------------------------------------------------------------------------- /scripts/video_feature_extractor/how2/s3d.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/scripts/video_feature_extractor/how2/s3d.sh -------------------------------------------------------------------------------- /scripts/video_feature_extractor/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/scripts/video_feature_extractor/model.py -------------------------------------------------------------------------------- /scripts/video_feature_extractor/pathbuilder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/scripts/video_feature_extractor/pathbuilder.py -------------------------------------------------------------------------------- /scripts/video_feature_extractor/preprocessing.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/scripts/video_feature_extractor/preprocessing.py -------------------------------------------------------------------------------- /scripts/video_feature_extractor/random_sequence_shuffler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/scripts/video_feature_extractor/random_sequence_shuffler.py -------------------------------------------------------------------------------- /scripts/video_feature_extractor/shard_feature.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/scripts/video_feature_extractor/shard_feature.py -------------------------------------------------------------------------------- /scripts/video_feature_extractor/videoreader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/scripts/video_feature_extractor/videoreader.py -------------------------------------------------------------------------------- /train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/XLearning-SCU/2024-ICLR-Norton/HEAD/train.py --------------------------------------------------------------------------------