├── .gitattributes ├── LICENSE ├── README.md ├── data ├── LICENSE ├── README.md ├── highlight_test_release.jsonl ├── highlight_train_release.jsonl ├── highlight_val_release.jsonl ├── subs_train.jsonl └── tvsum │ ├── tvsum_train.jsonl │ └── tvsum_val.jsonl ├── feedbacks.png ├── pipeline.png ├── results ├── best_model_hd │ ├── best_hl_val_preds.jsonl │ ├── best_hl_val_preds_metrics.json │ ├── eval.log.txt │ ├── hl_val_submission.jsonl │ ├── hl_val_submission_metrics.json │ ├── latest_hl_val_preds.jsonl │ ├── latest_hl_val_preds_metrics.json │ ├── opt.json │ ├── tensorboard_log │ │ └── events.out.tfevents.1698656585.GPUserver.94248.0 │ └── train.log.txt └── best_model_mr │ ├── best_hl_val_preds.jsonl │ ├── best_hl_val_preds_metrics.json │ ├── eval.log.txt │ ├── hl_test_submission.jsonl │ ├── hl_val_submission.jsonl │ ├── hl_val_submission_metrics.json │ ├── latest_hl_val_preds.jsonl │ ├── latest_hl_val_preds_metrics.json │ ├── model_best.ckpt │ ├── opt.json │ ├── tensorboard_log │ └── events.out.tfevents.1697767302.GPUserver.93330.0 │ └── train.log.txt ├── run_on_video ├── README.md ├── clip │ ├── __init__.py │ ├── bpe_simple_vocab_16e6.txt.gz │ ├── clip.py │ ├── model.py │ └── simple_tokenizer.py ├── data_utils.py ├── example │ ├── RoripwjYFp8_60.0_210.0.mp4 │ └── queries.jsonl ├── model_utils.py └── run.py ├── standalone_eval ├── README.md ├── eval.py ├── eval_sample.sh ├── sample_val_preds.jsonl ├── sample_val_preds_metrics_raw.json └── utils.py ├── taskweave ├── __init__.py ├── attention.py ├── config.py ├── inference.py ├── matcher.py ├── misc.py ├── model.py ├── position_encoding.py ├── postprocessing_taskweave.py ├── scripts │ ├── inference.sh │ ├── inference_audio.sh │ ├── pretrain.sh │ ├── train.sh │ ├── train_audio.sh │ ├── train_audio_umt.sh │ └── tvsum │ │ ├── train_tvsum.sh │ │ └── train_tvsum_audio.sh ├── span_utils.py ├── start_end_dataset.py ├── start_end_dataset_audio.py ├── text_encoder.py ├── train.py └── transformer.py └── utils ├── basic_utils.py ├── model_utils.py ├── temporal_nms.py ├── tensor_utils.py └── windows_utils.py /.gitattributes: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/.gitattributes -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/README.md -------------------------------------------------------------------------------- /data/LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/data/LICENSE -------------------------------------------------------------------------------- /data/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/data/README.md -------------------------------------------------------------------------------- /data/highlight_test_release.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/data/highlight_test_release.jsonl -------------------------------------------------------------------------------- /data/highlight_train_release.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/data/highlight_train_release.jsonl -------------------------------------------------------------------------------- /data/highlight_val_release.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/data/highlight_val_release.jsonl -------------------------------------------------------------------------------- /data/subs_train.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/data/subs_train.jsonl -------------------------------------------------------------------------------- /data/tvsum/tvsum_train.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/data/tvsum/tvsum_train.jsonl -------------------------------------------------------------------------------- /data/tvsum/tvsum_val.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/data/tvsum/tvsum_val.jsonl -------------------------------------------------------------------------------- /feedbacks.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/feedbacks.png -------------------------------------------------------------------------------- /pipeline.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/pipeline.png -------------------------------------------------------------------------------- /results/best_model_hd/best_hl_val_preds.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/results/best_model_hd/best_hl_val_preds.jsonl -------------------------------------------------------------------------------- /results/best_model_hd/best_hl_val_preds_metrics.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/results/best_model_hd/best_hl_val_preds_metrics.json -------------------------------------------------------------------------------- /results/best_model_hd/eval.log.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/results/best_model_hd/eval.log.txt -------------------------------------------------------------------------------- /results/best_model_hd/hl_val_submission.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/results/best_model_hd/hl_val_submission.jsonl -------------------------------------------------------------------------------- /results/best_model_hd/hl_val_submission_metrics.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/results/best_model_hd/hl_val_submission_metrics.json -------------------------------------------------------------------------------- /results/best_model_hd/latest_hl_val_preds.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/results/best_model_hd/latest_hl_val_preds.jsonl -------------------------------------------------------------------------------- /results/best_model_hd/latest_hl_val_preds_metrics.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/results/best_model_hd/latest_hl_val_preds_metrics.json -------------------------------------------------------------------------------- /results/best_model_hd/opt.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/results/best_model_hd/opt.json -------------------------------------------------------------------------------- /results/best_model_hd/tensorboard_log/events.out.tfevents.1698656585.GPUserver.94248.0: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/results/best_model_hd/tensorboard_log/events.out.tfevents.1698656585.GPUserver.94248.0 -------------------------------------------------------------------------------- /results/best_model_hd/train.log.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/results/best_model_hd/train.log.txt -------------------------------------------------------------------------------- /results/best_model_mr/best_hl_val_preds.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/results/best_model_mr/best_hl_val_preds.jsonl -------------------------------------------------------------------------------- /results/best_model_mr/best_hl_val_preds_metrics.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/results/best_model_mr/best_hl_val_preds_metrics.json -------------------------------------------------------------------------------- /results/best_model_mr/eval.log.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/results/best_model_mr/eval.log.txt -------------------------------------------------------------------------------- /results/best_model_mr/hl_test_submission.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/results/best_model_mr/hl_test_submission.jsonl -------------------------------------------------------------------------------- /results/best_model_mr/hl_val_submission.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/results/best_model_mr/hl_val_submission.jsonl -------------------------------------------------------------------------------- /results/best_model_mr/hl_val_submission_metrics.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/results/best_model_mr/hl_val_submission_metrics.json -------------------------------------------------------------------------------- /results/best_model_mr/latest_hl_val_preds.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/results/best_model_mr/latest_hl_val_preds.jsonl -------------------------------------------------------------------------------- /results/best_model_mr/latest_hl_val_preds_metrics.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/results/best_model_mr/latest_hl_val_preds_metrics.json -------------------------------------------------------------------------------- /results/best_model_mr/model_best.ckpt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/results/best_model_mr/model_best.ckpt -------------------------------------------------------------------------------- /results/best_model_mr/opt.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/results/best_model_mr/opt.json -------------------------------------------------------------------------------- /results/best_model_mr/tensorboard_log/events.out.tfevents.1697767302.GPUserver.93330.0: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/results/best_model_mr/tensorboard_log/events.out.tfevents.1697767302.GPUserver.93330.0 -------------------------------------------------------------------------------- /results/best_model_mr/train.log.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/results/best_model_mr/train.log.txt -------------------------------------------------------------------------------- /run_on_video/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/run_on_video/README.md -------------------------------------------------------------------------------- /run_on_video/clip/__init__.py: -------------------------------------------------------------------------------- 1 | from .clip import * 2 | -------------------------------------------------------------------------------- /run_on_video/clip/bpe_simple_vocab_16e6.txt.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/run_on_video/clip/bpe_simple_vocab_16e6.txt.gz -------------------------------------------------------------------------------- /run_on_video/clip/clip.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/run_on_video/clip/clip.py -------------------------------------------------------------------------------- /run_on_video/clip/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/run_on_video/clip/model.py -------------------------------------------------------------------------------- /run_on_video/clip/simple_tokenizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/run_on_video/clip/simple_tokenizer.py -------------------------------------------------------------------------------- /run_on_video/data_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/run_on_video/data_utils.py -------------------------------------------------------------------------------- /run_on_video/example/RoripwjYFp8_60.0_210.0.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/run_on_video/example/RoripwjYFp8_60.0_210.0.mp4 -------------------------------------------------------------------------------- /run_on_video/example/queries.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/run_on_video/example/queries.jsonl -------------------------------------------------------------------------------- /run_on_video/model_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/run_on_video/model_utils.py -------------------------------------------------------------------------------- /run_on_video/run.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/run_on_video/run.py -------------------------------------------------------------------------------- /standalone_eval/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/standalone_eval/README.md -------------------------------------------------------------------------------- /standalone_eval/eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/standalone_eval/eval.py -------------------------------------------------------------------------------- /standalone_eval/eval_sample.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/standalone_eval/eval_sample.sh -------------------------------------------------------------------------------- /standalone_eval/sample_val_preds.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/standalone_eval/sample_val_preds.jsonl -------------------------------------------------------------------------------- /standalone_eval/sample_val_preds_metrics_raw.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/standalone_eval/sample_val_preds_metrics_raw.json -------------------------------------------------------------------------------- /standalone_eval/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/standalone_eval/utils.py -------------------------------------------------------------------------------- /taskweave/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /taskweave/attention.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/taskweave/attention.py -------------------------------------------------------------------------------- /taskweave/config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/taskweave/config.py -------------------------------------------------------------------------------- /taskweave/inference.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/taskweave/inference.py -------------------------------------------------------------------------------- /taskweave/matcher.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/taskweave/matcher.py -------------------------------------------------------------------------------- /taskweave/misc.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/taskweave/misc.py -------------------------------------------------------------------------------- /taskweave/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/taskweave/model.py -------------------------------------------------------------------------------- /taskweave/position_encoding.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/taskweave/position_encoding.py -------------------------------------------------------------------------------- /taskweave/postprocessing_taskweave.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/taskweave/postprocessing_taskweave.py -------------------------------------------------------------------------------- /taskweave/scripts/inference.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/taskweave/scripts/inference.sh -------------------------------------------------------------------------------- /taskweave/scripts/inference_audio.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/taskweave/scripts/inference_audio.sh -------------------------------------------------------------------------------- /taskweave/scripts/pretrain.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/taskweave/scripts/pretrain.sh -------------------------------------------------------------------------------- /taskweave/scripts/train.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/taskweave/scripts/train.sh -------------------------------------------------------------------------------- /taskweave/scripts/train_audio.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/taskweave/scripts/train_audio.sh -------------------------------------------------------------------------------- /taskweave/scripts/train_audio_umt.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/taskweave/scripts/train_audio_umt.sh -------------------------------------------------------------------------------- /taskweave/scripts/tvsum/train_tvsum.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/taskweave/scripts/tvsum/train_tvsum.sh -------------------------------------------------------------------------------- /taskweave/scripts/tvsum/train_tvsum_audio.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/taskweave/scripts/tvsum/train_tvsum_audio.sh -------------------------------------------------------------------------------- /taskweave/span_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/taskweave/span_utils.py -------------------------------------------------------------------------------- /taskweave/start_end_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/taskweave/start_end_dataset.py -------------------------------------------------------------------------------- /taskweave/start_end_dataset_audio.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/taskweave/start_end_dataset_audio.py -------------------------------------------------------------------------------- /taskweave/text_encoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/taskweave/text_encoder.py -------------------------------------------------------------------------------- /taskweave/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/taskweave/train.py -------------------------------------------------------------------------------- /taskweave/transformer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/taskweave/transformer.py -------------------------------------------------------------------------------- /utils/basic_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/utils/basic_utils.py -------------------------------------------------------------------------------- /utils/model_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/utils/model_utils.py -------------------------------------------------------------------------------- /utils/temporal_nms.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/utils/temporal_nms.py -------------------------------------------------------------------------------- /utils/tensor_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/utils/tensor_utils.py -------------------------------------------------------------------------------- /utils/windows_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EdenGabriel/TaskWeave/HEAD/utils/windows_utils.py --------------------------------------------------------------------------------