├── .gitignore ├── README.md ├── VideoLLaMA2-LICENSE ├── assets ├── cat_and_chicken.mp4 ├── sora.mp4 ├── trace-demo.png └── trace-overview.png ├── install_requirements-npu.sh ├── requirements.txt ├── scripts ├── inference │ └── inference.py ├── train │ ├── pretrain-128.sh │ ├── sft-128.sh │ └── sft-youcook2.sh └── zero3.json └── trace ├── __init__.py ├── constants.py ├── conversation.py ├── eval ├── eval.sh ├── evaluate.py ├── mvbench │ ├── eval.sh │ └── evaluate.py ├── reformat_dvc.py ├── reformat_tvg.py ├── reformat_vhd.py └── videomme │ ├── eval.sh │ └── evaluate.py ├── metrics ├── README.md ├── dvc │ ├── SODA │ │ ├── LICENSE │ │ ├── README.md │ │ ├── dataset.py │ │ ├── nlpeval │ │ │ ├── bert_f_score.py │ │ │ ├── bert_r_score.py │ │ │ └── mover.py │ │ ├── requirements.txt │ │ ├── soda.py │ │ └── utils.py │ ├── __init__.py │ ├── eval_dvc.py │ ├── eval_dvc_anet.py │ ├── eval_soda.py │ └── metrics │ │ ├── README.md │ │ ├── cider.py │ │ ├── cider_scorer.py │ │ ├── eval_soda.py │ │ ├── meteor-1.5.jar │ │ ├── meteor.py │ │ ├── ptbtokenizer.py │ │ └── stanford-corenlp-3.4.1.jar ├── tvg │ ├── eval_tvg.py │ └── eval_tvg.sh └── vhd │ ├── eval_highlights.sh │ ├── eval_vhd.py │ └── utils.py ├── mm_utils.py ├── model ├── __init__.py ├── builder.py ├── language_model │ └── trace_mistral.py ├── multimodal_encoder │ ├── builder.py │ ├── clip_encoder.py │ ├── score_encoder.py │ ├── sync_encoder.py │ └── time_encoder.py ├── multimodal_projector │ ├── __init__.py │ └── builder.py └── trace_arch.py ├── prompts ├── dvc-anet-ft.txt ├── dvc-anet.txt ├── dvc.txt ├── mr.txt └── vhd.txt ├── trace_trainer.py ├── train_mt.py ├── train_mt_npu.py └── utils.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/.gitignore -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/README.md -------------------------------------------------------------------------------- /VideoLLaMA2-LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/VideoLLaMA2-LICENSE -------------------------------------------------------------------------------- /assets/cat_and_chicken.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/assets/cat_and_chicken.mp4 -------------------------------------------------------------------------------- /assets/sora.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/assets/sora.mp4 -------------------------------------------------------------------------------- /assets/trace-demo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/assets/trace-demo.png -------------------------------------------------------------------------------- /assets/trace-overview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/assets/trace-overview.png -------------------------------------------------------------------------------- /install_requirements-npu.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/install_requirements-npu.sh -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/requirements.txt -------------------------------------------------------------------------------- /scripts/inference/inference.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/scripts/inference/inference.py -------------------------------------------------------------------------------- /scripts/train/pretrain-128.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/scripts/train/pretrain-128.sh -------------------------------------------------------------------------------- /scripts/train/sft-128.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/scripts/train/sft-128.sh -------------------------------------------------------------------------------- /scripts/train/sft-youcook2.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/scripts/train/sft-youcook2.sh -------------------------------------------------------------------------------- /scripts/zero3.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/scripts/zero3.json -------------------------------------------------------------------------------- /trace/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/trace/__init__.py -------------------------------------------------------------------------------- /trace/constants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/trace/constants.py -------------------------------------------------------------------------------- /trace/conversation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/trace/conversation.py -------------------------------------------------------------------------------- /trace/eval/eval.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/trace/eval/eval.sh -------------------------------------------------------------------------------- /trace/eval/evaluate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/trace/eval/evaluate.py -------------------------------------------------------------------------------- /trace/eval/mvbench/eval.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/trace/eval/mvbench/eval.sh -------------------------------------------------------------------------------- /trace/eval/mvbench/evaluate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/trace/eval/mvbench/evaluate.py -------------------------------------------------------------------------------- /trace/eval/reformat_dvc.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/trace/eval/reformat_dvc.py -------------------------------------------------------------------------------- /trace/eval/reformat_tvg.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/trace/eval/reformat_tvg.py -------------------------------------------------------------------------------- /trace/eval/reformat_vhd.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/trace/eval/reformat_vhd.py -------------------------------------------------------------------------------- /trace/eval/videomme/eval.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/trace/eval/videomme/eval.sh -------------------------------------------------------------------------------- /trace/eval/videomme/evaluate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/trace/eval/videomme/evaluate.py -------------------------------------------------------------------------------- /trace/metrics/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/trace/metrics/README.md -------------------------------------------------------------------------------- /trace/metrics/dvc/SODA/LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/trace/metrics/dvc/SODA/LICENSE -------------------------------------------------------------------------------- /trace/metrics/dvc/SODA/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/trace/metrics/dvc/SODA/README.md -------------------------------------------------------------------------------- /trace/metrics/dvc/SODA/dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/trace/metrics/dvc/SODA/dataset.py -------------------------------------------------------------------------------- /trace/metrics/dvc/SODA/nlpeval/bert_f_score.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/trace/metrics/dvc/SODA/nlpeval/bert_f_score.py -------------------------------------------------------------------------------- /trace/metrics/dvc/SODA/nlpeval/bert_r_score.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/trace/metrics/dvc/SODA/nlpeval/bert_r_score.py -------------------------------------------------------------------------------- /trace/metrics/dvc/SODA/nlpeval/mover.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/trace/metrics/dvc/SODA/nlpeval/mover.py -------------------------------------------------------------------------------- /trace/metrics/dvc/SODA/requirements.txt: -------------------------------------------------------------------------------- 1 | numpy==1.19.1 2 | tqdm==4.48.2 3 | -------------------------------------------------------------------------------- /trace/metrics/dvc/SODA/soda.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/trace/metrics/dvc/SODA/soda.py -------------------------------------------------------------------------------- /trace/metrics/dvc/SODA/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/trace/metrics/dvc/SODA/utils.py -------------------------------------------------------------------------------- /trace/metrics/dvc/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/trace/metrics/dvc/__init__.py -------------------------------------------------------------------------------- /trace/metrics/dvc/eval_dvc.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/trace/metrics/dvc/eval_dvc.py -------------------------------------------------------------------------------- /trace/metrics/dvc/eval_dvc_anet.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/trace/metrics/dvc/eval_dvc_anet.py -------------------------------------------------------------------------------- /trace/metrics/dvc/eval_soda.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/trace/metrics/dvc/eval_soda.py -------------------------------------------------------------------------------- /trace/metrics/dvc/metrics/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/trace/metrics/dvc/metrics/README.md -------------------------------------------------------------------------------- /trace/metrics/dvc/metrics/cider.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/trace/metrics/dvc/metrics/cider.py -------------------------------------------------------------------------------- /trace/metrics/dvc/metrics/cider_scorer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/trace/metrics/dvc/metrics/cider_scorer.py -------------------------------------------------------------------------------- /trace/metrics/dvc/metrics/eval_soda.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/trace/metrics/dvc/metrics/eval_soda.py -------------------------------------------------------------------------------- /trace/metrics/dvc/metrics/meteor-1.5.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/trace/metrics/dvc/metrics/meteor-1.5.jar -------------------------------------------------------------------------------- /trace/metrics/dvc/metrics/meteor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/trace/metrics/dvc/metrics/meteor.py -------------------------------------------------------------------------------- /trace/metrics/dvc/metrics/ptbtokenizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/trace/metrics/dvc/metrics/ptbtokenizer.py -------------------------------------------------------------------------------- /trace/metrics/dvc/metrics/stanford-corenlp-3.4.1.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/trace/metrics/dvc/metrics/stanford-corenlp-3.4.1.jar -------------------------------------------------------------------------------- /trace/metrics/tvg/eval_tvg.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/trace/metrics/tvg/eval_tvg.py -------------------------------------------------------------------------------- /trace/metrics/tvg/eval_tvg.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/trace/metrics/tvg/eval_tvg.sh -------------------------------------------------------------------------------- /trace/metrics/vhd/eval_highlights.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/trace/metrics/vhd/eval_highlights.sh -------------------------------------------------------------------------------- /trace/metrics/vhd/eval_vhd.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/trace/metrics/vhd/eval_vhd.py -------------------------------------------------------------------------------- /trace/metrics/vhd/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/trace/metrics/vhd/utils.py -------------------------------------------------------------------------------- /trace/mm_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/trace/mm_utils.py -------------------------------------------------------------------------------- /trace/model/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/trace/model/__init__.py -------------------------------------------------------------------------------- /trace/model/builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/trace/model/builder.py -------------------------------------------------------------------------------- /trace/model/language_model/trace_mistral.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/trace/model/language_model/trace_mistral.py -------------------------------------------------------------------------------- /trace/model/multimodal_encoder/builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/trace/model/multimodal_encoder/builder.py -------------------------------------------------------------------------------- /trace/model/multimodal_encoder/clip_encoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/trace/model/multimodal_encoder/clip_encoder.py -------------------------------------------------------------------------------- /trace/model/multimodal_encoder/score_encoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/trace/model/multimodal_encoder/score_encoder.py -------------------------------------------------------------------------------- /trace/model/multimodal_encoder/sync_encoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/trace/model/multimodal_encoder/sync_encoder.py -------------------------------------------------------------------------------- /trace/model/multimodal_encoder/time_encoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/trace/model/multimodal_encoder/time_encoder.py -------------------------------------------------------------------------------- /trace/model/multimodal_projector/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/trace/model/multimodal_projector/__init__.py -------------------------------------------------------------------------------- /trace/model/multimodal_projector/builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/trace/model/multimodal_projector/builder.py -------------------------------------------------------------------------------- /trace/model/trace_arch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/trace/model/trace_arch.py -------------------------------------------------------------------------------- /trace/prompts/dvc-anet-ft.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/trace/prompts/dvc-anet-ft.txt -------------------------------------------------------------------------------- /trace/prompts/dvc-anet.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/trace/prompts/dvc-anet.txt -------------------------------------------------------------------------------- /trace/prompts/dvc.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/trace/prompts/dvc.txt -------------------------------------------------------------------------------- /trace/prompts/mr.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/trace/prompts/mr.txt -------------------------------------------------------------------------------- /trace/prompts/vhd.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/trace/prompts/vhd.txt -------------------------------------------------------------------------------- /trace/trace_trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/trace/trace_trainer.py -------------------------------------------------------------------------------- /trace/train_mt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/trace/train_mt.py -------------------------------------------------------------------------------- /trace/train_mt_npu.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/trace/train_mt_npu.py -------------------------------------------------------------------------------- /trace/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gyxxyg/TRACE/HEAD/trace/utils.py --------------------------------------------------------------------------------