├── .gitignore ├── README.md ├── assets ├── example.drawio.png ├── model.drawio.png ├── results_c.JPG └── results_n.JPG ├── evaluation ├── __init__.py ├── evaluation.py └── gpt_eval.py └── llama_adapter_v2_multimodal7b ├── README.md ├── __pycache__ ├── engine_finetune.cpython-38.pyc └── engine_pretrain.cpython-38.pyc ├── data ├── __pycache__ │ └── dataset.cpython-38.pyc └── dataset.py ├── demo.py ├── docs ├── eval.md └── train.md ├── encoder └── models │ ├── __init__.py │ ├── __pycache__ │ ├── __init__.cpython-38.pyc │ ├── helpers.cpython-38.pyc │ ├── imagebind_model.cpython-38.pyc │ ├── multimodal_preprocessors.cpython-38.pyc │ └── transformer.cpython-38.pyc │ ├── helpers.py │ ├── imagebind_model.py │ ├── multimodal_preprocessors.py │ └── transformer.py ├── engine_finetune.py ├── engine_pretrain.py ├── exps ├── finetune.sh └── pretrain.sh ├── finetune_data_config.yaml ├── llama ├── __init__.py ├── __pycache__ │ ├── __init__.cpython-38.pyc │ ├── llama.cpython-38.pyc │ ├── llama_adapter.cpython-38.pyc │ ├── tokenizer.cpython-38.pyc │ └── utils.cpython-38.pyc ├── llama.py ├── llama_adapter.py ├── tokenizer.py └── utils.py ├── main_finetune.py ├── main_pretrain.py ├── requirements.txt ├── save_weights.py └── util ├── __pycache__ ├── extract_adapter_from_checkpoint.cpython-38.pyc ├── lr_sched.cpython-38.pyc └── misc.cpython-38.pyc ├── evaluate_mme.py ├── extract_adapter_from_checkpoint.py ├── lr_sched.py └── misc.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbzuai-oryx/TrackingMeetsLMM/HEAD/.gitignore -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbzuai-oryx/TrackingMeetsLMM/HEAD/README.md -------------------------------------------------------------------------------- /assets/example.drawio.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbzuai-oryx/TrackingMeetsLMM/HEAD/assets/example.drawio.png -------------------------------------------------------------------------------- /assets/model.drawio.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbzuai-oryx/TrackingMeetsLMM/HEAD/assets/model.drawio.png -------------------------------------------------------------------------------- /assets/results_c.JPG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbzuai-oryx/TrackingMeetsLMM/HEAD/assets/results_c.JPG -------------------------------------------------------------------------------- /assets/results_n.JPG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbzuai-oryx/TrackingMeetsLMM/HEAD/assets/results_n.JPG -------------------------------------------------------------------------------- /evaluation/__init__.py: -------------------------------------------------------------------------------- 1 | from .gpt_eval import GPTEvaluation -------------------------------------------------------------------------------- /evaluation/evaluation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbzuai-oryx/TrackingMeetsLMM/HEAD/evaluation/evaluation.py -------------------------------------------------------------------------------- /evaluation/gpt_eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbzuai-oryx/TrackingMeetsLMM/HEAD/evaluation/gpt_eval.py -------------------------------------------------------------------------------- /llama_adapter_v2_multimodal7b/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbzuai-oryx/TrackingMeetsLMM/HEAD/llama_adapter_v2_multimodal7b/README.md -------------------------------------------------------------------------------- /llama_adapter_v2_multimodal7b/__pycache__/engine_finetune.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbzuai-oryx/TrackingMeetsLMM/HEAD/llama_adapter_v2_multimodal7b/__pycache__/engine_finetune.cpython-38.pyc -------------------------------------------------------------------------------- /llama_adapter_v2_multimodal7b/__pycache__/engine_pretrain.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbzuai-oryx/TrackingMeetsLMM/HEAD/llama_adapter_v2_multimodal7b/__pycache__/engine_pretrain.cpython-38.pyc -------------------------------------------------------------------------------- /llama_adapter_v2_multimodal7b/data/__pycache__/dataset.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbzuai-oryx/TrackingMeetsLMM/HEAD/llama_adapter_v2_multimodal7b/data/__pycache__/dataset.cpython-38.pyc -------------------------------------------------------------------------------- /llama_adapter_v2_multimodal7b/data/dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbzuai-oryx/TrackingMeetsLMM/HEAD/llama_adapter_v2_multimodal7b/data/dataset.py -------------------------------------------------------------------------------- /llama_adapter_v2_multimodal7b/demo.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbzuai-oryx/TrackingMeetsLMM/HEAD/llama_adapter_v2_multimodal7b/demo.py -------------------------------------------------------------------------------- /llama_adapter_v2_multimodal7b/docs/eval.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbzuai-oryx/TrackingMeetsLMM/HEAD/llama_adapter_v2_multimodal7b/docs/eval.md -------------------------------------------------------------------------------- /llama_adapter_v2_multimodal7b/docs/train.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbzuai-oryx/TrackingMeetsLMM/HEAD/llama_adapter_v2_multimodal7b/docs/train.md -------------------------------------------------------------------------------- /llama_adapter_v2_multimodal7b/encoder/models/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /llama_adapter_v2_multimodal7b/encoder/models/__pycache__/__init__.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbzuai-oryx/TrackingMeetsLMM/HEAD/llama_adapter_v2_multimodal7b/encoder/models/__pycache__/__init__.cpython-38.pyc -------------------------------------------------------------------------------- /llama_adapter_v2_multimodal7b/encoder/models/__pycache__/helpers.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbzuai-oryx/TrackingMeetsLMM/HEAD/llama_adapter_v2_multimodal7b/encoder/models/__pycache__/helpers.cpython-38.pyc -------------------------------------------------------------------------------- /llama_adapter_v2_multimodal7b/encoder/models/__pycache__/imagebind_model.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbzuai-oryx/TrackingMeetsLMM/HEAD/llama_adapter_v2_multimodal7b/encoder/models/__pycache__/imagebind_model.cpython-38.pyc -------------------------------------------------------------------------------- /llama_adapter_v2_multimodal7b/encoder/models/__pycache__/multimodal_preprocessors.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbzuai-oryx/TrackingMeetsLMM/HEAD/llama_adapter_v2_multimodal7b/encoder/models/__pycache__/multimodal_preprocessors.cpython-38.pyc -------------------------------------------------------------------------------- /llama_adapter_v2_multimodal7b/encoder/models/__pycache__/transformer.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbzuai-oryx/TrackingMeetsLMM/HEAD/llama_adapter_v2_multimodal7b/encoder/models/__pycache__/transformer.cpython-38.pyc -------------------------------------------------------------------------------- /llama_adapter_v2_multimodal7b/encoder/models/helpers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbzuai-oryx/TrackingMeetsLMM/HEAD/llama_adapter_v2_multimodal7b/encoder/models/helpers.py -------------------------------------------------------------------------------- /llama_adapter_v2_multimodal7b/encoder/models/imagebind_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbzuai-oryx/TrackingMeetsLMM/HEAD/llama_adapter_v2_multimodal7b/encoder/models/imagebind_model.py -------------------------------------------------------------------------------- /llama_adapter_v2_multimodal7b/encoder/models/multimodal_preprocessors.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbzuai-oryx/TrackingMeetsLMM/HEAD/llama_adapter_v2_multimodal7b/encoder/models/multimodal_preprocessors.py -------------------------------------------------------------------------------- /llama_adapter_v2_multimodal7b/encoder/models/transformer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbzuai-oryx/TrackingMeetsLMM/HEAD/llama_adapter_v2_multimodal7b/encoder/models/transformer.py -------------------------------------------------------------------------------- /llama_adapter_v2_multimodal7b/engine_finetune.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbzuai-oryx/TrackingMeetsLMM/HEAD/llama_adapter_v2_multimodal7b/engine_finetune.py -------------------------------------------------------------------------------- /llama_adapter_v2_multimodal7b/engine_pretrain.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbzuai-oryx/TrackingMeetsLMM/HEAD/llama_adapter_v2_multimodal7b/engine_pretrain.py -------------------------------------------------------------------------------- /llama_adapter_v2_multimodal7b/exps/finetune.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbzuai-oryx/TrackingMeetsLMM/HEAD/llama_adapter_v2_multimodal7b/exps/finetune.sh -------------------------------------------------------------------------------- /llama_adapter_v2_multimodal7b/exps/pretrain.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbzuai-oryx/TrackingMeetsLMM/HEAD/llama_adapter_v2_multimodal7b/exps/pretrain.sh -------------------------------------------------------------------------------- /llama_adapter_v2_multimodal7b/finetune_data_config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbzuai-oryx/TrackingMeetsLMM/HEAD/llama_adapter_v2_multimodal7b/finetune_data_config.yaml -------------------------------------------------------------------------------- /llama_adapter_v2_multimodal7b/llama/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbzuai-oryx/TrackingMeetsLMM/HEAD/llama_adapter_v2_multimodal7b/llama/__init__.py -------------------------------------------------------------------------------- /llama_adapter_v2_multimodal7b/llama/__pycache__/__init__.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbzuai-oryx/TrackingMeetsLMM/HEAD/llama_adapter_v2_multimodal7b/llama/__pycache__/__init__.cpython-38.pyc -------------------------------------------------------------------------------- /llama_adapter_v2_multimodal7b/llama/__pycache__/llama.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbzuai-oryx/TrackingMeetsLMM/HEAD/llama_adapter_v2_multimodal7b/llama/__pycache__/llama.cpython-38.pyc -------------------------------------------------------------------------------- /llama_adapter_v2_multimodal7b/llama/__pycache__/llama_adapter.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbzuai-oryx/TrackingMeetsLMM/HEAD/llama_adapter_v2_multimodal7b/llama/__pycache__/llama_adapter.cpython-38.pyc -------------------------------------------------------------------------------- /llama_adapter_v2_multimodal7b/llama/__pycache__/tokenizer.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbzuai-oryx/TrackingMeetsLMM/HEAD/llama_adapter_v2_multimodal7b/llama/__pycache__/tokenizer.cpython-38.pyc -------------------------------------------------------------------------------- /llama_adapter_v2_multimodal7b/llama/__pycache__/utils.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbzuai-oryx/TrackingMeetsLMM/HEAD/llama_adapter_v2_multimodal7b/llama/__pycache__/utils.cpython-38.pyc -------------------------------------------------------------------------------- /llama_adapter_v2_multimodal7b/llama/llama.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbzuai-oryx/TrackingMeetsLMM/HEAD/llama_adapter_v2_multimodal7b/llama/llama.py -------------------------------------------------------------------------------- /llama_adapter_v2_multimodal7b/llama/llama_adapter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbzuai-oryx/TrackingMeetsLMM/HEAD/llama_adapter_v2_multimodal7b/llama/llama_adapter.py -------------------------------------------------------------------------------- /llama_adapter_v2_multimodal7b/llama/tokenizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbzuai-oryx/TrackingMeetsLMM/HEAD/llama_adapter_v2_multimodal7b/llama/tokenizer.py -------------------------------------------------------------------------------- /llama_adapter_v2_multimodal7b/llama/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbzuai-oryx/TrackingMeetsLMM/HEAD/llama_adapter_v2_multimodal7b/llama/utils.py -------------------------------------------------------------------------------- /llama_adapter_v2_multimodal7b/main_finetune.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbzuai-oryx/TrackingMeetsLMM/HEAD/llama_adapter_v2_multimodal7b/main_finetune.py -------------------------------------------------------------------------------- /llama_adapter_v2_multimodal7b/main_pretrain.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbzuai-oryx/TrackingMeetsLMM/HEAD/llama_adapter_v2_multimodal7b/main_pretrain.py -------------------------------------------------------------------------------- /llama_adapter_v2_multimodal7b/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbzuai-oryx/TrackingMeetsLMM/HEAD/llama_adapter_v2_multimodal7b/requirements.txt -------------------------------------------------------------------------------- /llama_adapter_v2_multimodal7b/save_weights.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbzuai-oryx/TrackingMeetsLMM/HEAD/llama_adapter_v2_multimodal7b/save_weights.py -------------------------------------------------------------------------------- /llama_adapter_v2_multimodal7b/util/__pycache__/extract_adapter_from_checkpoint.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbzuai-oryx/TrackingMeetsLMM/HEAD/llama_adapter_v2_multimodal7b/util/__pycache__/extract_adapter_from_checkpoint.cpython-38.pyc -------------------------------------------------------------------------------- /llama_adapter_v2_multimodal7b/util/__pycache__/lr_sched.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbzuai-oryx/TrackingMeetsLMM/HEAD/llama_adapter_v2_multimodal7b/util/__pycache__/lr_sched.cpython-38.pyc -------------------------------------------------------------------------------- /llama_adapter_v2_multimodal7b/util/__pycache__/misc.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbzuai-oryx/TrackingMeetsLMM/HEAD/llama_adapter_v2_multimodal7b/util/__pycache__/misc.cpython-38.pyc -------------------------------------------------------------------------------- /llama_adapter_v2_multimodal7b/util/evaluate_mme.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbzuai-oryx/TrackingMeetsLMM/HEAD/llama_adapter_v2_multimodal7b/util/evaluate_mme.py -------------------------------------------------------------------------------- /llama_adapter_v2_multimodal7b/util/extract_adapter_from_checkpoint.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbzuai-oryx/TrackingMeetsLMM/HEAD/llama_adapter_v2_multimodal7b/util/extract_adapter_from_checkpoint.py -------------------------------------------------------------------------------- /llama_adapter_v2_multimodal7b/util/lr_sched.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbzuai-oryx/TrackingMeetsLMM/HEAD/llama_adapter_v2_multimodal7b/util/lr_sched.py -------------------------------------------------------------------------------- /llama_adapter_v2_multimodal7b/util/misc.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbzuai-oryx/TrackingMeetsLMM/HEAD/llama_adapter_v2_multimodal7b/util/misc.py --------------------------------------------------------------------------------