├── .gitignore ├── LICENSE ├── README.md ├── data └── prepare_question_file.py ├── docs ├── DATASET.md ├── Vision-Action_Instruction_Pre-training.md ├── Vision-Action_Instruction_Tuning.md ├── dataset.jpg ├── logo.png ├── teaser.jpg └── teaser.png ├── llava ├── __init__.py ├── constants.py ├── conversation.py ├── eval │ ├── model_vqa.py │ └── model_vqa_loader.py ├── mm_utils.py ├── model │ ├── __init__.py │ ├── apply_delta.py │ ├── builder.py │ ├── consolidate.py │ ├── language_model │ │ ├── llava_llama.py │ │ ├── llava_mistral.py │ │ └── llava_mpt.py │ ├── llava_arch.py │ ├── make_delta.py │ ├── multimodal_encoder │ │ ├── builder.py │ │ └── clip_encoder.py │ ├── multimodal_projector │ │ └── builder.py │ └── utils.py ├── train │ ├── llama_flash_attn_monkey_patch.py │ ├── llama_xformers_attn_monkey_patch.py │ ├── llava_trainer.py │ ├── train.py │ ├── train_mem.py │ └── train_xformers.py └── utils.py ├── pyproject.toml ├── scripts ├── merge_lora_weights.py ├── v1_5 │ ├── vision-action_instruction_pretraining_lora.sh │ └── vision-action_instruction_tuning_rlbench.sh ├── zero2.json ├── zero3.json └── zero3_offload.json └── sim ├── LICENSE ├── agents ├── __init__.py └── llarva_bc │ ├── __init__.py │ ├── launch_utils.py │ └── llarva_agent.py ├── conf └── eval.yaml ├── data └── generate_vision-action-tuning_anns.py ├── eval.py ├── helpers ├── __init__.py ├── custom_rlbench_env.py ├── demo_loading_utils.py ├── preprocess_agent.py └── utils.py ├── requirements.txt └── setup.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dantong88/LLARVA/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dantong88/LLARVA/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dantong88/LLARVA/HEAD/README.md -------------------------------------------------------------------------------- /data/prepare_question_file.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dantong88/LLARVA/HEAD/data/prepare_question_file.py -------------------------------------------------------------------------------- /docs/DATASET.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dantong88/LLARVA/HEAD/docs/DATASET.md -------------------------------------------------------------------------------- /docs/Vision-Action_Instruction_Pre-training.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dantong88/LLARVA/HEAD/docs/Vision-Action_Instruction_Pre-training.md -------------------------------------------------------------------------------- /docs/Vision-Action_Instruction_Tuning.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dantong88/LLARVA/HEAD/docs/Vision-Action_Instruction_Tuning.md -------------------------------------------------------------------------------- /docs/dataset.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dantong88/LLARVA/HEAD/docs/dataset.jpg -------------------------------------------------------------------------------- /docs/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dantong88/LLARVA/HEAD/docs/logo.png -------------------------------------------------------------------------------- /docs/teaser.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dantong88/LLARVA/HEAD/docs/teaser.jpg -------------------------------------------------------------------------------- /docs/teaser.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dantong88/LLARVA/HEAD/docs/teaser.png -------------------------------------------------------------------------------- /llava/__init__.py: -------------------------------------------------------------------------------- 1 | from .model import LlavaLlamaForCausalLM 2 | -------------------------------------------------------------------------------- /llava/constants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dantong88/LLARVA/HEAD/llava/constants.py -------------------------------------------------------------------------------- /llava/conversation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dantong88/LLARVA/HEAD/llava/conversation.py -------------------------------------------------------------------------------- /llava/eval/model_vqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dantong88/LLARVA/HEAD/llava/eval/model_vqa.py -------------------------------------------------------------------------------- /llava/eval/model_vqa_loader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dantong88/LLARVA/HEAD/llava/eval/model_vqa_loader.py -------------------------------------------------------------------------------- /llava/mm_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dantong88/LLARVA/HEAD/llava/mm_utils.py -------------------------------------------------------------------------------- /llava/model/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dantong88/LLARVA/HEAD/llava/model/__init__.py -------------------------------------------------------------------------------- /llava/model/apply_delta.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dantong88/LLARVA/HEAD/llava/model/apply_delta.py -------------------------------------------------------------------------------- /llava/model/builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dantong88/LLARVA/HEAD/llava/model/builder.py -------------------------------------------------------------------------------- /llava/model/consolidate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dantong88/LLARVA/HEAD/llava/model/consolidate.py -------------------------------------------------------------------------------- /llava/model/language_model/llava_llama.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dantong88/LLARVA/HEAD/llava/model/language_model/llava_llama.py -------------------------------------------------------------------------------- /llava/model/language_model/llava_mistral.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dantong88/LLARVA/HEAD/llava/model/language_model/llava_mistral.py -------------------------------------------------------------------------------- /llava/model/language_model/llava_mpt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dantong88/LLARVA/HEAD/llava/model/language_model/llava_mpt.py -------------------------------------------------------------------------------- /llava/model/llava_arch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dantong88/LLARVA/HEAD/llava/model/llava_arch.py -------------------------------------------------------------------------------- /llava/model/make_delta.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dantong88/LLARVA/HEAD/llava/model/make_delta.py -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dantong88/LLARVA/HEAD/llava/model/multimodal_encoder/builder.py -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/clip_encoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dantong88/LLARVA/HEAD/llava/model/multimodal_encoder/clip_encoder.py -------------------------------------------------------------------------------- /llava/model/multimodal_projector/builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dantong88/LLARVA/HEAD/llava/model/multimodal_projector/builder.py -------------------------------------------------------------------------------- /llava/model/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dantong88/LLARVA/HEAD/llava/model/utils.py -------------------------------------------------------------------------------- /llava/train/llama_flash_attn_monkey_patch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dantong88/LLARVA/HEAD/llava/train/llama_flash_attn_monkey_patch.py -------------------------------------------------------------------------------- /llava/train/llama_xformers_attn_monkey_patch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dantong88/LLARVA/HEAD/llava/train/llama_xformers_attn_monkey_patch.py -------------------------------------------------------------------------------- /llava/train/llava_trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dantong88/LLARVA/HEAD/llava/train/llava_trainer.py -------------------------------------------------------------------------------- /llava/train/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dantong88/LLARVA/HEAD/llava/train/train.py -------------------------------------------------------------------------------- /llava/train/train_mem.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dantong88/LLARVA/HEAD/llava/train/train_mem.py -------------------------------------------------------------------------------- /llava/train/train_xformers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dantong88/LLARVA/HEAD/llava/train/train_xformers.py -------------------------------------------------------------------------------- /llava/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dantong88/LLARVA/HEAD/llava/utils.py -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dantong88/LLARVA/HEAD/pyproject.toml -------------------------------------------------------------------------------- /scripts/merge_lora_weights.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dantong88/LLARVA/HEAD/scripts/merge_lora_weights.py -------------------------------------------------------------------------------- /scripts/v1_5/vision-action_instruction_pretraining_lora.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dantong88/LLARVA/HEAD/scripts/v1_5/vision-action_instruction_pretraining_lora.sh -------------------------------------------------------------------------------- /scripts/v1_5/vision-action_instruction_tuning_rlbench.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dantong88/LLARVA/HEAD/scripts/v1_5/vision-action_instruction_tuning_rlbench.sh -------------------------------------------------------------------------------- /scripts/zero2.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dantong88/LLARVA/HEAD/scripts/zero2.json -------------------------------------------------------------------------------- /scripts/zero3.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dantong88/LLARVA/HEAD/scripts/zero3.json -------------------------------------------------------------------------------- /scripts/zero3_offload.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dantong88/LLARVA/HEAD/scripts/zero3_offload.json -------------------------------------------------------------------------------- /sim/LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dantong88/LLARVA/HEAD/sim/LICENSE -------------------------------------------------------------------------------- /sim/agents/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sim/agents/llarva_bc/__init__.py: -------------------------------------------------------------------------------- 1 | import agents.llarva_bc.launch_utils -------------------------------------------------------------------------------- /sim/agents/llarva_bc/launch_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dantong88/LLARVA/HEAD/sim/agents/llarva_bc/launch_utils.py -------------------------------------------------------------------------------- /sim/agents/llarva_bc/llarva_agent.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dantong88/LLARVA/HEAD/sim/agents/llarva_bc/llarva_agent.py -------------------------------------------------------------------------------- /sim/conf/eval.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dantong88/LLARVA/HEAD/sim/conf/eval.yaml -------------------------------------------------------------------------------- /sim/data/generate_vision-action-tuning_anns.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dantong88/LLARVA/HEAD/sim/data/generate_vision-action-tuning_anns.py -------------------------------------------------------------------------------- /sim/eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dantong88/LLARVA/HEAD/sim/eval.py -------------------------------------------------------------------------------- /sim/helpers/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /sim/helpers/custom_rlbench_env.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dantong88/LLARVA/HEAD/sim/helpers/custom_rlbench_env.py -------------------------------------------------------------------------------- /sim/helpers/demo_loading_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dantong88/LLARVA/HEAD/sim/helpers/demo_loading_utils.py -------------------------------------------------------------------------------- /sim/helpers/preprocess_agent.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dantong88/LLARVA/HEAD/sim/helpers/preprocess_agent.py -------------------------------------------------------------------------------- /sim/helpers/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dantong88/LLARVA/HEAD/sim/helpers/utils.py -------------------------------------------------------------------------------- /sim/requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dantong88/LLARVA/HEAD/sim/requirements.txt -------------------------------------------------------------------------------- /sim/setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Dantong88/LLARVA/HEAD/sim/setup.py --------------------------------------------------------------------------------