├── README.md ├── assets └── teaser.gif ├── config ├── co-training_data.yaml ├── vln_dagger.yaml └── vln_r2r.yaml ├── llava ├── __init__.py ├── constants.py ├── conversation.py ├── eval │ ├── evaluate_interleave.py │ └── model_vqa.py ├── mm_utils.py ├── model │ ├── __init__.py │ ├── apply_delta.py │ ├── builder.py │ ├── consolidate.py │ ├── language_model │ │ ├── llava_gemma.py │ │ ├── llava_llama.py │ │ ├── llava_mistral.py │ │ ├── llava_mixtral.py │ │ ├── llava_mpt.py │ │ ├── llava_qwen.py │ │ ├── llava_qwen_moe.py │ │ └── modeling_llama.py │ ├── llava_arch.py │ ├── make_delta.py │ ├── multimodal_encoder │ │ ├── builder.py │ │ ├── clip_encoder.py │ │ ├── dev_eva_clip │ │ │ ├── eva_clip │ │ │ │ ├── __init__.py │ │ │ │ ├── bpe_simple_vocab_16e6.txt.gz │ │ │ │ ├── constants.py │ │ │ │ ├── eva_vit_model.py │ │ │ │ ├── factory.py │ │ │ │ ├── hf_configs.py │ │ │ │ ├── hf_model.py │ │ │ │ ├── loss.py │ │ │ │ ├── model.py │ │ │ │ ├── model_configs │ │ │ │ │ ├── EVA-CLIP-18B.json │ │ │ │ │ ├── EVA-CLIP-8B-plus.json │ │ │ │ │ ├── EVA-CLIP-8B.json │ │ │ │ │ ├── EVA01-CLIP-B-16.json │ │ │ │ │ ├── EVA01-CLIP-g-14-plus.json │ │ │ │ │ ├── EVA01-CLIP-g-14.json │ │ │ │ │ ├── EVA02-CLIP-B-16.json │ │ │ │ │ ├── EVA02-CLIP-L-14-336.json │ │ │ │ │ ├── EVA02-CLIP-L-14.json │ │ │ │ │ ├── EVA02-CLIP-bigE-14-plus.json │ │ │ │ │ ├── EVA02-CLIP-bigE-14.json │ │ │ │ │ ├── Internal-EVA02-CLIP-10B-14-448.json │ │ │ │ │ └── Internal-EVA02-CLIP-10B-14.json │ │ │ │ ├── modified_resnet.py │ │ │ │ ├── openai.py │ │ │ │ ├── pretrained.py │ │ │ │ ├── rope.py │ │ │ │ ├── timm_model.py │ │ │ │ ├── tokenizer.py │ │ │ │ ├── transform.py │ │ │ │ ├── transformer.py │ │ │ │ └── utils.py │ │ │ └── eva_vit.py │ │ ├── eva_clip │ │ │ ├── eva_clip_encoder.py │ │ │ ├── eva_clip_processors.py │ │ │ ├── eva_vit.py │ │ │ ├── factory.py │ │ │ └── model_configs │ │ │ │ ├── EVA-CLIP-18B.json │ │ │ │ ├── EVA-CLIP-8B-plus.json │ │ │ │ ├── EVA-CLIP-8B.json │ │ │ │ ├── EVA01-CLIP-B-16.json │ │ │ │ ├── EVA01-CLIP-g-14-plus.json │ │ │ │ ├── EVA01-CLIP-g-14.json │ │ │ │ ├── EVA02-CLIP-B-16.json │ │ │ │ ├── EVA02-CLIP-L-14-336.json │ │ │ │ ├── EVA02-CLIP-L-14.json │ │ │ │ ├── EVA02-CLIP-bigE-14-plus.json │ │ │ │ ├── EVA02-CLIP-bigE-14.json │ │ │ │ ├── Internal-EVA02-CLIP-10B-14-448.json │ │ │ │ └── Internal-EVA02-CLIP-10B-14.json │ │ ├── hf_vision.py │ │ ├── imagebind.py │ │ ├── mlcd │ │ │ └── vit_rope2d_hf.py │ │ ├── mlcd_encoder.py │ │ ├── open_clip_encoder.py │ │ └── siglip_encoder.py │ ├── multimodal_projector │ │ ├── builder.py │ │ └── pooler_projector.py │ ├── multimodal_resampler │ │ ├── builder.py │ │ ├── masked_drop.py │ │ ├── perceiver.py │ │ ├── qformer.py │ │ └── spatial_pool.py │ └── utils.py ├── serve │ ├── __init__.py │ ├── cli.py │ ├── controller.py │ ├── examples │ │ ├── extreme_ironing.jpg │ │ └── waterview.jpg │ ├── gradio_multi_image.py │ ├── gradio_web_server.py │ ├── model_worker.py │ ├── register_worker.py │ ├── sglang_worker.py │ └── test_message.py ├── train │ ├── llama_flash_attn_monkey_patch.py │ ├── llava_trainer.py │ ├── llava_trainer_eval.py │ ├── train.py │ ├── train_dpo.py │ └── train_mem.py └── utils.py ├── realworld ├── go2_vln_client.py ├── pid_controller.py ├── realworld.md └── utils.py ├── requirements.txt ├── scripts ├── streamvln_dagger_collect.sh ├── streamvln_eval_multi_gpu.sh ├── streamvln_stage_two_train_slurm.sh ├── streamvln_train_slurm.sh ├── streamvln_trajectory_generation.sh ├── zero2.json └── zero3.json ├── streamvln ├── args.py ├── dataset │ ├── mmc4_dataset.py │ └── vln_action_dataset.py ├── habitat_extensions │ ├── maps.py │ └── measures.py ├── http_realworld_server.py ├── model │ └── stream_video_vln.py ├── streamvln_agent.py ├── streamvln_dagger.py ├── streamvln_eval.py ├── streamvln_train.py ├── streamvln_trajectory_generation.py └── utils │ ├── dist.py │ └── utils.py └── trl ├── __init__.py ├── core.py ├── environment ├── __init__.py └── base_environment.py ├── extras ├── __init__.py ├── best_of_n_sampler.py └── dataset_formatting.py ├── import_utils.py ├── models ├── __init__.py ├── modeling_base.py ├── modeling_sd_base.py ├── modeling_value_head.py └── utils.py └── trainer ├── __init__.py ├── base.py ├── ddpo_config.py ├── ddpo_trainer.py ├── dpo_trainer.py ├── iterative_sft_trainer.py ├── model_config.py ├── ppo_config.py ├── ppo_trainer.py ├── reward_config.py ├── reward_trainer.py ├── sft_trainer.py └── utils.py /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/README.md -------------------------------------------------------------------------------- /assets/teaser.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/assets/teaser.gif -------------------------------------------------------------------------------- /config/co-training_data.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/config/co-training_data.yaml -------------------------------------------------------------------------------- /config/vln_dagger.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/config/vln_dagger.yaml -------------------------------------------------------------------------------- /config/vln_r2r.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/config/vln_r2r.yaml -------------------------------------------------------------------------------- /llava/__init__.py: -------------------------------------------------------------------------------- 1 | from .model import LlavaLlamaForCausalLM 2 | -------------------------------------------------------------------------------- /llava/constants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/constants.py -------------------------------------------------------------------------------- /llava/conversation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/conversation.py -------------------------------------------------------------------------------- /llava/eval/evaluate_interleave.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/eval/evaluate_interleave.py -------------------------------------------------------------------------------- /llava/eval/model_vqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/eval/model_vqa.py -------------------------------------------------------------------------------- /llava/mm_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/mm_utils.py -------------------------------------------------------------------------------- /llava/model/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/__init__.py -------------------------------------------------------------------------------- /llava/model/apply_delta.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/apply_delta.py -------------------------------------------------------------------------------- /llava/model/builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/builder.py -------------------------------------------------------------------------------- /llava/model/consolidate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/consolidate.py -------------------------------------------------------------------------------- /llava/model/language_model/llava_gemma.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/language_model/llava_gemma.py -------------------------------------------------------------------------------- /llava/model/language_model/llava_llama.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/language_model/llava_llama.py -------------------------------------------------------------------------------- /llava/model/language_model/llava_mistral.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/language_model/llava_mistral.py -------------------------------------------------------------------------------- /llava/model/language_model/llava_mixtral.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/language_model/llava_mixtral.py -------------------------------------------------------------------------------- /llava/model/language_model/llava_mpt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/language_model/llava_mpt.py -------------------------------------------------------------------------------- /llava/model/language_model/llava_qwen.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/language_model/llava_qwen.py -------------------------------------------------------------------------------- /llava/model/language_model/llava_qwen_moe.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/language_model/llava_qwen_moe.py -------------------------------------------------------------------------------- /llava/model/language_model/modeling_llama.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/language_model/modeling_llama.py -------------------------------------------------------------------------------- /llava/model/llava_arch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/llava_arch.py -------------------------------------------------------------------------------- /llava/model/make_delta.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/make_delta.py -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_encoder/builder.py -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/clip_encoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_encoder/clip_encoder.py -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/dev_eva_clip/eva_clip/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_encoder/dev_eva_clip/eva_clip/__init__.py -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/dev_eva_clip/eva_clip/bpe_simple_vocab_16e6.txt.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_encoder/dev_eva_clip/eva_clip/bpe_simple_vocab_16e6.txt.gz -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/dev_eva_clip/eva_clip/constants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_encoder/dev_eva_clip/eva_clip/constants.py -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/dev_eva_clip/eva_clip/eva_vit_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_encoder/dev_eva_clip/eva_clip/eva_vit_model.py -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/dev_eva_clip/eva_clip/factory.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_encoder/dev_eva_clip/eva_clip/factory.py -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/dev_eva_clip/eva_clip/hf_configs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_encoder/dev_eva_clip/eva_clip/hf_configs.py -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/dev_eva_clip/eva_clip/hf_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_encoder/dev_eva_clip/eva_clip/hf_model.py -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/dev_eva_clip/eva_clip/loss.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_encoder/dev_eva_clip/eva_clip/loss.py -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/dev_eva_clip/eva_clip/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_encoder/dev_eva_clip/eva_clip/model.py -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/dev_eva_clip/eva_clip/model_configs/EVA-CLIP-18B.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_encoder/dev_eva_clip/eva_clip/model_configs/EVA-CLIP-18B.json -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/dev_eva_clip/eva_clip/model_configs/EVA-CLIP-8B-plus.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_encoder/dev_eva_clip/eva_clip/model_configs/EVA-CLIP-8B-plus.json -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/dev_eva_clip/eva_clip/model_configs/EVA-CLIP-8B.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_encoder/dev_eva_clip/eva_clip/model_configs/EVA-CLIP-8B.json -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/dev_eva_clip/eva_clip/model_configs/EVA01-CLIP-B-16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_encoder/dev_eva_clip/eva_clip/model_configs/EVA01-CLIP-B-16.json -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/dev_eva_clip/eva_clip/model_configs/EVA01-CLIP-g-14-plus.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_encoder/dev_eva_clip/eva_clip/model_configs/EVA01-CLIP-g-14-plus.json -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/dev_eva_clip/eva_clip/model_configs/EVA01-CLIP-g-14.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_encoder/dev_eva_clip/eva_clip/model_configs/EVA01-CLIP-g-14.json -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/dev_eva_clip/eva_clip/model_configs/EVA02-CLIP-B-16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_encoder/dev_eva_clip/eva_clip/model_configs/EVA02-CLIP-B-16.json -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/dev_eva_clip/eva_clip/model_configs/EVA02-CLIP-L-14-336.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_encoder/dev_eva_clip/eva_clip/model_configs/EVA02-CLIP-L-14-336.json -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/dev_eva_clip/eva_clip/model_configs/EVA02-CLIP-L-14.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_encoder/dev_eva_clip/eva_clip/model_configs/EVA02-CLIP-L-14.json -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/dev_eva_clip/eva_clip/model_configs/EVA02-CLIP-bigE-14-plus.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_encoder/dev_eva_clip/eva_clip/model_configs/EVA02-CLIP-bigE-14-plus.json -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/dev_eva_clip/eva_clip/model_configs/EVA02-CLIP-bigE-14.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_encoder/dev_eva_clip/eva_clip/model_configs/EVA02-CLIP-bigE-14.json -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/dev_eva_clip/eva_clip/model_configs/Internal-EVA02-CLIP-10B-14-448.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_encoder/dev_eva_clip/eva_clip/model_configs/Internal-EVA02-CLIP-10B-14-448.json -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/dev_eva_clip/eva_clip/model_configs/Internal-EVA02-CLIP-10B-14.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_encoder/dev_eva_clip/eva_clip/model_configs/Internal-EVA02-CLIP-10B-14.json -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/dev_eva_clip/eva_clip/modified_resnet.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_encoder/dev_eva_clip/eva_clip/modified_resnet.py -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/dev_eva_clip/eva_clip/openai.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_encoder/dev_eva_clip/eva_clip/openai.py -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/dev_eva_clip/eva_clip/pretrained.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_encoder/dev_eva_clip/eva_clip/pretrained.py -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/dev_eva_clip/eva_clip/rope.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_encoder/dev_eva_clip/eva_clip/rope.py -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/dev_eva_clip/eva_clip/timm_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_encoder/dev_eva_clip/eva_clip/timm_model.py -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/dev_eva_clip/eva_clip/tokenizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_encoder/dev_eva_clip/eva_clip/tokenizer.py -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/dev_eva_clip/eva_clip/transform.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_encoder/dev_eva_clip/eva_clip/transform.py -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/dev_eva_clip/eva_clip/transformer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_encoder/dev_eva_clip/eva_clip/transformer.py -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/dev_eva_clip/eva_clip/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_encoder/dev_eva_clip/eva_clip/utils.py -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/dev_eva_clip/eva_vit.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_encoder/dev_eva_clip/eva_vit.py -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/eva_clip/eva_clip_encoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_encoder/eva_clip/eva_clip_encoder.py -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/eva_clip/eva_clip_processors.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_encoder/eva_clip/eva_clip_processors.py -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/eva_clip/eva_vit.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_encoder/eva_clip/eva_vit.py -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/eva_clip/factory.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_encoder/eva_clip/factory.py -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/eva_clip/model_configs/EVA-CLIP-18B.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_encoder/eva_clip/model_configs/EVA-CLIP-18B.json -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/eva_clip/model_configs/EVA-CLIP-8B-plus.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_encoder/eva_clip/model_configs/EVA-CLIP-8B-plus.json -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/eva_clip/model_configs/EVA-CLIP-8B.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_encoder/eva_clip/model_configs/EVA-CLIP-8B.json -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/eva_clip/model_configs/EVA01-CLIP-B-16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_encoder/eva_clip/model_configs/EVA01-CLIP-B-16.json -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/eva_clip/model_configs/EVA01-CLIP-g-14-plus.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_encoder/eva_clip/model_configs/EVA01-CLIP-g-14-plus.json -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/eva_clip/model_configs/EVA01-CLIP-g-14.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_encoder/eva_clip/model_configs/EVA01-CLIP-g-14.json -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/eva_clip/model_configs/EVA02-CLIP-B-16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_encoder/eva_clip/model_configs/EVA02-CLIP-B-16.json -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/eva_clip/model_configs/EVA02-CLIP-L-14-336.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_encoder/eva_clip/model_configs/EVA02-CLIP-L-14-336.json -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/eva_clip/model_configs/EVA02-CLIP-L-14.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_encoder/eva_clip/model_configs/EVA02-CLIP-L-14.json -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/eva_clip/model_configs/EVA02-CLIP-bigE-14-plus.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_encoder/eva_clip/model_configs/EVA02-CLIP-bigE-14-plus.json -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/eva_clip/model_configs/EVA02-CLIP-bigE-14.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_encoder/eva_clip/model_configs/EVA02-CLIP-bigE-14.json -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/eva_clip/model_configs/Internal-EVA02-CLIP-10B-14-448.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_encoder/eva_clip/model_configs/Internal-EVA02-CLIP-10B-14-448.json -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/eva_clip/model_configs/Internal-EVA02-CLIP-10B-14.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_encoder/eva_clip/model_configs/Internal-EVA02-CLIP-10B-14.json -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/hf_vision.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_encoder/hf_vision.py -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/imagebind.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_encoder/imagebind.py -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/mlcd/vit_rope2d_hf.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_encoder/mlcd/vit_rope2d_hf.py -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/mlcd_encoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_encoder/mlcd_encoder.py -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/open_clip_encoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_encoder/open_clip_encoder.py -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/siglip_encoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_encoder/siglip_encoder.py -------------------------------------------------------------------------------- /llava/model/multimodal_projector/builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_projector/builder.py -------------------------------------------------------------------------------- /llava/model/multimodal_projector/pooler_projector.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_projector/pooler_projector.py -------------------------------------------------------------------------------- /llava/model/multimodal_resampler/builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_resampler/builder.py -------------------------------------------------------------------------------- /llava/model/multimodal_resampler/masked_drop.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_resampler/masked_drop.py -------------------------------------------------------------------------------- /llava/model/multimodal_resampler/perceiver.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_resampler/perceiver.py -------------------------------------------------------------------------------- /llava/model/multimodal_resampler/qformer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_resampler/qformer.py -------------------------------------------------------------------------------- /llava/model/multimodal_resampler/spatial_pool.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/multimodal_resampler/spatial_pool.py -------------------------------------------------------------------------------- /llava/model/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/model/utils.py -------------------------------------------------------------------------------- /llava/serve/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /llava/serve/cli.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/serve/cli.py -------------------------------------------------------------------------------- /llava/serve/controller.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/serve/controller.py -------------------------------------------------------------------------------- /llava/serve/examples/extreme_ironing.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/serve/examples/extreme_ironing.jpg -------------------------------------------------------------------------------- /llava/serve/examples/waterview.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/serve/examples/waterview.jpg -------------------------------------------------------------------------------- /llava/serve/gradio_multi_image.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/serve/gradio_multi_image.py -------------------------------------------------------------------------------- /llava/serve/gradio_web_server.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/serve/gradio_web_server.py -------------------------------------------------------------------------------- /llava/serve/model_worker.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/serve/model_worker.py -------------------------------------------------------------------------------- /llava/serve/register_worker.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/serve/register_worker.py -------------------------------------------------------------------------------- /llava/serve/sglang_worker.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/serve/sglang_worker.py -------------------------------------------------------------------------------- /llava/serve/test_message.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/serve/test_message.py -------------------------------------------------------------------------------- /llava/train/llama_flash_attn_monkey_patch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/train/llama_flash_attn_monkey_patch.py -------------------------------------------------------------------------------- /llava/train/llava_trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/train/llava_trainer.py -------------------------------------------------------------------------------- /llava/train/llava_trainer_eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/train/llava_trainer_eval.py -------------------------------------------------------------------------------- /llava/train/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/train/train.py -------------------------------------------------------------------------------- /llava/train/train_dpo.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/train/train_dpo.py -------------------------------------------------------------------------------- /llava/train/train_mem.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/train/train_mem.py -------------------------------------------------------------------------------- /llava/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/llava/utils.py -------------------------------------------------------------------------------- /realworld/go2_vln_client.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/realworld/go2_vln_client.py -------------------------------------------------------------------------------- /realworld/pid_controller.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/realworld/pid_controller.py -------------------------------------------------------------------------------- /realworld/realworld.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/realworld/realworld.md -------------------------------------------------------------------------------- /realworld/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/realworld/utils.py -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/requirements.txt -------------------------------------------------------------------------------- /scripts/streamvln_dagger_collect.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/scripts/streamvln_dagger_collect.sh -------------------------------------------------------------------------------- /scripts/streamvln_eval_multi_gpu.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/scripts/streamvln_eval_multi_gpu.sh -------------------------------------------------------------------------------- /scripts/streamvln_stage_two_train_slurm.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/scripts/streamvln_stage_two_train_slurm.sh -------------------------------------------------------------------------------- /scripts/streamvln_train_slurm.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/scripts/streamvln_train_slurm.sh -------------------------------------------------------------------------------- /scripts/streamvln_trajectory_generation.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/scripts/streamvln_trajectory_generation.sh -------------------------------------------------------------------------------- /scripts/zero2.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/scripts/zero2.json -------------------------------------------------------------------------------- /scripts/zero3.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/scripts/zero3.json -------------------------------------------------------------------------------- /streamvln/args.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/streamvln/args.py -------------------------------------------------------------------------------- /streamvln/dataset/mmc4_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/streamvln/dataset/mmc4_dataset.py -------------------------------------------------------------------------------- /streamvln/dataset/vln_action_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/streamvln/dataset/vln_action_dataset.py -------------------------------------------------------------------------------- /streamvln/habitat_extensions/maps.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/streamvln/habitat_extensions/maps.py -------------------------------------------------------------------------------- /streamvln/habitat_extensions/measures.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/streamvln/habitat_extensions/measures.py -------------------------------------------------------------------------------- /streamvln/http_realworld_server.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/streamvln/http_realworld_server.py -------------------------------------------------------------------------------- /streamvln/model/stream_video_vln.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/streamvln/model/stream_video_vln.py -------------------------------------------------------------------------------- /streamvln/streamvln_agent.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/streamvln/streamvln_agent.py -------------------------------------------------------------------------------- /streamvln/streamvln_dagger.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/streamvln/streamvln_dagger.py -------------------------------------------------------------------------------- /streamvln/streamvln_eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/streamvln/streamvln_eval.py -------------------------------------------------------------------------------- /streamvln/streamvln_train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/streamvln/streamvln_train.py -------------------------------------------------------------------------------- /streamvln/streamvln_trajectory_generation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/streamvln/streamvln_trajectory_generation.py -------------------------------------------------------------------------------- /streamvln/utils/dist.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/streamvln/utils/dist.py -------------------------------------------------------------------------------- /streamvln/utils/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/streamvln/utils/utils.py -------------------------------------------------------------------------------- /trl/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/trl/__init__.py -------------------------------------------------------------------------------- /trl/core.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/trl/core.py -------------------------------------------------------------------------------- /trl/environment/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/trl/environment/__init__.py -------------------------------------------------------------------------------- /trl/environment/base_environment.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/trl/environment/base_environment.py -------------------------------------------------------------------------------- /trl/extras/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/trl/extras/__init__.py -------------------------------------------------------------------------------- /trl/extras/best_of_n_sampler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/trl/extras/best_of_n_sampler.py -------------------------------------------------------------------------------- /trl/extras/dataset_formatting.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/trl/extras/dataset_formatting.py -------------------------------------------------------------------------------- /trl/import_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/trl/import_utils.py -------------------------------------------------------------------------------- /trl/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/trl/models/__init__.py -------------------------------------------------------------------------------- /trl/models/modeling_base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/trl/models/modeling_base.py -------------------------------------------------------------------------------- /trl/models/modeling_sd_base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/trl/models/modeling_sd_base.py -------------------------------------------------------------------------------- /trl/models/modeling_value_head.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/trl/models/modeling_value_head.py -------------------------------------------------------------------------------- /trl/models/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/trl/models/utils.py -------------------------------------------------------------------------------- /trl/trainer/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/trl/trainer/__init__.py -------------------------------------------------------------------------------- /trl/trainer/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/trl/trainer/base.py -------------------------------------------------------------------------------- /trl/trainer/ddpo_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/trl/trainer/ddpo_config.py -------------------------------------------------------------------------------- /trl/trainer/ddpo_trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/trl/trainer/ddpo_trainer.py -------------------------------------------------------------------------------- /trl/trainer/dpo_trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/trl/trainer/dpo_trainer.py -------------------------------------------------------------------------------- /trl/trainer/iterative_sft_trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/trl/trainer/iterative_sft_trainer.py -------------------------------------------------------------------------------- /trl/trainer/model_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/trl/trainer/model_config.py -------------------------------------------------------------------------------- /trl/trainer/ppo_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/trl/trainer/ppo_config.py -------------------------------------------------------------------------------- /trl/trainer/ppo_trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/trl/trainer/ppo_trainer.py -------------------------------------------------------------------------------- /trl/trainer/reward_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/trl/trainer/reward_config.py -------------------------------------------------------------------------------- /trl/trainer/reward_trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/trl/trainer/reward_trainer.py -------------------------------------------------------------------------------- /trl/trainer/sft_trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/trl/trainer/sft_trainer.py -------------------------------------------------------------------------------- /trl/trainer/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/InternRobotics/StreamVLN/HEAD/trl/trainer/utils.py --------------------------------------------------------------------------------