├── .gitmodules ├── Dockerfile ├── LICENSE ├── README.md ├── data_prepare ├── LICENSE ├── README.md ├── coyo │ ├── coyo_downloader.py │ └── coyo_splitter.py ├── mmc4 │ ├── mmc4_downloader.py │ ├── mmc4_filter_and_counter.py │ └── mmc4_merger.py ├── panda70m.sh ├── panda_split.py ├── parallel_shards.sh └── sft │ ├── ART1_2.py │ ├── ESTVQA.py │ ├── LSVT.py │ ├── POIE.py │ ├── ReCTS.py │ ├── SROIE.py │ ├── merge_idefics2.py │ ├── merge_llava_onevision.py │ ├── merge_llava_onevision_eagle.py │ ├── mtwi.py │ ├── preprocess_art_shangy.py │ ├── preprocess_cambrian.py │ ├── preprocess_cambrian_eagle.py │ ├── preprocess_docreason.py │ ├── preprocess_flan.py │ ├── preprocess_idefics2.py │ ├── preprocess_idefics2_eagle.py │ ├── preprocess_kvqa.py │ ├── preprocess_llava_onevision.py │ ├── preprocess_m3it.py │ ├── preprocess_metamathqa.py │ ├── preprocess_viquae.py │ ├── split_vflan.py │ ├── unichart_pretrain.py │ └── unichart_sft.py ├── demo_images ├── LongVILA-pipeline.png ├── demo_img.png ├── demo_img_1.png ├── demo_img_2.png ├── demo_img_3.png ├── longvila-data.png ├── longvila-logo.png ├── longvila-mmsp.png ├── longvila-needle.png ├── longvila-results.png ├── nvila-logo.png └── vila-logo.jpg ├── demo_trt_llm └── README.md ├── environment_setup.sh ├── finetuning ├── README.md └── results-qwen2-full.png ├── llava ├── __init__.py ├── cli │ ├── eval.py │ ├── infer.py │ ├── run.py │ └── upload2hf.py ├── constants.py ├── conversation.py ├── data │ ├── __init__.py │ ├── base.py │ ├── builder.py │ ├── collate.py │ ├── dataset.py │ ├── dataset_impl │ │ ├── __init__.py │ │ ├── coyo_qa.py │ │ ├── coyo_recap.py │ │ ├── dummy.py │ │ ├── eagle_video_wds.py │ │ ├── eagle_wds.py │ │ ├── general_img_text.py │ │ ├── hiertext.py │ │ ├── lita.py │ │ ├── llava.py │ │ ├── llava_cot.py │ │ ├── panda70m.py │ │ ├── sam.py │ │ ├── textocr.py │ │ └── utils.py │ ├── datasets_mixture.py │ ├── hf_parquet.py │ ├── registry │ │ ├── datasets │ │ │ └── default.yaml │ │ └── mixtures.yaml │ ├── simple_vila_webdataset.py │ └── utils.py ├── entry.py ├── eval │ ├── __init__.py │ ├── aitz.py │ ├── alfred.py │ ├── cinepile.py │ ├── domain.py │ ├── egoschema.py │ ├── eval_model_loc.py │ ├── eval_refcoco.py │ ├── eventbench.py │ ├── finetuning │ │ ├── eval_aitz.py │ │ ├── eval_multi_view.py │ │ ├── eval_pathvqa.py │ │ ├── eval_pmcvqa.py │ │ ├── eval_scicap.py │ │ ├── eval_tallyqa.py │ │ ├── eval_widgetcap.py │ │ ├── model_refcoco.py │ │ ├── model_vqa_loader.py │ │ ├── model_vqa_loader_multi.py │ │ └── model_widgetcap.py │ ├── its.py │ ├── lmms │ │ ├── models │ │ │ ├── __init__.py │ │ │ └── vila_internal.py │ │ └── tasks │ │ │ ├── __init__.py │ │ │ └── videomme.py │ ├── m4c_evaluator.py │ ├── mathvista.py │ ├── mathvista_utils │ │ ├── calculate_score.py │ │ ├── extract_answer.py │ │ ├── prompts │ │ │ └── ext_ans.py │ │ └── utilities.py │ ├── mmmu_utils │ │ └── eval_utils.py │ ├── model_refcoco.py │ ├── model_vqa_video.py │ ├── model_vqa_videodemo.py │ ├── nuscenes.py │ ├── pathvqa.py │ ├── pmcvqa.py │ ├── refcoco.py │ ├── registry.yaml │ ├── rtl.py │ ├── rtl_nv_lita.py │ ├── scicap.py │ ├── tallyqa.py │ ├── textvqa.py │ ├── video │ │ ├── eval_benchmark_1_correctness.py │ │ ├── eval_benchmark_2_detailed_orientation.py │ │ ├── eval_benchmark_3_context.py │ │ ├── eval_benchmark_4_temporal.py │ │ ├── eval_benchmark_5_consistency.py │ │ ├── eval_video_qa.py │ │ ├── model_vqa_videodemo_benchmark.py │ │ └── utils.py │ ├── vision_niah_vila │ │ ├── eval_vision_niah.py │ │ ├── produce_haystack_embedding.py │ │ ├── produce_needle_embedding.py │ │ └── zigzag_ring_attn │ │ │ ├── modeling_qwen2.py │ │ │ ├── monkey_patch.py │ │ │ └── prepare_inputs.py │ ├── vnbench.py │ └── widgetcap.py ├── media.py ├── mm_utils.py ├── model │ ├── FloatPointQuantizeTorch.py │ ├── FloatPointQuantizeTriton.py │ ├── __init__.py │ ├── apply_delta.py │ ├── builder.py │ ├── coat │ │ ├── activation │ │ │ ├── __init__.py │ │ │ ├── fake_quantization │ │ │ │ ├── FloatPointQuantizeTorch.py │ │ │ │ ├── FloatPointQuantizeTriton.py │ │ │ │ ├── quantize_function.py │ │ │ │ └── utils.py │ │ │ ├── models │ │ │ │ ├── _fp8_quantization_config.py │ │ │ │ ├── _fp8_weightcache.py │ │ │ │ ├── _fp8manager.py │ │ │ │ ├── coat_llama.py │ │ │ │ ├── coat_llama_convert_from_hf.py │ │ │ │ └── coat_olmo.py │ │ │ ├── real_quantization │ │ │ │ ├── __init__.py │ │ │ │ ├── _dequantize.py │ │ │ │ ├── _division.py │ │ │ │ ├── _division_transpose.py │ │ │ │ ├── _memory_io.py │ │ │ │ ├── _quantize.py │ │ │ │ ├── _quantize_pertensor.py │ │ │ │ ├── _quantize_pertensor_transpose.py │ │ │ │ ├── _transpose.py │ │ │ │ ├── add_bwd.py │ │ │ │ ├── add_fwd.py │ │ │ │ ├── common.py │ │ │ │ ├── fp8linear.py │ │ │ │ ├── func_layernorm_noparam.py │ │ │ │ ├── func_quantize.py │ │ │ │ ├── func_rmsnorm.py │ │ │ │ ├── gelu_bwd.py │ │ │ │ ├── gelu_bwd_legacy.py │ │ │ │ ├── gelu_fwd.py │ │ │ │ ├── linear.py │ │ │ │ ├── mul_bwd.py │ │ │ │ ├── mul_bwd_legacy.py │ │ │ │ ├── mul_bwd_silu_fwd.py │ │ │ │ ├── mul_fwd.py │ │ │ │ ├── silu_bwd.py │ │ │ │ ├── silu_bwd_legacy.py │ │ │ │ └── silu_fwd.py │ │ │ └── utils.py │ │ ├── fp8_trainer.py │ │ └── optimizer │ │ │ ├── fp8_adamw.py │ │ │ └── kernels │ │ │ ├── bindings.cpp │ │ │ ├── build │ │ │ ├── lib.linux-x86_64-cpython-310 │ │ │ │ └── qoptim_cuda.cpython-310-x86_64-linux-gnu.so │ │ │ └── temp.linux-x86_64-cpython-310 │ │ │ │ ├── bindings.o │ │ │ │ ├── fp8_adamw_cuda.o │ │ │ │ ├── fp8_adamw_cuda_kernel.o │ │ │ │ ├── fp8_adamw_expand_cuda.o │ │ │ │ └── fp8_adamw_expand_cuda_kernel.o │ │ │ ├── csrc_expand_quantize │ │ │ ├── makefile │ │ │ └── nvcc_qoptim.cu │ │ │ ├── csrc_origin_quantize │ │ │ ├── makefile │ │ │ └── nvcc_qoptim.cu │ │ │ ├── fp8_adamw_cuda.cpp │ │ │ ├── fp8_adamw_cuda_kernel.cu │ │ │ ├── fp8_adamw_expand_cuda.cpp │ │ │ ├── fp8_adamw_expand_cuda_kernel.cu │ │ │ ├── include │ │ │ ├── fp8_adamw.h │ │ │ └── fp8_adamw_expand.h │ │ │ └── setup.py │ ├── configuration_llava.py │ ├── consolidate.py │ ├── deprecate_consolidate.py │ ├── encoders │ │ ├── __init__.py │ │ ├── base.py │ │ ├── image │ │ │ ├── __init__.py │ │ │ └── basic.py │ │ └── video │ │ │ ├── __init__.py │ │ │ ├── basic.py │ │ │ └── tsp.py │ ├── language_model │ │ ├── builder.py │ │ ├── chat_templates │ │ │ ├── mistral.jinja │ │ │ └── qwen2.jinja │ │ ├── configuration_quantize.py │ │ ├── fp8_qwen2_convert_from_hf.py │ │ ├── fp8activationqwen2.py │ │ ├── fp8activationresidualqwen2.py │ │ ├── fp8linearqwen2.py │ │ ├── llava_llama.py │ │ ├── llava_topdown_llama.py │ │ ├── qllama.py │ │ ├── qllava_qllama.py │ │ ├── qmemllama.py │ │ └── realqmemllama.py │ ├── liger │ │ ├── cross_entropy.py │ │ └── utils.py │ ├── llava_arch.py │ ├── loss.py │ ├── make_delta.py │ ├── multimodal_encoder │ │ ├── builder.py │ │ ├── clip_encoder.py │ │ ├── image_processor.py │ │ ├── intern │ │ │ ├── configuration_intern_vit.py │ │ │ ├── flash_attention.py │ │ │ └── modeling_intern_vit.py │ │ ├── intern_encoder.py │ │ ├── ps3_encoder.py │ │ ├── radio_encoder.py │ │ ├── radio_torchhub_encoder.py │ │ ├── siglip │ │ │ ├── __init__.py │ │ │ └── modeling_siglip.py │ │ ├── siglip_encoder.py │ │ ├── vision_encoder.py │ │ └── visualize_features.py │ ├── multimodal_projector │ │ ├── base_projector.py │ │ └── builder.py │ ├── qfunction.py │ ├── qlinear_te.py │ ├── quantization │ │ ├── FloatPointQuantizeTorch.py │ │ ├── FloatPointQuantizeTriton.py │ │ ├── QAct.py │ │ ├── QAdd.py │ │ ├── QFunction.py │ │ ├── QGELU.py │ │ ├── QIdentity.py │ │ ├── QLayerNorm.py │ │ ├── QLinear.py │ │ ├── QMul.py │ │ ├── Qconfig.py │ │ ├── __init__.py │ │ ├── debug.txt │ │ └── utils.py │ ├── qutils.py │ ├── realquantize │ │ ├── common.py │ │ ├── division.py │ │ ├── division_transpose.py │ │ ├── linear.py │ │ ├── quantize_and_transpose.py │ │ └── trans_grad_bias.py │ └── utils │ │ ├── __init__.py │ │ ├── packing.py │ │ └── utils.py ├── remote_code │ ├── INSTRUCTIONS.md │ ├── README.md │ ├── auto_processor.py │ ├── base_projector.py │ ├── builder.py │ ├── configuration_vila.py │ ├── constants.py │ ├── conversation.py │ ├── distributed.py │ ├── loss.py │ ├── media.py │ ├── media_encoder.py │ ├── mm_utils.py │ ├── model_utils_packing.py │ ├── modeling_vila.py │ ├── siglip_encoder.py │ ├── tokenizer_utils.py │ └── utils.py ├── train │ ├── __init__.py │ ├── args.py │ ├── callbacks │ │ └── autoresume_callback.py │ ├── deepspeed_replace │ │ └── runtime │ │ │ └── zero │ │ │ └── mics.py │ ├── llava_trainer.py │ ├── sequence_parallel │ │ ├── __init__.py │ │ ├── all_to_all.py │ │ ├── globals.py │ │ ├── hybrid_attn.py │ │ ├── input_utils.py │ │ ├── monkey_patch.py │ │ ├── ring │ │ │ ├── __init__.py │ │ │ ├── ring_flash_attn.py │ │ │ ├── ring_flash_attn_varlen.py │ │ │ ├── stripe_flash_attn.py │ │ │ ├── triton_utils.py │ │ │ ├── utils.py │ │ │ ├── zigzag_ring_flash_attn.py │ │ │ └── zigzag_ring_flash_attn_varlen.py │ │ └── ulysses_attn.py │ ├── short_video_filter.py │ ├── slurm_utils.py │ ├── train.py │ ├── train_hybrid.py │ ├── train_llm_to_long.py │ ├── train_ln.py │ ├── train_mem.py │ ├── train_mem_ln.py │ ├── transformer_normalize_monkey_patch.py │ └── utils.py ├── trl │ ├── __init__.py │ ├── core.py │ ├── environment │ │ ├── __init__.py │ │ └── base_environment.py │ ├── extras │ │ ├── __init__.py │ │ ├── best_of_n_sampler.py │ │ └── dataset_formatting.py │ ├── import_utils.py │ ├── models │ │ ├── __init__.py │ │ ├── modeling_base.py │ │ ├── modeling_sd_base.py │ │ ├── modeling_value_head.py │ │ └── utils.py │ └── trainer │ │ ├── __init__.py │ │ ├── base.py │ │ ├── ddpo_config.py │ │ ├── ddpo_trainer.py │ │ ├── dpo_trainer.py │ │ ├── iterative_sft_trainer.py │ │ ├── model_config.py │ │ ├── ppo_config.py │ │ ├── ppo_trainer.py │ │ ├── reward_config.py │ │ ├── reward_trainer.py │ │ ├── sft_trainer.py │ │ └── utils.py ├── utils │ ├── __init__.py │ ├── distributed.py │ ├── io.py │ ├── logging.py │ ├── media.py │ ├── merge_lora_weights_and_save_hf_model.py │ ├── tokenizer.py │ └── utils.py └── wids │ ├── __init__.py │ ├── wids.py │ ├── wids_bench.py │ ├── wids_cleanup.py │ ├── wids_dir.py │ ├── wids_dl.py │ ├── wids_index.py │ ├── wids_lru.py │ ├── wids_mmtar.py │ ├── wids_specs.py │ └── wids_tar.py ├── longvila ├── README.md └── train │ ├── 4_extend_llm_1M.sh │ ├── 4_extend_llm_256k.sh │ ├── 4_extend_llm_512k.sh │ ├── 4_extend_llm_64k.sh │ ├── 5_long_sft_1024frames.sh │ ├── 5_long_sft_2048frames.sh │ ├── 5_long_sft_256frames.sh │ └── 5_long_sft_512frames.sh ├── pyproject.toml ├── scripts ├── NVILA-Lite │ ├── align.sh │ ├── pretrain.sh │ ├── sft.sh │ └── stage15.sh ├── NVILA │ ├── stage15_9tile.sh │ ├── stage1_9tile.sh │ ├── stage2_9tile.sh │ ├── stage3_9tile.sh │ └── stage4.sh ├── deepspeed_inference.yaml ├── eval │ ├── activitynet-rtl.sh │ ├── cinepile.sh │ ├── egoschema.sh │ ├── eventbench.sh │ ├── lmms.sh │ ├── mathvista.sh │ ├── needle.sh │ └── vnbench.sh ├── fp8training │ ├── README.md │ ├── run.sh │ ├── sft_qwen_fp16.sh │ └── sft_qwen_fp8.sh ├── setups │ └── train.sh ├── zero2.json ├── zero3.json ├── zero3_gradient_clipping.json ├── zero3_mics_mini_fixed.json ├── zero3_offload_inference.json └── zero3pp.json ├── server.py ├── serving ├── README.md ├── query_nvila.py └── server.py └── vila_hd ├── README.md └── assets ├── 4kpro_results.png ├── av_example_1.jpg ├── teaser.png ├── vila_hd_general_results.png └── vila_hd_highres_results.png /.gitmodules: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/.gitmodules -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/Dockerfile -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/README.md -------------------------------------------------------------------------------- /data_prepare/LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/data_prepare/LICENSE -------------------------------------------------------------------------------- /data_prepare/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/data_prepare/README.md -------------------------------------------------------------------------------- /data_prepare/coyo/coyo_downloader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/data_prepare/coyo/coyo_downloader.py -------------------------------------------------------------------------------- /data_prepare/coyo/coyo_splitter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/data_prepare/coyo/coyo_splitter.py -------------------------------------------------------------------------------- /data_prepare/mmc4/mmc4_downloader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/data_prepare/mmc4/mmc4_downloader.py -------------------------------------------------------------------------------- /data_prepare/mmc4/mmc4_filter_and_counter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/data_prepare/mmc4/mmc4_filter_and_counter.py -------------------------------------------------------------------------------- /data_prepare/mmc4/mmc4_merger.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/data_prepare/mmc4/mmc4_merger.py -------------------------------------------------------------------------------- /data_prepare/panda70m.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/data_prepare/panda70m.sh -------------------------------------------------------------------------------- /data_prepare/panda_split.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/data_prepare/panda_split.py -------------------------------------------------------------------------------- /data_prepare/parallel_shards.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/data_prepare/parallel_shards.sh -------------------------------------------------------------------------------- /data_prepare/sft/ART1_2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/data_prepare/sft/ART1_2.py -------------------------------------------------------------------------------- /data_prepare/sft/ESTVQA.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/data_prepare/sft/ESTVQA.py -------------------------------------------------------------------------------- /data_prepare/sft/LSVT.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/data_prepare/sft/LSVT.py -------------------------------------------------------------------------------- /data_prepare/sft/POIE.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/data_prepare/sft/POIE.py -------------------------------------------------------------------------------- /data_prepare/sft/ReCTS.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/data_prepare/sft/ReCTS.py -------------------------------------------------------------------------------- /data_prepare/sft/SROIE.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/data_prepare/sft/SROIE.py -------------------------------------------------------------------------------- /data_prepare/sft/merge_idefics2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/data_prepare/sft/merge_idefics2.py -------------------------------------------------------------------------------- /data_prepare/sft/merge_llava_onevision.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/data_prepare/sft/merge_llava_onevision.py -------------------------------------------------------------------------------- /data_prepare/sft/merge_llava_onevision_eagle.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/data_prepare/sft/merge_llava_onevision_eagle.py -------------------------------------------------------------------------------- /data_prepare/sft/mtwi.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/data_prepare/sft/mtwi.py -------------------------------------------------------------------------------- /data_prepare/sft/preprocess_art_shangy.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/data_prepare/sft/preprocess_art_shangy.py -------------------------------------------------------------------------------- /data_prepare/sft/preprocess_cambrian.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/data_prepare/sft/preprocess_cambrian.py -------------------------------------------------------------------------------- /data_prepare/sft/preprocess_cambrian_eagle.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/data_prepare/sft/preprocess_cambrian_eagle.py -------------------------------------------------------------------------------- /data_prepare/sft/preprocess_docreason.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/data_prepare/sft/preprocess_docreason.py -------------------------------------------------------------------------------- /data_prepare/sft/preprocess_flan.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/data_prepare/sft/preprocess_flan.py -------------------------------------------------------------------------------- /data_prepare/sft/preprocess_idefics2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/data_prepare/sft/preprocess_idefics2.py -------------------------------------------------------------------------------- /data_prepare/sft/preprocess_idefics2_eagle.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/data_prepare/sft/preprocess_idefics2_eagle.py -------------------------------------------------------------------------------- /data_prepare/sft/preprocess_kvqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/data_prepare/sft/preprocess_kvqa.py -------------------------------------------------------------------------------- /data_prepare/sft/preprocess_llava_onevision.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/data_prepare/sft/preprocess_llava_onevision.py -------------------------------------------------------------------------------- /data_prepare/sft/preprocess_m3it.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/data_prepare/sft/preprocess_m3it.py -------------------------------------------------------------------------------- /data_prepare/sft/preprocess_metamathqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/data_prepare/sft/preprocess_metamathqa.py -------------------------------------------------------------------------------- /data_prepare/sft/preprocess_viquae.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/data_prepare/sft/preprocess_viquae.py -------------------------------------------------------------------------------- /data_prepare/sft/split_vflan.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/data_prepare/sft/split_vflan.py -------------------------------------------------------------------------------- /data_prepare/sft/unichart_pretrain.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/data_prepare/sft/unichart_pretrain.py -------------------------------------------------------------------------------- /data_prepare/sft/unichart_sft.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/data_prepare/sft/unichart_sft.py -------------------------------------------------------------------------------- /demo_images/LongVILA-pipeline.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/demo_images/LongVILA-pipeline.png -------------------------------------------------------------------------------- /demo_images/demo_img.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/demo_images/demo_img.png -------------------------------------------------------------------------------- /demo_images/demo_img_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/demo_images/demo_img_1.png -------------------------------------------------------------------------------- /demo_images/demo_img_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/demo_images/demo_img_2.png -------------------------------------------------------------------------------- /demo_images/demo_img_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/demo_images/demo_img_3.png -------------------------------------------------------------------------------- /demo_images/longvila-data.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/demo_images/longvila-data.png -------------------------------------------------------------------------------- /demo_images/longvila-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/demo_images/longvila-logo.png -------------------------------------------------------------------------------- /demo_images/longvila-mmsp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/demo_images/longvila-mmsp.png -------------------------------------------------------------------------------- /demo_images/longvila-needle.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/demo_images/longvila-needle.png -------------------------------------------------------------------------------- /demo_images/longvila-results.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/demo_images/longvila-results.png -------------------------------------------------------------------------------- /demo_images/nvila-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/demo_images/nvila-logo.png -------------------------------------------------------------------------------- /demo_images/vila-logo.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/demo_images/vila-logo.jpg -------------------------------------------------------------------------------- /demo_trt_llm/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/demo_trt_llm/README.md -------------------------------------------------------------------------------- /environment_setup.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/environment_setup.sh -------------------------------------------------------------------------------- /finetuning/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/finetuning/README.md -------------------------------------------------------------------------------- /finetuning/results-qwen2-full.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/finetuning/results-qwen2-full.png -------------------------------------------------------------------------------- /llava/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/__init__.py -------------------------------------------------------------------------------- /llava/cli/eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/cli/eval.py -------------------------------------------------------------------------------- /llava/cli/infer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/cli/infer.py -------------------------------------------------------------------------------- /llava/cli/run.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/cli/run.py -------------------------------------------------------------------------------- /llava/cli/upload2hf.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/cli/upload2hf.py -------------------------------------------------------------------------------- /llava/constants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/constants.py -------------------------------------------------------------------------------- /llava/conversation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/conversation.py -------------------------------------------------------------------------------- /llava/data/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/data/__init__.py -------------------------------------------------------------------------------- /llava/data/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/data/base.py -------------------------------------------------------------------------------- /llava/data/builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/data/builder.py -------------------------------------------------------------------------------- /llava/data/collate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/data/collate.py -------------------------------------------------------------------------------- /llava/data/dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/data/dataset.py -------------------------------------------------------------------------------- /llava/data/dataset_impl/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/data/dataset_impl/__init__.py -------------------------------------------------------------------------------- /llava/data/dataset_impl/coyo_qa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/data/dataset_impl/coyo_qa.py -------------------------------------------------------------------------------- /llava/data/dataset_impl/coyo_recap.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/data/dataset_impl/coyo_recap.py -------------------------------------------------------------------------------- /llava/data/dataset_impl/dummy.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/data/dataset_impl/dummy.py -------------------------------------------------------------------------------- /llava/data/dataset_impl/eagle_video_wds.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/data/dataset_impl/eagle_video_wds.py -------------------------------------------------------------------------------- /llava/data/dataset_impl/eagle_wds.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/data/dataset_impl/eagle_wds.py -------------------------------------------------------------------------------- /llava/data/dataset_impl/general_img_text.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/data/dataset_impl/general_img_text.py -------------------------------------------------------------------------------- /llava/data/dataset_impl/hiertext.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/data/dataset_impl/hiertext.py -------------------------------------------------------------------------------- /llava/data/dataset_impl/lita.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/data/dataset_impl/lita.py -------------------------------------------------------------------------------- /llava/data/dataset_impl/llava.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/data/dataset_impl/llava.py -------------------------------------------------------------------------------- /llava/data/dataset_impl/llava_cot.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/data/dataset_impl/llava_cot.py -------------------------------------------------------------------------------- /llava/data/dataset_impl/panda70m.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/data/dataset_impl/panda70m.py -------------------------------------------------------------------------------- /llava/data/dataset_impl/sam.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/data/dataset_impl/sam.py -------------------------------------------------------------------------------- /llava/data/dataset_impl/textocr.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/data/dataset_impl/textocr.py -------------------------------------------------------------------------------- /llava/data/dataset_impl/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/data/dataset_impl/utils.py -------------------------------------------------------------------------------- /llava/data/datasets_mixture.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/data/datasets_mixture.py -------------------------------------------------------------------------------- /llava/data/hf_parquet.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/data/hf_parquet.py -------------------------------------------------------------------------------- /llava/data/registry/datasets/default.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/data/registry/datasets/default.yaml -------------------------------------------------------------------------------- /llava/data/registry/mixtures.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/data/registry/mixtures.yaml -------------------------------------------------------------------------------- /llava/data/simple_vila_webdataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/data/simple_vila_webdataset.py -------------------------------------------------------------------------------- /llava/data/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/data/utils.py -------------------------------------------------------------------------------- /llava/entry.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/entry.py -------------------------------------------------------------------------------- /llava/eval/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/eval/__init__.py -------------------------------------------------------------------------------- /llava/eval/aitz.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/eval/aitz.py -------------------------------------------------------------------------------- /llava/eval/alfred.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/eval/alfred.py -------------------------------------------------------------------------------- /llava/eval/cinepile.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/eval/cinepile.py -------------------------------------------------------------------------------- /llava/eval/domain.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/eval/domain.py -------------------------------------------------------------------------------- /llava/eval/egoschema.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/eval/egoschema.py -------------------------------------------------------------------------------- /llava/eval/eval_model_loc.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/eval/eval_model_loc.py -------------------------------------------------------------------------------- /llava/eval/eval_refcoco.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/eval/eval_refcoco.py -------------------------------------------------------------------------------- /llava/eval/eventbench.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/eval/eventbench.py -------------------------------------------------------------------------------- /llava/eval/finetuning/eval_aitz.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/eval/finetuning/eval_aitz.py -------------------------------------------------------------------------------- /llava/eval/finetuning/eval_multi_view.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/eval/finetuning/eval_multi_view.py -------------------------------------------------------------------------------- /llava/eval/finetuning/eval_pathvqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/eval/finetuning/eval_pathvqa.py -------------------------------------------------------------------------------- /llava/eval/finetuning/eval_pmcvqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/eval/finetuning/eval_pmcvqa.py -------------------------------------------------------------------------------- /llava/eval/finetuning/eval_scicap.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/eval/finetuning/eval_scicap.py -------------------------------------------------------------------------------- /llava/eval/finetuning/eval_tallyqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/eval/finetuning/eval_tallyqa.py -------------------------------------------------------------------------------- /llava/eval/finetuning/eval_widgetcap.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/eval/finetuning/eval_widgetcap.py -------------------------------------------------------------------------------- /llava/eval/finetuning/model_refcoco.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/eval/finetuning/model_refcoco.py -------------------------------------------------------------------------------- /llava/eval/finetuning/model_vqa_loader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/eval/finetuning/model_vqa_loader.py -------------------------------------------------------------------------------- /llava/eval/finetuning/model_vqa_loader_multi.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/eval/finetuning/model_vqa_loader_multi.py -------------------------------------------------------------------------------- /llava/eval/finetuning/model_widgetcap.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/eval/finetuning/model_widgetcap.py -------------------------------------------------------------------------------- /llava/eval/its.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/eval/its.py -------------------------------------------------------------------------------- /llava/eval/lmms/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/eval/lmms/models/__init__.py -------------------------------------------------------------------------------- /llava/eval/lmms/models/vila_internal.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/eval/lmms/models/vila_internal.py -------------------------------------------------------------------------------- /llava/eval/lmms/tasks/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /llava/eval/lmms/tasks/videomme.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/eval/lmms/tasks/videomme.py -------------------------------------------------------------------------------- /llava/eval/m4c_evaluator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/eval/m4c_evaluator.py -------------------------------------------------------------------------------- /llava/eval/mathvista.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/eval/mathvista.py -------------------------------------------------------------------------------- /llava/eval/mathvista_utils/calculate_score.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/eval/mathvista_utils/calculate_score.py -------------------------------------------------------------------------------- /llava/eval/mathvista_utils/extract_answer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/eval/mathvista_utils/extract_answer.py -------------------------------------------------------------------------------- /llava/eval/mathvista_utils/prompts/ext_ans.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/eval/mathvista_utils/prompts/ext_ans.py -------------------------------------------------------------------------------- /llava/eval/mathvista_utils/utilities.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/eval/mathvista_utils/utilities.py -------------------------------------------------------------------------------- /llava/eval/mmmu_utils/eval_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/eval/mmmu_utils/eval_utils.py -------------------------------------------------------------------------------- /llava/eval/model_refcoco.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/eval/model_refcoco.py -------------------------------------------------------------------------------- /llava/eval/model_vqa_video.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/eval/model_vqa_video.py -------------------------------------------------------------------------------- /llava/eval/model_vqa_videodemo.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/eval/model_vqa_videodemo.py -------------------------------------------------------------------------------- /llava/eval/nuscenes.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/eval/nuscenes.py -------------------------------------------------------------------------------- /llava/eval/pathvqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/eval/pathvqa.py -------------------------------------------------------------------------------- /llava/eval/pmcvqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/eval/pmcvqa.py -------------------------------------------------------------------------------- /llava/eval/refcoco.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/eval/refcoco.py -------------------------------------------------------------------------------- /llava/eval/registry.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/eval/registry.yaml -------------------------------------------------------------------------------- /llava/eval/rtl.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/eval/rtl.py -------------------------------------------------------------------------------- /llava/eval/rtl_nv_lita.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/eval/rtl_nv_lita.py -------------------------------------------------------------------------------- /llava/eval/scicap.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/eval/scicap.py -------------------------------------------------------------------------------- /llava/eval/tallyqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/eval/tallyqa.py -------------------------------------------------------------------------------- /llava/eval/textvqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/eval/textvqa.py -------------------------------------------------------------------------------- /llava/eval/video/eval_benchmark_1_correctness.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/eval/video/eval_benchmark_1_correctness.py -------------------------------------------------------------------------------- /llava/eval/video/eval_benchmark_2_detailed_orientation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/eval/video/eval_benchmark_2_detailed_orientation.py -------------------------------------------------------------------------------- /llava/eval/video/eval_benchmark_3_context.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/eval/video/eval_benchmark_3_context.py -------------------------------------------------------------------------------- /llava/eval/video/eval_benchmark_4_temporal.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/eval/video/eval_benchmark_4_temporal.py -------------------------------------------------------------------------------- /llava/eval/video/eval_benchmark_5_consistency.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/eval/video/eval_benchmark_5_consistency.py -------------------------------------------------------------------------------- /llava/eval/video/eval_video_qa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/eval/video/eval_video_qa.py -------------------------------------------------------------------------------- /llava/eval/video/model_vqa_videodemo_benchmark.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/eval/video/model_vqa_videodemo_benchmark.py -------------------------------------------------------------------------------- /llava/eval/video/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/eval/video/utils.py -------------------------------------------------------------------------------- /llava/eval/vision_niah_vila/eval_vision_niah.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/eval/vision_niah_vila/eval_vision_niah.py -------------------------------------------------------------------------------- /llava/eval/vision_niah_vila/produce_haystack_embedding.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/eval/vision_niah_vila/produce_haystack_embedding.py -------------------------------------------------------------------------------- /llava/eval/vision_niah_vila/produce_needle_embedding.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/eval/vision_niah_vila/produce_needle_embedding.py -------------------------------------------------------------------------------- /llava/eval/vision_niah_vila/zigzag_ring_attn/modeling_qwen2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/eval/vision_niah_vila/zigzag_ring_attn/modeling_qwen2.py -------------------------------------------------------------------------------- /llava/eval/vision_niah_vila/zigzag_ring_attn/monkey_patch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/eval/vision_niah_vila/zigzag_ring_attn/monkey_patch.py -------------------------------------------------------------------------------- /llava/eval/vision_niah_vila/zigzag_ring_attn/prepare_inputs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/eval/vision_niah_vila/zigzag_ring_attn/prepare_inputs.py -------------------------------------------------------------------------------- /llava/eval/vnbench.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/eval/vnbench.py -------------------------------------------------------------------------------- /llava/eval/widgetcap.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/eval/widgetcap.py -------------------------------------------------------------------------------- /llava/media.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/media.py -------------------------------------------------------------------------------- /llava/mm_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/mm_utils.py -------------------------------------------------------------------------------- /llava/model/FloatPointQuantizeTorch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/FloatPointQuantizeTorch.py -------------------------------------------------------------------------------- /llava/model/FloatPointQuantizeTriton.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/FloatPointQuantizeTriton.py -------------------------------------------------------------------------------- /llava/model/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/__init__.py -------------------------------------------------------------------------------- /llava/model/apply_delta.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/apply_delta.py -------------------------------------------------------------------------------- /llava/model/builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/builder.py -------------------------------------------------------------------------------- /llava/model/coat/activation/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /llava/model/coat/activation/fake_quantization/FloatPointQuantizeTorch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/coat/activation/fake_quantization/FloatPointQuantizeTorch.py -------------------------------------------------------------------------------- /llava/model/coat/activation/fake_quantization/FloatPointQuantizeTriton.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/coat/activation/fake_quantization/FloatPointQuantizeTriton.py -------------------------------------------------------------------------------- /llava/model/coat/activation/fake_quantization/quantize_function.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/coat/activation/fake_quantization/quantize_function.py -------------------------------------------------------------------------------- /llava/model/coat/activation/fake_quantization/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/coat/activation/fake_quantization/utils.py -------------------------------------------------------------------------------- /llava/model/coat/activation/models/_fp8_quantization_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/coat/activation/models/_fp8_quantization_config.py -------------------------------------------------------------------------------- /llava/model/coat/activation/models/_fp8_weightcache.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/coat/activation/models/_fp8_weightcache.py -------------------------------------------------------------------------------- /llava/model/coat/activation/models/_fp8manager.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/coat/activation/models/_fp8manager.py -------------------------------------------------------------------------------- /llava/model/coat/activation/models/coat_llama.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/coat/activation/models/coat_llama.py -------------------------------------------------------------------------------- /llava/model/coat/activation/models/coat_llama_convert_from_hf.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/coat/activation/models/coat_llama_convert_from_hf.py -------------------------------------------------------------------------------- /llava/model/coat/activation/models/coat_olmo.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/coat/activation/models/coat_olmo.py -------------------------------------------------------------------------------- /llava/model/coat/activation/real_quantization/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/coat/activation/real_quantization/__init__.py -------------------------------------------------------------------------------- /llava/model/coat/activation/real_quantization/_dequantize.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/coat/activation/real_quantization/_dequantize.py -------------------------------------------------------------------------------- /llava/model/coat/activation/real_quantization/_division.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/coat/activation/real_quantization/_division.py -------------------------------------------------------------------------------- /llava/model/coat/activation/real_quantization/_division_transpose.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/coat/activation/real_quantization/_division_transpose.py -------------------------------------------------------------------------------- /llava/model/coat/activation/real_quantization/_memory_io.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/coat/activation/real_quantization/_memory_io.py -------------------------------------------------------------------------------- /llava/model/coat/activation/real_quantization/_quantize.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/coat/activation/real_quantization/_quantize.py -------------------------------------------------------------------------------- /llava/model/coat/activation/real_quantization/_quantize_pertensor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/coat/activation/real_quantization/_quantize_pertensor.py -------------------------------------------------------------------------------- /llava/model/coat/activation/real_quantization/_quantize_pertensor_transpose.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/coat/activation/real_quantization/_quantize_pertensor_transpose.py -------------------------------------------------------------------------------- /llava/model/coat/activation/real_quantization/_transpose.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/coat/activation/real_quantization/_transpose.py -------------------------------------------------------------------------------- /llava/model/coat/activation/real_quantization/add_bwd.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/coat/activation/real_quantization/add_bwd.py -------------------------------------------------------------------------------- /llava/model/coat/activation/real_quantization/add_fwd.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/coat/activation/real_quantization/add_fwd.py -------------------------------------------------------------------------------- /llava/model/coat/activation/real_quantization/common.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/coat/activation/real_quantization/common.py -------------------------------------------------------------------------------- /llava/model/coat/activation/real_quantization/fp8linear.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/coat/activation/real_quantization/fp8linear.py -------------------------------------------------------------------------------- /llava/model/coat/activation/real_quantization/func_layernorm_noparam.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/coat/activation/real_quantization/func_layernorm_noparam.py -------------------------------------------------------------------------------- /llava/model/coat/activation/real_quantization/func_quantize.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/coat/activation/real_quantization/func_quantize.py -------------------------------------------------------------------------------- /llava/model/coat/activation/real_quantization/func_rmsnorm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/coat/activation/real_quantization/func_rmsnorm.py -------------------------------------------------------------------------------- /llava/model/coat/activation/real_quantization/gelu_bwd.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/coat/activation/real_quantization/gelu_bwd.py -------------------------------------------------------------------------------- /llava/model/coat/activation/real_quantization/gelu_bwd_legacy.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/coat/activation/real_quantization/gelu_bwd_legacy.py -------------------------------------------------------------------------------- /llava/model/coat/activation/real_quantization/gelu_fwd.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/coat/activation/real_quantization/gelu_fwd.py -------------------------------------------------------------------------------- /llava/model/coat/activation/real_quantization/linear.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/coat/activation/real_quantization/linear.py -------------------------------------------------------------------------------- /llava/model/coat/activation/real_quantization/mul_bwd.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/coat/activation/real_quantization/mul_bwd.py -------------------------------------------------------------------------------- /llava/model/coat/activation/real_quantization/mul_bwd_legacy.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/coat/activation/real_quantization/mul_bwd_legacy.py -------------------------------------------------------------------------------- /llava/model/coat/activation/real_quantization/mul_bwd_silu_fwd.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/coat/activation/real_quantization/mul_bwd_silu_fwd.py -------------------------------------------------------------------------------- /llava/model/coat/activation/real_quantization/mul_fwd.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/coat/activation/real_quantization/mul_fwd.py -------------------------------------------------------------------------------- /llava/model/coat/activation/real_quantization/silu_bwd.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/coat/activation/real_quantization/silu_bwd.py -------------------------------------------------------------------------------- /llava/model/coat/activation/real_quantization/silu_bwd_legacy.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/coat/activation/real_quantization/silu_bwd_legacy.py -------------------------------------------------------------------------------- /llava/model/coat/activation/real_quantization/silu_fwd.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/coat/activation/real_quantization/silu_fwd.py -------------------------------------------------------------------------------- /llava/model/coat/activation/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/coat/activation/utils.py -------------------------------------------------------------------------------- /llava/model/coat/fp8_trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/coat/fp8_trainer.py -------------------------------------------------------------------------------- /llava/model/coat/optimizer/fp8_adamw.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/coat/optimizer/fp8_adamw.py -------------------------------------------------------------------------------- /llava/model/coat/optimizer/kernels/bindings.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/coat/optimizer/kernels/bindings.cpp -------------------------------------------------------------------------------- /llava/model/coat/optimizer/kernels/build/lib.linux-x86_64-cpython-310/qoptim_cuda.cpython-310-x86_64-linux-gnu.so: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/coat/optimizer/kernels/build/lib.linux-x86_64-cpython-310/qoptim_cuda.cpython-310-x86_64-linux-gnu.so -------------------------------------------------------------------------------- /llava/model/coat/optimizer/kernels/build/temp.linux-x86_64-cpython-310/bindings.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/coat/optimizer/kernels/build/temp.linux-x86_64-cpython-310/bindings.o -------------------------------------------------------------------------------- /llava/model/coat/optimizer/kernels/build/temp.linux-x86_64-cpython-310/fp8_adamw_cuda.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/coat/optimizer/kernels/build/temp.linux-x86_64-cpython-310/fp8_adamw_cuda.o -------------------------------------------------------------------------------- /llava/model/coat/optimizer/kernels/build/temp.linux-x86_64-cpython-310/fp8_adamw_cuda_kernel.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/coat/optimizer/kernels/build/temp.linux-x86_64-cpython-310/fp8_adamw_cuda_kernel.o -------------------------------------------------------------------------------- /llava/model/coat/optimizer/kernels/build/temp.linux-x86_64-cpython-310/fp8_adamw_expand_cuda.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/coat/optimizer/kernels/build/temp.linux-x86_64-cpython-310/fp8_adamw_expand_cuda.o -------------------------------------------------------------------------------- /llava/model/coat/optimizer/kernels/build/temp.linux-x86_64-cpython-310/fp8_adamw_expand_cuda_kernel.o: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/coat/optimizer/kernels/build/temp.linux-x86_64-cpython-310/fp8_adamw_expand_cuda_kernel.o -------------------------------------------------------------------------------- /llava/model/coat/optimizer/kernels/csrc_expand_quantize/makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/coat/optimizer/kernels/csrc_expand_quantize/makefile -------------------------------------------------------------------------------- /llava/model/coat/optimizer/kernels/csrc_expand_quantize/nvcc_qoptim.cu: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/coat/optimizer/kernels/csrc_expand_quantize/nvcc_qoptim.cu -------------------------------------------------------------------------------- /llava/model/coat/optimizer/kernels/csrc_origin_quantize/makefile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/coat/optimizer/kernels/csrc_origin_quantize/makefile -------------------------------------------------------------------------------- /llava/model/coat/optimizer/kernels/csrc_origin_quantize/nvcc_qoptim.cu: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/coat/optimizer/kernels/csrc_origin_quantize/nvcc_qoptim.cu -------------------------------------------------------------------------------- /llava/model/coat/optimizer/kernels/fp8_adamw_cuda.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/coat/optimizer/kernels/fp8_adamw_cuda.cpp -------------------------------------------------------------------------------- /llava/model/coat/optimizer/kernels/fp8_adamw_cuda_kernel.cu: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/coat/optimizer/kernels/fp8_adamw_cuda_kernel.cu -------------------------------------------------------------------------------- /llava/model/coat/optimizer/kernels/fp8_adamw_expand_cuda.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/coat/optimizer/kernels/fp8_adamw_expand_cuda.cpp -------------------------------------------------------------------------------- /llava/model/coat/optimizer/kernels/fp8_adamw_expand_cuda_kernel.cu: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/coat/optimizer/kernels/fp8_adamw_expand_cuda_kernel.cu -------------------------------------------------------------------------------- /llava/model/coat/optimizer/kernels/include/fp8_adamw.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/coat/optimizer/kernels/include/fp8_adamw.h -------------------------------------------------------------------------------- /llava/model/coat/optimizer/kernels/include/fp8_adamw_expand.h: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/coat/optimizer/kernels/include/fp8_adamw_expand.h -------------------------------------------------------------------------------- /llava/model/coat/optimizer/kernels/setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/coat/optimizer/kernels/setup.py -------------------------------------------------------------------------------- /llava/model/configuration_llava.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/configuration_llava.py -------------------------------------------------------------------------------- /llava/model/consolidate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/consolidate.py -------------------------------------------------------------------------------- /llava/model/deprecate_consolidate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/deprecate_consolidate.py -------------------------------------------------------------------------------- /llava/model/encoders/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/encoders/__init__.py -------------------------------------------------------------------------------- /llava/model/encoders/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/encoders/base.py -------------------------------------------------------------------------------- /llava/model/encoders/image/__init__.py: -------------------------------------------------------------------------------- 1 | from .basic import * 2 | -------------------------------------------------------------------------------- /llava/model/encoders/image/basic.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/encoders/image/basic.py -------------------------------------------------------------------------------- /llava/model/encoders/video/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/encoders/video/__init__.py -------------------------------------------------------------------------------- /llava/model/encoders/video/basic.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/encoders/video/basic.py -------------------------------------------------------------------------------- /llava/model/encoders/video/tsp.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/encoders/video/tsp.py -------------------------------------------------------------------------------- /llava/model/language_model/builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/language_model/builder.py -------------------------------------------------------------------------------- /llava/model/language_model/chat_templates/mistral.jinja: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/language_model/chat_templates/mistral.jinja -------------------------------------------------------------------------------- /llava/model/language_model/chat_templates/qwen2.jinja: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/language_model/chat_templates/qwen2.jinja -------------------------------------------------------------------------------- /llava/model/language_model/configuration_quantize.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/language_model/configuration_quantize.py -------------------------------------------------------------------------------- /llava/model/language_model/fp8_qwen2_convert_from_hf.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/language_model/fp8_qwen2_convert_from_hf.py -------------------------------------------------------------------------------- /llava/model/language_model/fp8activationqwen2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/language_model/fp8activationqwen2.py -------------------------------------------------------------------------------- /llava/model/language_model/fp8activationresidualqwen2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/language_model/fp8activationresidualqwen2.py -------------------------------------------------------------------------------- /llava/model/language_model/fp8linearqwen2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/language_model/fp8linearqwen2.py -------------------------------------------------------------------------------- /llava/model/language_model/llava_llama.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/language_model/llava_llama.py -------------------------------------------------------------------------------- /llava/model/language_model/llava_topdown_llama.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/language_model/llava_topdown_llama.py -------------------------------------------------------------------------------- /llava/model/language_model/qllama.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/language_model/qllama.py -------------------------------------------------------------------------------- /llava/model/language_model/qllava_qllama.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/language_model/qllava_qllama.py -------------------------------------------------------------------------------- /llava/model/language_model/qmemllama.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/language_model/qmemllama.py -------------------------------------------------------------------------------- /llava/model/language_model/realqmemllama.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/language_model/realqmemllama.py -------------------------------------------------------------------------------- /llava/model/liger/cross_entropy.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/liger/cross_entropy.py -------------------------------------------------------------------------------- /llava/model/liger/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/liger/utils.py -------------------------------------------------------------------------------- /llava/model/llava_arch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/llava_arch.py -------------------------------------------------------------------------------- /llava/model/loss.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/loss.py -------------------------------------------------------------------------------- /llava/model/make_delta.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/make_delta.py -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/multimodal_encoder/builder.py -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/clip_encoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/multimodal_encoder/clip_encoder.py -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/image_processor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/multimodal_encoder/image_processor.py -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/intern/configuration_intern_vit.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/multimodal_encoder/intern/configuration_intern_vit.py -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/intern/flash_attention.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/multimodal_encoder/intern/flash_attention.py -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/intern/modeling_intern_vit.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/multimodal_encoder/intern/modeling_intern_vit.py -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/intern_encoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/multimodal_encoder/intern_encoder.py -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/ps3_encoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/multimodal_encoder/ps3_encoder.py -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/radio_encoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/multimodal_encoder/radio_encoder.py -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/radio_torchhub_encoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/multimodal_encoder/radio_torchhub_encoder.py -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/siglip/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/multimodal_encoder/siglip/__init__.py -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/siglip/modeling_siglip.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/multimodal_encoder/siglip/modeling_siglip.py -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/siglip_encoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/multimodal_encoder/siglip_encoder.py -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/vision_encoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/multimodal_encoder/vision_encoder.py -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/visualize_features.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/multimodal_encoder/visualize_features.py -------------------------------------------------------------------------------- /llava/model/multimodal_projector/base_projector.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/multimodal_projector/base_projector.py -------------------------------------------------------------------------------- /llava/model/multimodal_projector/builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/multimodal_projector/builder.py -------------------------------------------------------------------------------- /llava/model/qfunction.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/qfunction.py -------------------------------------------------------------------------------- /llava/model/qlinear_te.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/qlinear_te.py -------------------------------------------------------------------------------- /llava/model/quantization/FloatPointQuantizeTorch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/quantization/FloatPointQuantizeTorch.py -------------------------------------------------------------------------------- /llava/model/quantization/FloatPointQuantizeTriton.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/quantization/FloatPointQuantizeTriton.py -------------------------------------------------------------------------------- /llava/model/quantization/QAct.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/quantization/QAct.py -------------------------------------------------------------------------------- /llava/model/quantization/QAdd.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/quantization/QAdd.py -------------------------------------------------------------------------------- /llava/model/quantization/QFunction.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/quantization/QFunction.py -------------------------------------------------------------------------------- /llava/model/quantization/QGELU.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/quantization/QGELU.py -------------------------------------------------------------------------------- /llava/model/quantization/QIdentity.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/quantization/QIdentity.py -------------------------------------------------------------------------------- /llava/model/quantization/QLayerNorm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/quantization/QLayerNorm.py -------------------------------------------------------------------------------- /llava/model/quantization/QLinear.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/quantization/QLinear.py -------------------------------------------------------------------------------- /llava/model/quantization/QMul.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/quantization/QMul.py -------------------------------------------------------------------------------- /llava/model/quantization/Qconfig.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/quantization/Qconfig.py -------------------------------------------------------------------------------- /llava/model/quantization/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/quantization/__init__.py -------------------------------------------------------------------------------- /llava/model/quantization/debug.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/quantization/debug.txt -------------------------------------------------------------------------------- /llava/model/quantization/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/quantization/utils.py -------------------------------------------------------------------------------- /llava/model/qutils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/qutils.py -------------------------------------------------------------------------------- /llava/model/realquantize/common.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/realquantize/common.py -------------------------------------------------------------------------------- /llava/model/realquantize/division.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/realquantize/division.py -------------------------------------------------------------------------------- /llava/model/realquantize/division_transpose.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/realquantize/division_transpose.py -------------------------------------------------------------------------------- /llava/model/realquantize/linear.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/realquantize/linear.py -------------------------------------------------------------------------------- /llava/model/realquantize/quantize_and_transpose.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/realquantize/quantize_and_transpose.py -------------------------------------------------------------------------------- /llava/model/realquantize/trans_grad_bias.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/realquantize/trans_grad_bias.py -------------------------------------------------------------------------------- /llava/model/utils/__init__.py: -------------------------------------------------------------------------------- 1 | from .utils import * 2 | -------------------------------------------------------------------------------- /llava/model/utils/packing.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/utils/packing.py -------------------------------------------------------------------------------- /llava/model/utils/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/model/utils/utils.py -------------------------------------------------------------------------------- /llava/remote_code/INSTRUCTIONS.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/remote_code/INSTRUCTIONS.md -------------------------------------------------------------------------------- /llava/remote_code/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/remote_code/README.md -------------------------------------------------------------------------------- /llava/remote_code/auto_processor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/remote_code/auto_processor.py -------------------------------------------------------------------------------- /llava/remote_code/base_projector.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/remote_code/base_projector.py -------------------------------------------------------------------------------- /llava/remote_code/builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/remote_code/builder.py -------------------------------------------------------------------------------- /llava/remote_code/configuration_vila.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/remote_code/configuration_vila.py -------------------------------------------------------------------------------- /llava/remote_code/constants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/remote_code/constants.py -------------------------------------------------------------------------------- /llava/remote_code/conversation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/remote_code/conversation.py -------------------------------------------------------------------------------- /llava/remote_code/distributed.py: -------------------------------------------------------------------------------- 1 | ../utils/distributed.py -------------------------------------------------------------------------------- /llava/remote_code/loss.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/remote_code/loss.py -------------------------------------------------------------------------------- /llava/remote_code/media.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/remote_code/media.py -------------------------------------------------------------------------------- /llava/remote_code/media_encoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/remote_code/media_encoder.py -------------------------------------------------------------------------------- /llava/remote_code/mm_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/remote_code/mm_utils.py -------------------------------------------------------------------------------- /llava/remote_code/model_utils_packing.py: -------------------------------------------------------------------------------- 1 | ../model/utils/packing.py -------------------------------------------------------------------------------- /llava/remote_code/modeling_vila.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/remote_code/modeling_vila.py -------------------------------------------------------------------------------- /llava/remote_code/siglip_encoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/remote_code/siglip_encoder.py -------------------------------------------------------------------------------- /llava/remote_code/tokenizer_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/remote_code/tokenizer_utils.py -------------------------------------------------------------------------------- /llava/remote_code/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/remote_code/utils.py -------------------------------------------------------------------------------- /llava/train/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /llava/train/args.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/train/args.py -------------------------------------------------------------------------------- /llava/train/callbacks/autoresume_callback.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/train/callbacks/autoresume_callback.py -------------------------------------------------------------------------------- /llava/train/deepspeed_replace/runtime/zero/mics.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/train/deepspeed_replace/runtime/zero/mics.py -------------------------------------------------------------------------------- /llava/train/llava_trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/train/llava_trainer.py -------------------------------------------------------------------------------- /llava/train/sequence_parallel/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/train/sequence_parallel/__init__.py -------------------------------------------------------------------------------- /llava/train/sequence_parallel/all_to_all.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/train/sequence_parallel/all_to_all.py -------------------------------------------------------------------------------- /llava/train/sequence_parallel/globals.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/train/sequence_parallel/globals.py -------------------------------------------------------------------------------- /llava/train/sequence_parallel/hybrid_attn.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/train/sequence_parallel/hybrid_attn.py -------------------------------------------------------------------------------- /llava/train/sequence_parallel/input_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/train/sequence_parallel/input_utils.py -------------------------------------------------------------------------------- /llava/train/sequence_parallel/monkey_patch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/train/sequence_parallel/monkey_patch.py -------------------------------------------------------------------------------- /llava/train/sequence_parallel/ring/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/train/sequence_parallel/ring/__init__.py -------------------------------------------------------------------------------- /llava/train/sequence_parallel/ring/ring_flash_attn.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/train/sequence_parallel/ring/ring_flash_attn.py -------------------------------------------------------------------------------- /llava/train/sequence_parallel/ring/ring_flash_attn_varlen.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/train/sequence_parallel/ring/ring_flash_attn_varlen.py -------------------------------------------------------------------------------- /llava/train/sequence_parallel/ring/stripe_flash_attn.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/train/sequence_parallel/ring/stripe_flash_attn.py -------------------------------------------------------------------------------- /llava/train/sequence_parallel/ring/triton_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/train/sequence_parallel/ring/triton_utils.py -------------------------------------------------------------------------------- /llava/train/sequence_parallel/ring/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/train/sequence_parallel/ring/utils.py -------------------------------------------------------------------------------- /llava/train/sequence_parallel/ring/zigzag_ring_flash_attn.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/train/sequence_parallel/ring/zigzag_ring_flash_attn.py -------------------------------------------------------------------------------- /llava/train/sequence_parallel/ring/zigzag_ring_flash_attn_varlen.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/train/sequence_parallel/ring/zigzag_ring_flash_attn_varlen.py -------------------------------------------------------------------------------- /llava/train/sequence_parallel/ulysses_attn.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/train/sequence_parallel/ulysses_attn.py -------------------------------------------------------------------------------- /llava/train/short_video_filter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/train/short_video_filter.py -------------------------------------------------------------------------------- /llava/train/slurm_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/train/slurm_utils.py -------------------------------------------------------------------------------- /llava/train/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/train/train.py -------------------------------------------------------------------------------- /llava/train/train_hybrid.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/train/train_hybrid.py -------------------------------------------------------------------------------- /llava/train/train_llm_to_long.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/train/train_llm_to_long.py -------------------------------------------------------------------------------- /llava/train/train_ln.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/train/train_ln.py -------------------------------------------------------------------------------- /llava/train/train_mem.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/train/train_mem.py -------------------------------------------------------------------------------- /llava/train/train_mem_ln.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/train/train_mem_ln.py -------------------------------------------------------------------------------- /llava/train/transformer_normalize_monkey_patch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/train/transformer_normalize_monkey_patch.py -------------------------------------------------------------------------------- /llava/train/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/train/utils.py -------------------------------------------------------------------------------- /llava/trl/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/trl/__init__.py -------------------------------------------------------------------------------- /llava/trl/core.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/trl/core.py -------------------------------------------------------------------------------- /llava/trl/environment/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/trl/environment/__init__.py -------------------------------------------------------------------------------- /llava/trl/environment/base_environment.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/trl/environment/base_environment.py -------------------------------------------------------------------------------- /llava/trl/extras/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/trl/extras/__init__.py -------------------------------------------------------------------------------- /llava/trl/extras/best_of_n_sampler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/trl/extras/best_of_n_sampler.py -------------------------------------------------------------------------------- /llava/trl/extras/dataset_formatting.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/trl/extras/dataset_formatting.py -------------------------------------------------------------------------------- /llava/trl/import_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/trl/import_utils.py -------------------------------------------------------------------------------- /llava/trl/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/trl/models/__init__.py -------------------------------------------------------------------------------- /llava/trl/models/modeling_base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/trl/models/modeling_base.py -------------------------------------------------------------------------------- /llava/trl/models/modeling_sd_base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/trl/models/modeling_sd_base.py -------------------------------------------------------------------------------- /llava/trl/models/modeling_value_head.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/trl/models/modeling_value_head.py -------------------------------------------------------------------------------- /llava/trl/models/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/trl/models/utils.py -------------------------------------------------------------------------------- /llava/trl/trainer/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/trl/trainer/__init__.py -------------------------------------------------------------------------------- /llava/trl/trainer/base.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/trl/trainer/base.py -------------------------------------------------------------------------------- /llava/trl/trainer/ddpo_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/trl/trainer/ddpo_config.py -------------------------------------------------------------------------------- /llava/trl/trainer/ddpo_trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/trl/trainer/ddpo_trainer.py -------------------------------------------------------------------------------- /llava/trl/trainer/dpo_trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/trl/trainer/dpo_trainer.py -------------------------------------------------------------------------------- /llava/trl/trainer/iterative_sft_trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/trl/trainer/iterative_sft_trainer.py -------------------------------------------------------------------------------- /llava/trl/trainer/model_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/trl/trainer/model_config.py -------------------------------------------------------------------------------- /llava/trl/trainer/ppo_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/trl/trainer/ppo_config.py -------------------------------------------------------------------------------- /llava/trl/trainer/ppo_trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/trl/trainer/ppo_trainer.py -------------------------------------------------------------------------------- /llava/trl/trainer/reward_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/trl/trainer/reward_config.py -------------------------------------------------------------------------------- /llava/trl/trainer/reward_trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/trl/trainer/reward_trainer.py -------------------------------------------------------------------------------- /llava/trl/trainer/sft_trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/trl/trainer/sft_trainer.py -------------------------------------------------------------------------------- /llava/trl/trainer/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/trl/trainer/utils.py -------------------------------------------------------------------------------- /llava/utils/__init__.py: -------------------------------------------------------------------------------- 1 | from .utils import * 2 | -------------------------------------------------------------------------------- /llava/utils/distributed.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/utils/distributed.py -------------------------------------------------------------------------------- /llava/utils/io.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/utils/io.py -------------------------------------------------------------------------------- /llava/utils/logging.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/utils/logging.py -------------------------------------------------------------------------------- /llava/utils/media.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/utils/media.py -------------------------------------------------------------------------------- /llava/utils/merge_lora_weights_and_save_hf_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/utils/merge_lora_weights_and_save_hf_model.py -------------------------------------------------------------------------------- /llava/utils/tokenizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/utils/tokenizer.py -------------------------------------------------------------------------------- /llava/utils/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/utils/utils.py -------------------------------------------------------------------------------- /llava/wids/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/wids/__init__.py -------------------------------------------------------------------------------- /llava/wids/wids.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/wids/wids.py -------------------------------------------------------------------------------- /llava/wids/wids_bench.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/wids/wids_bench.py -------------------------------------------------------------------------------- /llava/wids/wids_cleanup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/wids/wids_cleanup.py -------------------------------------------------------------------------------- /llava/wids/wids_dir.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/wids/wids_dir.py -------------------------------------------------------------------------------- /llava/wids/wids_dl.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/wids/wids_dl.py -------------------------------------------------------------------------------- /llava/wids/wids_index.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/wids/wids_index.py -------------------------------------------------------------------------------- /llava/wids/wids_lru.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/wids/wids_lru.py -------------------------------------------------------------------------------- /llava/wids/wids_mmtar.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/wids/wids_mmtar.py -------------------------------------------------------------------------------- /llava/wids/wids_specs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/wids/wids_specs.py -------------------------------------------------------------------------------- /llava/wids/wids_tar.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/llava/wids/wids_tar.py -------------------------------------------------------------------------------- /longvila/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/longvila/README.md -------------------------------------------------------------------------------- /longvila/train/4_extend_llm_1M.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/longvila/train/4_extend_llm_1M.sh -------------------------------------------------------------------------------- /longvila/train/4_extend_llm_256k.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/longvila/train/4_extend_llm_256k.sh -------------------------------------------------------------------------------- /longvila/train/4_extend_llm_512k.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/longvila/train/4_extend_llm_512k.sh -------------------------------------------------------------------------------- /longvila/train/4_extend_llm_64k.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/longvila/train/4_extend_llm_64k.sh -------------------------------------------------------------------------------- /longvila/train/5_long_sft_1024frames.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/longvila/train/5_long_sft_1024frames.sh -------------------------------------------------------------------------------- /longvila/train/5_long_sft_2048frames.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/longvila/train/5_long_sft_2048frames.sh -------------------------------------------------------------------------------- /longvila/train/5_long_sft_256frames.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/longvila/train/5_long_sft_256frames.sh -------------------------------------------------------------------------------- /longvila/train/5_long_sft_512frames.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/longvila/train/5_long_sft_512frames.sh -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/pyproject.toml -------------------------------------------------------------------------------- /scripts/NVILA-Lite/align.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/scripts/NVILA-Lite/align.sh -------------------------------------------------------------------------------- /scripts/NVILA-Lite/pretrain.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/scripts/NVILA-Lite/pretrain.sh -------------------------------------------------------------------------------- /scripts/NVILA-Lite/sft.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/scripts/NVILA-Lite/sft.sh -------------------------------------------------------------------------------- /scripts/NVILA-Lite/stage15.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/scripts/NVILA-Lite/stage15.sh -------------------------------------------------------------------------------- /scripts/NVILA/stage15_9tile.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/scripts/NVILA/stage15_9tile.sh -------------------------------------------------------------------------------- /scripts/NVILA/stage1_9tile.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/scripts/NVILA/stage1_9tile.sh -------------------------------------------------------------------------------- /scripts/NVILA/stage2_9tile.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/scripts/NVILA/stage2_9tile.sh -------------------------------------------------------------------------------- /scripts/NVILA/stage3_9tile.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/scripts/NVILA/stage3_9tile.sh -------------------------------------------------------------------------------- /scripts/NVILA/stage4.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/scripts/NVILA/stage4.sh -------------------------------------------------------------------------------- /scripts/deepspeed_inference.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/scripts/deepspeed_inference.yaml -------------------------------------------------------------------------------- /scripts/eval/activitynet-rtl.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/scripts/eval/activitynet-rtl.sh -------------------------------------------------------------------------------- /scripts/eval/cinepile.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/scripts/eval/cinepile.sh -------------------------------------------------------------------------------- /scripts/eval/egoschema.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/scripts/eval/egoschema.sh -------------------------------------------------------------------------------- /scripts/eval/eventbench.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/scripts/eval/eventbench.sh -------------------------------------------------------------------------------- /scripts/eval/lmms.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/scripts/eval/lmms.sh -------------------------------------------------------------------------------- /scripts/eval/mathvista.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/scripts/eval/mathvista.sh -------------------------------------------------------------------------------- /scripts/eval/needle.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/scripts/eval/needle.sh -------------------------------------------------------------------------------- /scripts/eval/vnbench.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/scripts/eval/vnbench.sh -------------------------------------------------------------------------------- /scripts/fp8training/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/scripts/fp8training/README.md -------------------------------------------------------------------------------- /scripts/fp8training/run.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/scripts/fp8training/run.sh -------------------------------------------------------------------------------- /scripts/fp8training/sft_qwen_fp16.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/scripts/fp8training/sft_qwen_fp16.sh -------------------------------------------------------------------------------- /scripts/fp8training/sft_qwen_fp8.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/scripts/fp8training/sft_qwen_fp8.sh -------------------------------------------------------------------------------- /scripts/setups/train.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/scripts/setups/train.sh -------------------------------------------------------------------------------- /scripts/zero2.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/scripts/zero2.json -------------------------------------------------------------------------------- /scripts/zero3.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/scripts/zero3.json -------------------------------------------------------------------------------- /scripts/zero3_gradient_clipping.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/scripts/zero3_gradient_clipping.json -------------------------------------------------------------------------------- /scripts/zero3_mics_mini_fixed.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/scripts/zero3_mics_mini_fixed.json -------------------------------------------------------------------------------- /scripts/zero3_offload_inference.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/scripts/zero3_offload_inference.json -------------------------------------------------------------------------------- /scripts/zero3pp.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/scripts/zero3pp.json -------------------------------------------------------------------------------- /server.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/server.py -------------------------------------------------------------------------------- /serving/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/serving/README.md -------------------------------------------------------------------------------- /serving/query_nvila.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/serving/query_nvila.py -------------------------------------------------------------------------------- /serving/server.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/serving/server.py -------------------------------------------------------------------------------- /vila_hd/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/vila_hd/README.md -------------------------------------------------------------------------------- /vila_hd/assets/4kpro_results.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/vila_hd/assets/4kpro_results.png -------------------------------------------------------------------------------- /vila_hd/assets/av_example_1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/vila_hd/assets/av_example_1.jpg -------------------------------------------------------------------------------- /vila_hd/assets/teaser.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/vila_hd/assets/teaser.png -------------------------------------------------------------------------------- /vila_hd/assets/vila_hd_general_results.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/vila_hd/assets/vila_hd_general_results.png -------------------------------------------------------------------------------- /vila_hd/assets/vila_hd_highres_results.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/NVlabs/VILA/HEAD/vila_hd/assets/vila_hd_highres_results.png --------------------------------------------------------------------------------