├── .gitignore ├── README.md ├── assets └── tree.jpg ├── data └── summary.py ├── data_generation ├── general │ ├── detail_23k_grounding.json │ ├── seeds │ │ ├── conv001.txt │ │ ├── conv002.txt │ │ ├── conv003.txt │ │ ├── conv004.txt │ │ ├── conv005.txt │ │ ├── conv006.txt │ │ ├── conv007.txt │ │ ├── conv008.txt │ │ ├── conv009.txt │ │ ├── conv010.txt │ │ ├── desc001.txt │ │ ├── desc002.txt │ │ ├── desc003.txt │ │ ├── desc004.txt │ │ ├── desc005.txt │ │ ├── desc006.txt │ │ ├── desc007.txt │ │ ├── desc008.txt │ │ ├── desc009.txt │ │ └── desc010.txt │ ├── source_file.jsonl │ └── system_message.txt └── task_specific │ ├── multi_turn │ ├── examples.jsonl │ ├── gen_answer_system_message.txt │ ├── gen_question_system_message.txt │ ├── seed_gen_answer_conv001.txt │ ├── seed_gen_answer_conv002.txt │ ├── seed_gen_answer_desc001.txt │ ├── seed_gen_answer_desc002.txt │ ├── seed_gen_question_conv001.txt │ ├── seed_gen_question_conv002.txt │ ├── seed_gen_question_desc001.txt │ └── seed_gen_question_desc002.txt │ └── single_turn │ ├── 1_seed_small_objects.txt │ ├── 1_small_objects.txt │ ├── 1_system_message_small_objects.txt │ ├── 2_object_relationship.txt │ ├── 2_seed_object_relationship.txt │ ├── 2_system_message_object_relationship.txt │ ├── 3_ocr.txt │ ├── 3_seed_ocr.txt │ ├── 3_system_message_ocr.txt │ ├── 4_object_attribute.txt │ ├── 4_seed_object_attribute.txt │ ├── 4_system_message_object_attribute.txt │ ├── 5_multi_objects.txt │ ├── 5_seed_multi_objects.txt │ └── 5_system_message_multi_objects.txt ├── demo ├── README.md └── home.py ├── fine_eval ├── gpt4roi.jsonl ├── images │ ├── 0000.jpg │ ├── 0001.jpg │ ├── 0002.jpg │ ├── 0003.jpg │ ├── 0004.jpg │ ├── 0005.jpg │ ├── 0006.jpg │ ├── 0007.jpg │ ├── 0008.jpg │ ├── 0009.jpg │ ├── 0010.jpg │ ├── 0011.jpg │ ├── 0012.jpg │ ├── 0013.jpg │ ├── 0014.jpg │ ├── 0015.jpg │ ├── 0016.jpg │ ├── 0017.jpg │ ├── 0018.jpg │ ├── 0019.jpg │ ├── 0020.jpg │ ├── 0021.jpg │ ├── 0022.jpg │ ├── 0023.jpg │ ├── 0024.jpg │ ├── 0025.jpg │ ├── 0026.jpg │ ├── 0027.jpg │ ├── 0028.jpg │ ├── 0029.jpg │ ├── 0030.jpg │ ├── 0031.jpg │ ├── 0032.jpg │ ├── 0033.jpg │ ├── 0034.jpg │ ├── 0035.jpg │ ├── 0036.jpg │ ├── 0037.jpg │ ├── 0038.jpg │ ├── 0039.jpg │ ├── 0040.jpg │ ├── 0041.jpg │ ├── 0042.jpg │ ├── 0043.jpg │ ├── 0044.jpg │ ├── 0045.jpg │ ├── 0046.jpg │ ├── 0047.jpg │ ├── 0048.jpg │ └── 0049.jpg ├── instructions.jsonl ├── llava.jsonl ├── pvit.jsonl └── shikra.jsonl ├── images └── example.jpg ├── pvit ├── __init__.py ├── __pycache__ │ ├── __init__.cpython-39.pyc │ ├── constants.cpython-39.pyc │ ├── conversation.cpython-39.pyc │ └── utils.cpython-39.pyc ├── constants.py ├── conversation.py ├── data │ ├── __init__.py │ ├── __pycache__ │ │ ├── __init__.cpython-39.pyc │ │ ├── inspect.cpython-39.pyc │ │ ├── multimodal_instruction.cpython-39.pyc │ │ ├── multimodal_instruction_prompt.cpython-39.pyc │ │ ├── sam_transforms.cpython-39.pyc │ │ ├── transforms.cpython-39.pyc │ │ └── utils.cpython-39.pyc │ ├── multimodal_instruction.py │ ├── transforms.py │ └── utils.py ├── model │ ├── __init__.py │ ├── __pycache__ │ │ ├── __init__.cpython-39.pyc │ │ ├── apply_delta.cpython-39.pyc │ │ ├── llava.cpython-39.pyc │ │ ├── prompt_model.cpython-39.pyc │ │ └── region_clip.cpython-39.pyc │ ├── apply_delta.py │ ├── llava.py │ ├── losses.py │ ├── make_delta.py │ ├── prompt_model.py │ ├── region_clip.py │ ├── region_clip_config.yaml │ └── utils.py ├── run_cli.py ├── run_fine_eval.py ├── serve │ ├── __init__.py │ ├── __pycache__ │ │ ├── __init__.cpython-39.pyc │ │ ├── controller.cpython-39.pyc │ │ ├── model_worker.cpython-39.pyc │ │ └── test_message.cpython-39.pyc │ ├── cli.py │ ├── controller.py │ ├── examples │ │ ├── extreme_ironing.jpg │ │ └── waterview.jpg │ ├── gateway │ │ ├── README.md │ │ └── nginx.conf │ ├── model_worker.py │ ├── register_worker.py │ └── test_message.py ├── train │ ├── __pycache__ │ │ ├── llama_flash_attn_monkey_patch.cpython-39.pyc │ │ ├── llava_trainer.cpython-39.pyc │ │ └── train.cpython-39.pyc │ ├── llama_flash_attn_monkey_patch.py │ ├── llava_trainer.py │ ├── train.py │ ├── train_lora.py │ └── train_mem.py └── utils.py ├── requirements.txt └── scripts ├── delta_apply.sh ├── delta_make.sh ├── model_up.sh ├── run_cli.sh ├── run_demo.sh ├── run_fine_eval.sh ├── train_stage_1.sh └── train_stage_2.sh /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/.gitignore -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/README.md -------------------------------------------------------------------------------- /assets/tree.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/assets/tree.jpg -------------------------------------------------------------------------------- /data/summary.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/data/summary.py -------------------------------------------------------------------------------- /data_generation/general/detail_23k_grounding.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/data_generation/general/detail_23k_grounding.json -------------------------------------------------------------------------------- /data_generation/general/seeds/conv001.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/data_generation/general/seeds/conv001.txt -------------------------------------------------------------------------------- /data_generation/general/seeds/conv002.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/data_generation/general/seeds/conv002.txt -------------------------------------------------------------------------------- /data_generation/general/seeds/conv003.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/data_generation/general/seeds/conv003.txt -------------------------------------------------------------------------------- /data_generation/general/seeds/conv004.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/data_generation/general/seeds/conv004.txt -------------------------------------------------------------------------------- /data_generation/general/seeds/conv005.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/data_generation/general/seeds/conv005.txt -------------------------------------------------------------------------------- /data_generation/general/seeds/conv006.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/data_generation/general/seeds/conv006.txt -------------------------------------------------------------------------------- /data_generation/general/seeds/conv007.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/data_generation/general/seeds/conv007.txt -------------------------------------------------------------------------------- /data_generation/general/seeds/conv008.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/data_generation/general/seeds/conv008.txt -------------------------------------------------------------------------------- /data_generation/general/seeds/conv009.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/data_generation/general/seeds/conv009.txt -------------------------------------------------------------------------------- /data_generation/general/seeds/conv010.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/data_generation/general/seeds/conv010.txt -------------------------------------------------------------------------------- /data_generation/general/seeds/desc001.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/data_generation/general/seeds/desc001.txt -------------------------------------------------------------------------------- /data_generation/general/seeds/desc002.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/data_generation/general/seeds/desc002.txt -------------------------------------------------------------------------------- /data_generation/general/seeds/desc003.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/data_generation/general/seeds/desc003.txt -------------------------------------------------------------------------------- /data_generation/general/seeds/desc004.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/data_generation/general/seeds/desc004.txt -------------------------------------------------------------------------------- /data_generation/general/seeds/desc005.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/data_generation/general/seeds/desc005.txt -------------------------------------------------------------------------------- /data_generation/general/seeds/desc006.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/data_generation/general/seeds/desc006.txt -------------------------------------------------------------------------------- /data_generation/general/seeds/desc007.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/data_generation/general/seeds/desc007.txt -------------------------------------------------------------------------------- /data_generation/general/seeds/desc008.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/data_generation/general/seeds/desc008.txt -------------------------------------------------------------------------------- /data_generation/general/seeds/desc009.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/data_generation/general/seeds/desc009.txt -------------------------------------------------------------------------------- /data_generation/general/seeds/desc010.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/data_generation/general/seeds/desc010.txt -------------------------------------------------------------------------------- /data_generation/general/source_file.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/data_generation/general/source_file.jsonl -------------------------------------------------------------------------------- /data_generation/general/system_message.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/data_generation/general/system_message.txt -------------------------------------------------------------------------------- /data_generation/task_specific/multi_turn/examples.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/data_generation/task_specific/multi_turn/examples.jsonl -------------------------------------------------------------------------------- /data_generation/task_specific/multi_turn/gen_answer_system_message.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/data_generation/task_specific/multi_turn/gen_answer_system_message.txt -------------------------------------------------------------------------------- /data_generation/task_specific/multi_turn/gen_question_system_message.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/data_generation/task_specific/multi_turn/gen_question_system_message.txt -------------------------------------------------------------------------------- /data_generation/task_specific/multi_turn/seed_gen_answer_conv001.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/data_generation/task_specific/multi_turn/seed_gen_answer_conv001.txt -------------------------------------------------------------------------------- /data_generation/task_specific/multi_turn/seed_gen_answer_conv002.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/data_generation/task_specific/multi_turn/seed_gen_answer_conv002.txt -------------------------------------------------------------------------------- /data_generation/task_specific/multi_turn/seed_gen_answer_desc001.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/data_generation/task_specific/multi_turn/seed_gen_answer_desc001.txt -------------------------------------------------------------------------------- /data_generation/task_specific/multi_turn/seed_gen_answer_desc002.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/data_generation/task_specific/multi_turn/seed_gen_answer_desc002.txt -------------------------------------------------------------------------------- /data_generation/task_specific/multi_turn/seed_gen_question_conv001.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/data_generation/task_specific/multi_turn/seed_gen_question_conv001.txt -------------------------------------------------------------------------------- /data_generation/task_specific/multi_turn/seed_gen_question_conv002.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/data_generation/task_specific/multi_turn/seed_gen_question_conv002.txt -------------------------------------------------------------------------------- /data_generation/task_specific/multi_turn/seed_gen_question_desc001.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/data_generation/task_specific/multi_turn/seed_gen_question_desc001.txt -------------------------------------------------------------------------------- /data_generation/task_specific/multi_turn/seed_gen_question_desc002.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/data_generation/task_specific/multi_turn/seed_gen_question_desc002.txt -------------------------------------------------------------------------------- /data_generation/task_specific/single_turn/1_seed_small_objects.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/data_generation/task_specific/single_turn/1_seed_small_objects.txt -------------------------------------------------------------------------------- /data_generation/task_specific/single_turn/1_small_objects.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/data_generation/task_specific/single_turn/1_small_objects.txt -------------------------------------------------------------------------------- /data_generation/task_specific/single_turn/1_system_message_small_objects.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/data_generation/task_specific/single_turn/1_system_message_small_objects.txt -------------------------------------------------------------------------------- /data_generation/task_specific/single_turn/2_object_relationship.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/data_generation/task_specific/single_turn/2_object_relationship.txt -------------------------------------------------------------------------------- /data_generation/task_specific/single_turn/2_seed_object_relationship.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/data_generation/task_specific/single_turn/2_seed_object_relationship.txt -------------------------------------------------------------------------------- /data_generation/task_specific/single_turn/2_system_message_object_relationship.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/data_generation/task_specific/single_turn/2_system_message_object_relationship.txt -------------------------------------------------------------------------------- /data_generation/task_specific/single_turn/3_ocr.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/data_generation/task_specific/single_turn/3_ocr.txt -------------------------------------------------------------------------------- /data_generation/task_specific/single_turn/3_seed_ocr.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/data_generation/task_specific/single_turn/3_seed_ocr.txt -------------------------------------------------------------------------------- /data_generation/task_specific/single_turn/3_system_message_ocr.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/data_generation/task_specific/single_turn/3_system_message_ocr.txt -------------------------------------------------------------------------------- /data_generation/task_specific/single_turn/4_object_attribute.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/data_generation/task_specific/single_turn/4_object_attribute.txt -------------------------------------------------------------------------------- /data_generation/task_specific/single_turn/4_seed_object_attribute.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/data_generation/task_specific/single_turn/4_seed_object_attribute.txt -------------------------------------------------------------------------------- /data_generation/task_specific/single_turn/4_system_message_object_attribute.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/data_generation/task_specific/single_turn/4_system_message_object_attribute.txt -------------------------------------------------------------------------------- /data_generation/task_specific/single_turn/5_multi_objects.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/data_generation/task_specific/single_turn/5_multi_objects.txt -------------------------------------------------------------------------------- /data_generation/task_specific/single_turn/5_seed_multi_objects.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/data_generation/task_specific/single_turn/5_seed_multi_objects.txt -------------------------------------------------------------------------------- /data_generation/task_specific/single_turn/5_system_message_multi_objects.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/data_generation/task_specific/single_turn/5_system_message_multi_objects.txt -------------------------------------------------------------------------------- /demo/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/demo/README.md -------------------------------------------------------------------------------- /demo/home.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/demo/home.py -------------------------------------------------------------------------------- /fine_eval/gpt4roi.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/fine_eval/gpt4roi.jsonl -------------------------------------------------------------------------------- /fine_eval/images/0000.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/fine_eval/images/0000.jpg -------------------------------------------------------------------------------- /fine_eval/images/0001.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/fine_eval/images/0001.jpg -------------------------------------------------------------------------------- /fine_eval/images/0002.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/fine_eval/images/0002.jpg -------------------------------------------------------------------------------- /fine_eval/images/0003.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/fine_eval/images/0003.jpg -------------------------------------------------------------------------------- /fine_eval/images/0004.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/fine_eval/images/0004.jpg -------------------------------------------------------------------------------- /fine_eval/images/0005.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/fine_eval/images/0005.jpg -------------------------------------------------------------------------------- /fine_eval/images/0006.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/fine_eval/images/0006.jpg -------------------------------------------------------------------------------- /fine_eval/images/0007.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/fine_eval/images/0007.jpg -------------------------------------------------------------------------------- /fine_eval/images/0008.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/fine_eval/images/0008.jpg -------------------------------------------------------------------------------- /fine_eval/images/0009.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/fine_eval/images/0009.jpg -------------------------------------------------------------------------------- /fine_eval/images/0010.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/fine_eval/images/0010.jpg -------------------------------------------------------------------------------- /fine_eval/images/0011.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/fine_eval/images/0011.jpg -------------------------------------------------------------------------------- /fine_eval/images/0012.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/fine_eval/images/0012.jpg -------------------------------------------------------------------------------- /fine_eval/images/0013.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/fine_eval/images/0013.jpg -------------------------------------------------------------------------------- /fine_eval/images/0014.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/fine_eval/images/0014.jpg -------------------------------------------------------------------------------- /fine_eval/images/0015.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/fine_eval/images/0015.jpg -------------------------------------------------------------------------------- /fine_eval/images/0016.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/fine_eval/images/0016.jpg -------------------------------------------------------------------------------- /fine_eval/images/0017.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/fine_eval/images/0017.jpg -------------------------------------------------------------------------------- /fine_eval/images/0018.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/fine_eval/images/0018.jpg -------------------------------------------------------------------------------- /fine_eval/images/0019.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/fine_eval/images/0019.jpg -------------------------------------------------------------------------------- /fine_eval/images/0020.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/fine_eval/images/0020.jpg -------------------------------------------------------------------------------- /fine_eval/images/0021.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/fine_eval/images/0021.jpg -------------------------------------------------------------------------------- /fine_eval/images/0022.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/fine_eval/images/0022.jpg -------------------------------------------------------------------------------- /fine_eval/images/0023.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/fine_eval/images/0023.jpg -------------------------------------------------------------------------------- /fine_eval/images/0024.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/fine_eval/images/0024.jpg -------------------------------------------------------------------------------- /fine_eval/images/0025.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/fine_eval/images/0025.jpg -------------------------------------------------------------------------------- /fine_eval/images/0026.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/fine_eval/images/0026.jpg -------------------------------------------------------------------------------- /fine_eval/images/0027.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/fine_eval/images/0027.jpg -------------------------------------------------------------------------------- /fine_eval/images/0028.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/fine_eval/images/0028.jpg -------------------------------------------------------------------------------- /fine_eval/images/0029.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/fine_eval/images/0029.jpg -------------------------------------------------------------------------------- /fine_eval/images/0030.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/fine_eval/images/0030.jpg -------------------------------------------------------------------------------- /fine_eval/images/0031.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/fine_eval/images/0031.jpg -------------------------------------------------------------------------------- /fine_eval/images/0032.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/fine_eval/images/0032.jpg -------------------------------------------------------------------------------- /fine_eval/images/0033.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/fine_eval/images/0033.jpg -------------------------------------------------------------------------------- /fine_eval/images/0034.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/fine_eval/images/0034.jpg -------------------------------------------------------------------------------- /fine_eval/images/0035.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/fine_eval/images/0035.jpg -------------------------------------------------------------------------------- /fine_eval/images/0036.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/fine_eval/images/0036.jpg -------------------------------------------------------------------------------- /fine_eval/images/0037.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/fine_eval/images/0037.jpg -------------------------------------------------------------------------------- /fine_eval/images/0038.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/fine_eval/images/0038.jpg -------------------------------------------------------------------------------- /fine_eval/images/0039.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/fine_eval/images/0039.jpg -------------------------------------------------------------------------------- /fine_eval/images/0040.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/fine_eval/images/0040.jpg -------------------------------------------------------------------------------- /fine_eval/images/0041.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/fine_eval/images/0041.jpg -------------------------------------------------------------------------------- /fine_eval/images/0042.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/fine_eval/images/0042.jpg -------------------------------------------------------------------------------- /fine_eval/images/0043.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/fine_eval/images/0043.jpg -------------------------------------------------------------------------------- /fine_eval/images/0044.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/fine_eval/images/0044.jpg -------------------------------------------------------------------------------- /fine_eval/images/0045.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/fine_eval/images/0045.jpg -------------------------------------------------------------------------------- /fine_eval/images/0046.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/fine_eval/images/0046.jpg -------------------------------------------------------------------------------- /fine_eval/images/0047.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/fine_eval/images/0047.jpg -------------------------------------------------------------------------------- /fine_eval/images/0048.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/fine_eval/images/0048.jpg -------------------------------------------------------------------------------- /fine_eval/images/0049.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/fine_eval/images/0049.jpg -------------------------------------------------------------------------------- /fine_eval/instructions.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/fine_eval/instructions.jsonl -------------------------------------------------------------------------------- /fine_eval/llava.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/fine_eval/llava.jsonl -------------------------------------------------------------------------------- /fine_eval/pvit.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/fine_eval/pvit.jsonl -------------------------------------------------------------------------------- /fine_eval/shikra.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/fine_eval/shikra.jsonl -------------------------------------------------------------------------------- /images/example.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/images/example.jpg -------------------------------------------------------------------------------- /pvit/__init__.py: -------------------------------------------------------------------------------- 1 | from .model import LlavaLlamaForCausalLM 2 | -------------------------------------------------------------------------------- /pvit/__pycache__/__init__.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/pvit/__pycache__/__init__.cpython-39.pyc -------------------------------------------------------------------------------- /pvit/__pycache__/constants.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/pvit/__pycache__/constants.cpython-39.pyc -------------------------------------------------------------------------------- /pvit/__pycache__/conversation.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/pvit/__pycache__/conversation.cpython-39.pyc -------------------------------------------------------------------------------- /pvit/__pycache__/utils.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/pvit/__pycache__/utils.cpython-39.pyc -------------------------------------------------------------------------------- /pvit/constants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/pvit/constants.py -------------------------------------------------------------------------------- /pvit/conversation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/pvit/conversation.py -------------------------------------------------------------------------------- /pvit/data/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pvit/data/__pycache__/__init__.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/pvit/data/__pycache__/__init__.cpython-39.pyc -------------------------------------------------------------------------------- /pvit/data/__pycache__/inspect.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/pvit/data/__pycache__/inspect.cpython-39.pyc -------------------------------------------------------------------------------- /pvit/data/__pycache__/multimodal_instruction.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/pvit/data/__pycache__/multimodal_instruction.cpython-39.pyc -------------------------------------------------------------------------------- /pvit/data/__pycache__/multimodal_instruction_prompt.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/pvit/data/__pycache__/multimodal_instruction_prompt.cpython-39.pyc -------------------------------------------------------------------------------- /pvit/data/__pycache__/sam_transforms.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/pvit/data/__pycache__/sam_transforms.cpython-39.pyc -------------------------------------------------------------------------------- /pvit/data/__pycache__/transforms.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/pvit/data/__pycache__/transforms.cpython-39.pyc -------------------------------------------------------------------------------- /pvit/data/__pycache__/utils.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/pvit/data/__pycache__/utils.cpython-39.pyc -------------------------------------------------------------------------------- /pvit/data/multimodal_instruction.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/pvit/data/multimodal_instruction.py -------------------------------------------------------------------------------- /pvit/data/transforms.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/pvit/data/transforms.py -------------------------------------------------------------------------------- /pvit/data/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/pvit/data/utils.py -------------------------------------------------------------------------------- /pvit/model/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/pvit/model/__init__.py -------------------------------------------------------------------------------- /pvit/model/__pycache__/__init__.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/pvit/model/__pycache__/__init__.cpython-39.pyc -------------------------------------------------------------------------------- /pvit/model/__pycache__/apply_delta.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/pvit/model/__pycache__/apply_delta.cpython-39.pyc -------------------------------------------------------------------------------- /pvit/model/__pycache__/llava.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/pvit/model/__pycache__/llava.cpython-39.pyc -------------------------------------------------------------------------------- /pvit/model/__pycache__/prompt_model.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/pvit/model/__pycache__/prompt_model.cpython-39.pyc -------------------------------------------------------------------------------- /pvit/model/__pycache__/region_clip.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/pvit/model/__pycache__/region_clip.cpython-39.pyc -------------------------------------------------------------------------------- /pvit/model/apply_delta.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/pvit/model/apply_delta.py -------------------------------------------------------------------------------- /pvit/model/llava.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/pvit/model/llava.py -------------------------------------------------------------------------------- /pvit/model/losses.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/pvit/model/losses.py -------------------------------------------------------------------------------- /pvit/model/make_delta.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/pvit/model/make_delta.py -------------------------------------------------------------------------------- /pvit/model/prompt_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/pvit/model/prompt_model.py -------------------------------------------------------------------------------- /pvit/model/region_clip.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/pvit/model/region_clip.py -------------------------------------------------------------------------------- /pvit/model/region_clip_config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/pvit/model/region_clip_config.yaml -------------------------------------------------------------------------------- /pvit/model/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/pvit/model/utils.py -------------------------------------------------------------------------------- /pvit/run_cli.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/pvit/run_cli.py -------------------------------------------------------------------------------- /pvit/run_fine_eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/pvit/run_fine_eval.py -------------------------------------------------------------------------------- /pvit/serve/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pvit/serve/__pycache__/__init__.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/pvit/serve/__pycache__/__init__.cpython-39.pyc -------------------------------------------------------------------------------- /pvit/serve/__pycache__/controller.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/pvit/serve/__pycache__/controller.cpython-39.pyc -------------------------------------------------------------------------------- /pvit/serve/__pycache__/model_worker.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/pvit/serve/__pycache__/model_worker.cpython-39.pyc -------------------------------------------------------------------------------- /pvit/serve/__pycache__/test_message.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/pvit/serve/__pycache__/test_message.cpython-39.pyc -------------------------------------------------------------------------------- /pvit/serve/cli.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/pvit/serve/cli.py -------------------------------------------------------------------------------- /pvit/serve/controller.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/pvit/serve/controller.py -------------------------------------------------------------------------------- /pvit/serve/examples/extreme_ironing.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/pvit/serve/examples/extreme_ironing.jpg -------------------------------------------------------------------------------- /pvit/serve/examples/waterview.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/pvit/serve/examples/waterview.jpg -------------------------------------------------------------------------------- /pvit/serve/gateway/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/pvit/serve/gateway/README.md -------------------------------------------------------------------------------- /pvit/serve/gateway/nginx.conf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/pvit/serve/gateway/nginx.conf -------------------------------------------------------------------------------- /pvit/serve/model_worker.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/pvit/serve/model_worker.py -------------------------------------------------------------------------------- /pvit/serve/register_worker.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/pvit/serve/register_worker.py -------------------------------------------------------------------------------- /pvit/serve/test_message.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/pvit/serve/test_message.py -------------------------------------------------------------------------------- /pvit/train/__pycache__/llama_flash_attn_monkey_patch.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/pvit/train/__pycache__/llama_flash_attn_monkey_patch.cpython-39.pyc -------------------------------------------------------------------------------- /pvit/train/__pycache__/llava_trainer.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/pvit/train/__pycache__/llava_trainer.cpython-39.pyc -------------------------------------------------------------------------------- /pvit/train/__pycache__/train.cpython-39.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/pvit/train/__pycache__/train.cpython-39.pyc -------------------------------------------------------------------------------- /pvit/train/llama_flash_attn_monkey_patch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/pvit/train/llama_flash_attn_monkey_patch.py -------------------------------------------------------------------------------- /pvit/train/llava_trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/pvit/train/llava_trainer.py -------------------------------------------------------------------------------- /pvit/train/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/pvit/train/train.py -------------------------------------------------------------------------------- /pvit/train/train_lora.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/pvit/train/train_lora.py -------------------------------------------------------------------------------- /pvit/train/train_mem.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/pvit/train/train_mem.py -------------------------------------------------------------------------------- /pvit/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/pvit/utils.py -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/requirements.txt -------------------------------------------------------------------------------- /scripts/delta_apply.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/scripts/delta_apply.sh -------------------------------------------------------------------------------- /scripts/delta_make.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/scripts/delta_make.sh -------------------------------------------------------------------------------- /scripts/model_up.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/scripts/model_up.sh -------------------------------------------------------------------------------- /scripts/run_cli.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/scripts/run_cli.sh -------------------------------------------------------------------------------- /scripts/run_demo.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/scripts/run_demo.sh -------------------------------------------------------------------------------- /scripts/run_fine_eval.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/scripts/run_fine_eval.sh -------------------------------------------------------------------------------- /scripts/train_stage_1.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/scripts/train_stage_1.sh -------------------------------------------------------------------------------- /scripts/train_stage_2.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PVIT-official/PVIT/HEAD/scripts/train_stage_2.sh --------------------------------------------------------------------------------