├── .gitignore ├── LICENSE ├── README.md ├── assets ├── data.png ├── demo_all.gif ├── demo_box.gif ├── demo_point.gif ├── framework.png ├── osprey.png ├── performance.png ├── qmsht.gif ├── qyqx.gif ├── table1.png ├── table2.png ├── table3.png ├── table4.png ├── table5.png ├── table6.png └── video_cover.png ├── dataset.md ├── demo ├── app.py ├── inference.py └── osprey_inference.py ├── osprey ├── __init__.py ├── configs │ ├── stage2.json │ └── stage3.json ├── constants.py ├── conversation.py ├── data_generation │ ├── ask_gpt.py │ ├── concise_qa │ │ ├── ask_example.txt │ │ ├── res_example.txt │ │ └── system_message.txt │ ├── conversation │ │ ├── ask_example.txt │ │ ├── res_example.txt │ │ └── system_message.txt │ ├── data_generation_pipeline.sh │ ├── description │ │ ├── ask_example.txt │ │ ├── res_example.txt │ │ └── system_message.txt │ ├── generate_gpt_prompt.py │ └── gpt_data_generation.py ├── datasets │ ├── data_modules.py │ ├── osprey_724k.py │ ├── stage2_data.py │ ├── vcr.py │ └── vg.py ├── eval │ ├── README.md │ ├── datasets │ │ ├── README.md │ │ ├── ade20k_instance_catid_mapping.txt │ │ ├── prepare_ade20k_ins_seg.py │ │ ├── prepare_ade20k_pan_seg.py │ │ └── prepare_ade20k_sem_seg.py │ ├── description │ │ ├── answers.json │ │ ├── prompt.json │ │ └── questions.json │ ├── eval_gpt.py │ ├── eval_open_vocab_seg_detectron2.py │ ├── ferret-bench │ │ ├── box_refer_caption.json │ │ └── box_refer_reason.json │ ├── ferret_bench_eval.py │ ├── gpt_eval.sh │ ├── lvis_paco_eval.py │ ├── osprey_generate_gpt_description_answer.py │ ├── pope │ │ └── evaluate.py │ ├── pope_eval.py │ ├── pope_eval.sh │ ├── refcocog_eval.py │ ├── rule.json │ ├── summarize_gpt_score.py │ └── utils │ │ ├── __init__.py │ │ ├── ade20k_150_with_prompt_eng.txt │ │ ├── cityscapes_with_prompt_eng.txt │ │ ├── instance_evaluation.py │ │ ├── openseg_classes.py │ │ ├── register_ade20k_panoptic.py │ │ └── register_cityscapes_panoptic.py ├── mm_utils.py ├── model │ ├── __init__.py │ ├── consolidate.py │ ├── language_model │ │ └── osprey_llama.py │ ├── layer.py │ ├── multimodal_encoder │ │ ├── builder.py │ │ ├── clip.py │ │ └── clip_encoder.py │ ├── multimodal_projector │ │ └── builder.py │ └── osprey_arch.py ├── train │ ├── llama_flash_attn_monkey_patch.py │ ├── osprey_trainer.py │ ├── train.py │ └── train_mem.py └── utils.py ├── pyproject.toml └── scripts ├── stage2.sh ├── stage3.sh ├── zero2.json ├── zero3.json └── zero3_offload.json /.gitignore: -------------------------------------------------------------------------------- 1 | *__pycache__ 2 | .DS_Store -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/README.md -------------------------------------------------------------------------------- /assets/data.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/assets/data.png -------------------------------------------------------------------------------- /assets/demo_all.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/assets/demo_all.gif -------------------------------------------------------------------------------- /assets/demo_box.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/assets/demo_box.gif -------------------------------------------------------------------------------- /assets/demo_point.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/assets/demo_point.gif -------------------------------------------------------------------------------- /assets/framework.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/assets/framework.png -------------------------------------------------------------------------------- /assets/osprey.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/assets/osprey.png -------------------------------------------------------------------------------- /assets/performance.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/assets/performance.png -------------------------------------------------------------------------------- /assets/qmsht.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/assets/qmsht.gif -------------------------------------------------------------------------------- /assets/qyqx.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/assets/qyqx.gif -------------------------------------------------------------------------------- /assets/table1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/assets/table1.png -------------------------------------------------------------------------------- /assets/table2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/assets/table2.png -------------------------------------------------------------------------------- /assets/table3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/assets/table3.png -------------------------------------------------------------------------------- /assets/table4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/assets/table4.png -------------------------------------------------------------------------------- /assets/table5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/assets/table5.png -------------------------------------------------------------------------------- /assets/table6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/assets/table6.png -------------------------------------------------------------------------------- /assets/video_cover.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/assets/video_cover.png -------------------------------------------------------------------------------- /dataset.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/dataset.md -------------------------------------------------------------------------------- /demo/app.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/demo/app.py -------------------------------------------------------------------------------- /demo/inference.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/demo/inference.py -------------------------------------------------------------------------------- /demo/osprey_inference.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/demo/osprey_inference.py -------------------------------------------------------------------------------- /osprey/__init__.py: -------------------------------------------------------------------------------- 1 | from .model import OspreyLlamaForCausalLM 2 | -------------------------------------------------------------------------------- /osprey/configs/stage2.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/configs/stage2.json -------------------------------------------------------------------------------- /osprey/configs/stage3.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/configs/stage3.json -------------------------------------------------------------------------------- /osprey/constants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/constants.py -------------------------------------------------------------------------------- /osprey/conversation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/conversation.py -------------------------------------------------------------------------------- /osprey/data_generation/ask_gpt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/data_generation/ask_gpt.py -------------------------------------------------------------------------------- /osprey/data_generation/concise_qa/ask_example.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/data_generation/concise_qa/ask_example.txt -------------------------------------------------------------------------------- /osprey/data_generation/concise_qa/res_example.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/data_generation/concise_qa/res_example.txt -------------------------------------------------------------------------------- /osprey/data_generation/concise_qa/system_message.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/data_generation/concise_qa/system_message.txt -------------------------------------------------------------------------------- /osprey/data_generation/conversation/ask_example.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/data_generation/conversation/ask_example.txt -------------------------------------------------------------------------------- /osprey/data_generation/conversation/res_example.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/data_generation/conversation/res_example.txt -------------------------------------------------------------------------------- /osprey/data_generation/conversation/system_message.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/data_generation/conversation/system_message.txt -------------------------------------------------------------------------------- /osprey/data_generation/data_generation_pipeline.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/data_generation/data_generation_pipeline.sh -------------------------------------------------------------------------------- /osprey/data_generation/description/ask_example.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/data_generation/description/ask_example.txt -------------------------------------------------------------------------------- /osprey/data_generation/description/res_example.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/data_generation/description/res_example.txt -------------------------------------------------------------------------------- /osprey/data_generation/description/system_message.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/data_generation/description/system_message.txt -------------------------------------------------------------------------------- /osprey/data_generation/generate_gpt_prompt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/data_generation/generate_gpt_prompt.py -------------------------------------------------------------------------------- /osprey/data_generation/gpt_data_generation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/data_generation/gpt_data_generation.py -------------------------------------------------------------------------------- /osprey/datasets/data_modules.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/datasets/data_modules.py -------------------------------------------------------------------------------- /osprey/datasets/osprey_724k.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/datasets/osprey_724k.py -------------------------------------------------------------------------------- /osprey/datasets/stage2_data.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/datasets/stage2_data.py -------------------------------------------------------------------------------- /osprey/datasets/vcr.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/datasets/vcr.py -------------------------------------------------------------------------------- /osprey/datasets/vg.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/datasets/vg.py -------------------------------------------------------------------------------- /osprey/eval/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/eval/README.md -------------------------------------------------------------------------------- /osprey/eval/datasets/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/eval/datasets/README.md -------------------------------------------------------------------------------- /osprey/eval/datasets/ade20k_instance_catid_mapping.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/eval/datasets/ade20k_instance_catid_mapping.txt -------------------------------------------------------------------------------- /osprey/eval/datasets/prepare_ade20k_ins_seg.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/eval/datasets/prepare_ade20k_ins_seg.py -------------------------------------------------------------------------------- /osprey/eval/datasets/prepare_ade20k_pan_seg.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/eval/datasets/prepare_ade20k_pan_seg.py -------------------------------------------------------------------------------- /osprey/eval/datasets/prepare_ade20k_sem_seg.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/eval/datasets/prepare_ade20k_sem_seg.py -------------------------------------------------------------------------------- /osprey/eval/description/answers.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/eval/description/answers.json -------------------------------------------------------------------------------- /osprey/eval/description/prompt.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/eval/description/prompt.json -------------------------------------------------------------------------------- /osprey/eval/description/questions.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/eval/description/questions.json -------------------------------------------------------------------------------- /osprey/eval/eval_gpt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/eval/eval_gpt.py -------------------------------------------------------------------------------- /osprey/eval/eval_open_vocab_seg_detectron2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/eval/eval_open_vocab_seg_detectron2.py -------------------------------------------------------------------------------- /osprey/eval/ferret-bench/box_refer_caption.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/eval/ferret-bench/box_refer_caption.json -------------------------------------------------------------------------------- /osprey/eval/ferret-bench/box_refer_reason.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/eval/ferret-bench/box_refer_reason.json -------------------------------------------------------------------------------- /osprey/eval/ferret_bench_eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/eval/ferret_bench_eval.py -------------------------------------------------------------------------------- /osprey/eval/gpt_eval.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/eval/gpt_eval.sh -------------------------------------------------------------------------------- /osprey/eval/lvis_paco_eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/eval/lvis_paco_eval.py -------------------------------------------------------------------------------- /osprey/eval/osprey_generate_gpt_description_answer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/eval/osprey_generate_gpt_description_answer.py -------------------------------------------------------------------------------- /osprey/eval/pope/evaluate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/eval/pope/evaluate.py -------------------------------------------------------------------------------- /osprey/eval/pope_eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/eval/pope_eval.py -------------------------------------------------------------------------------- /osprey/eval/pope_eval.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/eval/pope_eval.sh -------------------------------------------------------------------------------- /osprey/eval/refcocog_eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/eval/refcocog_eval.py -------------------------------------------------------------------------------- /osprey/eval/rule.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/eval/rule.json -------------------------------------------------------------------------------- /osprey/eval/summarize_gpt_score.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/eval/summarize_gpt_score.py -------------------------------------------------------------------------------- /osprey/eval/utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /osprey/eval/utils/ade20k_150_with_prompt_eng.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/eval/utils/ade20k_150_with_prompt_eng.txt -------------------------------------------------------------------------------- /osprey/eval/utils/cityscapes_with_prompt_eng.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/eval/utils/cityscapes_with_prompt_eng.txt -------------------------------------------------------------------------------- /osprey/eval/utils/instance_evaluation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/eval/utils/instance_evaluation.py -------------------------------------------------------------------------------- /osprey/eval/utils/openseg_classes.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/eval/utils/openseg_classes.py -------------------------------------------------------------------------------- /osprey/eval/utils/register_ade20k_panoptic.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/eval/utils/register_ade20k_panoptic.py -------------------------------------------------------------------------------- /osprey/eval/utils/register_cityscapes_panoptic.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/eval/utils/register_cityscapes_panoptic.py -------------------------------------------------------------------------------- /osprey/mm_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/mm_utils.py -------------------------------------------------------------------------------- /osprey/model/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/model/__init__.py -------------------------------------------------------------------------------- /osprey/model/consolidate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/model/consolidate.py -------------------------------------------------------------------------------- /osprey/model/language_model/osprey_llama.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/model/language_model/osprey_llama.py -------------------------------------------------------------------------------- /osprey/model/layer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/model/layer.py -------------------------------------------------------------------------------- /osprey/model/multimodal_encoder/builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/model/multimodal_encoder/builder.py -------------------------------------------------------------------------------- /osprey/model/multimodal_encoder/clip.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/model/multimodal_encoder/clip.py -------------------------------------------------------------------------------- /osprey/model/multimodal_encoder/clip_encoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/model/multimodal_encoder/clip_encoder.py -------------------------------------------------------------------------------- /osprey/model/multimodal_projector/builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/model/multimodal_projector/builder.py -------------------------------------------------------------------------------- /osprey/model/osprey_arch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/model/osprey_arch.py -------------------------------------------------------------------------------- /osprey/train/llama_flash_attn_monkey_patch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/train/llama_flash_attn_monkey_patch.py -------------------------------------------------------------------------------- /osprey/train/osprey_trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/train/osprey_trainer.py -------------------------------------------------------------------------------- /osprey/train/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/train/train.py -------------------------------------------------------------------------------- /osprey/train/train_mem.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/train/train_mem.py -------------------------------------------------------------------------------- /osprey/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/osprey/utils.py -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/pyproject.toml -------------------------------------------------------------------------------- /scripts/stage2.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/scripts/stage2.sh -------------------------------------------------------------------------------- /scripts/stage3.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/scripts/stage3.sh -------------------------------------------------------------------------------- /scripts/zero2.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/scripts/zero2.json -------------------------------------------------------------------------------- /scripts/zero3.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/scripts/zero3.json -------------------------------------------------------------------------------- /scripts/zero3_offload.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CircleRadon/Osprey/HEAD/scripts/zero3_offload.json --------------------------------------------------------------------------------