├── .gitignore ├── CUSTOM_DATA.md ├── LICENSE ├── README.md ├── assets ├── Libra_logo.png ├── Libra_logo_c.png ├── demo.gif ├── example_curent.jpg ├── example_prior.jpg ├── libra_architecture.png ├── result_chart.png └── x-ray-chest.svg ├── libra ├── __init__.py ├── constants.py ├── conversation.py ├── eval │ ├── __init__.py │ ├── eval_vqa_libra.py │ ├── radiology_report.py │ ├── run_libra.py │ └── temporal_f1.py ├── mm_utils.py ├── model │ ├── __init__.py │ ├── builder.py │ ├── language_model │ │ ├── libra_gemma.py │ │ ├── libra_llama.py │ │ ├── libra_mistral.py │ │ ├── libra_phi3.py │ │ ├── libra_qwen2.py │ │ └── libra_qwen3.py │ ├── libra_arch.py │ ├── multimodal_encoder │ │ ├── builder.py │ │ ├── clip_encoder.py │ │ ├── dino_encoder.py │ │ ├── open_clip_encoder │ │ │ ├── __init__.py │ │ │ ├── open_clip_encoder.py │ │ │ └── utils.py │ │ └── siglip_encoder.py │ └── multimodal_projector │ │ └── builder.py ├── serve │ ├── __init__.py │ ├── app.py │ └── cli.py ├── train │ ├── libra_trainer.py │ ├── llama2_flash_attn_monkey_patch.py │ ├── llama_flash_attn_monkey_patch.py │ ├── llama_xformers_attn_monkey_patch.py │ ├── train.py │ ├── train_mem.py │ └── train_xformers.py └── utils.py ├── pyproject.toml └── scripts ├── eval ├── get_eval_scores.sh └── libra_eval.sh ├── finetune.sh ├── finetune_lora.sh ├── merge_lora_weights.py ├── mimic-cxr ├── create_section_files.py └── section_parser.py ├── pretrain.sh ├── pretrain_xformers.sh ├── zero.json ├── zero2.json ├── zero3.json └── zero3_offload.json /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-iZhang/Libra/HEAD/.gitignore -------------------------------------------------------------------------------- /CUSTOM_DATA.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-iZhang/Libra/HEAD/CUSTOM_DATA.md -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-iZhang/Libra/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-iZhang/Libra/HEAD/README.md -------------------------------------------------------------------------------- /assets/Libra_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-iZhang/Libra/HEAD/assets/Libra_logo.png -------------------------------------------------------------------------------- /assets/Libra_logo_c.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-iZhang/Libra/HEAD/assets/Libra_logo_c.png -------------------------------------------------------------------------------- /assets/demo.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-iZhang/Libra/HEAD/assets/demo.gif -------------------------------------------------------------------------------- /assets/example_curent.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-iZhang/Libra/HEAD/assets/example_curent.jpg -------------------------------------------------------------------------------- /assets/example_prior.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-iZhang/Libra/HEAD/assets/example_prior.jpg -------------------------------------------------------------------------------- /assets/libra_architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-iZhang/Libra/HEAD/assets/libra_architecture.png -------------------------------------------------------------------------------- /assets/result_chart.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-iZhang/Libra/HEAD/assets/result_chart.png -------------------------------------------------------------------------------- /assets/x-ray-chest.svg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-iZhang/Libra/HEAD/assets/x-ray-chest.svg -------------------------------------------------------------------------------- /libra/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-iZhang/Libra/HEAD/libra/__init__.py -------------------------------------------------------------------------------- /libra/constants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-iZhang/Libra/HEAD/libra/constants.py -------------------------------------------------------------------------------- /libra/conversation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-iZhang/Libra/HEAD/libra/conversation.py -------------------------------------------------------------------------------- /libra/eval/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-iZhang/Libra/HEAD/libra/eval/__init__.py -------------------------------------------------------------------------------- /libra/eval/eval_vqa_libra.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-iZhang/Libra/HEAD/libra/eval/eval_vqa_libra.py -------------------------------------------------------------------------------- /libra/eval/radiology_report.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-iZhang/Libra/HEAD/libra/eval/radiology_report.py -------------------------------------------------------------------------------- /libra/eval/run_libra.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-iZhang/Libra/HEAD/libra/eval/run_libra.py -------------------------------------------------------------------------------- /libra/eval/temporal_f1.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-iZhang/Libra/HEAD/libra/eval/temporal_f1.py -------------------------------------------------------------------------------- /libra/mm_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-iZhang/Libra/HEAD/libra/mm_utils.py -------------------------------------------------------------------------------- /libra/model/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-iZhang/Libra/HEAD/libra/model/__init__.py -------------------------------------------------------------------------------- /libra/model/builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-iZhang/Libra/HEAD/libra/model/builder.py -------------------------------------------------------------------------------- /libra/model/language_model/libra_gemma.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-iZhang/Libra/HEAD/libra/model/language_model/libra_gemma.py -------------------------------------------------------------------------------- /libra/model/language_model/libra_llama.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-iZhang/Libra/HEAD/libra/model/language_model/libra_llama.py -------------------------------------------------------------------------------- /libra/model/language_model/libra_mistral.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-iZhang/Libra/HEAD/libra/model/language_model/libra_mistral.py -------------------------------------------------------------------------------- /libra/model/language_model/libra_phi3.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-iZhang/Libra/HEAD/libra/model/language_model/libra_phi3.py -------------------------------------------------------------------------------- /libra/model/language_model/libra_qwen2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-iZhang/Libra/HEAD/libra/model/language_model/libra_qwen2.py -------------------------------------------------------------------------------- /libra/model/language_model/libra_qwen3.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-iZhang/Libra/HEAD/libra/model/language_model/libra_qwen3.py -------------------------------------------------------------------------------- /libra/model/libra_arch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-iZhang/Libra/HEAD/libra/model/libra_arch.py -------------------------------------------------------------------------------- /libra/model/multimodal_encoder/builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-iZhang/Libra/HEAD/libra/model/multimodal_encoder/builder.py -------------------------------------------------------------------------------- /libra/model/multimodal_encoder/clip_encoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-iZhang/Libra/HEAD/libra/model/multimodal_encoder/clip_encoder.py -------------------------------------------------------------------------------- /libra/model/multimodal_encoder/dino_encoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-iZhang/Libra/HEAD/libra/model/multimodal_encoder/dino_encoder.py -------------------------------------------------------------------------------- /libra/model/multimodal_encoder/open_clip_encoder/__init__.py: -------------------------------------------------------------------------------- 1 | from .open_clip_encoder import OpenCLIPVisionTower -------------------------------------------------------------------------------- /libra/model/multimodal_encoder/open_clip_encoder/open_clip_encoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-iZhang/Libra/HEAD/libra/model/multimodal_encoder/open_clip_encoder/open_clip_encoder.py -------------------------------------------------------------------------------- /libra/model/multimodal_encoder/open_clip_encoder/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-iZhang/Libra/HEAD/libra/model/multimodal_encoder/open_clip_encoder/utils.py -------------------------------------------------------------------------------- /libra/model/multimodal_encoder/siglip_encoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-iZhang/Libra/HEAD/libra/model/multimodal_encoder/siglip_encoder.py -------------------------------------------------------------------------------- /libra/model/multimodal_projector/builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-iZhang/Libra/HEAD/libra/model/multimodal_projector/builder.py -------------------------------------------------------------------------------- /libra/serve/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /libra/serve/app.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-iZhang/Libra/HEAD/libra/serve/app.py -------------------------------------------------------------------------------- /libra/serve/cli.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-iZhang/Libra/HEAD/libra/serve/cli.py -------------------------------------------------------------------------------- /libra/train/libra_trainer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-iZhang/Libra/HEAD/libra/train/libra_trainer.py -------------------------------------------------------------------------------- /libra/train/llama2_flash_attn_monkey_patch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-iZhang/Libra/HEAD/libra/train/llama2_flash_attn_monkey_patch.py -------------------------------------------------------------------------------- /libra/train/llama_flash_attn_monkey_patch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-iZhang/Libra/HEAD/libra/train/llama_flash_attn_monkey_patch.py -------------------------------------------------------------------------------- /libra/train/llama_xformers_attn_monkey_patch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-iZhang/Libra/HEAD/libra/train/llama_xformers_attn_monkey_patch.py -------------------------------------------------------------------------------- /libra/train/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-iZhang/Libra/HEAD/libra/train/train.py -------------------------------------------------------------------------------- /libra/train/train_mem.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-iZhang/Libra/HEAD/libra/train/train_mem.py -------------------------------------------------------------------------------- /libra/train/train_xformers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-iZhang/Libra/HEAD/libra/train/train_xformers.py -------------------------------------------------------------------------------- /libra/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-iZhang/Libra/HEAD/libra/utils.py -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-iZhang/Libra/HEAD/pyproject.toml -------------------------------------------------------------------------------- /scripts/eval/get_eval_scores.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-iZhang/Libra/HEAD/scripts/eval/get_eval_scores.sh -------------------------------------------------------------------------------- /scripts/eval/libra_eval.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-iZhang/Libra/HEAD/scripts/eval/libra_eval.sh -------------------------------------------------------------------------------- /scripts/finetune.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-iZhang/Libra/HEAD/scripts/finetune.sh -------------------------------------------------------------------------------- /scripts/finetune_lora.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-iZhang/Libra/HEAD/scripts/finetune_lora.sh -------------------------------------------------------------------------------- /scripts/merge_lora_weights.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-iZhang/Libra/HEAD/scripts/merge_lora_weights.py -------------------------------------------------------------------------------- /scripts/mimic-cxr/create_section_files.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-iZhang/Libra/HEAD/scripts/mimic-cxr/create_section_files.py -------------------------------------------------------------------------------- /scripts/mimic-cxr/section_parser.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-iZhang/Libra/HEAD/scripts/mimic-cxr/section_parser.py -------------------------------------------------------------------------------- /scripts/pretrain.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-iZhang/Libra/HEAD/scripts/pretrain.sh -------------------------------------------------------------------------------- /scripts/pretrain_xformers.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-iZhang/Libra/HEAD/scripts/pretrain_xformers.sh -------------------------------------------------------------------------------- /scripts/zero.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-iZhang/Libra/HEAD/scripts/zero.json -------------------------------------------------------------------------------- /scripts/zero2.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-iZhang/Libra/HEAD/scripts/zero2.json -------------------------------------------------------------------------------- /scripts/zero3.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-iZhang/Libra/HEAD/scripts/zero3.json -------------------------------------------------------------------------------- /scripts/zero3_offload.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/X-iZhang/Libra/HEAD/scripts/zero3_offload.json --------------------------------------------------------------------------------