├── .gitattributes
├── .gitignore
├── DATASET.md
├── EMB.md
├── INSTALL.md
├── LICENSE
├── OMG_Seg_README.md
├── README.md
├── demo
├── README.md
├── configs
│ ├── m2_convl.py
│ ├── m2_convl_vid.py
│ └── names
│ │ └── th139_st101.py
├── image_demo.py
├── images
│ ├── 350_6L1vA-xJt-M
│ │ ├── 00002020.jpg
│ │ ├── 00002023.jpg
│ │ ├── 00002026.jpg
│ │ ├── 00002029.jpg
│ │ ├── 00002032.jpg
│ │ ├── 00002035.jpg
│ │ ├── 00002038.jpg
│ │ ├── 00002041.jpg
│ │ ├── 00002044.jpg
│ │ ├── 00002047.jpg
│ │ ├── 00002050.jpg
│ │ ├── 00002053.jpg
│ │ ├── 00002056.jpg
│ │ ├── 00002059.jpg
│ │ └── 00002062.jpg
│ └── sa_1002.jpg
└── video_demo.py
├── ext
├── cityscapes_scripts
│ ├── createPanopticImgs.py
│ └── helpers
│ │ ├── __init__.py
│ │ ├── annotation.py
│ │ ├── csHelpers.py
│ │ ├── labels.py
│ │ ├── labels_cityPersons.py
│ │ └── version.py
├── class_names
│ └── VIPSeg.py
├── davis2017
│ ├── __init__.py
│ ├── davis.py
│ ├── evaluation.py
│ ├── metrics.py
│ ├── results.py
│ └── utils.py
├── meta
│ └── sam_meta.py
├── open_clip
│ ├── __init__.py
│ ├── bpe_simple_vocab_16e6.txt.gz
│ ├── coca_model.py
│ ├── constants.py
│ ├── factory.py
│ ├── generation_utils.py
│ ├── hf_configs.py
│ ├── hf_model.py
│ ├── loss.py
│ ├── model.py
│ ├── model_configs
│ │ ├── EVA01-g-14-plus.json
│ │ ├── EVA01-g-14.json
│ │ ├── EVA02-B-16.json
│ │ ├── EVA02-E-14-plus.json
│ │ ├── EVA02-E-14.json
│ │ ├── EVA02-L-14-336.json
│ │ ├── EVA02-L-14.json
│ │ ├── RN101-quickgelu.json
│ │ ├── RN101.json
│ │ ├── RN50-quickgelu.json
│ │ ├── RN50.json
│ │ ├── RN50x16.json
│ │ ├── RN50x4.json
│ │ ├── RN50x64.json
│ │ ├── ViT-B-16-plus-240.json
│ │ ├── ViT-B-16-plus.json
│ │ ├── ViT-B-16.json
│ │ ├── ViT-B-32-plus-256.json
│ │ ├── ViT-B-32-quickgelu.json
│ │ ├── ViT-B-32.json
│ │ ├── ViT-H-14.json
│ │ ├── ViT-H-16.json
│ │ ├── ViT-L-14-280.json
│ │ ├── ViT-L-14-336.json
│ │ ├── ViT-L-14.json
│ │ ├── ViT-L-16-320.json
│ │ ├── ViT-L-16.json
│ │ ├── ViT-M-16-alt.json
│ │ ├── ViT-M-16.json
│ │ ├── ViT-M-32-alt.json
│ │ ├── ViT-M-32.json
│ │ ├── ViT-S-16-alt.json
│ │ ├── ViT-S-16.json
│ │ ├── ViT-S-32-alt.json
│ │ ├── ViT-S-32.json
│ │ ├── ViT-bigG-14.json
│ │ ├── ViT-e-14.json
│ │ ├── ViT-g-14.json
│ │ ├── coca_ViT-B-32.json
│ │ ├── coca_ViT-L-14.json
│ │ ├── coca_base.json
│ │ ├── coca_roberta-ViT-B-32.json
│ │ ├── convnext_base.json
│ │ ├── convnext_base_w.json
│ │ ├── convnext_base_w_320.json
│ │ ├── convnext_large.json
│ │ ├── convnext_large_d.json
│ │ ├── convnext_large_d_320.json
│ │ ├── convnext_small.json
│ │ ├── convnext_tiny.json
│ │ ├── convnext_xlarge.json
│ │ ├── convnext_xxlarge.json
│ │ ├── convnext_xxlarge_320.json
│ │ ├── mt5-base-ViT-B-32.json
│ │ ├── mt5-xl-ViT-H-14.json
│ │ ├── roberta-ViT-B-32.json
│ │ ├── swin_base_patch4_window7_224.json
│ │ ├── vit_medium_patch16_gap_256.json
│ │ ├── vit_relpos_medium_patch16_cls_224.json
│ │ ├── xlm-roberta-base-ViT-B-32.json
│ │ └── xlm-roberta-large-ViT-H-14.json
│ ├── modified_resnet.py
│ ├── openai.py
│ ├── pretrained.py
│ ├── push_to_hf_hub.py
│ ├── timm_model.py
│ ├── tokenizer.py
│ ├── transform.py
│ ├── transformer.py
│ ├── utils.py
│ ├── version.py
│ ├── zero_shot_classifier.py
│ └── zero_shot_metadata.py
├── sam
│ ├── __init__.py
│ ├── common.py
│ ├── image_encoder.py
│ ├── mask_decoder.py
│ ├── prompt_encoder.py
│ └── transformer.py
└── templates
│ ├── __init__.py
│ └── vild.py
├── figs
├── method_comparison.jpg
└── omg_teaser.jpg
├── omg_llava
├── .owners.yml
├── .pre-commit-config-zh-cn.yaml
├── .pre-commit-config.yaml
├── INSTALL.md
├── MANIFEST.in
├── README.md
├── figs
│ └── omg_llava.png
├── omg_llava
│ ├── __init__.py
│ ├── configs
│ │ ├── __init__.py
│ │ ├── finetune
│ │ │ ├── omg_llava_7b_finetune_8gpus.py
│ │ │ └── specific_tasks_finetune
│ │ │ │ ├── finetune_gcg.py
│ │ │ │ └── finetune_refseg.py
│ │ └── pretrain
│ │ │ └── omg_llava_7b_pretrain_8gpus.py
│ ├── dataset
│ │ ├── CombineDataset.py
│ │ ├── DecoupledGCGDataset.py
│ │ ├── GCGDataset.py
│ │ ├── LlavaDataset.py
│ │ ├── MDPVPointsDataset.py
│ │ ├── ReferringSegDataset.py
│ │ ├── RegionCaptionDataset.py
│ │ ├── SemanticSegDataset.py
│ │ ├── __init__.py
│ │ ├── collect_fns
│ │ │ ├── __init__.py
│ │ │ └── omg_llava_collate_fn.py
│ │ ├── process_functions
│ │ │ ├── __init__.py
│ │ │ ├── decoupled_gcg_process.py
│ │ │ ├── gcg_process.py
│ │ │ ├── mdpv_points_process.py
│ │ │ ├── referring_seg_process.py
│ │ │ ├── region_caption_process.py
│ │ │ └── semantic_seg_process.py
│ │ └── utils
│ │ │ ├── __init__.py
│ │ │ ├── ade20k_classes.json
│ │ │ ├── cocostuff_classes.txt
│ │ │ ├── grefer.py
│ │ │ ├── refcoco_refer.py
│ │ │ └── utils.py
│ ├── engine
│ │ ├── __init__.py
│ │ ├── dataset_info_hook.py
│ │ └── evaluate_chat_hook.py
│ ├── model
│ │ ├── __init__.py
│ │ ├── convnext_clip
│ │ │ ├── __init__.py
│ │ │ ├── open_clip
│ │ │ │ ├── __init__.py
│ │ │ │ ├── bpe_simple_vocab_16e6.txt.gz
│ │ │ │ ├── coca_model.py
│ │ │ │ ├── constants.py
│ │ │ │ ├── factory.py
│ │ │ │ ├── generation_utils.py
│ │ │ │ ├── hf_configs.py
│ │ │ │ ├── hf_model.py
│ │ │ │ ├── loss.py
│ │ │ │ ├── model.py
│ │ │ │ ├── model_configs
│ │ │ │ │ ├── EVA01-g-14-plus.json
│ │ │ │ │ ├── EVA01-g-14.json
│ │ │ │ │ ├── EVA02-B-16.json
│ │ │ │ │ ├── EVA02-E-14-plus.json
│ │ │ │ │ ├── EVA02-E-14.json
│ │ │ │ │ ├── EVA02-L-14-336.json
│ │ │ │ │ ├── EVA02-L-14.json
│ │ │ │ │ ├── RN101-quickgelu.json
│ │ │ │ │ ├── RN101.json
│ │ │ │ │ ├── RN50-quickgelu.json
│ │ │ │ │ ├── RN50.json
│ │ │ │ │ ├── RN50x16.json
│ │ │ │ │ ├── RN50x4.json
│ │ │ │ │ ├── RN50x64.json
│ │ │ │ │ ├── ViT-B-16-plus-240.json
│ │ │ │ │ ├── ViT-B-16-plus.json
│ │ │ │ │ ├── ViT-B-16.json
│ │ │ │ │ ├── ViT-B-32-plus-256.json
│ │ │ │ │ ├── ViT-B-32-quickgelu.json
│ │ │ │ │ ├── ViT-B-32.json
│ │ │ │ │ ├── ViT-H-14.json
│ │ │ │ │ ├── ViT-H-16.json
│ │ │ │ │ ├── ViT-L-14-280.json
│ │ │ │ │ ├── ViT-L-14-336.json
│ │ │ │ │ ├── ViT-L-14.json
│ │ │ │ │ ├── ViT-L-16-320.json
│ │ │ │ │ ├── ViT-L-16.json
│ │ │ │ │ ├── ViT-M-16-alt.json
│ │ │ │ │ ├── ViT-M-16.json
│ │ │ │ │ ├── ViT-M-32-alt.json
│ │ │ │ │ ├── ViT-M-32.json
│ │ │ │ │ ├── ViT-S-16-alt.json
│ │ │ │ │ ├── ViT-S-16.json
│ │ │ │ │ ├── ViT-S-32-alt.json
│ │ │ │ │ ├── ViT-S-32.json
│ │ │ │ │ ├── ViT-bigG-14.json
│ │ │ │ │ ├── ViT-e-14.json
│ │ │ │ │ ├── ViT-g-14.json
│ │ │ │ │ ├── coca_ViT-B-32.json
│ │ │ │ │ ├── coca_ViT-L-14.json
│ │ │ │ │ ├── coca_base.json
│ │ │ │ │ ├── coca_roberta-ViT-B-32.json
│ │ │ │ │ ├── convnext_base.json
│ │ │ │ │ ├── convnext_base_w.json
│ │ │ │ │ ├── convnext_base_w_320.json
│ │ │ │ │ ├── convnext_large.json
│ │ │ │ │ ├── convnext_large_d.json
│ │ │ │ │ ├── convnext_large_d_320.json
│ │ │ │ │ ├── convnext_small.json
│ │ │ │ │ ├── convnext_tiny.json
│ │ │ │ │ ├── convnext_xlarge.json
│ │ │ │ │ ├── convnext_xxlarge.json
│ │ │ │ │ ├── convnext_xxlarge_320.json
│ │ │ │ │ ├── mt5-base-ViT-B-32.json
│ │ │ │ │ ├── mt5-xl-ViT-H-14.json
│ │ │ │ │ ├── roberta-ViT-B-32.json
│ │ │ │ │ ├── swin_base_patch4_window7_224.json
│ │ │ │ │ ├── vit_medium_patch16_gap_256.json
│ │ │ │ │ ├── vit_relpos_medium_patch16_cls_224.json
│ │ │ │ │ ├── xlm-roberta-base-ViT-B-32.json
│ │ │ │ │ └── xlm-roberta-large-ViT-H-14.json
│ │ │ │ ├── modified_resnet.py
│ │ │ │ ├── openai.py
│ │ │ │ ├── pretrained.py
│ │ │ │ ├── push_to_hf_hub.py
│ │ │ │ ├── timm_model.py
│ │ │ │ ├── tokenizer.py
│ │ │ │ ├── transform.py
│ │ │ │ ├── transformer.py
│ │ │ │ ├── utils.py
│ │ │ │ ├── version.py
│ │ │ │ ├── zero_shot_classifier.py
│ │ │ │ └── zero_shot_metadata.py
│ │ │ └── openclip_backbone.py
│ │ ├── modules
│ │ │ ├── __init__.py
│ │ │ └── projector
│ │ │ │ ├── __init__.py
│ │ │ │ ├── configuration_projector.py
│ │ │ │ └── modeling_projector.py
│ │ ├── omg_llava.py
│ │ ├── omg_seg
│ │ │ ├── __init__.py
│ │ │ ├── mask2former_vid.py
│ │ │ ├── mask2former_vid_semanticsam.py
│ │ │ ├── omg_seg_visual_encoder.py
│ │ │ └── utils.py
│ │ └── utils.py
│ └── tools
│ │ ├── __init__.py
│ │ ├── app.py
│ │ ├── app_utils.py
│ │ ├── chat_omg_llava.py
│ │ ├── chat_omg_llava_msseg.py
│ │ ├── convert_deepspeed2pth.py
│ │ ├── evaluate_gcg.py
│ │ ├── evaluate_region_cap.py
│ │ ├── gcg_omg_seg_llava.py
│ │ ├── mmbench_omg_seg_llava.py
│ │ ├── refcoco_omg_seg_llava.py
│ │ ├── region_cap_mask_omg_seg_llava.py
│ │ └── utils_refcoco.py
├── requirements.txt
├── requirements
│ ├── deepspeed.txt
│ ├── docs.txt
│ ├── modelscope.txt
│ └── runtime.txt
├── setup.cfg
├── setup.py
├── test.jpg
└── xtuner
│ ├── __init__.py
│ ├── apis
│ ├── __init__.py
│ ├── datasets
│ │ ├── __init__.py
│ │ ├── alpaca.py
│ │ ├── arxiv.py
│ │ ├── code_alpaca.py
│ │ ├── colorist.py
│ │ ├── lawyer.py
│ │ ├── medical.py
│ │ ├── moss_003_sft.py
│ │ ├── oasst1.py
│ │ ├── open_orca.py
│ │ ├── sql.py
│ │ ├── tiny_codes.py
│ │ └── wizardlm.py
│ ├── model.py
│ └── training_args.py
│ ├── configs
│ ├── __init__.py
│ ├── baichuan
│ │ ├── baichuan2_13b_base
│ │ │ ├── baichuan2_13b_base_qlora_alpaca_e3.py
│ │ │ ├── baichuan2_13b_base_qlora_alpaca_enzh_e3.py
│ │ │ ├── baichuan2_13b_base_qlora_alpaca_enzh_oasst1_e3.py
│ │ │ ├── baichuan2_13b_base_qlora_alpaca_zh_e3.py
│ │ │ ├── baichuan2_13b_base_qlora_arxiv_gentitle_e3.py
│ │ │ ├── baichuan2_13b_base_qlora_code_alpaca_e3.py
│ │ │ ├── baichuan2_13b_base_qlora_colorist_e5.py
│ │ │ ├── baichuan2_13b_base_qlora_lawyer_e3.py
│ │ │ ├── baichuan2_13b_base_qlora_oasst1_512_e3.py
│ │ │ ├── baichuan2_13b_base_qlora_oasst1_e3.py
│ │ │ ├── baichuan2_13b_base_qlora_open_platypus_e3.py
│ │ │ └── baichuan2_13b_base_qlora_sql_e3.py
│ │ ├── baichuan2_13b_chat
│ │ │ ├── baichuan2_13b_chat_qlora_alpaca_e3.py
│ │ │ ├── baichuan2_13b_chat_qlora_alpaca_enzh_e3.py
│ │ │ ├── baichuan2_13b_chat_qlora_alpaca_enzh_oasst1_e3.py
│ │ │ ├── baichuan2_13b_chat_qlora_alpaca_zh_e3.py
│ │ │ ├── baichuan2_13b_chat_qlora_code_alpaca_e3.py
│ │ │ ├── baichuan2_13b_chat_qlora_lawyer_e3.py
│ │ │ ├── baichuan2_13b_chat_qlora_oasst1_512_e3.py
│ │ │ ├── baichuan2_13b_chat_qlora_oasst1_e3.py
│ │ │ └── baichuan2_13b_chat_qlora_open_platypus_e3.py
│ │ ├── baichuan2_7b_base
│ │ │ ├── baichuan2_7b_base_qlora_alpaca_e3.py
│ │ │ ├── baichuan2_7b_base_qlora_alpaca_enzh_e3.py
│ │ │ ├── baichuan2_7b_base_qlora_alpaca_enzh_oasst1_e3.py
│ │ │ ├── baichuan2_7b_base_qlora_alpaca_zh_e3.py
│ │ │ ├── baichuan2_7b_base_qlora_arxiv_gentitle_e3.py
│ │ │ ├── baichuan2_7b_base_qlora_code_alpaca_e3.py
│ │ │ ├── baichuan2_7b_base_qlora_colorist_e5.py
│ │ │ ├── baichuan2_7b_base_qlora_lawyer_e3.py
│ │ │ ├── baichuan2_7b_base_qlora_oasst1_512_e3.py
│ │ │ ├── baichuan2_7b_base_qlora_oasst1_e3.py
│ │ │ ├── baichuan2_7b_base_qlora_open_platypus_e3.py
│ │ │ └── baichuan2_7b_base_qlora_sql_e3.py
│ │ ├── baichuan2_7b_chat
│ │ │ ├── baichuan2_7b_chat_qlora_alpaca_e3.py
│ │ │ ├── baichuan2_7b_chat_qlora_alpaca_enzh_e3.py
│ │ │ ├── baichuan2_7b_chat_qlora_alpaca_enzh_oasst1_e3.py
│ │ │ ├── baichuan2_7b_chat_qlora_alpaca_zh_e3.py
│ │ │ ├── baichuan2_7b_chat_qlora_code_alpaca_e3.py
│ │ │ ├── baichuan2_7b_chat_qlora_lawyer_e3.py
│ │ │ ├── baichuan2_7b_chat_qlora_oasst1_512_e3.py
│ │ │ ├── baichuan2_7b_chat_qlora_oasst1_e3.py
│ │ │ └── baichuan2_7b_chat_qlora_open_platypus_e3.py
│ │ ├── baichuan_13b_base
│ │ │ ├── baichuan_13b_base_qlora_alpaca_e3.py
│ │ │ ├── baichuan_13b_base_qlora_alpaca_enzh_e3.py
│ │ │ ├── baichuan_13b_base_qlora_alpaca_enzh_oasst1_e3.py
│ │ │ ├── baichuan_13b_base_qlora_alpaca_zh_e3.py
│ │ │ ├── baichuan_13b_base_qlora_arxiv_gentitle_e3.py
│ │ │ ├── baichuan_13b_base_qlora_code_alpaca_e3.py
│ │ │ ├── baichuan_13b_base_qlora_colorist_e5.py
│ │ │ ├── baichuan_13b_base_qlora_lawyer_e3.py
│ │ │ ├── baichuan_13b_base_qlora_medical_e1.py
│ │ │ ├── baichuan_13b_base_qlora_moss_sft_all_e1.py
│ │ │ ├── baichuan_13b_base_qlora_moss_sft_all_e2_gpu8.py
│ │ │ ├── baichuan_13b_base_qlora_moss_sft_plugins_e1.py
│ │ │ ├── baichuan_13b_base_qlora_oasst1_512_e3.py
│ │ │ ├── baichuan_13b_base_qlora_oasst1_e3.py
│ │ │ ├── baichuan_13b_base_qlora_open_platypus_e3.py
│ │ │ ├── baichuan_13b_base_qlora_openorca_e1.py
│ │ │ ├── baichuan_13b_base_qlora_sql_e3.py
│ │ │ └── baichuan_13b_base_qlora_tiny_codes_e1.py
│ │ ├── baichuan_13b_chat
│ │ │ ├── baichuan_13b_chat_qlora_alpaca_e3.py
│ │ │ ├── baichuan_13b_chat_qlora_alpaca_enzh_e3.py
│ │ │ ├── baichuan_13b_chat_qlora_alpaca_enzh_oasst1_e3.py
│ │ │ ├── baichuan_13b_chat_qlora_alpaca_zh_e3.py
│ │ │ ├── baichuan_13b_chat_qlora_arxiv_gentitle_e3.py
│ │ │ ├── baichuan_13b_chat_qlora_code_alpaca_e3.py
│ │ │ ├── baichuan_13b_chat_qlora_colorist_e5.py
│ │ │ ├── baichuan_13b_chat_qlora_lawyer_e3.py
│ │ │ ├── baichuan_13b_chat_qlora_medical_e1.py
│ │ │ ├── baichuan_13b_chat_qlora_oasst1_512_e3.py
│ │ │ ├── baichuan_13b_chat_qlora_oasst1_e3.py
│ │ │ ├── baichuan_13b_chat_qlora_open_platypus_e3.py
│ │ │ ├── baichuan_13b_chat_qlora_openorca_e1.py
│ │ │ ├── baichuan_13b_chat_qlora_sql_e3.py
│ │ │ └── baichuan_13b_chat_qlora_tiny_codes_e1.py
│ │ └── baichuan_7b
│ │ │ ├── baichuan_7b_qlora_alpaca_e3.py
│ │ │ ├── baichuan_7b_qlora_alpaca_enzh_e3.py
│ │ │ ├── baichuan_7b_qlora_alpaca_enzh_oasst1_e3.py
│ │ │ ├── baichuan_7b_qlora_alpaca_zh_e3.py
│ │ │ ├── baichuan_7b_qlora_arxiv_gentitle_e3.py
│ │ │ ├── baichuan_7b_qlora_code_alpaca_e3.py
│ │ │ ├── baichuan_7b_qlora_colorist_e5.py
│ │ │ ├── baichuan_7b_qlora_lawyer_e3.py
│ │ │ ├── baichuan_7b_qlora_medical_e1.py
│ │ │ ├── baichuan_7b_qlora_moss_sft_all_e1.py
│ │ │ ├── baichuan_7b_qlora_moss_sft_all_e2_gpu8.py
│ │ │ ├── baichuan_7b_qlora_moss_sft_plugins_e1.py
│ │ │ ├── baichuan_7b_qlora_oasst1_512_e3.py
│ │ │ ├── baichuan_7b_qlora_oasst1_e3.py
│ │ │ ├── baichuan_7b_qlora_open_platypus_e3.py
│ │ │ ├── baichuan_7b_qlora_openorca_e1.py
│ │ │ ├── baichuan_7b_qlora_sql_e3.py
│ │ │ └── baichuan_7b_qlora_tiny_codes_e1.py
│ ├── chatglm
│ │ ├── chatglm2_6b
│ │ │ ├── chatglm2_6b_qlora_alpaca_e3.py
│ │ │ ├── chatglm2_6b_qlora_alpaca_enzh_e3.py
│ │ │ ├── chatglm2_6b_qlora_alpaca_enzh_oasst1_e3.py
│ │ │ ├── chatglm2_6b_qlora_alpaca_zh_e3.py
│ │ │ ├── chatglm2_6b_qlora_arxiv_gentitle_e3.py
│ │ │ ├── chatglm2_6b_qlora_code_alpaca_e3.py
│ │ │ ├── chatglm2_6b_qlora_colorist_e5.py
│ │ │ ├── chatglm2_6b_qlora_lawyer_e3.py
│ │ │ ├── chatglm2_6b_qlora_medical_e1.py
│ │ │ ├── chatglm2_6b_qlora_oasst1_512_e3.py
│ │ │ ├── chatglm2_6b_qlora_oasst1_e3.py
│ │ │ ├── chatglm2_6b_qlora_open_platypus_e3.py
│ │ │ ├── chatglm2_6b_qlora_openorca_e1.py
│ │ │ ├── chatglm2_6b_qlora_sql_e3.py
│ │ │ └── chatglm2_6b_qlora_tiny_codes_e1.py
│ │ ├── chatglm3_6b
│ │ │ ├── chatglm3_6b_qlora_alpaca_e3.py
│ │ │ ├── chatglm3_6b_qlora_alpaca_enzh_e3.py
│ │ │ ├── chatglm3_6b_qlora_alpaca_enzh_oasst1_e3.py
│ │ │ ├── chatglm3_6b_qlora_alpaca_zh_e3.py
│ │ │ ├── chatglm3_6b_qlora_arxiv_gentitle_e3.py
│ │ │ ├── chatglm3_6b_qlora_code_alpaca_e3.py
│ │ │ ├── chatglm3_6b_qlora_colorist_e5.py
│ │ │ ├── chatglm3_6b_qlora_lawyer_e3.py
│ │ │ ├── chatglm3_6b_qlora_medical_e1.py
│ │ │ ├── chatglm3_6b_qlora_oasst1_512_e3.py
│ │ │ ├── chatglm3_6b_qlora_oasst1_e3.py
│ │ │ ├── chatglm3_6b_qlora_open_platypus_e3.py
│ │ │ ├── chatglm3_6b_qlora_openorca_e1.py
│ │ │ ├── chatglm3_6b_qlora_sql_e3.py
│ │ │ └── chatglm3_6b_qlora_tiny_codes_e1.py
│ │ └── chatglm3_6b_base
│ │ │ ├── chatglm3_6b_base_qlora_alpaca_e3.py
│ │ │ ├── chatglm3_6b_base_qlora_alpaca_enzh_e3.py
│ │ │ ├── chatglm3_6b_base_qlora_alpaca_enzh_oasst1_e3.py
│ │ │ ├── chatglm3_6b_base_qlora_alpaca_zh_e3.py
│ │ │ ├── chatglm3_6b_base_qlora_arxiv_gentitle_e3.py
│ │ │ ├── chatglm3_6b_base_qlora_code_alpaca_e3.py
│ │ │ ├── chatglm3_6b_base_qlora_colorist_e5.py
│ │ │ ├── chatglm3_6b_base_qlora_lawyer_e3.py
│ │ │ ├── chatglm3_6b_base_qlora_medical_e1.py
│ │ │ ├── chatglm3_6b_base_qlora_oasst1_512_e3.py
│ │ │ ├── chatglm3_6b_base_qlora_oasst1_e3.py
│ │ │ ├── chatglm3_6b_base_qlora_open_platypus_e3.py
│ │ │ ├── chatglm3_6b_base_qlora_openorca_e1.py
│ │ │ ├── chatglm3_6b_base_qlora_sql_e3.py
│ │ │ └── chatglm3_6b_base_qlora_tiny_codes_e1.py
│ ├── cohere
│ │ ├── README.md
│ │ └── cohere_104b
│ │ │ └── cohere_100b_128k_sp32.py
│ ├── custom_dataset
│ │ ├── pretrain
│ │ │ ├── baichuan
│ │ │ │ ├── baichuan2_13b_base_full_custom_pretrain_e1.py
│ │ │ │ └── baichuan2_7b_base_full_custom_pretrain_e1.py
│ │ │ ├── chatglm
│ │ │ │ ├── chatglm2_6b_full_custom_pretrain_e1.py
│ │ │ │ └── chatglm3_6b_full_custom_pretrain_e1.py
│ │ │ ├── deepseek
│ │ │ │ └── deepseek_moe_16b_base_full_custom_pretrain_e1.py
│ │ │ ├── gemma
│ │ │ │ ├── gemma_2b_full_custom_pretrain_e1.py
│ │ │ │ └── gemma_7b_full_custom_pretrain_e1.py
│ │ │ ├── internlm
│ │ │ │ ├── internlm2_1_8b_full_custom_pretrain_e1.py
│ │ │ │ ├── internlm2_20b_full_custom_pretrain_e1.py
│ │ │ │ └── internlm2_7b_full_custom_pretrain_e1.py
│ │ │ ├── llama
│ │ │ │ ├── llama2_70b_full_custom_pretrain_e1.py
│ │ │ │ └── llama2_7b_full_custom_pretrain_e1.py
│ │ │ ├── mistral
│ │ │ │ └── mistral_7b_full_custom_pretrain_e1.py
│ │ │ ├── mixtral
│ │ │ │ └── mixtral_8x7b_full_custom_pretrain_e1.py
│ │ │ ├── qwen
│ │ │ │ ├── qwen1_5_0_5b_full_custom_pretrain_e1.py
│ │ │ │ ├── qwen1_5_14b_full_custom_pretrain_e1.py
│ │ │ │ ├── qwen1_5_1_8b_full_custom_pretrain_e1.py
│ │ │ │ ├── qwen1_5_4b_full_custom_pretrain_e1.py
│ │ │ │ ├── qwen1_5_72b_full_custom_pretrain_e1.py
│ │ │ │ ├── qwen1_5_7b_full_custom_pretrain_e1.py
│ │ │ │ ├── qwen_1_8b_full_custom_pretrain_e1.py
│ │ │ │ ├── qwen_72b_full_custom_pretrain_e1.py
│ │ │ │ └── qwen_7b_full_custom_pretrain_e1.py
│ │ │ ├── starcoder
│ │ │ │ └── starcoder_full_custom_pretrain_e1.py
│ │ │ ├── yi
│ │ │ │ ├── yi_34b_full_custom_pretrain_e1.py
│ │ │ │ └── yi_6b_full_custom_pretrain_e1.py
│ │ │ └── zephyr
│ │ │ │ └── zephyr_7b_beta_full_custom_pretrain_e1.py
│ │ └── sft
│ │ │ ├── baichuan
│ │ │ ├── baichuan2_13b_chat_qlora_custom_sft_e1.py
│ │ │ ├── baichuan2_7b_chat_qlora_custom_sft_e1.py
│ │ │ ├── baichuan_13b_chat_qlora_custom_sft_e1.py
│ │ │ └── baichuan_7b_qlora_custom_sft_e1.py
│ │ │ ├── chatglm
│ │ │ ├── chatglm2_6b_qlora_custom_sft_e1.py
│ │ │ └── chatglm3_6b_qlora_custom_sft_e1.py
│ │ │ ├── deepseek
│ │ │ ├── deepseek_moe_16b_chat_qlora_custom_sft_e1.py
│ │ │ └── deepseekcoder_6_7b_instruct_qlora_custom_sft_e1.py
│ │ │ ├── gemma
│ │ │ ├── gemma_2b_it_qlora_custom_sft_e1.py
│ │ │ ├── gemma_2b_qlora_custom_sft_e1.py
│ │ │ ├── gemma_7b_it_qlora_custom_sft_e1.py
│ │ │ └── gemma_7b_qlora_custom_sft_e1.py
│ │ │ ├── internlm
│ │ │ ├── internlm2_chat_1_8b_qlora_custom_sft_e1.py
│ │ │ ├── internlm2_chat_20b_qlora_custom_sft_e1.py
│ │ │ └── internlm2_chat_7b_qlora_custom_sft_e1.py
│ │ │ ├── llama
│ │ │ ├── llama2_70b_qlora_custom_sft_e1.py
│ │ │ └── llama2_7b_chat_qlora_custom_sft_e1.py
│ │ │ ├── mistral
│ │ │ └── mistral_7b_full_finetune_custom_sft_e1.py
│ │ │ ├── mixtral
│ │ │ └── mixtral_8x7b_instruct_qlora_custom_sft_e1.py
│ │ │ ├── qwen
│ │ │ ├── qwen1_5_0_5b_chat_qlora_custom_sft_e1.py
│ │ │ ├── qwen1_5_14b_chat_qlora_custom_sft_e1.py
│ │ │ ├── qwen1_5_1_8b_chat_qlora_custom_sft_e1.py
│ │ │ ├── qwen1_5_4b_chat_qlora_custom_sft_e1.py
│ │ │ ├── qwen1_5_72b_chat_qlora_custom_sft_e1.py
│ │ │ ├── qwen1_5_7b_chat_qlora_custom_sft_e1.py
│ │ │ ├── qwen_1_8b_chat_qlora_custom_sft_e1.py
│ │ │ ├── qwen_72b_qlora_custom_sft_e1.py
│ │ │ └── qwen_7b_chat_qlora_custom_sft_e1.py
│ │ │ ├── starcoder
│ │ │ └── starcoder_qlora_custom_sft_e1.py
│ │ │ ├── yi
│ │ │ ├── yi_34b_qlora_custom_sft_e1.py
│ │ │ └── yi_6b_qlora_custom_sft_e1.py
│ │ │ └── zephyr
│ │ │ └── zephyr_7b_beta_qlora_custom_sft_e1.py
│ ├── deepseek
│ │ ├── README.md
│ │ ├── deepseek_coder_6_7b_base
│ │ │ └── deepseek_coder_6_7b_base_qlora_code_alpaca_e3.py
│ │ ├── deepseek_coder_6_7b_instruct
│ │ │ └── deepseekcoder_6_7b_instruct_qlora_code_alpaca_e3.py
│ │ ├── deepseek_moe_16b_base
│ │ │ ├── deepseek_moe_16b_base_full_oasst1_e3.py
│ │ │ └── deepseek_moe_16b_base_qlora_oasst1_e3.py
│ │ ├── deepseek_moe_16b_chat
│ │ │ ├── deepseek_moe_16b_chat_full_oasst1_e3.py
│ │ │ └── deepseek_moe_16b_chat_qlora_oasst1_e3.py
│ │ ├── deepseek_v2_chat
│ │ │ └── deepseek_v2_chat_full_alpaca_e3.py
│ │ └── deepseek_v2_lite_chat
│ │ │ ├── deepseek_v2_lite_chat_full_alpaca_e3.py
│ │ │ └── deepseek_v2_lite_chat_full_alpaca_e3_32k_varlen.py
│ ├── deepspeed
│ │ ├── deepspeed_zero1.json
│ │ ├── deepspeed_zero2.json
│ │ ├── deepspeed_zero2_offload.json
│ │ ├── deepspeed_zero3.json
│ │ └── deepspeed_zero3_offload.json
│ ├── dpo
│ │ ├── internlm
│ │ │ ├── internlm2_chat_1_8b_dpo_full.py
│ │ │ ├── internlm2_chat_1_8b_dpo_full_varlenattn.py
│ │ │ ├── internlm2_chat_1_8b_dpo_full_varlenattn_jsonl_dataset.py
│ │ │ └── internlm2_chat_7b_dpo_qlora_varlenattn.py
│ │ └── llama
│ │ │ └── llama3_8b_instruct_dpo_qlora_varlenattn.py
│ ├── gemma
│ │ ├── gemma_2b
│ │ │ ├── gemma_2b_full_alpaca_e3.py
│ │ │ └── gemma_2b_qlora_alpaca_e3.py
│ │ ├── gemma_2b_it
│ │ │ ├── gemma_2b_it_full_alpaca_e3.py
│ │ │ └── gemma_2b_it_qlora_alpaca_e3.py
│ │ ├── gemma_7b
│ │ │ ├── gemma_7b_full_alpaca_e3.py
│ │ │ └── gemma_7b_qlora_alpaca_e3.py
│ │ └── gemma_7b_it
│ │ │ ├── gemma_7b_it_full_alpaca_e3.py
│ │ │ └── gemma_7b_it_qlora_alpaca_e3.py
│ ├── internlm
│ │ ├── internlm2_1_8b
│ │ │ ├── internlm2_1_8b_full_alpaca_e3.py
│ │ │ └── internlm2_1_8b_qlora_alpaca_e3.py
│ │ ├── internlm2_20b
│ │ │ ├── internlm2_20b_full_finetune_custom_dataset_e1.py
│ │ │ ├── internlm2_20b_qlora_alpaca_e3.py
│ │ │ ├── internlm2_20b_qlora_arxiv_gentitle_e3.py
│ │ │ ├── internlm2_20b_qlora_code_alpaca_e3.py
│ │ │ ├── internlm2_20b_qlora_colorist_e5.py
│ │ │ ├── internlm2_20b_qlora_lawyer_e3.py
│ │ │ ├── internlm2_20b_qlora_msagent_react_e3_gpu8.py
│ │ │ ├── internlm2_20b_qlora_oasst1_512_e3.py
│ │ │ ├── internlm2_20b_qlora_oasst1_e3.py
│ │ │ └── internlm2_20b_qlora_sql_e3.py
│ │ ├── internlm2_7b
│ │ │ ├── internlm2_7b_full_finetune_custom_dataset_e1.py
│ │ │ ├── internlm2_7b_full_finetune_custom_dataset_e1_sequence_parallel_4.py
│ │ │ ├── internlm2_7b_qlora_alpaca_e3.py
│ │ │ ├── internlm2_7b_qlora_arxiv_gentitle_e3.py
│ │ │ ├── internlm2_7b_qlora_code_alpaca_e3.py
│ │ │ ├── internlm2_7b_qlora_colorist_e5.py
│ │ │ ├── internlm2_7b_qlora_json_e3.py
│ │ │ ├── internlm2_7b_qlora_lawyer_e3.py
│ │ │ ├── internlm2_7b_qlora_msagent_react_e3_gpu8.py
│ │ │ ├── internlm2_7b_qlora_oasst1_512_e3.py
│ │ │ ├── internlm2_7b_qlora_oasst1_e3.py
│ │ │ ├── internlm2_7b_qlora_sql_e3.py
│ │ │ ├── internlm2_7b_w_internevo_dataset.py
│ │ │ ├── internlm2_7b_w_tokenized_dataset.py
│ │ │ └── internlm2_7b_w_untokenized_dataset.py
│ │ ├── internlm2_chat_1_8b
│ │ │ ├── internlm2_chat_1_8b_full_alpaca_e3.py
│ │ │ └── internlm2_chat_1_8b_qlora_alpaca_e3.py
│ │ ├── internlm2_chat_20b
│ │ │ ├── internlm2_chat_20b_full_finetune_custom_dataset_e1.py
│ │ │ ├── internlm2_chat_20b_qlora_alpaca_e3.py
│ │ │ ├── internlm2_chat_20b_qlora_code_alpaca_e3.py
│ │ │ ├── internlm2_chat_20b_qlora_lawyer_e3.py
│ │ │ ├── internlm2_chat_20b_qlora_oasst1_512_e3.py
│ │ │ └── internlm2_chat_20b_qlora_oasst1_e3.py
│ │ ├── internlm2_chat_7b
│ │ │ ├── internlm2_chat_7b_full_finetune_custom_dataset_e1.py
│ │ │ ├── internlm2_chat_7b_qlora_alpaca_e3.py
│ │ │ ├── internlm2_chat_7b_qlora_code_alpaca_e3.py
│ │ │ ├── internlm2_chat_7b_qlora_lawyer_e3.py
│ │ │ ├── internlm2_chat_7b_qlora_oasst1_512_e3.py
│ │ │ └── internlm2_chat_7b_qlora_oasst1_e3.py
│ │ ├── internlm_20b
│ │ │ ├── internlm_20b_qlora_alpaca_e3.py
│ │ │ ├── internlm_20b_qlora_alpaca_enzh_e3.py
│ │ │ ├── internlm_20b_qlora_alpaca_enzh_oasst1_e3.py
│ │ │ ├── internlm_20b_qlora_alpaca_zh_e3.py
│ │ │ ├── internlm_20b_qlora_arxiv_gentitle_e3.py
│ │ │ ├── internlm_20b_qlora_code_alpaca_e3.py
│ │ │ ├── internlm_20b_qlora_colorist_e5.py
│ │ │ ├── internlm_20b_qlora_lawyer_e3.py
│ │ │ ├── internlm_20b_qlora_msagent_react_e3_gpu8.py
│ │ │ ├── internlm_20b_qlora_oasst1_512_e3.py
│ │ │ ├── internlm_20b_qlora_oasst1_e3.py
│ │ │ ├── internlm_20b_qlora_open_platypus_e3.py
│ │ │ └── internlm_20b_qlora_sql_e3.py
│ │ ├── internlm_7b
│ │ │ ├── internlm_7b_full_alpaca_e3.py
│ │ │ ├── internlm_7b_full_alpaca_enzh_e3.py
│ │ │ ├── internlm_7b_full_alpaca_enzh_oasst1_e3.py
│ │ │ ├── internlm_7b_full_alpaca_zh_e3.py
│ │ │ ├── internlm_7b_full_intern_repo_dataset_template.py
│ │ │ ├── internlm_7b_full_oasst1_e3.py
│ │ │ ├── internlm_7b_qlora_alpaca_e3.py
│ │ │ ├── internlm_7b_qlora_alpaca_enzh_e3.py
│ │ │ ├── internlm_7b_qlora_alpaca_enzh_oasst1_e3.py
│ │ │ ├── internlm_7b_qlora_alpaca_zh_e3.py
│ │ │ ├── internlm_7b_qlora_arxiv_gentitle_e3.py
│ │ │ ├── internlm_7b_qlora_code_alpaca_e3.py
│ │ │ ├── internlm_7b_qlora_colorist_e5.py
│ │ │ ├── internlm_7b_qlora_json_e3.py
│ │ │ ├── internlm_7b_qlora_lawyer_e3.py
│ │ │ ├── internlm_7b_qlora_medical_e1.py
│ │ │ ├── internlm_7b_qlora_moss_sft_all_e1.py
│ │ │ ├── internlm_7b_qlora_moss_sft_all_e2_gpu8.py
│ │ │ ├── internlm_7b_qlora_moss_sft_plugins_e1.py
│ │ │ ├── internlm_7b_qlora_msagent_react_e3_gpu8.py
│ │ │ ├── internlm_7b_qlora_oasst1_512_e3.py
│ │ │ ├── internlm_7b_qlora_oasst1_e3.py
│ │ │ ├── internlm_7b_qlora_oasst1_e3_hf.py
│ │ │ ├── internlm_7b_qlora_oasst1_mmlu_e3.py
│ │ │ ├── internlm_7b_qlora_open_platypus_e3.py
│ │ │ ├── internlm_7b_qlora_openorca_e1.py
│ │ │ ├── internlm_7b_qlora_sql_e3.py
│ │ │ └── internlm_7b_qlora_tiny_codes_e1.py
│ │ ├── internlm_chat_20b
│ │ │ ├── internlm_chat_20b_qlora_alpaca_e3.py
│ │ │ ├── internlm_chat_20b_qlora_alpaca_enzh_e3.py
│ │ │ ├── internlm_chat_20b_qlora_alpaca_enzh_oasst1_e3.py
│ │ │ ├── internlm_chat_20b_qlora_alpaca_zh_e3.py
│ │ │ ├── internlm_chat_20b_qlora_code_alpaca_e3.py
│ │ │ ├── internlm_chat_20b_qlora_lawyer_e3.py
│ │ │ ├── internlm_chat_20b_qlora_oasst1_512_e3.py
│ │ │ ├── internlm_chat_20b_qlora_oasst1_e3.py
│ │ │ └── internlm_chat_20b_qlora_open_platypus_e3.py
│ │ └── internlm_chat_7b
│ │ │ ├── internlm_chat_7b_qlora_alpaca_e3.py
│ │ │ ├── internlm_chat_7b_qlora_alpaca_enzh_e3.py
│ │ │ ├── internlm_chat_7b_qlora_alpaca_enzh_oasst1_e3.py
│ │ │ ├── internlm_chat_7b_qlora_alpaca_zh_e3.py
│ │ │ ├── internlm_chat_7b_qlora_arxiv_gentitle_e3.py
│ │ │ ├── internlm_chat_7b_qlora_code_alpaca_e3.py
│ │ │ ├── internlm_chat_7b_qlora_colorist_e5.py
│ │ │ ├── internlm_chat_7b_qlora_lawyer_e3.py
│ │ │ ├── internlm_chat_7b_qlora_medical_e1.py
│ │ │ ├── internlm_chat_7b_qlora_oasst1_512_e3.py
│ │ │ ├── internlm_chat_7b_qlora_oasst1_e3.py
│ │ │ ├── internlm_chat_7b_qlora_open_platypus_e3.py
│ │ │ ├── internlm_chat_7b_qlora_openorca_e1.py
│ │ │ ├── internlm_chat_7b_qlora_sql_e3.py
│ │ │ └── internlm_chat_7b_qlora_tiny_codes_e1.py
│ ├── llama
│ │ ├── llama2_70b
│ │ │ ├── llama2_70b_full_wizardlm_e1.py
│ │ │ ├── llama2_70b_int8_lora_open_platypus_e1.py
│ │ │ ├── llama2_70b_int8_lora_open_platypus_e1_hf.py
│ │ │ ├── llama2_70b_qlora_open_platypus_e1.py
│ │ │ └── llama2_70b_qlora_open_platypus_e1_hf.py
│ │ ├── llama2_7b
│ │ │ ├── llama2_7b_full_pgbooks_400iters_sp1.py
│ │ │ ├── llama2_7b_full_pgbooks_400iters_sp4.py
│ │ │ ├── llama2_7b_full_wizardlm_e1.py
│ │ │ ├── llama2_7b_qlora_alpaca_e3.py
│ │ │ ├── llama2_7b_qlora_alpaca_enzh_e3.py
│ │ │ ├── llama2_7b_qlora_alpaca_enzh_oasst1_e3.py
│ │ │ ├── llama2_7b_qlora_alpaca_zh_e3.py
│ │ │ ├── llama2_7b_qlora_arxiv_gentitle_e3.py
│ │ │ ├── llama2_7b_qlora_code_alpaca_e3.py
│ │ │ ├── llama2_7b_qlora_colorist_e5.py
│ │ │ ├── llama2_7b_qlora_lawyer_e3.py
│ │ │ ├── llama2_7b_qlora_medical_e1.py
│ │ │ ├── llama2_7b_qlora_moss_sft_all_e1.py
│ │ │ ├── llama2_7b_qlora_moss_sft_all_e2_gpu8.py
│ │ │ ├── llama2_7b_qlora_moss_sft_plugins_e1.py
│ │ │ ├── llama2_7b_qlora_msagent_react_e3_gpu8.py
│ │ │ ├── llama2_7b_qlora_oasst1_512_e3.py
│ │ │ ├── llama2_7b_qlora_oasst1_e3.py
│ │ │ ├── llama2_7b_qlora_open_platypus_e3.py
│ │ │ ├── llama2_7b_qlora_openorca_e1.py
│ │ │ ├── llama2_7b_qlora_sql_e3.py
│ │ │ └── llama2_7b_qlora_tiny_codes_e1.py
│ │ ├── llama2_7b_chat
│ │ │ ├── llama2_7b_chat_qlora_alpaca_e3.py
│ │ │ ├── llama2_7b_chat_qlora_alpaca_enzh_e3.py
│ │ │ ├── llama2_7b_chat_qlora_alpaca_enzh_oasst1_e3.py
│ │ │ ├── llama2_7b_chat_qlora_alpaca_zh_e3.py
│ │ │ ├── llama2_7b_chat_qlora_arxiv_gentitle_e3.py
│ │ │ ├── llama2_7b_chat_qlora_code_alpaca_e3.py
│ │ │ ├── llama2_7b_chat_qlora_colorist_e5.py
│ │ │ ├── llama2_7b_chat_qlora_lawyer_e3.py
│ │ │ ├── llama2_7b_chat_qlora_medical_e1.py
│ │ │ ├── llama2_7b_chat_qlora_oasst1_512_e3.py
│ │ │ ├── llama2_7b_chat_qlora_oasst1_e3.py
│ │ │ ├── llama2_7b_chat_qlora_open_platypus_e3.py
│ │ │ ├── llama2_7b_chat_qlora_openorca_e1.py
│ │ │ ├── llama2_7b_chat_qlora_sql_e3.py
│ │ │ └── llama2_7b_chat_qlora_tiny_codes_e1.py
│ │ ├── llama3_70b_instruct
│ │ │ └── llama3_70b_instruct_qlora_alpaca_e3_2k_gpu8.py
│ │ ├── llama3_8b
│ │ │ ├── README.md
│ │ │ └── llama3_8b_full_alpaca_e3.py
│ │ ├── llama3_8b_instruct
│ │ │ ├── llama3_8b_instruct_full_alpaca_e3.py
│ │ │ └── llama3_8b_instruct_qlora_alpaca_e3.py
│ │ └── llama_7b
│ │ │ ├── llama_7b_qlora_alpaca_e3.py
│ │ │ ├── llama_7b_qlora_alpaca_enzh_e3.py
│ │ │ ├── llama_7b_qlora_alpaca_enzh_oasst1_e3.py
│ │ │ ├── llama_7b_qlora_alpaca_zh_e3.py
│ │ │ ├── llama_7b_qlora_arxiv_gentitle_e3.py
│ │ │ ├── llama_7b_qlora_code_alpaca_e3.py
│ │ │ ├── llama_7b_qlora_colorist_e5.py
│ │ │ ├── llama_7b_qlora_lawyer_e3.py
│ │ │ ├── llama_7b_qlora_medical_e1.py
│ │ │ ├── llama_7b_qlora_moss_sft_all_e1.py
│ │ │ ├── llama_7b_qlora_moss_sft_all_e2_gpu8.py
│ │ │ ├── llama_7b_qlora_moss_sft_plugins_e1.py
│ │ │ ├── llama_7b_qlora_oasst1_512_e3.py
│ │ │ ├── llama_7b_qlora_oasst1_e3.py
│ │ │ ├── llama_7b_qlora_open_platypus_e3.py
│ │ │ ├── llama_7b_qlora_openorca_e1.py
│ │ │ ├── llama_7b_qlora_sql_e3.py
│ │ │ └── llama_7b_qlora_tiny_codes_e1.py
│ ├── llama_speed_benchmark
│ │ ├── llama2_70b
│ │ │ ├── llama2_70b_full_alpaca_enzh_128k_sp8.py
│ │ │ ├── llama2_70b_full_alpaca_enzh_256k_sp16.py
│ │ │ ├── llama2_70b_full_alpaca_enzh_32k_sp4.py
│ │ │ └── llama2_70b_full_alpaca_enzh_8k_sp1.py
│ │ ├── llama2_7b
│ │ │ ├── llama2_7b_full_alpaca_enzh_128k_sp8.py
│ │ │ ├── llama2_7b_full_alpaca_enzh_1M_sp16.py
│ │ │ ├── llama2_7b_full_alpaca_enzh_256k_sp8.py
│ │ │ ├── llama2_7b_full_alpaca_enzh_32k_sp1.py
│ │ │ └── llama2_7b_full_alpaca_enzh_8k_sp1.py
│ │ └── yi_34b
│ │ │ ├── yi_34b_200k_full_alpaca_enzh_128k_sp8.py
│ │ │ ├── yi_34b_200k_full_alpaca_enzh_256k_sp8.py
│ │ │ ├── yi_34b_200k_full_alpaca_enzh_32k_sp2.py
│ │ │ └── yi_34b_200k_full_alpaca_enzh_8k_sp1.py
│ ├── llava
│ │ ├── README.md
│ │ ├── README_zh-CN.md
│ │ ├── internlm2_chat_1_8b_clip_vit_large_p14_336
│ │ │ ├── finetune
│ │ │ │ └── llava_internlm2_chat_1_8b_qlora_clip_vit_large_p14_336_lora_e1_gpu8_finetune.py
│ │ │ └── pretrain
│ │ │ │ └── llava_internlm2_chat_1_8b_clip_vit_large_p14_336_e1_gpu8_pretrain.py
│ │ ├── internlm2_chat_20b_clip_vit_large_p14_336
│ │ │ ├── finetune
│ │ │ │ ├── llava_internlm2_chat_20b_clip_vit_large_p14_336_e1_gpu8_finetune.py
│ │ │ │ └── llava_internlm2_chat_20b_qlora_clip_vit_large_p14_336_lora_e1_gpu8_finetune.py
│ │ │ └── pretrain
│ │ │ │ └── llava_internlm2_chat_20b_clip_vit_large_p14_336_e1_gpu8_pretrain.py
│ │ ├── internlm2_chat_7b_clip_vit_large_p14_336
│ │ │ ├── finetune
│ │ │ │ ├── llava_internlm2_chat_7b_clip_vit_large_p14_336_e1_gpu8_finetune.py
│ │ │ │ └── llava_internlm2_chat_7b_qlora_clip_vit_large_p14_336_lora_e1_gpu8_finetune.py
│ │ │ └── pretrain
│ │ │ │ └── llava_internlm2_chat_7b_clip_vit_large_p14_336_e1_gpu8_pretrain.py
│ │ ├── internlm_chat_7b_clip_vit_large_p14_336
│ │ │ ├── finetune
│ │ │ │ └── llava_internlm_chat_7b_qlora_clip_vit_large_p14_336_lora_e1_gpu8_finetune.py
│ │ │ └── pretrain
│ │ │ │ └── llava_internlm_chat_7b_clip_vit_large_p14_336_e1_gpu8_pretrain.py
│ │ ├── llama3_70b_instruct_clip_vit_large_p14_336
│ │ │ └── pretrain
│ │ │ │ └── llava_llama3_70b_instruct_quant_clip_vit_large_p14_336_e1_gpu8_pretrain.py
│ │ ├── llama3_8b_instruct_clip_vit_large_p14_336
│ │ │ ├── README.md
│ │ │ ├── convert_xtuner_weights_to_hf.py
│ │ │ ├── convert_xtuner_weights_to_llava.py
│ │ │ ├── finetune
│ │ │ │ ├── llava_llama3_8b_instruct_full_clip_vit_large_p14_336_e1_gpu8_finetune.py
│ │ │ │ ├── llava_llama3_8b_instruct_full_clip_vit_large_p14_336_lora_e1_gpu8_finetune.py
│ │ │ │ ├── llava_llama3_8b_instruct_full_clip_vit_large_p14_336_lora_e1_gpu8_internvl_finetune.py
│ │ │ │ └── llava_llama3_8b_instruct_qlora_clip_vit_large_p14_336_e1_gpu1_finetune.py
│ │ │ └── pretrain
│ │ │ │ ├── llava_llama3_8b_instruct_clip_vit_large_p14_336_e1_gpu8_pretrain.py
│ │ │ │ ├── llava_llama3_8b_instruct_clip_vit_large_p14_336_e1_gpu8_sharegpt4v_pretrain.py
│ │ │ │ └── llava_llama3_8b_instruct_quant_clip_vit_large_p14_336_e1_gpu1_pretrain.py
│ │ ├── official
│ │ │ ├── llava_v15_13b
│ │ │ │ ├── llava_v15_13b_finetune.py
│ │ │ │ ├── llava_v15_13b_finetune_lora.py
│ │ │ │ └── llava_v15_13b_pretrain.py
│ │ │ └── llava_v15_7b
│ │ │ │ ├── llava_v15_7b_finetune.py
│ │ │ │ ├── llava_v15_7b_finetune_lora.py
│ │ │ │ └── llava_v15_7b_pretrain.py
│ │ ├── phi3_mini_4k_instruct_clip_vit_large_p14_336
│ │ │ ├── README.md
│ │ │ ├── convert_phi_to_llama.py
│ │ │ ├── convert_xtuner_weights_to_hf.py
│ │ │ ├── convert_xtuner_weights_to_llava.py
│ │ │ ├── finetune
│ │ │ │ ├── llava_phi3_mini_4k_instruct_full_clip_vit_large_p14_336_e1_gpu8_finetune.py
│ │ │ │ └── llava_phi3_mini_4k_instruct_full_clip_vit_large_p14_336_full_e2_gpu8_internvl_finetune.py
│ │ │ └── pretrain
│ │ │ │ ├── llava_phi3_mini_4k_instruct_clip_vit_large_p14_336_e1_gpu8_pretrain.py
│ │ │ │ └── llava_phi3_mini_4k_instruct_clip_vit_large_p14_336_e1_gpu8_sharegpt4v_pretrain.py
│ │ ├── vicuna_13b_v15_clip_vit_large_p14_336
│ │ │ ├── finetune
│ │ │ │ └── llava_vicuna_13b_v15_qlora_clip_vit_large_p14_336_lora_e1_gpu8_finetune.py
│ │ │ └── pretrain
│ │ │ │ └── llava_vicuna_13b_v15_clip_vit_large_p14_336_e1_gpu8_pretrain.py
│ │ └── vicuna_7b_v15_clip_vit_large_p14_336
│ │ │ ├── finetune
│ │ │ ├── llava_vicuna_7b_v15_qlora_clip_vit_large_p14_336_lora_e1_gpu8_finetune.py
│ │ │ └── llava_vicuna_7b_v15_qlora_clip_vit_large_p14_336_lora_e1_gpu8_finetune_refcoco.py
│ │ │ └── pretrain
│ │ │ └── llava_vicuna_7b_v15_clip_vit_large_p14_336_e1_gpu8_pretrain.py
│ ├── mistral
│ │ ├── mistral_7b_full_finetune_custom_dataset_e1.py
│ │ ├── mistral_7b_qlora_skypile_pretrain_e1.py
│ │ ├── mistral_7b_w_tokenized_dataset.py
│ │ └── mistral_7b_w_untokenized_dataset.py
│ ├── mixtral
│ │ ├── README.md
│ │ ├── mixtral_8x7b
│ │ │ ├── mixtral_8x7b_full_oasst1_e3.py
│ │ │ └── mixtral_8x7b_qlora_oasst1_e3.py
│ │ └── mixtral_8x7b_instruct
│ │ │ ├── mixtral_8x7b_instruct_full_oasst1_e3.py
│ │ │ └── mixtral_8x7b_instruct_qlora_oasst1_e3.py
│ ├── orpo
│ │ ├── internlm
│ │ │ ├── internlm2_chat_1_8b_orpo_full.py
│ │ │ ├── internlm2_chat_1_8b_orpo_full_varlenattn.py
│ │ │ ├── internlm2_chat_1_8b_orpo_full_varlenattn_jsonl_dataset.py
│ │ │ └── internlm2_chat_7b_orpo_qlora_varlenattn_ultrafeedback_e5.py
│ │ └── llama
│ │ │ └── llama3_8b_instruct_orpo_qlora_varlenattn_ultrafeedback_e5.py
│ ├── phi
│ │ └── phi3
│ │ │ ├── phi3_mini_128k_instruct_full_alpaca_e3.py
│ │ │ ├── phi3_mini_128k_instruct_qlora_alpaca_e3.py
│ │ │ ├── phi3_mini_4k_instruct_full_alpaca_e3.py
│ │ │ └── phi3_mini_4k_instruct_qlora_alpaca_e3.py
│ ├── qwen
│ │ ├── qwen1
│ │ │ ├── qwen_1_8b
│ │ │ │ ├── qwen_1_8b_qlora_alpaca_e3.py
│ │ │ │ ├── qwen_1_8b_qlora_alpaca_enzh_e3.py
│ │ │ │ ├── qwen_1_8b_qlora_alpaca_enzh_oasst1_e3.py
│ │ │ │ ├── qwen_1_8b_qlora_alpaca_zh_e3.py
│ │ │ │ └── qwen_1_8b_qlora_code_alpaca_e3.py
│ │ │ ├── qwen_1_8b_chat
│ │ │ │ ├── qwen_1_8b_chat_qlora_alpaca_e3.py
│ │ │ │ ├── qwen_1_8b_chat_qlora_alpaca_enzh_e3.py
│ │ │ │ ├── qwen_1_8b_chat_qlora_alpaca_enzh_oasst1_e3.py
│ │ │ │ ├── qwen_1_8b_chat_qlora_alpaca_zh_e3.py
│ │ │ │ └── qwen_1_8b_chat_qlora_code_alpaca_e3.py
│ │ │ ├── qwen_72b
│ │ │ │ ├── qwen_72b_qlora_alpaca_e3.py
│ │ │ │ ├── qwen_72b_qlora_alpaca_enzh_e3.py
│ │ │ │ ├── qwen_72b_qlora_alpaca_enzh_oasst1_e3.py
│ │ │ │ ├── qwen_72b_qlora_alpaca_zh_e3.py
│ │ │ │ └── qwen_72b_qlora_code_alpaca_e3.py
│ │ │ ├── qwen_7b
│ │ │ │ ├── qwen_7b_qlora_alpaca_e3.py
│ │ │ │ ├── qwen_7b_qlora_alpaca_enzh_e3.py
│ │ │ │ ├── qwen_7b_qlora_alpaca_enzh_oasst1_e3.py
│ │ │ │ ├── qwen_7b_qlora_alpaca_zh_e3.py
│ │ │ │ ├── qwen_7b_qlora_arxiv_gentitle_e3.py
│ │ │ │ ├── qwen_7b_qlora_code_alpaca_e3.py
│ │ │ │ ├── qwen_7b_qlora_colorist_e5.py
│ │ │ │ ├── qwen_7b_qlora_lawyer_e3.py
│ │ │ │ ├── qwen_7b_qlora_medical_e1.py
│ │ │ │ ├── qwen_7b_qlora_moss_sft_all_e1.py
│ │ │ │ ├── qwen_7b_qlora_moss_sft_all_e2_gpu8.py
│ │ │ │ ├── qwen_7b_qlora_moss_sft_plugins_e1.py
│ │ │ │ ├── qwen_7b_qlora_oasst1_512_e3.py
│ │ │ │ ├── qwen_7b_qlora_oasst1_e3.py
│ │ │ │ ├── qwen_7b_qlora_open_platypus_e3.py
│ │ │ │ ├── qwen_7b_qlora_openorca_e1.py
│ │ │ │ ├── qwen_7b_qlora_sql_e3.py
│ │ │ │ └── qwen_7b_qlora_tiny_codes_e1.py
│ │ │ └── qwen_7b_chat
│ │ │ │ ├── qwen_7b_chat_qlora_alpaca_e3.py
│ │ │ │ ├── qwen_7b_chat_qlora_alpaca_enzh_e3.py
│ │ │ │ ├── qwen_7b_chat_qlora_alpaca_enzh_oasst1_e3.py
│ │ │ │ ├── qwen_7b_chat_qlora_alpaca_zh_e3.py
│ │ │ │ ├── qwen_7b_chat_qlora_arxiv_gentitle_e3.py
│ │ │ │ ├── qwen_7b_chat_qlora_code_alpaca_e3.py
│ │ │ │ ├── qwen_7b_chat_qlora_colorist_e5.py
│ │ │ │ ├── qwen_7b_chat_qlora_lawyer_e3.py
│ │ │ │ ├── qwen_7b_chat_qlora_medical_e1.py
│ │ │ │ ├── qwen_7b_chat_qlora_oasst1_512_e3.py
│ │ │ │ ├── qwen_7b_chat_qlora_oasst1_e3.py
│ │ │ │ ├── qwen_7b_chat_qlora_open_platypus_e3.py
│ │ │ │ ├── qwen_7b_chat_qlora_openorca_e1.py
│ │ │ │ ├── qwen_7b_chat_qlora_sql_e3.py
│ │ │ │ └── qwen_7b_chat_qlora_tiny_codes_e1.py
│ │ └── qwen1_5
│ │ │ ├── qwen1_5_0_5b
│ │ │ ├── qwen1_5_0_5b_full_alpaca_e3.py
│ │ │ └── qwen1_5_0_5b_qlora_alpaca_e3.py
│ │ │ ├── qwen1_5_0_5b_chat
│ │ │ ├── qwen1_5_0_5b_chat_full_alpaca_e3.py
│ │ │ └── qwen1_5_0_5b_chat_qlora_alpaca_e3.py
│ │ │ ├── qwen1_5_110b
│ │ │ ├── qwen1_5_110b_full_alpaca_e3.py
│ │ │ └── qwen1_5_110b_qlora_alpaca_e3.py
│ │ │ ├── qwen1_5_110b_chat
│ │ │ ├── README.md
│ │ │ ├── qwen1_5_110b_chat_full_alpaca_e3.py
│ │ │ ├── qwen1_5_110b_chat_qlora_alpaca_e3.py
│ │ │ └── qwen1_5_110b_chat_qlora_alpaca_e3_16k_2gpus.py
│ │ │ ├── qwen1_5_14b
│ │ │ ├── qwen1_5_14b_full_alpaca_e3.py
│ │ │ └── qwen1_5_14b_qlora_alpaca_e3.py
│ │ │ ├── qwen1_5_14b_chat
│ │ │ ├── qwen1_5_14b_chat_full_alpaca_e3.py
│ │ │ └── qwen1_5_14b_chat_qlora_alpaca_e3.py
│ │ │ ├── qwen1_5_1_8b
│ │ │ ├── qwen1_5_1_8b_full_alpaca_e3.py
│ │ │ └── qwen1_5_1_8b_qlora_alpaca_e3.py
│ │ │ ├── qwen1_5_1_8b_chat
│ │ │ ├── qwen1_5_1_8b_chat_full_alpaca_e3.py
│ │ │ └── qwen1_5_1_8b_chat_qlora_alpaca_e3.py
│ │ │ ├── qwen1_5_4b
│ │ │ ├── qwen1_5_4b_full_alpaca_e3.py
│ │ │ └── qwen1_5_4b_qlora_alpaca_e3.py
│ │ │ ├── qwen1_5_4b_chat
│ │ │ ├── qwen1_5_4b_chat_full_alpaca_e3.py
│ │ │ └── qwen1_5_4b_chat_qlora_alpaca_e3.py
│ │ │ ├── qwen1_5_72b
│ │ │ ├── qwen1_5_72b_full_alpaca_e3.py
│ │ │ └── qwen1_5_72b_qlora_alpaca_e3.py
│ │ │ ├── qwen1_5_72b_chat
│ │ │ ├── qwen1_5_72b_chat_full_alpaca_e3.py
│ │ │ └── qwen1_5_72b_chat_qlora_alpaca_e3.py
│ │ │ ├── qwen1_5_7b
│ │ │ ├── qwen1_5_7b_full_alpaca_e3.py
│ │ │ └── qwen1_5_7b_qlora_alpaca_e3.py
│ │ │ └── qwen1_5_7b_chat
│ │ │ ├── qwen1_5_7b_chat_full_alpaca_e3.py
│ │ │ └── qwen1_5_7b_chat_qlora_alpaca_e3.py
│ ├── qwen_moe
│ │ └── qwen1_5
│ │ │ └── qwen1_5_moe_a2_7_b_chat
│ │ │ └── qwen1_5_moe_a2_7_b_chat_full_alpaca_e3.py
│ ├── reward_model
│ │ ├── internlm
│ │ │ ├── internlm2_chat_1_8b_reward_full_ultrafeedback.py
│ │ │ ├── internlm2_chat_1_8b_reward_full_varlenattn_jsonl_dataset.py
│ │ │ ├── internlm2_chat_1_8b_reward_full_varlenattn_ultrafeedback.py
│ │ │ └── internlm2_chat_1_8b_reward_qlora_varlenattn_ultrafeedback.py
│ │ └── llama
│ │ │ └── llama3_8b_instruct_reward_full_varlenattn_ultrafeedback.py
│ ├── starcoder
│ │ └── starcoder_qlora_stack_exchange_example.py
│ ├── yi
│ │ ├── yi_34b
│ │ │ └── yi_34b_qlora_alpaca_enzh_e3.py
│ │ └── yi_6b
│ │ │ └── yi_6b_qlora_alpaca_enzh_e3.py
│ └── zephyr
│ │ └── zephyr_7b_beta_qlora_alpaca_e3.py
│ ├── dataset
│ ├── __init__.py
│ ├── collate_fns
│ │ ├── __init__.py
│ │ ├── default_collate_fn.py
│ │ ├── mmlu_collate_fn.py
│ │ └── preference_collate_fn.py
│ ├── concat_dataset.py
│ ├── huggingface.py
│ ├── intern_repo.py
│ ├── json_dataset.py
│ ├── llava.py
│ ├── map_fns
│ │ ├── __init__.py
│ │ ├── dataset_map_fns
│ │ │ ├── __init__.py
│ │ │ ├── alpaca_map_fn.py
│ │ │ ├── alpaca_zh_map_fn.py
│ │ │ ├── arxiv_map_fn.py
│ │ │ ├── code_alpaca_map_fn.py
│ │ │ ├── colors_map_fn.py
│ │ │ ├── crime_kg_assitant_map_fn.py
│ │ │ ├── default_map_fn.py
│ │ │ ├── law_reference_map_fn.py
│ │ │ ├── llava_map_fn.py
│ │ │ ├── medical_map_fn.py
│ │ │ ├── msagent_map_fn.py
│ │ │ ├── oasst1_map_fn.py
│ │ │ ├── openai_map_fn.py
│ │ │ ├── openorca_map_fn.py
│ │ │ ├── pretrain_map_fn.py
│ │ │ ├── sql_map_fn.py
│ │ │ ├── stack_exchange_map_fn.py
│ │ │ ├── tiny_codes_map_fn.py
│ │ │ └── wizardlm_map_fn.py
│ │ └── template_map_fn.py
│ ├── modelscope.py
│ ├── moss_sft.py
│ ├── preference_dataset.py
│ ├── refcoco_json.py
│ ├── samplers
│ │ ├── __init__.py
│ │ ├── intern_repo.py
│ │ └── length_grouped.py
│ └── utils.py
│ ├── engine
│ ├── __init__.py
│ ├── _strategy
│ │ ├── __init__.py
│ │ └── deepspeed.py
│ ├── hooks
│ │ ├── __init__.py
│ │ ├── dataset_info_hook.py
│ │ ├── evaluate_chat_hook.py
│ │ ├── hf_checkpoint_hook.py
│ │ ├── throughput_hook.py
│ │ └── varlen_attn_args_to_messagehub_hook.py
│ └── runner
│ │ ├── __init__.py
│ │ └── loops.py
│ ├── entry_point.py
│ ├── evaluation
│ ├── __init__.py
│ └── metrics
│ │ ├── __init__.py
│ │ ├── mmlu_metric.py
│ │ └── reward_metric.py
│ ├── model
│ ├── __init__.py
│ ├── dpo.py
│ ├── llava.py
│ ├── modules
│ │ ├── __init__.py
│ │ ├── dispatch
│ │ │ ├── __init__.py
│ │ │ ├── attention.py
│ │ │ ├── baichuan.py
│ │ │ ├── cohere.py
│ │ │ ├── deepseek_v2.py
│ │ │ ├── internlm.py
│ │ │ ├── internlm2.py
│ │ │ ├── llama.py
│ │ │ ├── mistral.py
│ │ │ ├── phi3.py
│ │ │ ├── qwen2.py
│ │ │ ├── triton_kernels
│ │ │ │ ├── __init__.py
│ │ │ │ ├── layer_norm.py
│ │ │ │ ├── rms_norm.py
│ │ │ │ └── rotary.py
│ │ │ ├── utils.py
│ │ │ └── yi.py
│ │ └── projector
│ │ │ ├── __init__.py
│ │ │ ├── configuration_projector.py
│ │ │ └── modeling_projector.py
│ ├── orpo.py
│ ├── reward.py
│ ├── sft.py
│ ├── transformers_models
│ │ ├── __init__.py
│ │ ├── deepseek_v2
│ │ │ ├── __init__.py
│ │ │ ├── configuration_deepseek.py
│ │ │ ├── modeling_deepseek.py
│ │ │ └── tokenization_deepseek_fast.py
│ │ └── mixtral
│ │ │ ├── __init__.py
│ │ │ ├── configuration_mixtral.py
│ │ │ └── modeling_mixtral.py
│ └── utils.py
│ ├── parallel
│ ├── __init__.py
│ └── sequence
│ │ ├── __init__.py
│ │ ├── attention.py
│ │ ├── comm.py
│ │ ├── data_collate.py
│ │ ├── reduce_loss.py
│ │ ├── sampler.py
│ │ └── setup_distributed.py
│ ├── registry.py
│ ├── tools
│ ├── chat.py
│ ├── check_custom_dataset.py
│ ├── copy_cfg.py
│ ├── data_preprocess
│ │ ├── arxiv.py
│ │ └── convert_refcoco.py
│ ├── eval_refcoco.py
│ ├── get_data_order.py
│ ├── list_cfg.py
│ ├── list_dataset_format.py
│ ├── log_dataset.py
│ ├── mmbench.py
│ ├── model_converters
│ │ ├── merge.py
│ │ ├── modeling_internlm2_reward
│ │ │ ├── __init__.py
│ │ │ ├── configuration_internlm2.py
│ │ │ └── modeling_internlm2.py
│ │ ├── pth_to_hf.py
│ │ └── split.py
│ ├── plugins
│ │ ├── __init__.py
│ │ ├── api.py
│ │ ├── calculate.py
│ │ ├── search.py
│ │ └── solve.py
│ ├── process_untokenized_datasets.py
│ ├── process_untokenized_datasets_legacy.py
│ ├── process_untokenized_llava_data.py
│ ├── test.py
│ ├── tokenize_ftdp_datasets.py
│ ├── train.py
│ └── utils.py
│ ├── utils
│ ├── __init__.py
│ ├── constants.py
│ ├── fileio.py
│ ├── handle_moe_load_and_save.py
│ ├── stop_criteria.py
│ ├── templates.py
│ └── zero_to_any_dtype.py
│ └── version.py
├── seg
├── configs
│ ├── _base_
│ │ ├── datasets
│ │ │ ├── ade_panoptic.py
│ │ │ ├── ade_panoptic_ov.py
│ │ │ ├── ade_panoptic_ov_720p.py
│ │ │ ├── cityscapes_panoptic.py
│ │ │ ├── cityscapes_panoptic_720p.py
│ │ │ ├── coco_panoptic_lsj.py
│ │ │ ├── coco_panoptic_lsj_sam.py
│ │ │ ├── coco_panoptic_lsj_sam_720p.py
│ │ │ ├── coco_panoptic_video_ade_yt19_yt21_vip_cityscapes.py
│ │ │ ├── coco_panoptic_video_lsj.py
│ │ │ ├── coco_panoptic_video_yt19_vip_cityscapes_cocopansam.py
│ │ │ ├── coco_panoptic_video_yt19_vip_cocopansam.py
│ │ │ ├── coco_panoptic_video_yt19_yt21_vip_cityscapes_cocopansam.py
│ │ │ ├── davis.py
│ │ │ ├── joint_dataset.py
│ │ │ ├── objects365v2_detection_lsj.py
│ │ │ ├── objects365v2_instance_lsj.py
│ │ │ ├── vipseg.py
│ │ │ ├── youtube_vis_2019.py
│ │ │ ├── youtube_vis_2021.py
│ │ │ └── youtube_vis_ovis.py
│ │ ├── default_runtime.py
│ │ └── schedules
│ │ │ ├── schedule_12e.py
│ │ │ └── schedule_24e.py
│ ├── m2_train_close_set
│ │ └── omg_convl_coco_vid_ade_yt19_yt21_vip_city.py
│ ├── m2ov_train
│ │ ├── omg_convl_vlm_fix_12e_ov_coco_vid_yt19_vip_city_cocopansam.py
│ │ └── omg_convl_vlm_fix_12e_ov_coco_vid_yt19_y21_vip_city_cocopansam.py
│ └── m2ov_val
│ │ ├── datasets
│ │ ├── ade.py
│ │ ├── cityscapes.py
│ │ ├── coco.py
│ │ ├── coco_pan_point.py
│ │ ├── davis.py
│ │ ├── vipseg.py
│ │ ├── y19.py
│ │ └── y21.py
│ │ ├── eval_m2_convl_300q_ov_ade.py
│ │ ├── eval_m2_convl_300q_ov_cityscapes.py
│ │ ├── eval_m2_convl_300q_ov_coco.py
│ │ ├── eval_m2_convl_300q_ov_davis.py
│ │ ├── eval_m2_convl_300q_ov_vipseg.py
│ │ ├── eval_m2_convl_300q_ov_y19.py
│ │ ├── eval_m2_convl_300q_ov_y21.py
│ │ ├── eval_m2_convl_ov_coco_pan_point.py
│ │ └── models
│ │ └── m2_convl_300q.py
├── datasets
│ ├── ade_ov.py
│ ├── cityscapes.py
│ ├── coco_ins_ov.py
│ ├── coco_ov.py
│ ├── coco_pan_sam.py
│ ├── concat_dataset.py
│ ├── davis.py
│ ├── pipelines
│ │ ├── formatting.py
│ │ ├── frame_copy.py
│ │ ├── frame_sampling.py
│ │ ├── loading.py
│ │ └── transforms.py
│ ├── samplers
│ │ ├── batch_sampler.py
│ │ └── multi_dataset_sampler.py
│ ├── vipseg.py
│ └── youtube_vis_dataset.py
├── evaluation
│ ├── hooks
│ │ └── visual_hook.py
│ └── metrics
│ │ ├── cityscapes_panoptic_metric.py
│ │ ├── ins_cls_iou_metric.py
│ │ ├── vip_seg_metric.py
│ │ └── vos_metric.py
└── models
│ ├── backbones
│ ├── __init__.py
│ └── openclip_backbone.py
│ ├── data_preprocessor
│ ├── __init__.py
│ ├── ovsam_preprocessor.py
│ └── vidseg_data_preprocessor.py
│ ├── detectors
│ ├── __init__.py
│ ├── mask2former_vid.py
│ └── mask2former_vid_minvis.py
│ ├── fusion_head
│ ├── __init__.py
│ └── omgseg_fusionhead.py
│ ├── heads
│ ├── __init__.py
│ └── mask2former_vid.py
│ ├── task_modules
│ └── cost.py
│ └── utils
│ ├── __init__.py
│ ├── class_overlapping.py
│ ├── load_checkpoint.py
│ ├── mask_pool.py
│ ├── no_obj.py
│ ├── offline_video_metrics.py
│ ├── online_pq_utils.py
│ ├── pan_seg_transform.py
│ └── video_gt_preprocess.py
└── tools
├── dataset_convert
└── vis_to_coco.py
├── dist.sh
├── eval_scripts
├── eval_davis.py
└── eval_video.py
├── gen_cls.py
├── slurm.sh
├── test.py
└── train.py
/.gitattributes:
--------------------------------------------------------------------------------
1 | *.jpg filter=lfs diff=lfs merge=lfs -text
2 |
--------------------------------------------------------------------------------
/EMB.md:
--------------------------------------------------------------------------------
1 | ## Generate text embedding for each dataset and Download the pretrained models.
2 |
3 | ### For Separate Dataset. (Mainly For Evaluation)
4 |
5 | We adopt the separate dataset embedding for testing.
6 |
7 |
8 | ```commandline
9 | ./tools/dist.sh gen_cls seg/configs/m2ov_val/eval_m2_convl_300q_ov_coco.py 1
10 | ```
11 | ```commandline
12 | ./tools/dist.sh gen_cls seg/configs/m2ov_val/eval_m2_convl_300q_ov_ade.py 1
13 | ```
14 |
15 | ```commandline
16 | ./tools/dist.sh gen_cls seg/configs/m2ov_val/eval_m2_convl_300q_ov_cityscapes.py 1
17 | ```
18 |
19 | ```commandline
20 | ./tools/dist.sh gen_cls seg/configs/m2ov_val/eval_m2_convl_300q_ov_vipseg.py 1
21 | ```
22 |
23 | ```commandline
24 | ./tools/dist.sh gen_cls seg/configs/m2ov_val/eval_m2_convl_300q_ov_y19.py 1
25 | ```
26 |
27 | ```commandline
28 | ./tools/dist.sh gen_cls seg/configs/m2ov_val/eval_m2_convl_300q_ov_y21.py 1
29 | ```
30 |
31 | ### For Merged Dataset Training. (Mainly For Co-Training)
32 |
33 | We adopt the merged dataset embedding for training.
34 |
35 | ```commandline
36 | ./tools/dist.sh gen_cls seg/configs/m2ov_train/omg_convl_vlm_fix_24e_ov_coco_vid_yt19_vip_city_cocopansam.py 1
37 | ```
38 |
39 | Once you finish converting the embedding, you will obtain the embedding file in your cache folder.
40 |
41 | ### Download Pre-trained Open-ClIP models.
42 |
43 | When generating the class embedding classifier, the scripts will automatically download the pre-trained CLIP models.
44 |
45 | If you are in China, you can use [HF-Mirror](https://hf-mirror.com/). Follow the step to set the default path.
46 |
47 | ```commandline
48 | pip install -U huggingface_hub
49 | ```
50 |
51 | ```commandline
52 | export HF_ENDPOINT=https://hf-mirror.com
53 | ```
54 |
55 |
56 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | This project is licensed under the MIT license.
2 | Copyrights are respective of each contributor listed at the beginning of each definition file.
3 |
4 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
5 |
6 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
7 |
8 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
--------------------------------------------------------------------------------
/demo/README.md:
--------------------------------------------------------------------------------
1 | # OMG-Seg Demo
2 |
3 | We provide a single-file demo in this folder to facilitate getting started. Supposing that you are in the root directory of this project.
4 |
5 | ## Embedding Generation
6 | To use the demo, you need first generate a class name list to tell the OMG-Seg model all possible categories as the vocabulary dictionary. We have already provided a sample vocabulary list in `demo/configs/names/th139_st101.py`.
7 |
8 | Then, we need to generate the class embeddings based on the names. You can do this by the following command:
9 | ```commandline
10 | PYTHONPATH=. python tools/gen_cls.py demo/configs/m2_convl.py
11 | ```
12 | The script will automatically read the class list, which is imported in `demo/configs/m2_convl.py` (please refer to `CLASSES` and `DATASET_NAME`), and generate the embeddings.
13 |
14 | ## Run the Demo
15 | After generating the embeddings, you can run the demo by:
16 | ```commandline
17 | PYTHONPATH=. python demo/image_demo.py
18 | ```
19 | for image; and
20 | ```commandline
21 | PYTHONPATH=. python demo/video_demo.py
22 | ```
23 | for video.
24 |
25 | Please refer to `test_image` and `test_video` for the visualization of the outputs.
26 |
27 | ## Customization
28 | If you want to try your own images or videos, please change the `IMG_PATH`, `VID_PATH`, and `MODEL_PATH`.
29 |
30 | If you want to customize our model, please refer to the config scripts (`demo/configs/m2_convl.py` and `demo/configs/m2_convl_vid.py`) for details.
31 |
32 | Note that all the model-related code have been imported in the config file. You need to find the corresponding path to find the model implementation details.
--------------------------------------------------------------------------------
/demo/images/350_6L1vA-xJt-M/00002020.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lxtGH/OMG-Seg/1ae2e7d446a607fddaefabf4ace312a2cdf7ab55/demo/images/350_6L1vA-xJt-M/00002020.jpg
--------------------------------------------------------------------------------
/demo/images/350_6L1vA-xJt-M/00002023.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lxtGH/OMG-Seg/1ae2e7d446a607fddaefabf4ace312a2cdf7ab55/demo/images/350_6L1vA-xJt-M/00002023.jpg
--------------------------------------------------------------------------------
/demo/images/350_6L1vA-xJt-M/00002026.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lxtGH/OMG-Seg/1ae2e7d446a607fddaefabf4ace312a2cdf7ab55/demo/images/350_6L1vA-xJt-M/00002026.jpg
--------------------------------------------------------------------------------
/demo/images/350_6L1vA-xJt-M/00002029.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lxtGH/OMG-Seg/1ae2e7d446a607fddaefabf4ace312a2cdf7ab55/demo/images/350_6L1vA-xJt-M/00002029.jpg
--------------------------------------------------------------------------------
/demo/images/350_6L1vA-xJt-M/00002032.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lxtGH/OMG-Seg/1ae2e7d446a607fddaefabf4ace312a2cdf7ab55/demo/images/350_6L1vA-xJt-M/00002032.jpg
--------------------------------------------------------------------------------
/demo/images/350_6L1vA-xJt-M/00002035.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lxtGH/OMG-Seg/1ae2e7d446a607fddaefabf4ace312a2cdf7ab55/demo/images/350_6L1vA-xJt-M/00002035.jpg
--------------------------------------------------------------------------------
/demo/images/350_6L1vA-xJt-M/00002038.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lxtGH/OMG-Seg/1ae2e7d446a607fddaefabf4ace312a2cdf7ab55/demo/images/350_6L1vA-xJt-M/00002038.jpg
--------------------------------------------------------------------------------
/demo/images/350_6L1vA-xJt-M/00002041.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lxtGH/OMG-Seg/1ae2e7d446a607fddaefabf4ace312a2cdf7ab55/demo/images/350_6L1vA-xJt-M/00002041.jpg
--------------------------------------------------------------------------------
/demo/images/350_6L1vA-xJt-M/00002044.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lxtGH/OMG-Seg/1ae2e7d446a607fddaefabf4ace312a2cdf7ab55/demo/images/350_6L1vA-xJt-M/00002044.jpg
--------------------------------------------------------------------------------
/demo/images/350_6L1vA-xJt-M/00002047.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lxtGH/OMG-Seg/1ae2e7d446a607fddaefabf4ace312a2cdf7ab55/demo/images/350_6L1vA-xJt-M/00002047.jpg
--------------------------------------------------------------------------------
/demo/images/350_6L1vA-xJt-M/00002050.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lxtGH/OMG-Seg/1ae2e7d446a607fddaefabf4ace312a2cdf7ab55/demo/images/350_6L1vA-xJt-M/00002050.jpg
--------------------------------------------------------------------------------
/demo/images/350_6L1vA-xJt-M/00002053.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lxtGH/OMG-Seg/1ae2e7d446a607fddaefabf4ace312a2cdf7ab55/demo/images/350_6L1vA-xJt-M/00002053.jpg
--------------------------------------------------------------------------------
/demo/images/350_6L1vA-xJt-M/00002056.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lxtGH/OMG-Seg/1ae2e7d446a607fddaefabf4ace312a2cdf7ab55/demo/images/350_6L1vA-xJt-M/00002056.jpg
--------------------------------------------------------------------------------
/demo/images/350_6L1vA-xJt-M/00002059.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lxtGH/OMG-Seg/1ae2e7d446a607fddaefabf4ace312a2cdf7ab55/demo/images/350_6L1vA-xJt-M/00002059.jpg
--------------------------------------------------------------------------------
/demo/images/350_6L1vA-xJt-M/00002062.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lxtGH/OMG-Seg/1ae2e7d446a607fddaefabf4ace312a2cdf7ab55/demo/images/350_6L1vA-xJt-M/00002062.jpg
--------------------------------------------------------------------------------
/demo/images/sa_1002.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lxtGH/OMG-Seg/1ae2e7d446a607fddaefabf4ace312a2cdf7ab55/demo/images/sa_1002.jpg
--------------------------------------------------------------------------------
/ext/cityscapes_scripts/helpers/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lxtGH/OMG-Seg/1ae2e7d446a607fddaefabf4ace312a2cdf7ab55/ext/cityscapes_scripts/helpers/__init__.py
--------------------------------------------------------------------------------
/ext/cityscapes_scripts/helpers/version.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env python
2 |
3 | import os
4 |
5 | with open(os.path.join(os.path.dirname(__file__), '..', 'VERSION')) as f:
6 | version = f.read().strip()
7 |
8 | if __name__ == "__main__":
9 | print(version)
10 |
--------------------------------------------------------------------------------
/ext/davis2017/__init__.py:
--------------------------------------------------------------------------------
1 | from __future__ import absolute_import
2 |
3 | __version__ = '0.1.0'
4 |
--------------------------------------------------------------------------------
/ext/meta/sam_meta.py:
--------------------------------------------------------------------------------
1 | meta_dict = {
2 | 'vit_h': dict(
3 | encoder_embed_dim=1280,
4 | encoder_depth=32,
5 | encoder_num_heads=16,
6 | encoder_global_attn_indexes=[7, 15, 23, 31],
7 | # common
8 | prompt_embed_dim=256,
9 | image_size=1024,
10 | vit_patch_size=16,
11 | image_embedding_size=64
12 | ),
13 | 'vit_l': dict(
14 | encoder_embed_dim=1024,
15 | encoder_depth=24,
16 | encoder_num_heads=16,
17 | encoder_global_attn_indexes=[5, 11, 17, 23],
18 | # common
19 | prompt_embed_dim=256,
20 | image_size=1024,
21 | vit_patch_size=16,
22 | image_embedding_size=64
23 | ),
24 | 'vit_b': dict(
25 | encoder_embed_dim=768,
26 | encoder_depth=12,
27 | encoder_num_heads=12,
28 | encoder_global_attn_indexes=[2, 5, 8, 11],
29 | # common
30 | prompt_embed_dim=256,
31 | image_size=1024,
32 | vit_patch_size=16,
33 | image_embedding_size=64
34 | )
35 | }
36 |
37 | checkpoint_dict = {
38 | 'vit_h': 'https://dl.fbaipublicfiles.com/segment_anything/sam_vit_h_4b8939.pth',
39 | 'vit_l': 'https://dl.fbaipublicfiles.com/segment_anything/sam_vit_l_0b3195.pth',
40 | 'vit_b': 'https://dl.fbaipublicfiles.com/segment_anything/sam_vit_b_01ec64.pth',
41 | }
42 |
--------------------------------------------------------------------------------
/ext/open_clip/__init__.py:
--------------------------------------------------------------------------------
1 | from .coca_model import CoCa
2 | from .constants import OPENAI_DATASET_MEAN, OPENAI_DATASET_STD
3 | from .factory import create_model, create_model_and_transforms, create_model_from_pretrained, get_tokenizer, create_loss
4 | from .factory import list_models, add_model_config, get_model_config, load_checkpoint
5 | from .loss import ClipLoss, DistillClipLoss, CoCaLoss
6 | from .model import CLIP, CustomTextCLIP, CLIPTextCfg, CLIPVisionCfg, \
7 | convert_weights_to_lp, convert_weights_to_fp16, trace_model, get_cast_dtype, get_input_dtype
8 | from .openai import load_openai_model, list_openai_models
9 | from .pretrained import list_pretrained, list_pretrained_models_by_tag, list_pretrained_tags_by_model, \
10 | get_pretrained_url, download_pretrained_from_url, is_pretrained_cfg, get_pretrained_cfg, download_pretrained
11 | from .push_to_hf_hub import push_pretrained_to_hf_hub, push_to_hf_hub
12 | from .tokenizer import SimpleTokenizer, tokenize, decode
13 | from .transform import image_transform, AugmentationCfg
14 | from .zero_shot_classifier import build_zero_shot_classifier, build_zero_shot_classifier_legacy
15 | from .zero_shot_metadata import OPENAI_IMAGENET_TEMPLATES, SIMPLE_IMAGENET_TEMPLATES, IMAGENET_CLASSNAMES
16 |
--------------------------------------------------------------------------------
/ext/open_clip/bpe_simple_vocab_16e6.txt.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lxtGH/OMG-Seg/1ae2e7d446a607fddaefabf4ace312a2cdf7ab55/ext/open_clip/bpe_simple_vocab_16e6.txt.gz
--------------------------------------------------------------------------------
/ext/open_clip/constants.py:
--------------------------------------------------------------------------------
1 | OPENAI_DATASET_MEAN = (0.48145466, 0.4578275, 0.40821073)
2 | OPENAI_DATASET_STD = (0.26862954, 0.26130258, 0.27577711)
3 |
--------------------------------------------------------------------------------
/ext/open_clip/generation_utils.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lxtGH/OMG-Seg/1ae2e7d446a607fddaefabf4ace312a2cdf7ab55/ext/open_clip/generation_utils.py
--------------------------------------------------------------------------------
/ext/open_clip/model_configs/EVA01-g-14-plus.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 1024,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "timm_model_name": "eva_giant_patch14_224",
6 | "timm_model_pretrained": false,
7 | "timm_pool": "token",
8 | "timm_proj": null
9 | },
10 | "text_cfg": {
11 | "context_length": 77,
12 | "vocab_size": 49408,
13 | "width": 1024,
14 | "heads": 16,
15 | "layers": 24
16 | },
17 | "custom_text": true
18 | }
--------------------------------------------------------------------------------
/ext/open_clip/model_configs/EVA01-g-14.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 1024,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "timm_model_name": "eva_giant_patch14_224",
6 | "timm_model_pretrained": false,
7 | "timm_pool": "token",
8 | "timm_proj": null
9 | },
10 | "text_cfg": {
11 | "context_length": 77,
12 | "vocab_size": 49408,
13 | "width": 768,
14 | "heads": 12,
15 | "layers": 12
16 | },
17 | "custom_text": true
18 | }
--------------------------------------------------------------------------------
/ext/open_clip/model_configs/EVA02-B-16.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 512,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "timm_model_name": "eva02_base_patch16_clip_224",
6 | "timm_model_pretrained": false,
7 | "timm_pool": "token",
8 | "timm_proj": null
9 | },
10 | "text_cfg": {
11 | "context_length": 77,
12 | "vocab_size": 49408,
13 | "width": 512,
14 | "heads": 8,
15 | "layers": 12
16 | },
17 | "custom_text": true
18 | }
--------------------------------------------------------------------------------
/ext/open_clip/model_configs/EVA02-E-14-plus.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 1024,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "timm_model_name": "eva02_enormous_patch14_clip_224",
6 | "timm_model_pretrained": false,
7 | "timm_pool": "token",
8 | "timm_proj": null
9 | },
10 | "text_cfg": {
11 | "context_length": 77,
12 | "vocab_size": 49408,
13 | "width": 1280,
14 | "heads": 20,
15 | "layers": 32
16 | },
17 | "custom_text": true
18 | }
--------------------------------------------------------------------------------
/ext/open_clip/model_configs/EVA02-E-14.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 1024,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "timm_model_name": "eva02_enormous_patch14_clip_224",
6 | "timm_model_pretrained": false,
7 | "timm_pool": "token",
8 | "timm_proj": null
9 | },
10 | "text_cfg": {
11 | "context_length": 77,
12 | "vocab_size": 49408,
13 | "width": 1024,
14 | "heads": 16,
15 | "layers": 24
16 | },
17 | "custom_text": true
18 | }
--------------------------------------------------------------------------------
/ext/open_clip/model_configs/EVA02-L-14-336.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 768,
3 | "vision_cfg": {
4 | "image_size": 336,
5 | "timm_model_name": "eva02_large_patch14_clip_336",
6 | "timm_model_pretrained": false,
7 | "timm_pool": "token",
8 | "timm_proj": null
9 | },
10 | "text_cfg": {
11 | "context_length": 77,
12 | "vocab_size": 49408,
13 | "width": 768,
14 | "heads": 12,
15 | "layers": 12
16 | },
17 | "custom_text": true
18 | }
--------------------------------------------------------------------------------
/ext/open_clip/model_configs/EVA02-L-14.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 768,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "timm_model_name": "eva02_large_patch14_clip_224",
6 | "timm_model_pretrained": false,
7 | "timm_pool": "token",
8 | "timm_proj": null
9 | },
10 | "text_cfg": {
11 | "context_length": 77,
12 | "vocab_size": 49408,
13 | "width": 768,
14 | "heads": 12,
15 | "layers": 12
16 | },
17 | "custom_text": true
18 | }
--------------------------------------------------------------------------------
/ext/open_clip/model_configs/RN101-quickgelu.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 512,
3 | "quick_gelu": true,
4 | "vision_cfg": {
5 | "image_size": 224,
6 | "layers": [
7 | 3,
8 | 4,
9 | 23,
10 | 3
11 | ],
12 | "width": 64,
13 | "patch_size": null
14 | },
15 | "text_cfg": {
16 | "context_length": 77,
17 | "vocab_size": 49408,
18 | "width": 512,
19 | "heads": 8,
20 | "layers": 12
21 | }
22 | }
--------------------------------------------------------------------------------
/ext/open_clip/model_configs/RN101.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 512,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "layers": [
6 | 3,
7 | 4,
8 | 23,
9 | 3
10 | ],
11 | "width": 64,
12 | "patch_size": null
13 | },
14 | "text_cfg": {
15 | "context_length": 77,
16 | "vocab_size": 49408,
17 | "width": 512,
18 | "heads": 8,
19 | "layers": 12
20 | }
21 | }
--------------------------------------------------------------------------------
/ext/open_clip/model_configs/RN50-quickgelu.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 1024,
3 | "quick_gelu": true,
4 | "vision_cfg": {
5 | "image_size": 224,
6 | "layers": [
7 | 3,
8 | 4,
9 | 6,
10 | 3
11 | ],
12 | "width": 64,
13 | "patch_size": null
14 | },
15 | "text_cfg": {
16 | "context_length": 77,
17 | "vocab_size": 49408,
18 | "width": 512,
19 | "heads": 8,
20 | "layers": 12
21 | }
22 | }
23 |
--------------------------------------------------------------------------------
/ext/open_clip/model_configs/RN50.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 1024,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "layers": [
6 | 3,
7 | 4,
8 | 6,
9 | 3
10 | ],
11 | "width": 64,
12 | "patch_size": null
13 | },
14 | "text_cfg": {
15 | "context_length": 77,
16 | "vocab_size": 49408,
17 | "width": 512,
18 | "heads": 8,
19 | "layers": 12
20 | }
21 | }
--------------------------------------------------------------------------------
/ext/open_clip/model_configs/RN50x16.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 768,
3 | "vision_cfg": {
4 | "image_size": 384,
5 | "layers": [
6 | 6,
7 | 8,
8 | 18,
9 | 8
10 | ],
11 | "width": 96,
12 | "patch_size": null
13 | },
14 | "text_cfg": {
15 | "context_length": 77,
16 | "vocab_size": 49408,
17 | "width": 768,
18 | "heads": 12,
19 | "layers": 12
20 | }
21 | }
--------------------------------------------------------------------------------
/ext/open_clip/model_configs/RN50x4.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 640,
3 | "vision_cfg": {
4 | "image_size": 288,
5 | "layers": [
6 | 4,
7 | 6,
8 | 10,
9 | 6
10 | ],
11 | "width": 80,
12 | "patch_size": null
13 | },
14 | "text_cfg": {
15 | "context_length": 77,
16 | "vocab_size": 49408,
17 | "width": 640,
18 | "heads": 10,
19 | "layers": 12
20 | }
21 | }
--------------------------------------------------------------------------------
/ext/open_clip/model_configs/RN50x64.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 1024,
3 | "vision_cfg": {
4 | "image_size": 448,
5 | "layers": [
6 | 3,
7 | 15,
8 | 36,
9 | 10
10 | ],
11 | "width": 128,
12 | "patch_size": null
13 | },
14 | "text_cfg": {
15 | "context_length": 77,
16 | "vocab_size": 49408,
17 | "width": 1024,
18 | "heads": 16,
19 | "layers": 12
20 | }
21 | }
--------------------------------------------------------------------------------
/ext/open_clip/model_configs/ViT-B-16-plus-240.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 640,
3 | "vision_cfg": {
4 | "image_size": 240,
5 | "layers": 12,
6 | "width": 896,
7 | "patch_size": 16
8 | },
9 | "text_cfg": {
10 | "context_length": 77,
11 | "vocab_size": 49408,
12 | "width": 640,
13 | "heads": 10,
14 | "layers": 12
15 | }
16 | }
--------------------------------------------------------------------------------
/ext/open_clip/model_configs/ViT-B-16-plus.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 640,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "layers": 12,
6 | "width": 896,
7 | "patch_size": 16
8 | },
9 | "text_cfg": {
10 | "context_length": 77,
11 | "vocab_size": 49408,
12 | "width": 640,
13 | "heads": 10,
14 | "layers": 12
15 | }
16 | }
--------------------------------------------------------------------------------
/ext/open_clip/model_configs/ViT-B-16.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 512,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "layers": 12,
6 | "width": 768,
7 | "patch_size": 16
8 | },
9 | "text_cfg": {
10 | "context_length": 77,
11 | "vocab_size": 49408,
12 | "width": 512,
13 | "heads": 8,
14 | "layers": 12
15 | }
16 | }
--------------------------------------------------------------------------------
/ext/open_clip/model_configs/ViT-B-32-plus-256.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 640,
3 | "vision_cfg": {
4 | "image_size": 256,
5 | "layers": 12,
6 | "width": 896,
7 | "patch_size": 32
8 | },
9 | "text_cfg": {
10 | "context_length": 77,
11 | "vocab_size": 49408,
12 | "width": 640,
13 | "heads": 10,
14 | "layers": 12
15 | }
16 | }
--------------------------------------------------------------------------------
/ext/open_clip/model_configs/ViT-B-32-quickgelu.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 512,
3 | "quick_gelu": true,
4 | "vision_cfg": {
5 | "image_size": 224,
6 | "layers": 12,
7 | "width": 768,
8 | "patch_size": 32
9 | },
10 | "text_cfg": {
11 | "context_length": 77,
12 | "vocab_size": 49408,
13 | "width": 512,
14 | "heads": 8,
15 | "layers": 12
16 | }
17 | }
--------------------------------------------------------------------------------
/ext/open_clip/model_configs/ViT-B-32.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 512,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "layers": 12,
6 | "width": 768,
7 | "patch_size": 32
8 | },
9 | "text_cfg": {
10 | "context_length": 77,
11 | "vocab_size": 49408,
12 | "width": 512,
13 | "heads": 8,
14 | "layers": 12
15 | }
16 | }
--------------------------------------------------------------------------------
/ext/open_clip/model_configs/ViT-H-14.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 1024,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "layers": 32,
6 | "width": 1280,
7 | "head_width": 80,
8 | "patch_size": 14
9 | },
10 | "text_cfg": {
11 | "context_length": 77,
12 | "vocab_size": 49408,
13 | "width": 1024,
14 | "heads": 16,
15 | "layers": 24
16 | }
17 | }
--------------------------------------------------------------------------------
/ext/open_clip/model_configs/ViT-H-16.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 1024,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "layers": 32,
6 | "width": 1280,
7 | "head_width": 80,
8 | "patch_size": 16
9 | },
10 | "text_cfg": {
11 | "context_length": 77,
12 | "vocab_size": 49408,
13 | "width": 1024,
14 | "heads": 16,
15 | "layers": 24
16 | }
17 | }
--------------------------------------------------------------------------------
/ext/open_clip/model_configs/ViT-L-14-280.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 768,
3 | "vision_cfg": {
4 | "image_size": 280,
5 | "layers": 24,
6 | "width": 1024,
7 | "patch_size": 14
8 | },
9 | "text_cfg": {
10 | "context_length": 77,
11 | "vocab_size": 49408,
12 | "width": 768,
13 | "heads": 12,
14 | "layers": 12
15 | }
16 | }
--------------------------------------------------------------------------------
/ext/open_clip/model_configs/ViT-L-14-336.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 768,
3 | "vision_cfg": {
4 | "image_size": 336,
5 | "layers": 24,
6 | "width": 1024,
7 | "patch_size": 14
8 | },
9 | "text_cfg": {
10 | "context_length": 77,
11 | "vocab_size": 49408,
12 | "width": 768,
13 | "heads": 12,
14 | "layers": 12
15 | }
16 | }
--------------------------------------------------------------------------------
/ext/open_clip/model_configs/ViT-L-14.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 768,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "layers": 24,
6 | "width": 1024,
7 | "patch_size": 14
8 | },
9 | "text_cfg": {
10 | "context_length": 77,
11 | "vocab_size": 49408,
12 | "width": 768,
13 | "heads": 12,
14 | "layers": 12
15 | }
16 | }
--------------------------------------------------------------------------------
/ext/open_clip/model_configs/ViT-L-16-320.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 768,
3 | "vision_cfg": {
4 | "image_size": 320,
5 | "layers": 24,
6 | "width": 1024,
7 | "patch_size": 16
8 | },
9 | "text_cfg": {
10 | "context_length": 77,
11 | "vocab_size": 49408,
12 | "width": 768,
13 | "heads": 12,
14 | "layers": 12
15 | }
16 | }
--------------------------------------------------------------------------------
/ext/open_clip/model_configs/ViT-L-16.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 768,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "layers": 24,
6 | "width": 1024,
7 | "patch_size": 16
8 | },
9 | "text_cfg": {
10 | "context_length": 77,
11 | "vocab_size": 49408,
12 | "width": 768,
13 | "heads": 12,
14 | "layers": 12
15 | }
16 | }
--------------------------------------------------------------------------------
/ext/open_clip/model_configs/ViT-M-16-alt.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 384,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "layers": 12,
6 | "width": 512,
7 | "patch_size": 16,
8 | "ls_init_value": 1e-4
9 | },
10 | "text_cfg": {
11 | "context_length": 77,
12 | "vocab_size": 49408,
13 | "width": 384,
14 | "heads": 6,
15 | "layers": 12
16 | }
17 | }
--------------------------------------------------------------------------------
/ext/open_clip/model_configs/ViT-M-16.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 512,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "layers": 12,
6 | "width": 512,
7 | "patch_size": 16
8 | },
9 | "text_cfg": {
10 | "context_length": 77,
11 | "vocab_size": 49408,
12 | "width": 512,
13 | "heads": 8,
14 | "layers": 12
15 | }
16 | }
--------------------------------------------------------------------------------
/ext/open_clip/model_configs/ViT-M-32-alt.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 384,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "layers": 12,
6 | "width": 512,
7 | "patch_size": 32
8 | },
9 | "text_cfg": {
10 | "context_length": 77,
11 | "vocab_size": 49408,
12 | "width": 384,
13 | "heads": 6,
14 | "layers": 12
15 | }
16 | }
--------------------------------------------------------------------------------
/ext/open_clip/model_configs/ViT-M-32.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 512,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "layers": 12,
6 | "width": 512,
7 | "patch_size": 32
8 | },
9 | "text_cfg": {
10 | "context_length": 77,
11 | "vocab_size": 49408,
12 | "width": 512,
13 | "heads": 8,
14 | "layers": 12
15 | }
16 | }
--------------------------------------------------------------------------------
/ext/open_clip/model_configs/ViT-S-16-alt.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 256,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "layers": 12,
6 | "width": 384,
7 | "patch_size": 16
8 | },
9 | "text_cfg": {
10 | "context_length": 77,
11 | "vocab_size": 49408,
12 | "width": 256,
13 | "heads": 4,
14 | "layers": 10
15 | }
16 | }
--------------------------------------------------------------------------------
/ext/open_clip/model_configs/ViT-S-16.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 384,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "layers": 12,
6 | "width": 384,
7 | "patch_size": 16
8 | },
9 | "text_cfg": {
10 | "context_length": 77,
11 | "vocab_size": 49408,
12 | "width": 384,
13 | "heads": 6,
14 | "layers": 12
15 | }
16 | }
--------------------------------------------------------------------------------
/ext/open_clip/model_configs/ViT-S-32-alt.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 256,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "layers": 12,
6 | "width": 384,
7 | "patch_size": 32
8 | },
9 | "text_cfg": {
10 | "context_length": 77,
11 | "vocab_size": 49408,
12 | "width": 256,
13 | "heads": 4,
14 | "layers": 10
15 | }
16 | }
--------------------------------------------------------------------------------
/ext/open_clip/model_configs/ViT-S-32.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 384,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "layers": 12,
6 | "width": 384,
7 | "patch_size": 32
8 | },
9 | "text_cfg": {
10 | "context_length": 77,
11 | "vocab_size": 49408,
12 | "width": 384,
13 | "heads": 6,
14 | "layers": 12
15 | }
16 | }
--------------------------------------------------------------------------------
/ext/open_clip/model_configs/ViT-bigG-14.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 1280,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "layers": 48,
6 | "width": 1664,
7 | "head_width": 104,
8 | "mlp_ratio": 4.9231,
9 | "patch_size": 14
10 | },
11 | "text_cfg": {
12 | "context_length": 77,
13 | "vocab_size": 49408,
14 | "width": 1280,
15 | "heads": 20,
16 | "layers": 32
17 | }
18 | }
--------------------------------------------------------------------------------
/ext/open_clip/model_configs/ViT-e-14.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 1280,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "layers": 56,
6 | "width": 1792,
7 | "head_width": 112,
8 | "mlp_ratio": 8.5715,
9 | "patch_size": 14
10 | },
11 | "text_cfg": {
12 | "context_length": 77,
13 | "vocab_size": 49408,
14 | "width": 1280,
15 | "heads": 20,
16 | "layers": 36
17 | }
18 | }
--------------------------------------------------------------------------------
/ext/open_clip/model_configs/ViT-g-14.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 1024,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "layers": 40,
6 | "width": 1408,
7 | "head_width": 88,
8 | "mlp_ratio": 4.3637,
9 | "patch_size": 14
10 | },
11 | "text_cfg": {
12 | "context_length": 77,
13 | "vocab_size": 49408,
14 | "width": 1024,
15 | "heads": 16,
16 | "layers": 24
17 | }
18 | }
--------------------------------------------------------------------------------
/ext/open_clip/model_configs/coca_ViT-B-32.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 512,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "layers": 12,
6 | "width": 768,
7 | "patch_size": 32,
8 | "attentional_pool": true,
9 | "attn_pooler_heads": 8,
10 | "output_tokens": true
11 | },
12 | "text_cfg": {
13 | "context_length": 76,
14 | "vocab_size": 49408,
15 | "width": 512,
16 | "heads": 8,
17 | "layers": 12,
18 | "embed_cls": true,
19 | "output_tokens": true
20 | },
21 | "multimodal_cfg": {
22 | "context_length": 76,
23 | "vocab_size": 49408,
24 | "width": 512,
25 | "heads": 8,
26 | "layers": 12,
27 | "attn_pooler_heads": 8
28 | },
29 | "custom_text": true
30 | }
--------------------------------------------------------------------------------
/ext/open_clip/model_configs/coca_ViT-L-14.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 768,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "layers": 24,
6 | "width": 1024,
7 | "patch_size": 14,
8 | "attentional_pool": true,
9 | "attn_pooler_heads": 8,
10 | "output_tokens": true
11 | },
12 | "text_cfg": {
13 | "context_length": 76,
14 | "vocab_size": 49408,
15 | "width": 768,
16 | "heads": 12,
17 | "layers": 12,
18 | "embed_cls": true,
19 | "output_tokens": true
20 | },
21 | "multimodal_cfg": {
22 | "context_length": 76,
23 | "vocab_size": 49408,
24 | "width": 768,
25 | "heads": 12,
26 | "layers": 12,
27 | "attn_pooler_heads": 12
28 | },
29 | "custom_text": true
30 | }
31 |
--------------------------------------------------------------------------------
/ext/open_clip/model_configs/coca_base.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 512,
3 | "multimodal_cfg": {
4 | "width": 768,
5 | "context_length": 76,
6 | "vocab_size": 64000,
7 | "mlp_ratio": 4,
8 | "layers": 12,
9 | "dim_head": 64,
10 | "heads": 12,
11 | "n_queries": 256,
12 | "attn_pooler_heads": 8
13 | },
14 | "vision_cfg": {
15 | "image_size": 288,
16 | "layers": 12,
17 | "width": 768,
18 | "patch_size": 18,
19 | "output_tokens": true
20 | },
21 | "text_cfg": {
22 | "context_length": 76,
23 | "vocab_size": 64000,
24 | "layers": 12,
25 | "heads": 12,
26 | "width": 768,
27 | "embed_cls": true,
28 | "output_tokens": true
29 | },
30 | "custom_text": true
31 | }
--------------------------------------------------------------------------------
/ext/open_clip/model_configs/coca_roberta-ViT-B-32.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 512,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "layers": 12,
6 | "width": 768,
7 | "patch_size": 32,
8 | "output_tokens": true
9 | },
10 | "text_cfg": {
11 | "hf_model_name": "roberta-base",
12 | "hf_tokenizer_name": "roberta-base",
13 | "proj": "linear",
14 | "width": 768,
15 | "output_tokens": true
16 | },
17 | "multimodal_cfg": {
18 | "context_length": 76,
19 | "width": 768,
20 | "heads": 8,
21 | "layers": 12
22 | },
23 | "custom_text": true
24 | }
25 |
--------------------------------------------------------------------------------
/ext/open_clip/model_configs/convnext_base.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 512,
3 | "vision_cfg": {
4 | "timm_model_name": "convnext_base",
5 | "timm_model_pretrained": false,
6 | "timm_pool": "",
7 | "timm_proj": "linear",
8 | "timm_drop": 0.0,
9 | "timm_drop_path": 0.1,
10 | "image_size": 224
11 | },
12 | "text_cfg": {
13 | "context_length": 77,
14 | "vocab_size": 49408,
15 | "width": 512,
16 | "heads": 8,
17 | "layers": 12
18 | }
19 | }
--------------------------------------------------------------------------------
/ext/open_clip/model_configs/convnext_base_w.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 640,
3 | "vision_cfg": {
4 | "timm_model_name": "convnext_base",
5 | "timm_model_pretrained": false,
6 | "timm_pool": "",
7 | "timm_proj": "linear",
8 | "timm_drop": 0.0,
9 | "timm_drop_path": 0.1,
10 | "image_size": 256
11 | },
12 | "text_cfg": {
13 | "context_length": 77,
14 | "vocab_size": 49408,
15 | "width": 640,
16 | "heads": 10,
17 | "layers": 12
18 | }
19 | }
--------------------------------------------------------------------------------
/ext/open_clip/model_configs/convnext_base_w_320.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 640,
3 | "vision_cfg": {
4 | "timm_model_name": "convnext_base",
5 | "timm_model_pretrained": false,
6 | "timm_pool": "",
7 | "timm_proj": "linear",
8 | "timm_drop": 0.0,
9 | "timm_drop_path": 0.1,
10 | "image_size": 320
11 | },
12 | "text_cfg": {
13 | "context_length": 77,
14 | "vocab_size": 49408,
15 | "width": 640,
16 | "heads": 10,
17 | "layers": 12
18 | }
19 | }
--------------------------------------------------------------------------------
/ext/open_clip/model_configs/convnext_large.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 768,
3 | "vision_cfg": {
4 | "timm_model_name": "convnext_large",
5 | "timm_model_pretrained": false,
6 | "timm_pool": "",
7 | "timm_proj": "linear",
8 | "timm_drop": 0.0,
9 | "timm_drop_path": 0.1,
10 | "image_size": 224
11 | },
12 | "text_cfg": {
13 | "context_length": 77,
14 | "vocab_size": 49408,
15 | "width": 768,
16 | "heads": 12,
17 | "layers": 12
18 | }
19 | }
--------------------------------------------------------------------------------
/ext/open_clip/model_configs/convnext_large_d.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 768,
3 | "vision_cfg": {
4 | "timm_model_name": "convnext_large",
5 | "timm_model_pretrained": false,
6 | "timm_pool": "",
7 | "timm_proj": "mlp",
8 | "timm_drop": 0.0,
9 | "timm_drop_path": 0.1,
10 | "image_size": 256
11 | },
12 | "text_cfg": {
13 | "context_length": 77,
14 | "vocab_size": 49408,
15 | "width": 768,
16 | "heads": 12,
17 | "layers": 16
18 | }
19 | }
--------------------------------------------------------------------------------
/ext/open_clip/model_configs/convnext_large_d_320.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 768,
3 | "vision_cfg": {
4 | "timm_model_name": "convnext_large",
5 | "timm_model_pretrained": false,
6 | "timm_pool": "",
7 | "timm_proj": "mlp",
8 | "timm_drop": 0.0,
9 | "timm_drop_path": 0.1,
10 | "image_size": 320
11 | },
12 | "text_cfg": {
13 | "context_length": 77,
14 | "vocab_size": 49408,
15 | "width": 768,
16 | "heads": 12,
17 | "layers": 16
18 | }
19 | }
--------------------------------------------------------------------------------
/ext/open_clip/model_configs/convnext_small.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 512,
3 | "vision_cfg": {
4 | "timm_model_name": "convnext_small",
5 | "timm_model_pretrained": false,
6 | "timm_pool": "",
7 | "timm_proj": "linear",
8 | "timm_drop": 0.0,
9 | "timm_drop_path": 0.1,
10 | "image_size": 224
11 | },
12 | "text_cfg": {
13 | "context_length": 77,
14 | "vocab_size": 49408,
15 | "width": 512,
16 | "heads": 8,
17 | "layers": 12
18 | }
19 | }
--------------------------------------------------------------------------------
/ext/open_clip/model_configs/convnext_tiny.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 1024,
3 | "vision_cfg": {
4 | "timm_model_name": "convnext_tiny",
5 | "timm_model_pretrained": false,
6 | "timm_pool": "",
7 | "timm_proj": "linear",
8 | "timm_drop": 0.0,
9 | "timm_drop_path": 0.1,
10 | "image_size": 224
11 | },
12 | "text_cfg": {
13 | "context_length": 77,
14 | "vocab_size": 49408,
15 | "width": 512,
16 | "heads": 8,
17 | "layers": 12
18 | }
19 | }
--------------------------------------------------------------------------------
/ext/open_clip/model_configs/convnext_xlarge.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 1024,
3 | "vision_cfg": {
4 | "timm_model_name": "convnext_xlarge",
5 | "timm_model_pretrained": false,
6 | "timm_pool": "",
7 | "timm_proj": "linear",
8 | "timm_drop": 0.0,
9 | "timm_drop_path": 0.1,
10 | "image_size": 256
11 | },
12 | "text_cfg": {
13 | "context_length": 77,
14 | "vocab_size": 49408,
15 | "width": 1024,
16 | "heads": 16,
17 | "layers": 20
18 | }
19 | }
--------------------------------------------------------------------------------
/ext/open_clip/model_configs/convnext_xxlarge.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 1024,
3 | "vision_cfg": {
4 | "timm_model_name": "convnext_xxlarge",
5 | "timm_model_pretrained": false,
6 | "timm_pool": "",
7 | "timm_proj": "linear",
8 | "timm_drop": 0.0,
9 | "timm_drop_path": 0.1,
10 | "image_size": 256
11 | },
12 | "text_cfg": {
13 | "context_length": 77,
14 | "vocab_size": 49408,
15 | "width": 1024,
16 | "heads": 16,
17 | "layers": 24
18 | }
19 | }
--------------------------------------------------------------------------------
/ext/open_clip/model_configs/convnext_xxlarge_320.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 1024,
3 | "vision_cfg": {
4 | "timm_model_name": "convnext_xxlarge",
5 | "timm_model_pretrained": false,
6 | "timm_pool": "",
7 | "timm_proj": "linear",
8 | "timm_drop": 0.0,
9 | "timm_drop_path": 0.1,
10 | "image_size": 320
11 | },
12 | "text_cfg": {
13 | "context_length": 77,
14 | "vocab_size": 49408,
15 | "width": 1024,
16 | "heads": 16,
17 | "layers": 24
18 | }
19 | }
--------------------------------------------------------------------------------
/ext/open_clip/model_configs/mt5-base-ViT-B-32.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 512,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "layers": 12,
6 | "width": 768,
7 | "patch_size": 32
8 | },
9 | "text_cfg": {
10 | "hf_model_name": "google/mt5-base",
11 | "hf_tokenizer_name": "google/mt5-base",
12 | "proj": "mlp",
13 | "pooler_type": "mean_pooler"
14 | }
15 | }
16 |
--------------------------------------------------------------------------------
/ext/open_clip/model_configs/mt5-xl-ViT-H-14.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 1024,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "layers": 32,
6 | "width": 1280,
7 | "head_width": 80,
8 | "patch_size": 14
9 | },
10 | "text_cfg": {
11 | "hf_model_name": "google/mt5-xl",
12 | "hf_tokenizer_name": "google/mt5-xl",
13 | "proj": "mlp",
14 | "pooler_type": "mean_pooler"
15 | }
16 | }
17 |
--------------------------------------------------------------------------------
/ext/open_clip/model_configs/roberta-ViT-B-32.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 512,
3 | "quick_gelu": true,
4 | "vision_cfg": {
5 | "image_size": 224,
6 | "layers": 12,
7 | "width": 768,
8 | "patch_size": 32
9 | },
10 | "text_cfg": {
11 | "hf_model_name": "roberta-base",
12 | "hf_tokenizer_name": "roberta-base",
13 | "proj": "mlp",
14 | "pooler_type": "mean_pooler"
15 | }
16 | }
17 |
--------------------------------------------------------------------------------
/ext/open_clip/model_configs/swin_base_patch4_window7_224.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 640,
3 | "vision_cfg": {
4 | "timm_model_name": "swin_base_patch4_window7_224",
5 | "timm_model_pretrained": false,
6 | "timm_pool": "",
7 | "timm_proj": "linear",
8 | "image_size": 224
9 | },
10 | "text_cfg": {
11 | "context_length": 77,
12 | "vocab_size": 49408,
13 | "width": 640,
14 | "heads": 10,
15 | "layers": 12
16 | }
17 | }
--------------------------------------------------------------------------------
/ext/open_clip/model_configs/vit_medium_patch16_gap_256.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 512,
3 | "vision_cfg": {
4 | "timm_model_name": "vit_medium_patch16_gap_256",
5 | "timm_model_pretrained": false,
6 | "timm_pool": "",
7 | "timm_proj": "linear",
8 | "image_size": 256
9 | },
10 | "text_cfg": {
11 | "context_length": 77,
12 | "vocab_size": 49408,
13 | "width": 512,
14 | "heads": 8,
15 | "layers": 12
16 | }
17 | }
--------------------------------------------------------------------------------
/ext/open_clip/model_configs/vit_relpos_medium_patch16_cls_224.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 512,
3 | "vision_cfg": {
4 | "timm_model_name": "vit_relpos_medium_patch16_cls_224",
5 | "timm_model_pretrained": false,
6 | "timm_pool": "",
7 | "timm_proj": "linear",
8 | "image_size": 224
9 | },
10 | "text_cfg": {
11 | "context_length": 77,
12 | "vocab_size": 49408,
13 | "width": 512,
14 | "heads": 8,
15 | "layers": 12
16 | }
17 | }
--------------------------------------------------------------------------------
/ext/open_clip/model_configs/xlm-roberta-base-ViT-B-32.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 512,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "layers": 12,
6 | "width": 768,
7 | "patch_size": 32
8 | },
9 | "text_cfg": {
10 | "hf_model_name": "xlm-roberta-base",
11 | "hf_tokenizer_name": "xlm-roberta-base",
12 | "proj": "mlp",
13 | "pooler_type": "mean_pooler"
14 | }
15 | }
16 |
--------------------------------------------------------------------------------
/ext/open_clip/model_configs/xlm-roberta-large-ViT-H-14.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 1024,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "layers": 32,
6 | "width": 1280,
7 | "head_width": 80,
8 | "patch_size": 14
9 | },
10 | "text_cfg": {
11 | "hf_model_name": "xlm-roberta-large",
12 | "hf_tokenizer_name": "xlm-roberta-large",
13 | "proj": "mlp",
14 | "pooler_type": "mean_pooler"
15 | }
16 | }
17 |
--------------------------------------------------------------------------------
/ext/open_clip/version.py:
--------------------------------------------------------------------------------
1 | __version__ = '2.20.0'
2 |
--------------------------------------------------------------------------------
/ext/sam/__init__.py:
--------------------------------------------------------------------------------
1 | from .image_encoder import ImageEncoderViT
2 | from .prompt_encoder import PromptEncoder
3 | from .mask_decoder import MaskDecoder
4 |
--------------------------------------------------------------------------------
/ext/sam/common.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) Meta Platforms, Inc. and affiliates.
2 | # All rights reserved.
3 |
4 | # This source code is licensed under the license found in the
5 | # LICENSE file in the root directory of this source tree.
6 |
7 | import torch
8 | import torch.nn as nn
9 |
10 | from typing import Type
11 |
12 |
13 | class MLPBlock(nn.Module):
14 | def __init__(
15 | self,
16 | embedding_dim: int,
17 | mlp_dim: int,
18 | act: Type[nn.Module] = nn.GELU,
19 | ) -> None:
20 | super().__init__()
21 | self.lin1 = nn.Linear(embedding_dim, mlp_dim)
22 | self.lin2 = nn.Linear(mlp_dim, embedding_dim)
23 | self.act = act()
24 |
25 | def forward(self, x: torch.Tensor) -> torch.Tensor:
26 | return self.lin2(self.act(self.lin1(x)))
27 |
28 |
29 | # From https://github.com/facebookresearch/detectron2/blob/main/detectron2/layers/batch_norm.py # noqa
30 | # Itself from https://github.com/facebookresearch/ConvNeXt/blob/d1fa8f6fef0a165b27399986cc2bdacc92777e40/models/convnext.py#L119 # noqa
31 | class LayerNorm2d(nn.Module):
32 | def __init__(self, num_channels: int, eps: float = 1e-6) -> None:
33 | super().__init__()
34 | self.weight = nn.Parameter(torch.ones(num_channels))
35 | self.bias = nn.Parameter(torch.zeros(num_channels))
36 | self.eps = eps
37 |
38 | def forward(self, x: torch.Tensor) -> torch.Tensor:
39 | u = x.mean(1, keepdim=True)
40 | s = (x - u).pow(2).mean(1, keepdim=True)
41 | x = (x - u) / torch.sqrt(s + self.eps)
42 | x = self.weight[:, None, None] * x + self.bias[:, None, None]
43 | return x
44 |
--------------------------------------------------------------------------------
/ext/templates/__init__.py:
--------------------------------------------------------------------------------
1 | from .vild import VILD_PROMPT
2 |
--------------------------------------------------------------------------------
/ext/templates/vild.py:
--------------------------------------------------------------------------------
1 | # https://github.com/bytedance/fc-clip/blob/93f3122518e8a3ef98926e5ea761a776d5050430/fcclip/fcclip.py#L26C1-L41C2
2 | VILD_PROMPT = [
3 | "a photo of a {}.",
4 | "This is a photo of a {}",
5 | "There is a {} in the scene",
6 | "There is the {} in the scene",
7 | "a photo of a {} in the scene",
8 | "a photo of a small {}.",
9 | "a photo of a medium {}.",
10 | "a photo of a large {}.",
11 | "This is a photo of a small {}.",
12 | "This is a photo of a medium {}.",
13 | "This is a photo of a large {}.",
14 | "There is a small {} in the scene.",
15 | "There is a medium {} in the scene.",
16 | "There is a large {} in the scene.",
17 | ]
18 |
--------------------------------------------------------------------------------
/figs/method_comparison.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lxtGH/OMG-Seg/1ae2e7d446a607fddaefabf4ace312a2cdf7ab55/figs/method_comparison.jpg
--------------------------------------------------------------------------------
/figs/omg_teaser.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lxtGH/OMG-Seg/1ae2e7d446a607fddaefabf4ace312a2cdf7ab55/figs/omg_teaser.jpg
--------------------------------------------------------------------------------
/omg_llava/.owners.yml:
--------------------------------------------------------------------------------
1 | assign:
2 | issues: disabled
3 | pull_requests: disabled
4 | strategy:
5 | random
6 | # daily-shift-based
7 | schedule:
8 | '*/1 * * * *'
9 |
--------------------------------------------------------------------------------
/omg_llava/.pre-commit-config-zh-cn.yaml:
--------------------------------------------------------------------------------
1 | exclude: ^tests/data/
2 | repos:
3 | - repo: https://gitee.com/openmmlab/mirrors-flake8
4 | rev: 5.0.4
5 | hooks:
6 | - id: flake8
7 | - repo: https://gitee.com/openmmlab/mirrors-isort
8 | rev: 5.11.5
9 | hooks:
10 | - id: isort
11 | - repo: https://gitee.com/openmmlab/mirrors-yapf
12 | rev: v0.32.0
13 | hooks:
14 | - id: yapf
15 | - repo: https://gitee.com/openmmlab/mirrors-pre-commit-hooks
16 | rev: v4.3.0
17 | hooks:
18 | - id: trailing-whitespace
19 | - id: check-yaml
20 | - id: end-of-file-fixer
21 | - id: requirements-txt-fixer
22 | - id: double-quote-string-fixer
23 | - id: check-merge-conflict
24 | - id: fix-encoding-pragma
25 | args: ["--remove"]
26 | - id: mixed-line-ending
27 | args: ["--fix=lf"]
28 | - repo: https://gitee.com/openmmlab/mirrors-codespell
29 | rev: v2.2.1
30 | hooks:
31 | - id: codespell
32 | - repo: https://gitee.com/openmmlab/mirrors-mdformat
33 | rev: 0.7.9
34 | hooks:
35 | - id: mdformat
36 | args: ["--number"]
37 | additional_dependencies:
38 | - mdformat-openmmlab
39 | - mdformat_frontmatter
40 | - linkify-it-py
41 | - repo: https://gitee.com/openmmlab/mirrors-docformatter
42 | rev: v1.3.1
43 | hooks:
44 | - id: docformatter
45 | args: ["--in-place", "--wrap-descriptions", "79"]
46 | - repo: https://github.com/asottile/pyupgrade
47 | rev: v3.0.0
48 | hooks:
49 | - id: pyupgrade
50 | args: ["--py36-plus"]
51 |
--------------------------------------------------------------------------------
/omg_llava/.pre-commit-config.yaml:
--------------------------------------------------------------------------------
1 | exclude: ^tests/data/
2 | repos:
3 | - repo: https://github.com/PyCQA/flake8
4 | rev: 5.0.4
5 | hooks:
6 | - id: flake8
7 | - repo: https://github.com/PyCQA/isort
8 | rev: 5.11.5
9 | hooks:
10 | - id: isort
11 | - repo: https://github.com/pre-commit/mirrors-yapf
12 | rev: v0.32.0
13 | hooks:
14 | - id: yapf
15 | exclude: 'xtuner/parallel/sequence/__init__.py'
16 | - repo: https://github.com/pre-commit/pre-commit-hooks
17 | rev: v4.3.0
18 | hooks:
19 | - id: trailing-whitespace
20 | - id: check-yaml
21 | - id: end-of-file-fixer
22 | - id: requirements-txt-fixer
23 | - id: double-quote-string-fixer
24 | - id: check-merge-conflict
25 | - id: fix-encoding-pragma
26 | args: ["--remove"]
27 | - id: mixed-line-ending
28 | args: ["--fix=lf"]
29 | - repo: https://github.com/codespell-project/codespell
30 | rev: v2.2.1
31 | hooks:
32 | - id: codespell
33 | - repo: https://github.com/executablebooks/mdformat
34 | rev: 0.7.9
35 | hooks:
36 | - id: mdformat
37 | args: ["--number"]
38 | additional_dependencies:
39 | - mdformat-openmmlab
40 | - mdformat_frontmatter
41 | - linkify-it-py
42 | exclude: 'docs/zh_cn/user_guides/sequence_parallel.md'
43 | - repo: https://github.com/myint/docformatter
44 | rev: v1.3.1
45 | hooks:
46 | - id: docformatter
47 | args: ["--in-place", "--wrap-descriptions", "79"]
48 | - repo: https://github.com/asottile/pyupgrade
49 | rev: v3.0.0
50 | hooks:
51 | - id: pyupgrade
52 | args: ["--py36-plus"]
53 |
--------------------------------------------------------------------------------
/omg_llava/MANIFEST.in:
--------------------------------------------------------------------------------
1 | recursive-include xtuner/configs *.py *.yml *.json
2 | recursive-include xtuner/tools *.sh *.py
3 |
--------------------------------------------------------------------------------
/omg_llava/figs/omg_llava.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lxtGH/OMG-Seg/1ae2e7d446a607fddaefabf4ace312a2cdf7ab55/omg_llava/figs/omg_llava.png
--------------------------------------------------------------------------------
/omg_llava/omg_llava/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lxtGH/OMG-Seg/1ae2e7d446a607fddaefabf4ace312a2cdf7ab55/omg_llava/omg_llava/__init__.py
--------------------------------------------------------------------------------
/omg_llava/omg_llava/configs/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lxtGH/OMG-Seg/1ae2e7d446a607fddaefabf4ace312a2cdf7ab55/omg_llava/omg_llava/configs/__init__.py
--------------------------------------------------------------------------------
/omg_llava/omg_llava/dataset/collect_fns/__init__.py:
--------------------------------------------------------------------------------
1 | from .omg_llava_collate_fn import omg_llava_collate_fn
--------------------------------------------------------------------------------
/omg_llava/omg_llava/dataset/process_functions/__init__.py:
--------------------------------------------------------------------------------
1 | from .gcg_process import glamm_refcocog_map_fn, glamm_granf_map_fn, glamm_openpsg_map_fn, glamm_flickr_map_fn
2 | from .mdpv_points_process import mdpv_points_map_fn
3 | from .referring_seg_process import referring_seg_map_fn, referring_seg_gcg_format_map_fn
4 | from .region_caption_process import osprey_region_caption_map_fn, osprey_region_caption_gcg_format_map_fn, osprey_region_conversation_map_fn
5 | from .semantic_seg_process import semantic_seg_map_fn, pascal_part_map_fn, semantic_seg_gcg_format_map_fn, pascal_part_gcg_format_map_fn
6 | from .decoupled_gcg_process import glamm_openpsg_decoupled_given_objects_map_fn, glamm_openpsg_decoupled_given_description_map_fn,\
7 | glamm_flickr_decoupled_given_objects_map_fn, glamm_flickr_decoupled_given_description_map_fn,\
8 | glamm_granf_decoupled_given_objects_map_fn, glamm_granf_decoupled_given_description_map_fn,\
9 | glamm_refcocog_decoupled_given_description_map_fn, glamm_refcocog_decoupled_given_objects_map_fn
--------------------------------------------------------------------------------
/omg_llava/omg_llava/dataset/utils/__init__.py:
--------------------------------------------------------------------------------
1 | from .utils import expand2square, expand2square_mask, expand2square_points, expand2square_bbox
--------------------------------------------------------------------------------
/omg_llava/omg_llava/dataset/utils/ade20k_classes.json:
--------------------------------------------------------------------------------
1 | [
2 | "wall", "building", "sky", "floor", "tree", "ceiling", "road",
3 | "bed", "windowpane", "grass", "cabinet", "sidewalk",
4 | "person", "earth", "door", "table", "mountain", "plant",
5 | "curtain", "chair", "car", "water", "painting", "sofa",
6 | "shelf", "house", "sea", "mirror", "rug", "field", "armchair",
7 | "seat", "fence", "desk", "rock", "wardrobe", "lamp",
8 | "bathtub", "railing", "cushion", "base", "box", "column",
9 | "signboard", "chest of drawers", "counter", "sand", "sink",
10 | "skyscraper", "fireplace", "refrigerator", "grandstand",
11 | "path", "stairs", "runway", "case", "pool table", "pillow",
12 | "screen door", "stairway", "river", "bridge", "bookcase",
13 | "blind", "coffee table", "toilet", "flower", "book", "hill",
14 | "bench", "countertop", "stove", "palm", "kitchen island",
15 | "computer", "swivel chair", "boat", "bar", "arcade machine",
16 | "hovel", "bus", "towel", "light", "truck", "tower",
17 | "chandelier", "awning", "streetlight", "booth",
18 | "television receiver", "airplane", "dirt track", "apparel",
19 | "pole", "land", "bannister", "escalator", "ottoman", "bottle",
20 | "buffet", "poster", "stage", "van", "ship", "fountain",
21 | "conveyer belt", "canopy", "washer", "plaything",
22 | "swimming pool", "stool", "barrel", "basket", "waterfall",
23 | "tent", "bag", "minibike", "cradle", "oven", "ball", "food",
24 | "step", "tank", "trade name", "microwave", "pot", "animal",
25 | "bicycle", "lake", "dishwasher", "screen", "blanket",
26 | "sculpture", "hood", "sconce", "vase", "traffic light",
27 | "tray", "ashcan", "fan", "pier", "crt screen", "plate",
28 | "monitor", "bulletin board", "shower", "radiator", "glass",
29 | "clock", "flag"
30 | ]
--------------------------------------------------------------------------------
/omg_llava/omg_llava/engine/__init__.py:
--------------------------------------------------------------------------------
1 | from .dataset_info_hook import DatasetInfoHook_withSpecoalTokens
2 | from .evaluate_chat_hook import EvaluateChatHook_withSpecialTokens
--------------------------------------------------------------------------------
/omg_llava/omg_llava/engine/dataset_info_hook.py:
--------------------------------------------------------------------------------
1 | from xtuner.registry import BUILDER
2 | from xtuner.engine.hooks import DatasetInfoHook
3 |
4 | class DatasetInfoHook_withSpecoalTokens(DatasetInfoHook):
5 | def __init__(self, tokenizer, is_intern_repo_dataset=False):
6 | self.tokenizer = BUILDER.build(tokenizer)
7 | self.is_intern_repo_dataset = is_intern_repo_dataset
8 | # add special tokens
9 | # Adding special tokens for pixel grounding
10 | segmentation_tokens = ['[SEG]']
11 | # Adding tokens for GCG
12 | phrase_tokens = ['', '
']
13 | # add for visual prompt
14 | region_tokens = ['']
15 | point_tokens = ['']
16 | special_tokens = segmentation_tokens + phrase_tokens + region_tokens + point_tokens
17 | self.tokenizer.add_tokens(special_tokens, special_tokens=True)
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/__init__.py:
--------------------------------------------------------------------------------
1 | from .convnext_clip import OpenCLIPBackbone, OpenCLIPBackbone_omgseg
2 | from .modules import ProjectorConfig_OMG_LLaVA, ProjectorModel_OMG_LLaVA
3 | from .omg_seg import OMGSegVisualEncoder, Mask2FormerVideoSemSamHead
4 | from .omg_llava import OMG_LLaVA
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/__init__.py:
--------------------------------------------------------------------------------
1 | from .openclip_backbone import OpenCLIPBackbone, OpenCLIPBackbone_omgseg
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/__init__.py:
--------------------------------------------------------------------------------
1 | from .coca_model import CoCa
2 | from .constants import OPENAI_DATASET_MEAN, OPENAI_DATASET_STD
3 | from .factory import create_model, create_model_and_transforms, create_model_from_pretrained, get_tokenizer, create_loss
4 | from .factory import list_models, add_model_config, get_model_config, load_checkpoint
5 | from .loss import ClipLoss, DistillClipLoss, CoCaLoss
6 | from .model import CLIP, CustomTextCLIP, CLIPTextCfg, CLIPVisionCfg, \
7 | convert_weights_to_lp, convert_weights_to_fp16, trace_model, get_cast_dtype, get_input_dtype
8 | from .openai import load_openai_model, list_openai_models
9 | from .pretrained import list_pretrained, list_pretrained_models_by_tag, list_pretrained_tags_by_model, \
10 | get_pretrained_url, download_pretrained_from_url, is_pretrained_cfg, get_pretrained_cfg, download_pretrained
11 | from .push_to_hf_hub import push_pretrained_to_hf_hub, push_to_hf_hub
12 | from .tokenizer import SimpleTokenizer, tokenize, decode
13 | from .transform import image_transform, AugmentationCfg
14 | from .zero_shot_classifier import build_zero_shot_classifier, build_zero_shot_classifier_legacy
15 | from .zero_shot_metadata import OPENAI_IMAGENET_TEMPLATES, SIMPLE_IMAGENET_TEMPLATES, IMAGENET_CLASSNAMES
16 |
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/bpe_simple_vocab_16e6.txt.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lxtGH/OMG-Seg/1ae2e7d446a607fddaefabf4ace312a2cdf7ab55/omg_llava/omg_llava/model/convnext_clip/open_clip/bpe_simple_vocab_16e6.txt.gz
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/constants.py:
--------------------------------------------------------------------------------
1 | OPENAI_DATASET_MEAN = (0.48145466, 0.4578275, 0.40821073)
2 | OPENAI_DATASET_STD = (0.26862954, 0.26130258, 0.27577711)
3 |
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/generation_utils.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lxtGH/OMG-Seg/1ae2e7d446a607fddaefabf4ace312a2cdf7ab55/omg_llava/omg_llava/model/convnext_clip/open_clip/generation_utils.py
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/model_configs/EVA01-g-14-plus.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 1024,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "timm_model_name": "eva_giant_patch14_224",
6 | "timm_model_pretrained": false,
7 | "timm_pool": "token",
8 | "timm_proj": null
9 | },
10 | "text_cfg": {
11 | "context_length": 77,
12 | "vocab_size": 49408,
13 | "width": 1024,
14 | "heads": 16,
15 | "layers": 24
16 | },
17 | "custom_text": true
18 | }
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/model_configs/EVA01-g-14.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 1024,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "timm_model_name": "eva_giant_patch14_224",
6 | "timm_model_pretrained": false,
7 | "timm_pool": "token",
8 | "timm_proj": null
9 | },
10 | "text_cfg": {
11 | "context_length": 77,
12 | "vocab_size": 49408,
13 | "width": 768,
14 | "heads": 12,
15 | "layers": 12
16 | },
17 | "custom_text": true
18 | }
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/model_configs/EVA02-B-16.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 512,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "timm_model_name": "eva02_base_patch16_clip_224",
6 | "timm_model_pretrained": false,
7 | "timm_pool": "token",
8 | "timm_proj": null
9 | },
10 | "text_cfg": {
11 | "context_length": 77,
12 | "vocab_size": 49408,
13 | "width": 512,
14 | "heads": 8,
15 | "layers": 12
16 | },
17 | "custom_text": true
18 | }
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/model_configs/EVA02-E-14-plus.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 1024,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "timm_model_name": "eva02_enormous_patch14_clip_224",
6 | "timm_model_pretrained": false,
7 | "timm_pool": "token",
8 | "timm_proj": null
9 | },
10 | "text_cfg": {
11 | "context_length": 77,
12 | "vocab_size": 49408,
13 | "width": 1280,
14 | "heads": 20,
15 | "layers": 32
16 | },
17 | "custom_text": true
18 | }
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/model_configs/EVA02-E-14.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 1024,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "timm_model_name": "eva02_enormous_patch14_clip_224",
6 | "timm_model_pretrained": false,
7 | "timm_pool": "token",
8 | "timm_proj": null
9 | },
10 | "text_cfg": {
11 | "context_length": 77,
12 | "vocab_size": 49408,
13 | "width": 1024,
14 | "heads": 16,
15 | "layers": 24
16 | },
17 | "custom_text": true
18 | }
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/model_configs/EVA02-L-14-336.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 768,
3 | "vision_cfg": {
4 | "image_size": 336,
5 | "timm_model_name": "eva02_large_patch14_clip_336",
6 | "timm_model_pretrained": false,
7 | "timm_pool": "token",
8 | "timm_proj": null
9 | },
10 | "text_cfg": {
11 | "context_length": 77,
12 | "vocab_size": 49408,
13 | "width": 768,
14 | "heads": 12,
15 | "layers": 12
16 | },
17 | "custom_text": true
18 | }
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/model_configs/EVA02-L-14.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 768,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "timm_model_name": "eva02_large_patch14_clip_224",
6 | "timm_model_pretrained": false,
7 | "timm_pool": "token",
8 | "timm_proj": null
9 | },
10 | "text_cfg": {
11 | "context_length": 77,
12 | "vocab_size": 49408,
13 | "width": 768,
14 | "heads": 12,
15 | "layers": 12
16 | },
17 | "custom_text": true
18 | }
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/model_configs/RN101-quickgelu.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 512,
3 | "quick_gelu": true,
4 | "vision_cfg": {
5 | "image_size": 224,
6 | "layers": [
7 | 3,
8 | 4,
9 | 23,
10 | 3
11 | ],
12 | "width": 64,
13 | "patch_size": null
14 | },
15 | "text_cfg": {
16 | "context_length": 77,
17 | "vocab_size": 49408,
18 | "width": 512,
19 | "heads": 8,
20 | "layers": 12
21 | }
22 | }
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/model_configs/RN101.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 512,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "layers": [
6 | 3,
7 | 4,
8 | 23,
9 | 3
10 | ],
11 | "width": 64,
12 | "patch_size": null
13 | },
14 | "text_cfg": {
15 | "context_length": 77,
16 | "vocab_size": 49408,
17 | "width": 512,
18 | "heads": 8,
19 | "layers": 12
20 | }
21 | }
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/model_configs/RN50-quickgelu.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 1024,
3 | "quick_gelu": true,
4 | "vision_cfg": {
5 | "image_size": 224,
6 | "layers": [
7 | 3,
8 | 4,
9 | 6,
10 | 3
11 | ],
12 | "width": 64,
13 | "patch_size": null
14 | },
15 | "text_cfg": {
16 | "context_length": 77,
17 | "vocab_size": 49408,
18 | "width": 512,
19 | "heads": 8,
20 | "layers": 12
21 | }
22 | }
23 |
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/model_configs/RN50.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 1024,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "layers": [
6 | 3,
7 | 4,
8 | 6,
9 | 3
10 | ],
11 | "width": 64,
12 | "patch_size": null
13 | },
14 | "text_cfg": {
15 | "context_length": 77,
16 | "vocab_size": 49408,
17 | "width": 512,
18 | "heads": 8,
19 | "layers": 12
20 | }
21 | }
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/model_configs/RN50x16.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 768,
3 | "vision_cfg": {
4 | "image_size": 384,
5 | "layers": [
6 | 6,
7 | 8,
8 | 18,
9 | 8
10 | ],
11 | "width": 96,
12 | "patch_size": null
13 | },
14 | "text_cfg": {
15 | "context_length": 77,
16 | "vocab_size": 49408,
17 | "width": 768,
18 | "heads": 12,
19 | "layers": 12
20 | }
21 | }
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/model_configs/RN50x4.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 640,
3 | "vision_cfg": {
4 | "image_size": 288,
5 | "layers": [
6 | 4,
7 | 6,
8 | 10,
9 | 6
10 | ],
11 | "width": 80,
12 | "patch_size": null
13 | },
14 | "text_cfg": {
15 | "context_length": 77,
16 | "vocab_size": 49408,
17 | "width": 640,
18 | "heads": 10,
19 | "layers": 12
20 | }
21 | }
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/model_configs/RN50x64.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 1024,
3 | "vision_cfg": {
4 | "image_size": 448,
5 | "layers": [
6 | 3,
7 | 15,
8 | 36,
9 | 10
10 | ],
11 | "width": 128,
12 | "patch_size": null
13 | },
14 | "text_cfg": {
15 | "context_length": 77,
16 | "vocab_size": 49408,
17 | "width": 1024,
18 | "heads": 16,
19 | "layers": 12
20 | }
21 | }
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/model_configs/ViT-B-16-plus-240.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 640,
3 | "vision_cfg": {
4 | "image_size": 240,
5 | "layers": 12,
6 | "width": 896,
7 | "patch_size": 16
8 | },
9 | "text_cfg": {
10 | "context_length": 77,
11 | "vocab_size": 49408,
12 | "width": 640,
13 | "heads": 10,
14 | "layers": 12
15 | }
16 | }
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/model_configs/ViT-B-16-plus.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 640,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "layers": 12,
6 | "width": 896,
7 | "patch_size": 16
8 | },
9 | "text_cfg": {
10 | "context_length": 77,
11 | "vocab_size": 49408,
12 | "width": 640,
13 | "heads": 10,
14 | "layers": 12
15 | }
16 | }
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/model_configs/ViT-B-16.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 512,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "layers": 12,
6 | "width": 768,
7 | "patch_size": 16
8 | },
9 | "text_cfg": {
10 | "context_length": 77,
11 | "vocab_size": 49408,
12 | "width": 512,
13 | "heads": 8,
14 | "layers": 12
15 | }
16 | }
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/model_configs/ViT-B-32-plus-256.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 640,
3 | "vision_cfg": {
4 | "image_size": 256,
5 | "layers": 12,
6 | "width": 896,
7 | "patch_size": 32
8 | },
9 | "text_cfg": {
10 | "context_length": 77,
11 | "vocab_size": 49408,
12 | "width": 640,
13 | "heads": 10,
14 | "layers": 12
15 | }
16 | }
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/model_configs/ViT-B-32-quickgelu.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 512,
3 | "quick_gelu": true,
4 | "vision_cfg": {
5 | "image_size": 224,
6 | "layers": 12,
7 | "width": 768,
8 | "patch_size": 32
9 | },
10 | "text_cfg": {
11 | "context_length": 77,
12 | "vocab_size": 49408,
13 | "width": 512,
14 | "heads": 8,
15 | "layers": 12
16 | }
17 | }
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/model_configs/ViT-B-32.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 512,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "layers": 12,
6 | "width": 768,
7 | "patch_size": 32
8 | },
9 | "text_cfg": {
10 | "context_length": 77,
11 | "vocab_size": 49408,
12 | "width": 512,
13 | "heads": 8,
14 | "layers": 12
15 | }
16 | }
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/model_configs/ViT-H-14.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 1024,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "layers": 32,
6 | "width": 1280,
7 | "head_width": 80,
8 | "patch_size": 14
9 | },
10 | "text_cfg": {
11 | "context_length": 77,
12 | "vocab_size": 49408,
13 | "width": 1024,
14 | "heads": 16,
15 | "layers": 24
16 | }
17 | }
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/model_configs/ViT-H-16.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 1024,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "layers": 32,
6 | "width": 1280,
7 | "head_width": 80,
8 | "patch_size": 16
9 | },
10 | "text_cfg": {
11 | "context_length": 77,
12 | "vocab_size": 49408,
13 | "width": 1024,
14 | "heads": 16,
15 | "layers": 24
16 | }
17 | }
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/model_configs/ViT-L-14-280.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 768,
3 | "vision_cfg": {
4 | "image_size": 280,
5 | "layers": 24,
6 | "width": 1024,
7 | "patch_size": 14
8 | },
9 | "text_cfg": {
10 | "context_length": 77,
11 | "vocab_size": 49408,
12 | "width": 768,
13 | "heads": 12,
14 | "layers": 12
15 | }
16 | }
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/model_configs/ViT-L-14-336.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 768,
3 | "vision_cfg": {
4 | "image_size": 336,
5 | "layers": 24,
6 | "width": 1024,
7 | "patch_size": 14
8 | },
9 | "text_cfg": {
10 | "context_length": 77,
11 | "vocab_size": 49408,
12 | "width": 768,
13 | "heads": 12,
14 | "layers": 12
15 | }
16 | }
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/model_configs/ViT-L-14.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 768,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "layers": 24,
6 | "width": 1024,
7 | "patch_size": 14
8 | },
9 | "text_cfg": {
10 | "context_length": 77,
11 | "vocab_size": 49408,
12 | "width": 768,
13 | "heads": 12,
14 | "layers": 12
15 | }
16 | }
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/model_configs/ViT-L-16-320.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 768,
3 | "vision_cfg": {
4 | "image_size": 320,
5 | "layers": 24,
6 | "width": 1024,
7 | "patch_size": 16
8 | },
9 | "text_cfg": {
10 | "context_length": 77,
11 | "vocab_size": 49408,
12 | "width": 768,
13 | "heads": 12,
14 | "layers": 12
15 | }
16 | }
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/model_configs/ViT-L-16.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 768,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "layers": 24,
6 | "width": 1024,
7 | "patch_size": 16
8 | },
9 | "text_cfg": {
10 | "context_length": 77,
11 | "vocab_size": 49408,
12 | "width": 768,
13 | "heads": 12,
14 | "layers": 12
15 | }
16 | }
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/model_configs/ViT-M-16-alt.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 384,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "layers": 12,
6 | "width": 512,
7 | "patch_size": 16,
8 | "ls_init_value": 1e-4
9 | },
10 | "text_cfg": {
11 | "context_length": 77,
12 | "vocab_size": 49408,
13 | "width": 384,
14 | "heads": 6,
15 | "layers": 12
16 | }
17 | }
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/model_configs/ViT-M-16.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 512,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "layers": 12,
6 | "width": 512,
7 | "patch_size": 16
8 | },
9 | "text_cfg": {
10 | "context_length": 77,
11 | "vocab_size": 49408,
12 | "width": 512,
13 | "heads": 8,
14 | "layers": 12
15 | }
16 | }
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/model_configs/ViT-M-32-alt.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 384,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "layers": 12,
6 | "width": 512,
7 | "patch_size": 32
8 | },
9 | "text_cfg": {
10 | "context_length": 77,
11 | "vocab_size": 49408,
12 | "width": 384,
13 | "heads": 6,
14 | "layers": 12
15 | }
16 | }
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/model_configs/ViT-M-32.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 512,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "layers": 12,
6 | "width": 512,
7 | "patch_size": 32
8 | },
9 | "text_cfg": {
10 | "context_length": 77,
11 | "vocab_size": 49408,
12 | "width": 512,
13 | "heads": 8,
14 | "layers": 12
15 | }
16 | }
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/model_configs/ViT-S-16-alt.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 256,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "layers": 12,
6 | "width": 384,
7 | "patch_size": 16
8 | },
9 | "text_cfg": {
10 | "context_length": 77,
11 | "vocab_size": 49408,
12 | "width": 256,
13 | "heads": 4,
14 | "layers": 10
15 | }
16 | }
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/model_configs/ViT-S-16.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 384,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "layers": 12,
6 | "width": 384,
7 | "patch_size": 16
8 | },
9 | "text_cfg": {
10 | "context_length": 77,
11 | "vocab_size": 49408,
12 | "width": 384,
13 | "heads": 6,
14 | "layers": 12
15 | }
16 | }
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/model_configs/ViT-S-32-alt.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 256,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "layers": 12,
6 | "width": 384,
7 | "patch_size": 32
8 | },
9 | "text_cfg": {
10 | "context_length": 77,
11 | "vocab_size": 49408,
12 | "width": 256,
13 | "heads": 4,
14 | "layers": 10
15 | }
16 | }
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/model_configs/ViT-S-32.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 384,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "layers": 12,
6 | "width": 384,
7 | "patch_size": 32
8 | },
9 | "text_cfg": {
10 | "context_length": 77,
11 | "vocab_size": 49408,
12 | "width": 384,
13 | "heads": 6,
14 | "layers": 12
15 | }
16 | }
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/model_configs/ViT-bigG-14.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 1280,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "layers": 48,
6 | "width": 1664,
7 | "head_width": 104,
8 | "mlp_ratio": 4.9231,
9 | "patch_size": 14
10 | },
11 | "text_cfg": {
12 | "context_length": 77,
13 | "vocab_size": 49408,
14 | "width": 1280,
15 | "heads": 20,
16 | "layers": 32
17 | }
18 | }
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/model_configs/ViT-e-14.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 1280,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "layers": 56,
6 | "width": 1792,
7 | "head_width": 112,
8 | "mlp_ratio": 8.5715,
9 | "patch_size": 14
10 | },
11 | "text_cfg": {
12 | "context_length": 77,
13 | "vocab_size": 49408,
14 | "width": 1280,
15 | "heads": 20,
16 | "layers": 36
17 | }
18 | }
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/model_configs/ViT-g-14.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 1024,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "layers": 40,
6 | "width": 1408,
7 | "head_width": 88,
8 | "mlp_ratio": 4.3637,
9 | "patch_size": 14
10 | },
11 | "text_cfg": {
12 | "context_length": 77,
13 | "vocab_size": 49408,
14 | "width": 1024,
15 | "heads": 16,
16 | "layers": 24
17 | }
18 | }
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/model_configs/coca_ViT-B-32.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 512,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "layers": 12,
6 | "width": 768,
7 | "patch_size": 32,
8 | "attentional_pool": true,
9 | "attn_pooler_heads": 8,
10 | "output_tokens": true
11 | },
12 | "text_cfg": {
13 | "context_length": 76,
14 | "vocab_size": 49408,
15 | "width": 512,
16 | "heads": 8,
17 | "layers": 12,
18 | "embed_cls": true,
19 | "output_tokens": true
20 | },
21 | "multimodal_cfg": {
22 | "context_length": 76,
23 | "vocab_size": 49408,
24 | "width": 512,
25 | "heads": 8,
26 | "layers": 12,
27 | "attn_pooler_heads": 8
28 | },
29 | "custom_text": true
30 | }
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/model_configs/coca_ViT-L-14.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 768,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "layers": 24,
6 | "width": 1024,
7 | "patch_size": 14,
8 | "attentional_pool": true,
9 | "attn_pooler_heads": 8,
10 | "output_tokens": true
11 | },
12 | "text_cfg": {
13 | "context_length": 76,
14 | "vocab_size": 49408,
15 | "width": 768,
16 | "heads": 12,
17 | "layers": 12,
18 | "embed_cls": true,
19 | "output_tokens": true
20 | },
21 | "multimodal_cfg": {
22 | "context_length": 76,
23 | "vocab_size": 49408,
24 | "width": 768,
25 | "heads": 12,
26 | "layers": 12,
27 | "attn_pooler_heads": 12
28 | },
29 | "custom_text": true
30 | }
31 |
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/model_configs/coca_base.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 512,
3 | "multimodal_cfg": {
4 | "width": 768,
5 | "context_length": 76,
6 | "vocab_size": 64000,
7 | "mlp_ratio": 4,
8 | "layers": 12,
9 | "dim_head": 64,
10 | "heads": 12,
11 | "n_queries": 256,
12 | "attn_pooler_heads": 8
13 | },
14 | "vision_cfg": {
15 | "image_size": 288,
16 | "layers": 12,
17 | "width": 768,
18 | "patch_size": 18,
19 | "output_tokens": true
20 | },
21 | "text_cfg": {
22 | "context_length": 76,
23 | "vocab_size": 64000,
24 | "layers": 12,
25 | "heads": 12,
26 | "width": 768,
27 | "embed_cls": true,
28 | "output_tokens": true
29 | },
30 | "custom_text": true
31 | }
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/model_configs/coca_roberta-ViT-B-32.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 512,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "layers": 12,
6 | "width": 768,
7 | "patch_size": 32,
8 | "output_tokens": true
9 | },
10 | "text_cfg": {
11 | "hf_model_name": "roberta-base",
12 | "hf_tokenizer_name": "roberta-base",
13 | "proj": "linear",
14 | "width": 768,
15 | "output_tokens": true
16 | },
17 | "multimodal_cfg": {
18 | "context_length": 76,
19 | "width": 768,
20 | "heads": 8,
21 | "layers": 12
22 | },
23 | "custom_text": true
24 | }
25 |
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/model_configs/convnext_base.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 512,
3 | "vision_cfg": {
4 | "timm_model_name": "convnext_base",
5 | "timm_model_pretrained": false,
6 | "timm_pool": "",
7 | "timm_proj": "linear",
8 | "timm_drop": 0.0,
9 | "timm_drop_path": 0.1,
10 | "image_size": 224
11 | },
12 | "text_cfg": {
13 | "context_length": 77,
14 | "vocab_size": 49408,
15 | "width": 512,
16 | "heads": 8,
17 | "layers": 12
18 | }
19 | }
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/model_configs/convnext_base_w.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 640,
3 | "vision_cfg": {
4 | "timm_model_name": "convnext_base",
5 | "timm_model_pretrained": false,
6 | "timm_pool": "",
7 | "timm_proj": "linear",
8 | "timm_drop": 0.0,
9 | "timm_drop_path": 0.1,
10 | "image_size": 256
11 | },
12 | "text_cfg": {
13 | "context_length": 77,
14 | "vocab_size": 49408,
15 | "width": 640,
16 | "heads": 10,
17 | "layers": 12
18 | }
19 | }
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/model_configs/convnext_base_w_320.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 640,
3 | "vision_cfg": {
4 | "timm_model_name": "convnext_base",
5 | "timm_model_pretrained": false,
6 | "timm_pool": "",
7 | "timm_proj": "linear",
8 | "timm_drop": 0.0,
9 | "timm_drop_path": 0.1,
10 | "image_size": 320
11 | },
12 | "text_cfg": {
13 | "context_length": 77,
14 | "vocab_size": 49408,
15 | "width": 640,
16 | "heads": 10,
17 | "layers": 12
18 | }
19 | }
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/model_configs/convnext_large.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 768,
3 | "vision_cfg": {
4 | "timm_model_name": "convnext_large",
5 | "timm_model_pretrained": false,
6 | "timm_pool": "",
7 | "timm_proj": "linear",
8 | "timm_drop": 0.0,
9 | "timm_drop_path": 0.1,
10 | "image_size": 224
11 | },
12 | "text_cfg": {
13 | "context_length": 77,
14 | "vocab_size": 49408,
15 | "width": 768,
16 | "heads": 12,
17 | "layers": 12
18 | }
19 | }
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/model_configs/convnext_large_d.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 768,
3 | "vision_cfg": {
4 | "timm_model_name": "convnext_large",
5 | "timm_model_pretrained": false,
6 | "timm_pool": "",
7 | "timm_proj": "mlp",
8 | "timm_drop": 0.0,
9 | "timm_drop_path": 0.1,
10 | "image_size": 256
11 | },
12 | "text_cfg": {
13 | "context_length": 77,
14 | "vocab_size": 49408,
15 | "width": 768,
16 | "heads": 12,
17 | "layers": 16
18 | }
19 | }
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/model_configs/convnext_large_d_320.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 768,
3 | "vision_cfg": {
4 | "timm_model_name": "convnext_large",
5 | "timm_model_pretrained": false,
6 | "timm_pool": "",
7 | "timm_proj": "mlp",
8 | "timm_drop": 0.0,
9 | "timm_drop_path": 0.1,
10 | "image_size": 320
11 | },
12 | "text_cfg": {
13 | "context_length": 77,
14 | "vocab_size": 49408,
15 | "width": 768,
16 | "heads": 12,
17 | "layers": 16
18 | }
19 | }
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/model_configs/convnext_small.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 512,
3 | "vision_cfg": {
4 | "timm_model_name": "convnext_small",
5 | "timm_model_pretrained": false,
6 | "timm_pool": "",
7 | "timm_proj": "linear",
8 | "timm_drop": 0.0,
9 | "timm_drop_path": 0.1,
10 | "image_size": 224
11 | },
12 | "text_cfg": {
13 | "context_length": 77,
14 | "vocab_size": 49408,
15 | "width": 512,
16 | "heads": 8,
17 | "layers": 12
18 | }
19 | }
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/model_configs/convnext_tiny.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 1024,
3 | "vision_cfg": {
4 | "timm_model_name": "convnext_tiny",
5 | "timm_model_pretrained": false,
6 | "timm_pool": "",
7 | "timm_proj": "linear",
8 | "timm_drop": 0.0,
9 | "timm_drop_path": 0.1,
10 | "image_size": 224
11 | },
12 | "text_cfg": {
13 | "context_length": 77,
14 | "vocab_size": 49408,
15 | "width": 512,
16 | "heads": 8,
17 | "layers": 12
18 | }
19 | }
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/model_configs/convnext_xlarge.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 1024,
3 | "vision_cfg": {
4 | "timm_model_name": "convnext_xlarge",
5 | "timm_model_pretrained": false,
6 | "timm_pool": "",
7 | "timm_proj": "linear",
8 | "timm_drop": 0.0,
9 | "timm_drop_path": 0.1,
10 | "image_size": 256
11 | },
12 | "text_cfg": {
13 | "context_length": 77,
14 | "vocab_size": 49408,
15 | "width": 1024,
16 | "heads": 16,
17 | "layers": 20
18 | }
19 | }
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/model_configs/convnext_xxlarge.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 1024,
3 | "vision_cfg": {
4 | "timm_model_name": "convnext_xxlarge",
5 | "timm_model_pretrained": false,
6 | "timm_pool": "",
7 | "timm_proj": "linear",
8 | "timm_drop": 0.0,
9 | "timm_drop_path": 0.1,
10 | "image_size": 256
11 | },
12 | "text_cfg": {
13 | "context_length": 77,
14 | "vocab_size": 49408,
15 | "width": 1024,
16 | "heads": 16,
17 | "layers": 24
18 | }
19 | }
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/model_configs/convnext_xxlarge_320.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 1024,
3 | "vision_cfg": {
4 | "timm_model_name": "convnext_xxlarge",
5 | "timm_model_pretrained": false,
6 | "timm_pool": "",
7 | "timm_proj": "linear",
8 | "timm_drop": 0.0,
9 | "timm_drop_path": 0.1,
10 | "image_size": 320
11 | },
12 | "text_cfg": {
13 | "context_length": 77,
14 | "vocab_size": 49408,
15 | "width": 1024,
16 | "heads": 16,
17 | "layers": 24
18 | }
19 | }
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/model_configs/mt5-base-ViT-B-32.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 512,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "layers": 12,
6 | "width": 768,
7 | "patch_size": 32
8 | },
9 | "text_cfg": {
10 | "hf_model_name": "google/mt5-base",
11 | "hf_tokenizer_name": "google/mt5-base",
12 | "proj": "mlp",
13 | "pooler_type": "mean_pooler"
14 | }
15 | }
16 |
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/model_configs/mt5-xl-ViT-H-14.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 1024,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "layers": 32,
6 | "width": 1280,
7 | "head_width": 80,
8 | "patch_size": 14
9 | },
10 | "text_cfg": {
11 | "hf_model_name": "google/mt5-xl",
12 | "hf_tokenizer_name": "google/mt5-xl",
13 | "proj": "mlp",
14 | "pooler_type": "mean_pooler"
15 | }
16 | }
17 |
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/model_configs/roberta-ViT-B-32.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 512,
3 | "quick_gelu": true,
4 | "vision_cfg": {
5 | "image_size": 224,
6 | "layers": 12,
7 | "width": 768,
8 | "patch_size": 32
9 | },
10 | "text_cfg": {
11 | "hf_model_name": "roberta-base",
12 | "hf_tokenizer_name": "roberta-base",
13 | "proj": "mlp",
14 | "pooler_type": "mean_pooler"
15 | }
16 | }
17 |
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/model_configs/swin_base_patch4_window7_224.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 640,
3 | "vision_cfg": {
4 | "timm_model_name": "swin_base_patch4_window7_224",
5 | "timm_model_pretrained": false,
6 | "timm_pool": "",
7 | "timm_proj": "linear",
8 | "image_size": 224
9 | },
10 | "text_cfg": {
11 | "context_length": 77,
12 | "vocab_size": 49408,
13 | "width": 640,
14 | "heads": 10,
15 | "layers": 12
16 | }
17 | }
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/model_configs/vit_medium_patch16_gap_256.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 512,
3 | "vision_cfg": {
4 | "timm_model_name": "vit_medium_patch16_gap_256",
5 | "timm_model_pretrained": false,
6 | "timm_pool": "",
7 | "timm_proj": "linear",
8 | "image_size": 256
9 | },
10 | "text_cfg": {
11 | "context_length": 77,
12 | "vocab_size": 49408,
13 | "width": 512,
14 | "heads": 8,
15 | "layers": 12
16 | }
17 | }
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/model_configs/vit_relpos_medium_patch16_cls_224.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 512,
3 | "vision_cfg": {
4 | "timm_model_name": "vit_relpos_medium_patch16_cls_224",
5 | "timm_model_pretrained": false,
6 | "timm_pool": "",
7 | "timm_proj": "linear",
8 | "image_size": 224
9 | },
10 | "text_cfg": {
11 | "context_length": 77,
12 | "vocab_size": 49408,
13 | "width": 512,
14 | "heads": 8,
15 | "layers": 12
16 | }
17 | }
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/model_configs/xlm-roberta-base-ViT-B-32.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 512,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "layers": 12,
6 | "width": 768,
7 | "patch_size": 32
8 | },
9 | "text_cfg": {
10 | "hf_model_name": "xlm-roberta-base",
11 | "hf_tokenizer_name": "xlm-roberta-base",
12 | "proj": "mlp",
13 | "pooler_type": "mean_pooler"
14 | }
15 | }
16 |
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/model_configs/xlm-roberta-large-ViT-H-14.json:
--------------------------------------------------------------------------------
1 | {
2 | "embed_dim": 1024,
3 | "vision_cfg": {
4 | "image_size": 224,
5 | "layers": 32,
6 | "width": 1280,
7 | "head_width": 80,
8 | "patch_size": 14
9 | },
10 | "text_cfg": {
11 | "hf_model_name": "xlm-roberta-large",
12 | "hf_tokenizer_name": "xlm-roberta-large",
13 | "proj": "mlp",
14 | "pooler_type": "mean_pooler"
15 | }
16 | }
17 |
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/convnext_clip/open_clip/version.py:
--------------------------------------------------------------------------------
1 | __version__ = '2.20.0'
2 |
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/modules/__init__.py:
--------------------------------------------------------------------------------
1 | from xtuner.model import *
2 | from .projector import ProjectorModel_OMG_LLaVA, ProjectorConfig_OMG_LLaVA
3 |
4 | __all__ = ['ProjectorConfig_OMG_LLaVA', 'ProjectorModel_OMG_LLaVA', ]
5 |
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/modules/projector/__init__.py:
--------------------------------------------------------------------------------
1 | from xtuner.model.modules.projector import *
2 | from transformers import AutoConfig, AutoModel
3 | from .configuration_projector import ProjectorConfig_OMG_LLaVA
4 | from .modeling_projector import ProjectorModel_OMG_LLaVA
5 |
6 | AutoConfig.register('projector', ProjectorConfig_OMG_LLaVA)
7 | AutoModel.register(ProjectorConfig_OMG_LLaVA, ProjectorModel_OMG_LLaVA)
8 |
9 | __all__ = ['ProjectorConfig_OMG_LLaVA', 'ProjectorModel_OMG_LLaVA']
10 |
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/modules/projector/configuration_projector.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) OpenMMLab. All rights reserved.
2 | from transformers import PretrainedConfig
3 |
4 | class ProjectorConfig_OMG_LLaVA(PretrainedConfig):
5 | model_type = 'projector'
6 | _auto_class = 'AutoConfig'
7 |
8 | def __init__(
9 | self,
10 | visual_hidden_size=4096,
11 | llm_hidden_size=4096,
12 | depth=2,
13 | hidden_act='gelu',
14 | bias=True,
15 | query_channels=256,
16 | feat_channels=1536,
17 | pixel_shuffle_ratio=None,
18 | additional_bg_tokens=10,
19 | visual_prompt_proj=False,
20 | add_cross_attn_layer=False,
21 | **kwargs,
22 | ):
23 | self.visual_hidden_size = visual_hidden_size
24 | self.llm_hidden_size = llm_hidden_size
25 | self.depth = depth
26 | self.hidden_act = hidden_act
27 | self.bias = bias
28 | self.query_channels=query_channels
29 | self.feat_channels=feat_channels
30 | if pixel_shuffle_ratio is not None:
31 | self.feat_channels = self.feat_channels * pixel_shuffle_ratio * pixel_shuffle_ratio
32 | self.additional_bg_tokens = additional_bg_tokens
33 | self.visual_prompt_proj = visual_prompt_proj
34 | self.add_cross_attn_layer = add_cross_attn_layer
35 | super().__init__(**kwargs)
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/omg_seg/__init__.py:
--------------------------------------------------------------------------------
1 | from .omg_seg_visual_encoder import OMGSegVisualEncoder
2 | from .mask2former_vid_semanticsam import Mask2FormerVideoSemSamHead
--------------------------------------------------------------------------------
/omg_llava/omg_llava/model/omg_seg/utils.py:
--------------------------------------------------------------------------------
1 | import torch
2 | import torch.nn.functional as F
3 |
4 | def mask_pool(x, mask):
5 | """
6 | Args:
7 | x: [B, C, H, W]
8 | mask: [B, Q, H, W]
9 | """
10 | if not x.shape[-2:] == mask.shape[-2:]:
11 | # reshape mask to x
12 | mask = F.interpolate(mask, size=x.shape[-2:], mode='bilinear', align_corners=False)
13 | with torch.no_grad():
14 | mask = mask.detach()
15 | mask = (mask > 0).to(mask.dtype)
16 | denorm = mask.sum(dim=(-1, -2), keepdim=True) + 1e-8
17 |
18 | mask_pooled_x = torch.einsum(
19 | "bchw,bqhw->bqc",
20 | x,
21 | mask / denorm,
22 | )
23 |
24 | return mask_pooled_x
25 |
26 |
27 |
28 |
--------------------------------------------------------------------------------
/omg_llava/omg_llava/tools/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lxtGH/OMG-Seg/1ae2e7d446a607fddaefabf4ace312a2cdf7ab55/omg_llava/omg_llava/tools/__init__.py
--------------------------------------------------------------------------------
/omg_llava/requirements.txt:
--------------------------------------------------------------------------------
1 | -r requirements/runtime.txt
2 | -r requirements/deepspeed.txt
3 | -r requirements/modelscope.txt
4 |
5 | gradio==4.37.2
6 | gradio-image-prompter
7 | pycocotools
8 | timm
9 | ftfy
10 | kornia
--------------------------------------------------------------------------------
/omg_llava/requirements/deepspeed.txt:
--------------------------------------------------------------------------------
1 | # Minimum 0.12.3, see https://github.com/microsoft/DeepSpeed/pull/4587
2 | deepspeed>=0.12.3
3 | mpi4py-mpich
4 |
--------------------------------------------------------------------------------
/omg_llava/requirements/docs.txt:
--------------------------------------------------------------------------------
1 | docutils
2 | myst-parser==2.0.0
3 | sphinx==6.2.1
4 | sphinx-argparse
5 | sphinx-book-theme==1.0.1
6 | sphinx-copybutton==0.5.2
7 | sphinx_markdown_tables
8 |
--------------------------------------------------------------------------------
/omg_llava/requirements/modelscope.txt:
--------------------------------------------------------------------------------
1 | modelscope
2 |
--------------------------------------------------------------------------------
/omg_llava/requirements/runtime.txt:
--------------------------------------------------------------------------------
1 | # Minimum 0.40.0.post4 to fix some 4-bit precision bugs
2 | bitsandbytes>=0.40.0.post4
3 | # Minimum 2.16.0 to fix some bugs, see https://github.com/huggingface/datasets/pull/6444
4 | datasets>=2.16.0
5 | einops
6 | # Minimum 0.1.2 to fix some bugs, see https://github.com/InternLM/lagent/pull/44
7 | lagent>=0.1.2
8 | # Minimum 0.10.3 to support distributed evaluation for MMBench
9 | # see https://github.com/open-mmlab/mmengine/pull/1469
10 | mmengine>=0.10.3
11 | openpyxl
12 | # Minimum 0.4.0 to support QLoRA, see https://github.com/huggingface/peft/pull/476
13 | triton==2.1.0
14 | peft>=0.4.0
15 | scikit-image
16 | scipy
17 | SentencePiece
18 | tiktoken
19 | torch
20 | torchvision
21 | # Minimum 4.36.0 to support `Cache` data structure used by KV Cache
22 | # Registering a causal mask in `LlamaModel` is not friendly for very large
23 | # `max_position_embeddings`. Refer to
24 | # https://github.com/huggingface/transformers/blob/v4.38.0/src/transformers/models/llama/modeling_llama.py#L921-L923
25 | # transformers>=4.36.0,!=4.38.0,!=4.38.1,!=4.38.2
26 | transformers==4.36.0
27 | transformers_stream_generator
28 |
--------------------------------------------------------------------------------
/omg_llava/setup.cfg:
--------------------------------------------------------------------------------
1 | [isort]
2 | line_length = 79
3 | multi_line_output = 0
4 | extra_standard_library = setuptools
5 | known_first_party = xtuner
6 | known_third_party = pytest,yaml
7 | no_lines_before = STDLIB,LOCALFOLDER
8 | default_section = THIRDPARTY
9 |
10 | [yapf]
11 | BASED_ON_STYLE = pep8
12 | BLANK_LINE_BEFORE_NESTED_CLASS_OR_DEF = true
13 | SPLIT_BEFORE_EXPRESSION_AFTER_OPENING_PAREN = true
14 |
15 | [codespell]
16 | ignore-words-list = nd, ba, warmup
17 |
--------------------------------------------------------------------------------
/omg_llava/test.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/lxtGH/OMG-Seg/1ae2e7d446a607fddaefabf4ace312a2cdf7ab55/omg_llava/test.jpg
--------------------------------------------------------------------------------
/omg_llava/xtuner/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) OpenMMLab. All rights reserved.
2 | import os
3 |
4 | from mmengine.utils import digit_version
5 |
6 | from .entry_point import cli
7 | from .version import __version__, version_info
8 |
9 | HF_CEPH_HUB = os.getenv('HF_CEPH_HUB', '')
10 | HF_USE_CEPH = os.getenv('HF_USE_CEPH', 0) or HF_CEPH_HUB != ''
11 | DS_CEPH_DIR = os.getenv('DS_CEPH_DIR', None)
12 | if HF_USE_CEPH:
13 | from .utils.fileio import (patch_hf_auto_from_pretrained,
14 | patch_hf_save_pretrained)
15 | patch_hf_auto_from_pretrained(HF_CEPH_HUB)
16 | patch_hf_save_pretrained()
17 |
18 | if DS_CEPH_DIR:
19 | from .utils.fileio import patch_deepspeed_engine
20 | patch_deepspeed_engine()
21 |
22 | __all__ = [
23 | '__version__', 'version_info', 'digit_version', 'cli', 'HF_USE_CEPH',
24 | 'DS_CEPH_DIR'
25 | ]
26 |
--------------------------------------------------------------------------------
/omg_llava/xtuner/apis/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) OpenMMLab. All rights reserved.
2 | from .datasets import * # noqa: F401, F403
3 | from .model import * # noqa: F401, F403
4 | from .training_args import * # noqa: F401, F403
5 |
--------------------------------------------------------------------------------
/omg_llava/xtuner/apis/datasets/arxiv.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) OpenMMLab. All rights reserved.
2 | from functools import partial
3 |
4 | from datasets import load_dataset
5 |
6 | from xtuner.dataset import process_hf_dataset
7 | from xtuner.dataset.collate_fns import default_collate_fn
8 | from xtuner.dataset.map_fns import arxiv_map_fn, template_map_fn_factory
9 | from xtuner.utils import PROMPT_TEMPLATE
10 |
11 |
12 | def arxiv_dataset(tokenizer,
13 | data_file=None,
14 | max_length=2048,
15 | prompt_template=PROMPT_TEMPLATE.default,
16 | remove_unused_columns=True,
17 | pack_to_max_length=True):
18 | template_map_fn = template_map_fn_factory(template=prompt_template)
19 | # 1. Download data from https://kaggle.com/datasets/Cornell-University/arxiv # noqa: E501
20 | # 2. Process data with `./tools/data_preprocess/arxiv.py`
21 | if data_file is None:
22 | data_file = './data/arxiv_postprocess_csAIcsCLcsCV_20200101.json'
23 | dataset_org = load_dataset(path='json', data_files=dict(train=data_file))
24 | dataset = process_hf_dataset(
25 | dataset=dataset_org,
26 | tokenizer=tokenizer,
27 | max_length=max_length,
28 | dataset_map_fn=arxiv_map_fn,
29 | template_map_fn=template_map_fn,
30 | remove_unused_columns=remove_unused_columns,
31 | shuffle_before_pack=True,
32 | pack_to_max_length=pack_to_max_length)
33 |
34 | return dataset
35 |
36 |
37 | def arxiv_data_collator(return_hf_format=False):
38 | return partial(default_collate_fn, return_hf_format=return_hf_format)
39 |
--------------------------------------------------------------------------------
/omg_llava/xtuner/apis/datasets/code_alpaca.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) OpenMMLab. All rights reserved.
2 | from functools import partial
3 |
4 | from datasets import load_dataset
5 |
6 | from xtuner.dataset import process_hf_dataset
7 | from xtuner.dataset.collate_fns import default_collate_fn
8 | from xtuner.dataset.map_fns import code_alpaca_map_fn, template_map_fn_factory
9 | from xtuner.utils import PROMPT_TEMPLATE
10 |
11 |
12 | def code_alpaca_dataset(tokenizer,
13 | path='HuggingFaceH4/CodeAlpaca_20K',
14 | max_length=2048,
15 | prompt_template=PROMPT_TEMPLATE.default,
16 | remove_unused_columns=True,
17 | pack_to_max_length=True):
18 | template_map_fn = template_map_fn_factory(template=prompt_template)
19 | dataset_org = load_dataset(path)
20 | dataset = process_hf_dataset(
21 | dataset=dataset_org,
22 | tokenizer=tokenizer,
23 | max_length=max_length,
24 | dataset_map_fn=code_alpaca_map_fn,
25 | template_map_fn=template_map_fn,
26 | remove_unused_columns=remove_unused_columns,
27 | shuffle_before_pack=True,
28 | pack_to_max_length=pack_to_max_length)
29 |
30 | return dataset
31 |
32 |
33 | def code_alpaca_data_collator(return_hf_format=False):
34 | return partial(default_collate_fn, return_hf_format=return_hf_format)
35 |
--------------------------------------------------------------------------------
/omg_llava/xtuner/apis/datasets/colorist.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) OpenMMLab. All rights reserved.
2 | from functools import partial
3 |
4 | from datasets import load_dataset
5 |
6 | from xtuner.dataset import process_hf_dataset
7 | from xtuner.dataset.collate_fns import default_collate_fn
8 | from xtuner.dataset.map_fns import colors_map_fn, template_map_fn_factory
9 | from xtuner.utils import PROMPT_TEMPLATE
10 |
11 |
12 | def colorist_dataset(tokenizer,
13 | path='burkelibbey/colors',
14 | max_length=2048,
15 | prompt_template=PROMPT_TEMPLATE.default,
16 | remove_unused_columns=True,
17 | pack_to_max_length=True):
18 | template_map_fn = template_map_fn_factory(template=prompt_template)
19 | dataset_org = load_dataset(path)
20 | dataset = process_hf_dataset(
21 | dataset=dataset_org,
22 | tokenizer=tokenizer,
23 | max_length=max_length,
24 | dataset_map_fn=colors_map_fn,
25 | template_map_fn=template_map_fn,
26 | remove_unused_columns=remove_unused_columns,
27 | shuffle_before_pack=True,
28 | pack_to_max_length=pack_to_max_length)
29 |
30 | return dataset
31 |
32 |
33 | def colorist_data_collator(return_hf_format=False):
34 | return partial(default_collate_fn, return_hf_format=return_hf_format)
35 |
--------------------------------------------------------------------------------
/omg_llava/xtuner/apis/datasets/medical.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) OpenMMLab. All rights reserved.
2 | from functools import partial
3 |
4 | from datasets import load_dataset
5 |
6 | from xtuner.dataset import process_hf_dataset
7 | from xtuner.dataset.collate_fns import default_collate_fn
8 | from xtuner.dataset.map_fns import medical_map_fn, template_map_fn_factory
9 | from xtuner.utils import PROMPT_TEMPLATE
10 |
11 |
12 | def medical_dataset(tokenizer,
13 | path='shibing624/medical',
14 | max_length=2048,
15 | prompt_template=PROMPT_TEMPLATE.default,
16 | remove_unused_columns=False,
17 | pack_to_max_length=True):
18 | template_map_fn = template_map_fn_factory(template=prompt_template)
19 | dataset_org = load_dataset(path)
20 | dataset = process_hf_dataset(
21 | dataset=dataset_org,
22 | tokenizer=tokenizer,
23 | max_length=max_length,
24 | dataset_map_fn=medical_map_fn,
25 | template_map_fn=template_map_fn,
26 | remove_unused_columns=remove_unused_columns,
27 | shuffle_before_pack=True,
28 | pack_to_max_length=pack_to_max_length)
29 |
30 | return dataset
31 |
32 |
33 | def medical_data_collator(return_hf_format=False):
34 | return partial(default_collate_fn, return_hf_format=return_hf_format)
35 |
--------------------------------------------------------------------------------
/omg_llava/xtuner/apis/datasets/oasst1.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) OpenMMLab. All rights reserved.
2 | from functools import partial
3 |
4 | from datasets import load_dataset
5 |
6 | from xtuner.dataset import process_hf_dataset
7 | from xtuner.dataset.collate_fns import default_collate_fn
8 | from xtuner.dataset.map_fns import oasst1_map_fn, template_map_fn_factory
9 | from xtuner.utils import PROMPT_TEMPLATE
10 |
11 |
12 | def oasst1_dataset(tokenizer,
13 | path='timdettmers/openassistant-guanaco',
14 | max_length=2048,
15 | prompt_template=PROMPT_TEMPLATE.default,
16 | remove_unused_columns=False,
17 | pack_to_max_length=True):
18 | template_map_fn = template_map_fn_factory(template=prompt_template)
19 | dataset_org = load_dataset(path)
20 | dataset = process_hf_dataset(
21 | dataset=dataset_org,
22 | tokenizer=tokenizer,
23 | max_length=max_length,
24 | dataset_map_fn=oasst1_map_fn,
25 | template_map_fn=template_map_fn,
26 | remove_unused_columns=remove_unused_columns,
27 | shuffle_before_pack=True,
28 | pack_to_max_length=pack_to_max_length)
29 |
30 | return dataset
31 |
32 |
33 | def oasst1_data_collator(return_hf_format=False):
34 | return partial(default_collate_fn, return_hf_format=return_hf_format)
35 |
--------------------------------------------------------------------------------
/omg_llava/xtuner/apis/datasets/open_orca.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) OpenMMLab. All rights reserved.
2 | from functools import partial
3 |
4 | from datasets import load_dataset
5 |
6 | from xtuner.dataset import process_hf_dataset
7 | from xtuner.dataset.collate_fns import default_collate_fn
8 | from xtuner.dataset.map_fns import openorca_map_fn, template_map_fn_factory
9 | from xtuner.utils import PROMPT_TEMPLATE
10 |
11 |
12 | def openorca_dataset(tokenizer,
13 | path='Open-Orca/OpenOrca',
14 | max_length=2048,
15 | prompt_template=PROMPT_TEMPLATE.default,
16 | remove_unused_columns=True,
17 | pack_to_max_length=True):
18 | template_map_fn = template_map_fn_factory(template=prompt_template)
19 | dataset_org = load_dataset(path)
20 | dataset = process_hf_dataset(
21 | dataset=dataset_org,
22 | tokenizer=tokenizer,
23 | max_length=max_length,
24 | dataset_map_fn=openorca_map_fn,
25 | template_map_fn=template_map_fn,
26 | remove_unused_columns=remove_unused_columns,
27 | shuffle_before_pack=True,
28 | pack_to_max_length=pack_to_max_length)
29 |
30 | return dataset
31 |
32 |
33 | def openorca_data_collator(return_hf_format=False):
34 | return partial(default_collate_fn, return_hf_format=return_hf_format)
35 |
--------------------------------------------------------------------------------
/omg_llava/xtuner/apis/datasets/sql.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) OpenMMLab. All rights reserved.
2 | from functools import partial
3 |
4 | from datasets import load_dataset
5 |
6 | from xtuner.dataset import process_hf_dataset
7 | from xtuner.dataset.collate_fns import default_collate_fn
8 | from xtuner.dataset.map_fns import sql_map_fn, template_map_fn_factory
9 | from xtuner.utils import PROMPT_TEMPLATE
10 |
11 |
12 | def sql_dataset(tokenizer,
13 | path='b-mc2/sql-create-context',
14 | max_length=2048,
15 | prompt_template=PROMPT_TEMPLATE.default,
16 | remove_unused_columns=True,
17 | pack_to_max_length=True):
18 | template_map_fn = template_map_fn_factory(template=prompt_template)
19 | dataset_org = load_dataset(path)
20 | dataset = process_hf_dataset(
21 | dataset=dataset_org,
22 | tokenizer=tokenizer,
23 | max_length=max_length,
24 | dataset_map_fn=sql_map_fn,
25 | template_map_fn=template_map_fn,
26 | remove_unused_columns=remove_unused_columns,
27 | shuffle_before_pack=True,
28 | pack_to_max_length=pack_to_max_length)
29 |
30 | return dataset
31 |
32 |
33 | def sql_data_collator(return_hf_format=False):
34 | return partial(default_collate_fn, return_hf_format=return_hf_format)
35 |
--------------------------------------------------------------------------------
/omg_llava/xtuner/apis/datasets/tiny_codes.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) OpenMMLab. All rights reserved.
2 | from functools import partial
3 |
4 | from datasets import load_dataset
5 |
6 | from xtuner.dataset import process_hf_dataset
7 | from xtuner.dataset.collate_fns import default_collate_fn
8 | from xtuner.dataset.map_fns import template_map_fn_factory, tiny_codes_map_fn
9 | from xtuner.utils import PROMPT_TEMPLATE
10 |
11 |
12 | def tiny_codes_dataset(tokenizer,
13 | path='nampdn-ai/tiny-codes',
14 | max_length=2048,
15 | prompt_template=PROMPT_TEMPLATE.default,
16 | remove_unused_columns=True,
17 | pack_to_max_length=True):
18 | template_map_fn = template_map_fn_factory(template=prompt_template)
19 | dataset_org = load_dataset(path)
20 | dataset = process_hf_dataset(
21 | dataset=dataset_org,
22 | tokenizer=tokenizer,
23 | max_length=max_length,
24 | dataset_map_fn=tiny_codes_map_fn,
25 | template_map_fn=template_map_fn,
26 | remove_unused_columns=remove_unused_columns,
27 | shuffle_before_pack=True,
28 | pack_to_max_length=pack_to_max_length)
29 |
30 | return dataset
31 |
32 |
33 | def tiny_codes_data_collator(return_hf_format=False):
34 | return partial(default_collate_fn, return_hf_format=return_hf_format)
35 |
--------------------------------------------------------------------------------
/omg_llava/xtuner/apis/datasets/wizardlm.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) OpenMMLab. All rights reserved.
2 | from functools import partial
3 |
4 | from datasets import load_dataset
5 |
6 | from xtuner.dataset import process_hf_dataset
7 | from xtuner.dataset.collate_fns import default_collate_fn
8 | from xtuner.dataset.map_fns import template_map_fn_factory, wizardlm_map_fn
9 | from xtuner.utils import PROMPT_TEMPLATE
10 |
11 |
12 | def wizardlm_dataset(tokenizer,
13 | path='WizardLM/WizardLM_evol_instruct_V2_196k',
14 | max_length=2048,
15 | prompt_template=PROMPT_TEMPLATE.default,
16 | remove_unused_columns=False,
17 | pack_to_max_length=True):
18 | template_map_fn = template_map_fn_factory(template=prompt_template)
19 | dataset_org = load_dataset(path)
20 | dataset = process_hf_dataset(
21 | dataset=dataset_org,
22 | tokenizer=tokenizer,
23 | max_length=max_length,
24 | dataset_map_fn=wizardlm_map_fn,
25 | template_map_fn=template_map_fn,
26 | remove_unused_columns=remove_unused_columns,
27 | shuffle_before_pack=True,
28 | pack_to_max_length=pack_to_max_length)
29 |
30 | return dataset
31 |
32 |
33 | def wizardlm_data_collator(return_hf_format=False):
34 | return partial(default_collate_fn, return_hf_format=return_hf_format)
35 |
--------------------------------------------------------------------------------
/omg_llava/xtuner/configs/__init__.py:
--------------------------------------------------------------------------------
1 | # Copyright (c) OpenMMLab. All rights reserved.
2 | import os
3 |
4 |
5 | def get_cfgs_name_path():
6 | path = os.path.dirname(__file__)
7 | mapping = {}
8 | for root, dirs, files in os.walk(path):
9 | for file_ in files:
10 | if file_.endswith(
11 | ('.py', '.json')
12 | ) and not file_.startswith('.') and not file_.startswith('_'):
13 | mapping[os.path.splitext(file_)[0]] = os.path.join(root, file_)
14 | return mapping
15 |
16 |
17 | cfgs_name_path = get_cfgs_name_path()
18 |
19 | __all__ = ['cfgs_name_path']
20 |
--------------------------------------------------------------------------------
/omg_llava/xtuner/configs/cohere/README.md:
--------------------------------------------------------------------------------
1 | # Cohere 8x7B
2 |
3 | ## Install
4 |
5 | ```bash
6 | # Install the latest xtuner
7 | pip install -U 'xtuner[deepspeed]'
8 |
9 | # Cohere requires the latest version of transformers.
10 | pip install git+https://github.com/huggingface/transformers.git
11 |
12 | # Sequence parallel requires flash-attn
13 | pip install flash-attn
14 | ```
15 |
16 | ## Full Parameter Fine-tune
17 |
18 | Full parameter fine-tune needs 64 A100-80G
19 |
20 | ### slurm
21 |
22 | Note: `$PARTITION` means the virtual partition of slurm.
23 |
24 | ```bash
25 | srun -p $PARTITION --job-name=Cohere --nodes=8 --gres=gpu:8 --ntasks-per-node=8 xtuner train cohere_100b_128k_sp32 --deepspeed deepspeed_zero3 --launcher slurm
26 | ```
27 |
28 | ### torchrun
29 |
30 | Note: `$NODE_0_ADDR` means the ip address of the node_0 machine.
31 |
32 | ```bash
33 | # excuete on node 0
34 | NPROC_PER_NODE=8 NNODES=8 PORT=29600 ADDR=$NODE_0_ADDR NODE_RANK=0 xtuner train cohere_100b_128k_sp32 --deepspeed deepspeed_zero3
35 |
36 | # excuete on node 1
37 | NPROC_PER_NODE=8 NNODES=8 PORT=29600 ADDR=$NODE_0_ADDR NODE_RANK=1 xtuner train cohere_100b_128k_sp32 --deepspeed deepspeed_zero3
38 | ```
39 |
40 | ### Speed
41 |
42 | 16 * A100 80G:
43 |
44 | | Model | Sequence Length | GPUs Number | Sequence Parallel World Size | Tokens per Second | TFLOPs |
45 | | :---------: | :-------------: | :---------: | :--------------------------: | :---------------: | :----: |
46 | | Cohere_100b | 128k | 64 | 32 | 97.3 | 173.4 |
47 | | Cohere_100b | 128k | 128 | 16 | 102.1 | 182.7 |
48 | | Cohere_100b | 128k | 256 | 16 | 101.3 | 181.3 |
49 |
--------------------------------------------------------------------------------
/omg_llava/xtuner/configs/deepspeed/deepspeed_zero1.json:
--------------------------------------------------------------------------------
1 | {
2 | "gradient_accumulation_steps": "auto",
3 | "train_micro_batch_size_per_gpu": "auto",
4 | "gradient_clipping": "auto",
5 | "zero_allow_untested_optimizer": true,
6 | "zero_force_ds_cpu_optimizer": false,
7 | "zero_optimization": {
8 | "stage": 1,
9 | "overlap_comm": true
10 | },
11 | "fp16": {
12 | "enabled": "auto",
13 | "initial_scale_power": 16
14 | },
15 | "bf16": {
16 | "enabled": "auto"
17 | }
18 | }
19 |
--------------------------------------------------------------------------------
/omg_llava/xtuner/configs/deepspeed/deepspeed_zero2.json:
--------------------------------------------------------------------------------
1 | {
2 | "gradient_accumulation_steps": "auto",
3 | "train_micro_batch_size_per_gpu": "auto",
4 | "gradient_clipping": "auto",
5 | "zero_allow_untested_optimizer": true,
6 | "zero_force_ds_cpu_optimizer": false,
7 | "zero_optimization": {
8 | "stage": 2,
9 | "overlap_comm": true
10 | },
11 | "fp16": {
12 | "enabled": "auto",
13 | "initial_scale_power": 16
14 | },
15 | "bf16": {
16 | "enabled": "auto"
17 | }
18 | }
19 |
--------------------------------------------------------------------------------
/omg_llava/xtuner/configs/deepspeed/deepspeed_zero2_offload.json:
--------------------------------------------------------------------------------
1 | {
2 | "gradient_accumulation_steps": "auto",
3 | "train_micro_batch_size_per_gpu": "auto",
4 | "gradient_clipping": "auto",
5 | "zero_allow_untested_optimizer": true,
6 | "zero_force_ds_cpu_optimizer": false,
7 | "zero_optimization": {
8 | "stage": 2,
9 | "overlap_comm": true,
10 | "offload_optimizer": {
11 | "device": "cpu",
12 | "pin_memory": true
13 | }
14 | },
15 | "fp16": {
16 | "enabled": "auto",
17 | "initial_scale_power": 16
18 | },
19 | "bf16": {
20 | "enabled": "auto"
21 | }
22 | }
23 |
--------------------------------------------------------------------------------
/omg_llava/xtuner/configs/deepspeed/deepspeed_zero3.json:
--------------------------------------------------------------------------------
1 | {
2 | "gradient_accumulation_steps": "auto",
3 | "train_micro_batch_size_per_gpu": "auto",
4 | "gradient_clipping": "auto",
5 | "zero_allow_untested_optimizer": true,
6 | "zero_force_ds_cpu_optimizer": false,
7 | "zero_optimization": {
8 | "stage": 3,
9 | "overlap_comm": true,
10 | "stage3_gather_16bit_weights_on_model_save": true
11 | },
12 | "fp16": {
13 | "enabled": "auto",
14 | "initial_scale_power": 16
15 | },
16 | "bf16": {
17 | "enabled": "auto"
18 | }
19 | }
20 |
--------------------------------------------------------------------------------
/omg_llava/xtuner/configs/deepspeed/deepspeed_zero3_offload.json:
--------------------------------------------------------------------------------
1 | {
2 | "gradient_accumulation_steps": "auto",
3 | "train_micro_batch_size_per_gpu": "auto",
4 | "gradient_clipping": "auto",
5 | "zero_allow_untested_optimizer": true,
6 | "zero_force_ds_cpu_optimizer": false,
7 | "zero_optimization": {
8 | "stage": 3,
9 | "overlap_comm": true,
10 | "offload_optimizer": {
11 | "device": "cpu",
12 | "pin_memory": true
13 | },
14 | "offload_param": {
15 | "device": "cpu",
16 | "pin_memory": true
17 | },
18 | "stage3_gather_16bit_weights_on_model_save": true
19 | },
20 | "fp16": {
21 | "enabled": "auto",
22 | "initial_scale_power": 16
23 | },
24 | "bf16": {
25 | "enabled": "auto"
26 | }
27 | }
28 |
--------------------------------------------------------------------------------
/omg_llava/xtuner/configs/qwen/qwen1_5/qwen1_5_110b_chat/README.md:
--------------------------------------------------------------------------------
1 | # Qwen 110B
2 |
3 | ## Install
4 |
5 | ```bash
6 | # Install the latest xtuner
7 | pip install -U 'xtuner[deepspeed]'
8 |
9 | # We recommend installing flash_attn
10 | # pip install flash-attn
11 |
12 | # install the latest transformers
13 | pip install -U transformers
14 | ```
15 |
16 | ## QLoRA Fine-tune
17 |
18 | Training Qwen 110B with 32k context capability requires only 2 * A100 80G.
19 |
20 | ```bash
21 | xtuner train xtuner/configs/qwen/qwen1_5/qwen1_5_110b_chat/qwen1_5_110b_chat_qlora_alpaca_e3_16k_2gpus.py --deepspeed deepspeed_zero3
22 | ```
23 |
24 |
25 |
26 |