├── LICENSE ├── README.md ├── assets ├── 06203.pdf ├── documents │ └── demo_paper.pdf ├── images │ └── Dolphins_overview.png └── videos │ └── demo.mp4 ├── configs ├── accelerate_config │ ├── eval_config.yaml │ └── train_config.yaml ├── dataset_config.py ├── lora_config.py └── prompt_tuning_config.py ├── constants.py ├── conversation.py ├── dataset_utils ├── __init__.py ├── alpaca_gpt4_dataset.py ├── aokvqa_dataset.py ├── arguments.py ├── baize_dataset.py ├── bddx_dataset.py ├── builder.py ├── cc_sbu_align_dataset.py ├── clevr_dataset.py ├── coco_caption_dataset.py ├── coco_ric_dataset.py ├── coco_sic_dataset.py ├── collater.py ├── dial_dataset.py ├── dolly_dataset.py ├── few_shot_dataset.py ├── gc_dataset.py ├── gqa_dataset.py ├── iconqa_dataset.py ├── instruct_dataset.py ├── instruct_interleaved_dataset.py ├── llava_dataset.py ├── lrv_instruction_dataset.py ├── meme_cap_dataset.py ├── nlvr_dataset.py ├── ocr_vqa_dataset.py ├── processors.py ├── rec_dataset.py ├── reg_dataset.py ├── samplers │ ├── __init__.py │ └── infinite_sampler.py ├── shikra_dataset.py ├── snli_ve_datasets.py ├── svit_bbox_dataset.py ├── svit_dataset.py ├── svit_dial_dataset.py ├── text_ocr_dataset.py ├── textvqa_dataset.py ├── vicuna_dataset.py ├── visual_dialog_dataset.py ├── vizwiz_vqa_dataset.py ├── vqa_dataset.py └── vsr_dataset.py ├── inference.py ├── mllm ├── flamingo │ ├── __init__.py │ ├── config.json │ ├── configuration_flamingo.py │ ├── converting_flamingo_to_bf16.py │ ├── converting_flamingo_to_hf.py │ ├── converting_flamingo_to_lora.py │ ├── falcon │ │ ├── __init__.py │ │ ├── configuration_RW.py │ │ └── modelling_RW.py │ ├── flamingo-falcon-7B.json │ ├── flamingo-llama2-chat-13B.json │ ├── flamingo-llama2-chat-7B.json │ ├── flamingo-mpt-1B-redpajama.json │ ├── flamingo-mpt-30B-bf16.json │ ├── flamingo-mpt-30B.json │ ├── flamingo-mpt-7B.json │ ├── flamingo-vicuna-33B-v1.3.json │ ├── flamingo-vicuna-7B-v1.3.json │ ├── injecting_falcon_into_flamingo.py │ ├── injecting_llama2_into_flamingo.py │ ├── injecting_mpt-1B-redpajama_into_flamingo.py │ ├── injecting_mpt_into_flamingo.py │ ├── injecting_vicuna_into_flamingo.py │ ├── modeling_flamingo.py │ ├── mpt │ │ ├── __init__.py │ │ ├── adapt_tokenizer.py │ │ ├── attention.py │ │ ├── blocks.py │ │ ├── configuration_mpt.py │ │ ├── custom_embedding.py │ │ ├── flash_attn_triton.py │ │ ├── hf_prefixlm_converter.py │ │ ├── meta_init_context.py │ │ ├── modeling_mpt.py │ │ ├── norm.py │ │ └── param_init_fns.py │ ├── mpt_redpajama │ │ ├── __init__.py │ │ ├── attention.py │ │ ├── configuration_mosaic_gpt.py │ │ ├── gpt_blocks.py │ │ ├── low_precision_layernorm.py │ │ ├── mosaic_gpt.py │ │ └── param_init_fns.py │ └── utils.py ├── otter │ ├── Otter-MPT7B-config.json │ ├── __init__.py │ ├── __pycache__ │ │ ├── __init__.cpython-38.pyc │ │ ├── configuration_otter.cpython-38.pyc │ │ └── modeling_otter.cpython-38.pyc │ ├── config.json │ ├── configuration_otter.py │ ├── converting_flamingo_to_otter.py │ ├── converting_otter_fp32_to_fp16.py │ ├── converting_otter_pt_to_hf.py │ ├── converting_otter_to_lora.py │ ├── flamingo_pt2otter_hf.py │ └── modeling_otter.py └── src │ ├── __init__.py │ ├── factory.py │ ├── flamingo.py │ ├── flamingo_lm.py │ ├── helpers.py │ ├── mpt_lora_patch │ ├── README.md │ ├── adapt_tokenizer.py │ ├── attention.py │ ├── blocks.py │ ├── configuration_mpt.py │ ├── export_hf_checkpoint.py │ ├── hf_prefixlm_converter.py │ ├── meta_init_context.py │ ├── modeling_mpt.py │ ├── norm.py │ └── param_init_fns.py │ └── utils.py ├── pipeline ├── __init__.py ├── train.py └── train_utils.py ├── playground └── videos │ └── 1.mp4 ├── requirements.txt └── serve ├── __init__.py ├── cli.py ├── controller.py ├── examples ├── 25.png ├── 26.png ├── 28.png ├── 29.png ├── 30.png ├── 31.png ├── 32.png ├── baseball.jpg ├── bathroom.jpg ├── cat.jpg ├── chinchilla.png ├── dinner.jpg ├── extreme_ironing.jpg ├── flamingo.png ├── shiba.png ├── soccer.png ├── temp_video.mp4 ├── tennis.jpg └── waterview.jpg ├── gradio_css.py ├── gradio_patch.py ├── gradio_web_server.py ├── gradio_web_server_video.py ├── model_worker.py ├── register_worker.py ├── run_demo.sh ├── serving_utils.py └── test_message.py /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/README.md -------------------------------------------------------------------------------- /assets/06203.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/assets/06203.pdf -------------------------------------------------------------------------------- /assets/documents/demo_paper.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/assets/documents/demo_paper.pdf -------------------------------------------------------------------------------- /assets/images/Dolphins_overview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/assets/images/Dolphins_overview.png -------------------------------------------------------------------------------- /assets/videos/demo.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/assets/videos/demo.mp4 -------------------------------------------------------------------------------- /configs/accelerate_config/eval_config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/configs/accelerate_config/eval_config.yaml -------------------------------------------------------------------------------- /configs/accelerate_config/train_config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/configs/accelerate_config/train_config.yaml -------------------------------------------------------------------------------- /configs/dataset_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/configs/dataset_config.py -------------------------------------------------------------------------------- /configs/lora_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/configs/lora_config.py -------------------------------------------------------------------------------- /configs/prompt_tuning_config.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/configs/prompt_tuning_config.py -------------------------------------------------------------------------------- /constants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/constants.py -------------------------------------------------------------------------------- /conversation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/conversation.py -------------------------------------------------------------------------------- /dataset_utils/__init__.py: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /dataset_utils/alpaca_gpt4_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/dataset_utils/alpaca_gpt4_dataset.py -------------------------------------------------------------------------------- /dataset_utils/aokvqa_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/dataset_utils/aokvqa_dataset.py -------------------------------------------------------------------------------- /dataset_utils/arguments.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/dataset_utils/arguments.py -------------------------------------------------------------------------------- /dataset_utils/baize_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/dataset_utils/baize_dataset.py -------------------------------------------------------------------------------- /dataset_utils/bddx_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/dataset_utils/bddx_dataset.py -------------------------------------------------------------------------------- /dataset_utils/builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/dataset_utils/builder.py -------------------------------------------------------------------------------- /dataset_utils/cc_sbu_align_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/dataset_utils/cc_sbu_align_dataset.py -------------------------------------------------------------------------------- /dataset_utils/clevr_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/dataset_utils/clevr_dataset.py -------------------------------------------------------------------------------- /dataset_utils/coco_caption_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/dataset_utils/coco_caption_dataset.py -------------------------------------------------------------------------------- /dataset_utils/coco_ric_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/dataset_utils/coco_ric_dataset.py -------------------------------------------------------------------------------- /dataset_utils/coco_sic_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/dataset_utils/coco_sic_dataset.py -------------------------------------------------------------------------------- /dataset_utils/collater.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/dataset_utils/collater.py -------------------------------------------------------------------------------- /dataset_utils/dial_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/dataset_utils/dial_dataset.py -------------------------------------------------------------------------------- /dataset_utils/dolly_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/dataset_utils/dolly_dataset.py -------------------------------------------------------------------------------- /dataset_utils/few_shot_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/dataset_utils/few_shot_dataset.py -------------------------------------------------------------------------------- /dataset_utils/gc_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/dataset_utils/gc_dataset.py -------------------------------------------------------------------------------- /dataset_utils/gqa_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/dataset_utils/gqa_dataset.py -------------------------------------------------------------------------------- /dataset_utils/iconqa_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/dataset_utils/iconqa_dataset.py -------------------------------------------------------------------------------- /dataset_utils/instruct_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/dataset_utils/instruct_dataset.py -------------------------------------------------------------------------------- /dataset_utils/instruct_interleaved_dataset.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dataset_utils/llava_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/dataset_utils/llava_dataset.py -------------------------------------------------------------------------------- /dataset_utils/lrv_instruction_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/dataset_utils/lrv_instruction_dataset.py -------------------------------------------------------------------------------- /dataset_utils/meme_cap_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/dataset_utils/meme_cap_dataset.py -------------------------------------------------------------------------------- /dataset_utils/nlvr_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/dataset_utils/nlvr_dataset.py -------------------------------------------------------------------------------- /dataset_utils/ocr_vqa_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/dataset_utils/ocr_vqa_dataset.py -------------------------------------------------------------------------------- /dataset_utils/processors.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/dataset_utils/processors.py -------------------------------------------------------------------------------- /dataset_utils/rec_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/dataset_utils/rec_dataset.py -------------------------------------------------------------------------------- /dataset_utils/reg_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/dataset_utils/reg_dataset.py -------------------------------------------------------------------------------- /dataset_utils/samplers/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/dataset_utils/samplers/__init__.py -------------------------------------------------------------------------------- /dataset_utils/samplers/infinite_sampler.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/dataset_utils/samplers/infinite_sampler.py -------------------------------------------------------------------------------- /dataset_utils/shikra_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/dataset_utils/shikra_dataset.py -------------------------------------------------------------------------------- /dataset_utils/snli_ve_datasets.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/dataset_utils/snli_ve_datasets.py -------------------------------------------------------------------------------- /dataset_utils/svit_bbox_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/dataset_utils/svit_bbox_dataset.py -------------------------------------------------------------------------------- /dataset_utils/svit_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/dataset_utils/svit_dataset.py -------------------------------------------------------------------------------- /dataset_utils/svit_dial_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/dataset_utils/svit_dial_dataset.py -------------------------------------------------------------------------------- /dataset_utils/text_ocr_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/dataset_utils/text_ocr_dataset.py -------------------------------------------------------------------------------- /dataset_utils/textvqa_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/dataset_utils/textvqa_dataset.py -------------------------------------------------------------------------------- /dataset_utils/vicuna_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/dataset_utils/vicuna_dataset.py -------------------------------------------------------------------------------- /dataset_utils/visual_dialog_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/dataset_utils/visual_dialog_dataset.py -------------------------------------------------------------------------------- /dataset_utils/vizwiz_vqa_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/dataset_utils/vizwiz_vqa_dataset.py -------------------------------------------------------------------------------- /dataset_utils/vqa_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/dataset_utils/vqa_dataset.py -------------------------------------------------------------------------------- /dataset_utils/vsr_dataset.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/dataset_utils/vsr_dataset.py -------------------------------------------------------------------------------- /inference.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/inference.py -------------------------------------------------------------------------------- /mllm/flamingo/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/flamingo/__init__.py -------------------------------------------------------------------------------- /mllm/flamingo/config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/flamingo/config.json -------------------------------------------------------------------------------- /mllm/flamingo/configuration_flamingo.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/flamingo/configuration_flamingo.py -------------------------------------------------------------------------------- /mllm/flamingo/converting_flamingo_to_bf16.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/flamingo/converting_flamingo_to_bf16.py -------------------------------------------------------------------------------- /mllm/flamingo/converting_flamingo_to_hf.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/flamingo/converting_flamingo_to_hf.py -------------------------------------------------------------------------------- /mllm/flamingo/converting_flamingo_to_lora.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/flamingo/converting_flamingo_to_lora.py -------------------------------------------------------------------------------- /mllm/flamingo/falcon/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /mllm/flamingo/falcon/configuration_RW.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/flamingo/falcon/configuration_RW.py -------------------------------------------------------------------------------- /mllm/flamingo/falcon/modelling_RW.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/flamingo/falcon/modelling_RW.py -------------------------------------------------------------------------------- /mllm/flamingo/flamingo-falcon-7B.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/flamingo/flamingo-falcon-7B.json -------------------------------------------------------------------------------- /mllm/flamingo/flamingo-llama2-chat-13B.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/flamingo/flamingo-llama2-chat-13B.json -------------------------------------------------------------------------------- /mllm/flamingo/flamingo-llama2-chat-7B.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/flamingo/flamingo-llama2-chat-7B.json -------------------------------------------------------------------------------- /mllm/flamingo/flamingo-mpt-1B-redpajama.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/flamingo/flamingo-mpt-1B-redpajama.json -------------------------------------------------------------------------------- /mllm/flamingo/flamingo-mpt-30B-bf16.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/flamingo/flamingo-mpt-30B-bf16.json -------------------------------------------------------------------------------- /mllm/flamingo/flamingo-mpt-30B.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/flamingo/flamingo-mpt-30B.json -------------------------------------------------------------------------------- /mllm/flamingo/flamingo-mpt-7B.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/flamingo/flamingo-mpt-7B.json -------------------------------------------------------------------------------- /mllm/flamingo/flamingo-vicuna-33B-v1.3.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/flamingo/flamingo-vicuna-33B-v1.3.json -------------------------------------------------------------------------------- /mllm/flamingo/flamingo-vicuna-7B-v1.3.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/flamingo/flamingo-vicuna-7B-v1.3.json -------------------------------------------------------------------------------- /mllm/flamingo/injecting_falcon_into_flamingo.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/flamingo/injecting_falcon_into_flamingo.py -------------------------------------------------------------------------------- /mllm/flamingo/injecting_llama2_into_flamingo.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/flamingo/injecting_llama2_into_flamingo.py -------------------------------------------------------------------------------- /mllm/flamingo/injecting_mpt-1B-redpajama_into_flamingo.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/flamingo/injecting_mpt-1B-redpajama_into_flamingo.py -------------------------------------------------------------------------------- /mllm/flamingo/injecting_mpt_into_flamingo.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/flamingo/injecting_mpt_into_flamingo.py -------------------------------------------------------------------------------- /mllm/flamingo/injecting_vicuna_into_flamingo.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/flamingo/injecting_vicuna_into_flamingo.py -------------------------------------------------------------------------------- /mllm/flamingo/modeling_flamingo.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/flamingo/modeling_flamingo.py -------------------------------------------------------------------------------- /mllm/flamingo/mpt/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /mllm/flamingo/mpt/adapt_tokenizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/flamingo/mpt/adapt_tokenizer.py -------------------------------------------------------------------------------- /mllm/flamingo/mpt/attention.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/flamingo/mpt/attention.py -------------------------------------------------------------------------------- /mllm/flamingo/mpt/blocks.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/flamingo/mpt/blocks.py -------------------------------------------------------------------------------- /mllm/flamingo/mpt/configuration_mpt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/flamingo/mpt/configuration_mpt.py -------------------------------------------------------------------------------- /mllm/flamingo/mpt/custom_embedding.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/flamingo/mpt/custom_embedding.py -------------------------------------------------------------------------------- /mllm/flamingo/mpt/flash_attn_triton.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/flamingo/mpt/flash_attn_triton.py -------------------------------------------------------------------------------- /mllm/flamingo/mpt/hf_prefixlm_converter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/flamingo/mpt/hf_prefixlm_converter.py -------------------------------------------------------------------------------- /mllm/flamingo/mpt/meta_init_context.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/flamingo/mpt/meta_init_context.py -------------------------------------------------------------------------------- /mllm/flamingo/mpt/modeling_mpt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/flamingo/mpt/modeling_mpt.py -------------------------------------------------------------------------------- /mllm/flamingo/mpt/norm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/flamingo/mpt/norm.py -------------------------------------------------------------------------------- /mllm/flamingo/mpt/param_init_fns.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/flamingo/mpt/param_init_fns.py -------------------------------------------------------------------------------- /mllm/flamingo/mpt_redpajama/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /mllm/flamingo/mpt_redpajama/attention.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/flamingo/mpt_redpajama/attention.py -------------------------------------------------------------------------------- /mllm/flamingo/mpt_redpajama/configuration_mosaic_gpt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/flamingo/mpt_redpajama/configuration_mosaic_gpt.py -------------------------------------------------------------------------------- /mllm/flamingo/mpt_redpajama/gpt_blocks.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/flamingo/mpt_redpajama/gpt_blocks.py -------------------------------------------------------------------------------- /mllm/flamingo/mpt_redpajama/low_precision_layernorm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/flamingo/mpt_redpajama/low_precision_layernorm.py -------------------------------------------------------------------------------- /mllm/flamingo/mpt_redpajama/mosaic_gpt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/flamingo/mpt_redpajama/mosaic_gpt.py -------------------------------------------------------------------------------- /mllm/flamingo/mpt_redpajama/param_init_fns.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/flamingo/mpt_redpajama/param_init_fns.py -------------------------------------------------------------------------------- /mllm/flamingo/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/flamingo/utils.py -------------------------------------------------------------------------------- /mllm/otter/Otter-MPT7B-config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/otter/Otter-MPT7B-config.json -------------------------------------------------------------------------------- /mllm/otter/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/otter/__init__.py -------------------------------------------------------------------------------- /mllm/otter/__pycache__/__init__.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/otter/__pycache__/__init__.cpython-38.pyc -------------------------------------------------------------------------------- /mllm/otter/__pycache__/configuration_otter.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/otter/__pycache__/configuration_otter.cpython-38.pyc -------------------------------------------------------------------------------- /mllm/otter/__pycache__/modeling_otter.cpython-38.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/otter/__pycache__/modeling_otter.cpython-38.pyc -------------------------------------------------------------------------------- /mllm/otter/config.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/otter/config.json -------------------------------------------------------------------------------- /mllm/otter/configuration_otter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/otter/configuration_otter.py -------------------------------------------------------------------------------- /mllm/otter/converting_flamingo_to_otter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/otter/converting_flamingo_to_otter.py -------------------------------------------------------------------------------- /mllm/otter/converting_otter_fp32_to_fp16.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/otter/converting_otter_fp32_to_fp16.py -------------------------------------------------------------------------------- /mllm/otter/converting_otter_pt_to_hf.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/otter/converting_otter_pt_to_hf.py -------------------------------------------------------------------------------- /mllm/otter/converting_otter_to_lora.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/otter/converting_otter_to_lora.py -------------------------------------------------------------------------------- /mllm/otter/flamingo_pt2otter_hf.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/otter/flamingo_pt2otter_hf.py -------------------------------------------------------------------------------- /mllm/otter/modeling_otter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/otter/modeling_otter.py -------------------------------------------------------------------------------- /mllm/src/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /mllm/src/factory.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/src/factory.py -------------------------------------------------------------------------------- /mllm/src/flamingo.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/src/flamingo.py -------------------------------------------------------------------------------- /mllm/src/flamingo_lm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/src/flamingo_lm.py -------------------------------------------------------------------------------- /mllm/src/helpers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/src/helpers.py -------------------------------------------------------------------------------- /mllm/src/mpt_lora_patch/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/src/mpt_lora_patch/README.md -------------------------------------------------------------------------------- /mllm/src/mpt_lora_patch/adapt_tokenizer.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/src/mpt_lora_patch/adapt_tokenizer.py -------------------------------------------------------------------------------- /mllm/src/mpt_lora_patch/attention.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/src/mpt_lora_patch/attention.py -------------------------------------------------------------------------------- /mllm/src/mpt_lora_patch/blocks.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/src/mpt_lora_patch/blocks.py -------------------------------------------------------------------------------- /mllm/src/mpt_lora_patch/configuration_mpt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/src/mpt_lora_patch/configuration_mpt.py -------------------------------------------------------------------------------- /mllm/src/mpt_lora_patch/export_hf_checkpoint.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/src/mpt_lora_patch/export_hf_checkpoint.py -------------------------------------------------------------------------------- /mllm/src/mpt_lora_patch/hf_prefixlm_converter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/src/mpt_lora_patch/hf_prefixlm_converter.py -------------------------------------------------------------------------------- /mllm/src/mpt_lora_patch/meta_init_context.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/src/mpt_lora_patch/meta_init_context.py -------------------------------------------------------------------------------- /mllm/src/mpt_lora_patch/modeling_mpt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/src/mpt_lora_patch/modeling_mpt.py -------------------------------------------------------------------------------- /mllm/src/mpt_lora_patch/norm.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/src/mpt_lora_patch/norm.py -------------------------------------------------------------------------------- /mllm/src/mpt_lora_patch/param_init_fns.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/src/mpt_lora_patch/param_init_fns.py -------------------------------------------------------------------------------- /mllm/src/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/mllm/src/utils.py -------------------------------------------------------------------------------- /pipeline/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pipeline/train.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/pipeline/train.py -------------------------------------------------------------------------------- /pipeline/train_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/pipeline/train_utils.py -------------------------------------------------------------------------------- /playground/videos/1.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/playground/videos/1.mp4 -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/requirements.txt -------------------------------------------------------------------------------- /serve/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /serve/cli.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/serve/cli.py -------------------------------------------------------------------------------- /serve/controller.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/serve/controller.py -------------------------------------------------------------------------------- /serve/examples/25.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/serve/examples/25.png -------------------------------------------------------------------------------- /serve/examples/26.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/serve/examples/26.png -------------------------------------------------------------------------------- /serve/examples/28.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/serve/examples/28.png -------------------------------------------------------------------------------- /serve/examples/29.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/serve/examples/29.png -------------------------------------------------------------------------------- /serve/examples/30.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/serve/examples/30.png -------------------------------------------------------------------------------- /serve/examples/31.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/serve/examples/31.png -------------------------------------------------------------------------------- /serve/examples/32.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/serve/examples/32.png -------------------------------------------------------------------------------- /serve/examples/baseball.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/serve/examples/baseball.jpg -------------------------------------------------------------------------------- /serve/examples/bathroom.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/serve/examples/bathroom.jpg -------------------------------------------------------------------------------- /serve/examples/cat.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/serve/examples/cat.jpg -------------------------------------------------------------------------------- /serve/examples/chinchilla.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/serve/examples/chinchilla.png -------------------------------------------------------------------------------- /serve/examples/dinner.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/serve/examples/dinner.jpg -------------------------------------------------------------------------------- /serve/examples/extreme_ironing.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/serve/examples/extreme_ironing.jpg -------------------------------------------------------------------------------- /serve/examples/flamingo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/serve/examples/flamingo.png -------------------------------------------------------------------------------- /serve/examples/shiba.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/serve/examples/shiba.png -------------------------------------------------------------------------------- /serve/examples/soccer.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/serve/examples/soccer.png -------------------------------------------------------------------------------- /serve/examples/temp_video.mp4: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /serve/examples/tennis.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/serve/examples/tennis.jpg -------------------------------------------------------------------------------- /serve/examples/waterview.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/serve/examples/waterview.jpg -------------------------------------------------------------------------------- /serve/gradio_css.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/serve/gradio_css.py -------------------------------------------------------------------------------- /serve/gradio_patch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/serve/gradio_patch.py -------------------------------------------------------------------------------- /serve/gradio_web_server.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/serve/gradio_web_server.py -------------------------------------------------------------------------------- /serve/gradio_web_server_video.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/serve/gradio_web_server_video.py -------------------------------------------------------------------------------- /serve/model_worker.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/serve/model_worker.py -------------------------------------------------------------------------------- /serve/register_worker.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/serve/register_worker.py -------------------------------------------------------------------------------- /serve/run_demo.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/serve/run_demo.sh -------------------------------------------------------------------------------- /serve/serving_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/serve/serving_utils.py -------------------------------------------------------------------------------- /serve/test_message.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/SaFo-Lab/Dolphins/HEAD/serve/test_message.py --------------------------------------------------------------------------------