├── .gitattributes ├── .idea └── GITA.iml ├── README.md ├── checkpoints └── Vision_Text │ └── GVLQA_BASE │ ├── connectivity │ ├── llava-v1.5-13b-lora(128, 256)-unfreeze_vit-False-epoch-1 │ │ ├── README.md │ │ ├── adapter_config.json │ │ ├── adapter_model.bin │ │ ├── config.json │ │ ├── non_lora_trainables.bin │ │ └── trainer_state.json │ └── llava-v1.5-7b-lora(128, 256)-unfreeze_vit-False-epoch-1 │ │ ├── README.md │ │ ├── adapter_config.json │ │ ├── adapter_model.bin │ │ ├── config.json │ │ ├── non_lora_trainables.bin │ │ └── trainer_state.json │ ├── cycle │ ├── llava-v1.5-13b-lora(128, 256)-unfreeze_vit-False-epoch-10 │ │ ├── README.md │ │ ├── adapter_config.json │ │ ├── adapter_model.bin │ │ ├── config.json │ │ ├── non_lora_trainables.bin │ │ └── trainer_state.json │ └── llava-v1.5-7b-lora(128, 256)-unfreeze_vit-False-epoch-20 │ │ ├── README.md │ │ ├── adapter_config.json │ │ ├── adapter_model.bin │ │ ├── config.json │ │ ├── non_lora_trainables.bin │ │ └── trainer_state.json │ ├── flow │ ├── llava-v1.5-13b-lora(128, 256)-unfreeze_vit-False-epoch-10 │ │ ├── README.md │ │ ├── adapter_config.json │ │ ├── adapter_model.bin │ │ ├── config.json │ │ ├── non_lora_trainables.bin │ │ └── trainer_state.json │ └── llava-v1.5-7b-lora(128, 256)-unfreeze_vit-False-epoch-20 │ │ ├── README.md │ │ ├── adapter_config.json │ │ ├── adapter_model.bin │ │ ├── config.json │ │ ├── non_lora_trainables.bin │ │ └── trainer_state.json │ ├── hamilton │ ├── llava-v1.5-13b-lora(128, 256)-unfreeze_vit-False-epoch-30 │ │ ├── README.md │ │ ├── adapter_config.json │ │ ├── adapter_model.bin │ │ ├── config.json │ │ ├── non_lora_trainables.bin │ │ └── trainer_state.json │ └── llava-v1.5-7b-lora(128, 256)-unfreeze_vit-False-epoch-30 │ │ ├── README.md │ │ ├── adapter_config.json │ │ ├── adapter_model.bin │ │ ├── config.json │ │ ├── non_lora_trainables.bin │ │ └── trainer_state.json │ ├── matching │ ├── llava-v1.5-13b-lora(128, 256)-unfreeze_vit-False-epoch-50 │ │ ├── README.md │ │ ├── adapter_config.json │ │ ├── adapter_model.bin │ │ ├── config.json │ │ ├── non_lora_trainables.bin │ │ └── trainer_state.json │ └── llava-v1.5-7b-lora(128, 256)-unfreeze_vit-False-epoch-5 │ │ ├── README.md │ │ ├── adapter_config.json │ │ ├── adapter_model.bin │ │ ├── config.json │ │ ├── non_lora_trainables.bin │ │ └── trainer_state.json │ ├── shortest_path │ ├── llava-v1.5-13b-lora(128, 256)-unfreeze_vit-False-epoch-10 │ │ ├── README.md │ │ ├── adapter_config.json │ │ ├── adapter_model.bin │ │ ├── config.json │ │ ├── non_lora_trainables.bin │ │ └── trainer_state.json │ └── llava-v1.5-7b-lora(128, 256)-unfreeze_vit-False-epoch-10 │ │ ├── README.md │ │ ├── adapter_config.json │ │ ├── adapter_model.bin │ │ ├── config.json │ │ ├── non_lora_trainables.bin │ │ └── trainer_state.json │ └── topology │ ├── llava-v1.5-13b-lora(128, 256)-unfreeze_vit-False-epoch-10 │ ├── README.md │ ├── adapter_config.json │ ├── adapter_model.bin │ ├── config.json │ ├── non_lora_trainables.bin │ └── trainer_state.json │ └── llava-v1.5-7b-lora(128, 256)-unfreeze_vit-False-epoch-10 │ ├── README.md │ ├── adapter_config.json │ ├── adapter_model.bin │ ├── config.json │ ├── non_lora_trainables.bin │ └── trainer_state.json ├── fastchat ├── __init__.py ├── __pycache__ │ ├── __init__.cpython-310.pyc │ ├── constants.cpython-310.pyc │ ├── conversation.cpython-310.pyc │ └── utils.cpython-310.pyc ├── constants.py ├── conversation.py ├── custom_eval │ ├── __pycache__ │ │ └── eval.cpython-310.pyc │ └── eval.py ├── data │ ├── __init__.py │ ├── clean_sharegpt.py │ ├── convert_alpaca.py │ ├── extract_gpt4_only.py │ ├── extract_single_round.py │ ├── filter_wrong_format.py │ ├── get_stats.py │ ├── hardcoded_questions.py │ ├── inspect_data.py │ ├── merge.py │ ├── optional_clean.py │ ├── optional_replace.py │ ├── prepare_all.py │ ├── pretty_json.py │ ├── sample.py │ ├── split_long_conversation.py │ └── split_train_test.py ├── linkpred_builder.py ├── llm_judge │ ├── README.md │ ├── clean_judgment.py │ ├── common.py │ ├── compute_agreement.py │ ├── data │ │ ├── judge_prompts.jsonl │ │ ├── mt_bench │ │ │ ├── misc │ │ │ │ └── radar.png │ │ │ ├── question.jsonl │ │ │ └── reference_answer │ │ │ │ └── gpt-4.jsonl │ │ └── vicuna_bench │ │ │ ├── question.jsonl │ │ │ └── reference_answer │ │ │ └── gpt-4.jsonl │ ├── download_mt_bench_pregenerated.py │ ├── gen_api_answer.py │ ├── gen_judgment.py │ ├── gen_model_answer.py │ ├── qa_browser.py │ └── show_result.py ├── model │ ├── __init__.py │ ├── __pycache__ │ │ ├── __init__.cpython-310.pyc │ │ ├── compression.cpython-310.pyc │ │ ├── llama_condense_monkey_patch.cpython-310.pyc │ │ ├── model_adapter.cpython-310.pyc │ │ ├── model_chatglm.cpython-310.pyc │ │ ├── model_codet5p.cpython-310.pyc │ │ ├── model_exllama.cpython-310.pyc │ │ ├── model_falcon.cpython-310.pyc │ │ ├── model_xfastertransformer.cpython-310.pyc │ │ └── monkey_patch_non_inplace.cpython-310.pyc │ ├── apply_delta.py │ ├── apply_lora.py │ ├── compression.py │ ├── convert_fp16.py │ ├── llama_condense_monkey_patch.py │ ├── make_delta.py │ ├── model_adapter.py │ ├── model_chatglm.py │ ├── model_codet5p.py │ ├── model_exllama.py │ ├── model_falcon.py │ ├── model_registry.py │ ├── model_xfastertransformer.py │ ├── monkey_patch_non_inplace.py │ ├── rwkv_model.py │ └── upload_hub.py ├── modules │ ├── __init__.py │ ├── __pycache__ │ │ ├── __init__.cpython-310.pyc │ │ ├── awq.cpython-310.pyc │ │ ├── exllama.cpython-310.pyc │ │ ├── gptq.cpython-310.pyc │ │ └── xfastertransformer.cpython-310.pyc │ ├── awq.py │ ├── exllama.py │ ├── gptq.py │ └── xfastertransformer.py ├── nodecls_builder.py ├── protocol │ ├── api_protocol.py │ └── openai_api_protocol.py ├── serve │ ├── __init__.py │ ├── __pycache__ │ │ ├── __init__.cpython-310.pyc │ │ ├── cli.cpython-310.pyc │ │ └── inference.cpython-310.pyc │ ├── api_provider.py │ ├── base_model_worker.py │ ├── cli.py │ ├── controller.py │ ├── gateway │ │ ├── README.md │ │ └── nginx.conf │ ├── gradio_block_arena_anony.py │ ├── gradio_block_arena_named.py │ ├── gradio_web_server.py │ ├── gradio_web_server_multi.py │ ├── huggingface_api.py │ ├── huggingface_api_worker.py │ ├── inference.py │ ├── launch_all_serve.py │ ├── model_worker.py │ ├── monitor │ │ ├── basic_stats.py │ │ ├── clean_battle_data.py │ │ ├── clean_chat_data.py │ │ ├── dataset_release_scripts │ │ │ ├── arena_33k │ │ │ │ ├── count_unique_users.py │ │ │ │ ├── filter_bad_conv.py │ │ │ │ ├── merge_field.py │ │ │ │ ├── sample.py │ │ │ │ └── upload_hf_dataset.py │ │ │ └── lmsys_chat_1m │ │ │ │ ├── approve_all.py │ │ │ │ ├── compute_stats.py │ │ │ │ ├── filter_bad_conv.py │ │ │ │ ├── final_post_processing.py │ │ │ │ ├── instructions.md │ │ │ │ ├── merge_oai_tag.py │ │ │ │ ├── process_all.sh │ │ │ │ ├── sample.py │ │ │ │ └── upload_hf_dataset.py │ │ ├── elo_analysis.py │ │ ├── inspect_conv.py │ │ ├── intersect_conv_file.py │ │ ├── leaderboard_csv_to_html.py │ │ ├── monitor.py │ │ ├── summarize_cluster.py │ │ ├── tag_openai_moderation.py │ │ └── topic_clustering.py │ ├── multi_model_worker.py │ ├── openai_api_server.py │ ├── register_worker.py │ ├── shutdown_serve.py │ ├── test_message.py │ ├── test_throughput.py │ └── vllm_worker.py ├── train │ ├── __pycache__ │ │ ├── llama2_flash_attn_monkey_patch.cpython-310.pyc │ │ ├── llama_flash_attn_monkey_patch.cpython-310.pyc │ │ └── train.cpython-310.pyc │ ├── linkpred_builder.py │ ├── llama2_flash_attn_monkey_patch.py │ ├── llama_flash_attn_monkey_patch.py │ ├── llama_xformers_attn_monkey_patch.py │ ├── nodecls_builder.py │ ├── train.py │ ├── train_baichuan.py │ ├── train_flant5.py │ ├── train_lora.py │ ├── train_lora_t5.py │ ├── train_mem.py │ └── train_xformers.py └── utils.py ├── llava ├── __init__.py ├── __pycache__ │ ├── __init__.cpython-310.pyc │ ├── __init__.cpython-311.pyc │ ├── constants.cpython-310.pyc │ ├── conversation.cpython-310.pyc │ ├── mm_utils.cpython-310.pyc │ └── utils.cpython-310.pyc ├── constants.py ├── conversation.py ├── custom_eval │ ├── __pycache__ │ │ └── eval.cpython-310.pyc │ └── eval.py ├── eval │ ├── eval_gpt_review.py │ ├── eval_gpt_review_bench.py │ ├── eval_gpt_review_visual.py │ ├── eval_pope.py │ ├── eval_science_qa.py │ ├── eval_science_qa_gpt4.py │ ├── eval_science_qa_gpt4_requery.py │ ├── eval_textvqa.py │ ├── generate_webpage_data_from_table.py │ ├── m4c_evaluator.py │ ├── model_qa.py │ ├── model_vqa.py │ ├── model_vqa_loader.py │ ├── model_vqa_mmbench.py │ ├── model_vqa_qbench.py │ ├── model_vqa_science.py │ ├── qa_baseline_gpt35.py │ ├── run_llava.py │ ├── summarize_gpt_review.py │ ├── table │ │ ├── answer │ │ │ ├── answer_alpaca-13b.jsonl │ │ │ ├── answer_bard.jsonl │ │ │ ├── answer_gpt35.jsonl │ │ │ ├── answer_llama-13b.jsonl │ │ │ └── answer_vicuna-13b.jsonl │ │ ├── caps_boxes_coco2014_val_80.jsonl │ │ ├── model.jsonl │ │ ├── prompt.jsonl │ │ ├── question.jsonl │ │ ├── results │ │ │ ├── test_sqa_llava_13b_v0.json │ │ │ └── test_sqa_llava_lcs_558k_sqa_12e_vicuna_v1_3_13b.json │ │ ├── review │ │ │ ├── review_alpaca-13b_vicuna-13b.jsonl │ │ │ ├── review_bard_vicuna-13b.jsonl │ │ │ ├── review_gpt35_vicuna-13b.jsonl │ │ │ └── review_llama-13b_vicuna-13b.jsonl │ │ ├── reviewer.jsonl │ │ └── rule.json │ └── webpage │ │ ├── figures │ │ ├── alpaca.png │ │ ├── bard.jpg │ │ ├── chatgpt.svg │ │ ├── llama.jpg │ │ ├── swords_FILL0_wght300_GRAD0_opsz48.svg │ │ └── vicuna.jpeg │ │ ├── index.html │ │ ├── script.js │ │ └── styles.css ├── mm_utils.py ├── model │ ├── __init__.py │ ├── __pycache__ │ │ ├── __init__.cpython-310.pyc │ │ ├── __init__.cpython-311.pyc │ │ ├── builder.cpython-310.pyc │ │ └── llava_arch.cpython-310.pyc │ ├── apply_delta.py │ ├── builder.py │ ├── consolidate.py │ ├── language_model │ │ ├── __pycache__ │ │ │ ├── llava_llama.cpython-310.pyc │ │ │ ├── llava_llama.cpython-311.pyc │ │ │ └── llava_mpt.cpython-310.pyc │ │ ├── llava_llama.py │ │ ├── llava_mpt.py │ │ └── mpt │ │ │ ├── __pycache__ │ │ │ ├── adapt_tokenizer.cpython-310.pyc │ │ │ ├── attention.cpython-310.pyc │ │ │ ├── blocks.cpython-310.pyc │ │ │ ├── configuration_mpt.cpython-310.pyc │ │ │ ├── custom_embedding.cpython-310.pyc │ │ │ ├── flash_attn_triton.cpython-310.pyc │ │ │ ├── hf_prefixlm_converter.cpython-310.pyc │ │ │ ├── meta_init_context.cpython-310.pyc │ │ │ ├── modeling_mpt.cpython-310.pyc │ │ │ ├── norm.cpython-310.pyc │ │ │ └── param_init_fns.cpython-310.pyc │ │ │ ├── adapt_tokenizer.py │ │ │ ├── attention.py │ │ │ ├── blocks.py │ │ │ ├── configuration_mpt.py │ │ │ ├── custom_embedding.py │ │ │ ├── flash_attn_triton.py │ │ │ ├── hf_prefixlm_converter.py │ │ │ ├── meta_init_context.py │ │ │ ├── modeling_mpt.py │ │ │ ├── norm.py │ │ │ └── param_init_fns.py │ ├── llava_arch.py │ ├── make_delta.py │ ├── multimodal_encoder │ │ ├── __pycache__ │ │ │ ├── builder.cpython-310.pyc │ │ │ └── clip_encoder.cpython-310.pyc │ │ ├── builder.py │ │ └── clip_encoder.py │ ├── multimodal_projector │ │ ├── __pycache__ │ │ │ └── builder.cpython-310.pyc │ │ └── builder.py │ └── utils.py ├── serve │ ├── __init__.py │ ├── cli.py │ ├── controller.py │ ├── examples │ │ ├── extreme_ironing.jpg │ │ └── waterview.jpg │ ├── gradio_web_server.py │ ├── model_worker.py │ ├── register_worker.py │ └── test_message.py ├── test_topo.py ├── train │ ├── __pycache__ │ │ ├── linkpred_builder.cpython-310.pyc │ │ ├── llama_flash_attn_monkey_patch.cpython-310.pyc │ │ ├── llava_trainer.cpython-310.pyc │ │ ├── nodecls_builder.cpython-310.pyc │ │ └── train.cpython-310.pyc │ ├── linkpred_builder.py │ ├── llama_flash_attn_monkey_patch.py │ ├── llama_xformers_attn_monkey_patch.py │ ├── llava_trainer.py │ ├── nodecls_builder.py │ ├── train.py │ ├── train_mem.py │ └── train_xformers.py └── utils.py ├── pyproject.toml └── scripts ├── eval ├── eval.sh └── eval_loop.sh ├── train ├── finetune_lora.sh └── finetune_lora_loop.sh ├── zero2.json ├── zero3.json └── zero3_offload.json /.gitattributes: -------------------------------------------------------------------------------- 1 | *.bin filter=lfs diff=lfs merge=lfs -text 2 | -------------------------------------------------------------------------------- /.idea/GITA.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 12 | -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/connectivity/llava-v1.5-13b-lora(128, 256)-unfreeze_vit-False-epoch-1/README.md: -------------------------------------------------------------------------------- 1 | --- 2 | library_name: peft 3 | --- 4 | ## Training procedure 5 | 6 | ### Framework versions 7 | 8 | 9 | - PEFT 0.4.0 10 | -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/connectivity/llava-v1.5-13b-lora(128, 256)-unfreeze_vit-False-epoch-1/adapter_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "auto_mapping": null, 3 | "base_model_name_or_path": "/mnt/sdb1/local_llm/llava-v1.5-13b", 4 | "bias": "none", 5 | "fan_in_fan_out": false, 6 | "inference_mode": true, 7 | "init_lora_weights": true, 8 | "layers_pattern": null, 9 | "layers_to_transform": null, 10 | "lora_alpha": 256, 11 | "lora_dropout": 0.05, 12 | "modules_to_save": null, 13 | "peft_type": "LORA", 14 | "r": 128, 15 | "revision": null, 16 | "target_modules": [ 17 | "v_proj", 18 | "q_proj", 19 | "up_proj", 20 | "o_proj", 21 | "gate_proj", 22 | "k_proj", 23 | "down_proj" 24 | ], 25 | "task_type": "CAUSAL_LM" 26 | } -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/connectivity/llava-v1.5-13b-lora(128, 256)-unfreeze_vit-False-epoch-1/adapter_model.bin: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:aa28e20884d0987e4006e9c08e6e1b0f7438638be5b136229a97a580f0a8868b 3 | size 1001584333 4 | -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/connectivity/llava-v1.5-13b-lora(128, 256)-unfreeze_vit-False-epoch-1/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "_name_or_path": "/mnt/sdb1/local_llm/llava-v1.5-13b", 3 | "architectures": [ 4 | "LlavaLlamaForCausalLM" 5 | ], 6 | "bos_token_id": 1, 7 | "eos_token_id": 2, 8 | "freeze_mm_mlp_adapter": false, 9 | "freeze_mm_vision_resampler": false, 10 | "hidden_act": "silu", 11 | "hidden_size": 5120, 12 | "image_aspect_ratio": "pad", 13 | "initializer_range": 0.02, 14 | "intermediate_size": 13824, 15 | "max_length": 4096, 16 | "max_position_embeddings": 4096, 17 | "mm_hidden_size": 1024, 18 | "mm_projector_lr": 2e-05, 19 | "mm_projector_type": "mlp2x_gelu", 20 | "mm_resampler_type": null, 21 | "mm_use_im_patch_token": false, 22 | "mm_use_im_start_end": false, 23 | "mm_vision_select_feature": "patch", 24 | "mm_vision_select_layer": -2, 25 | "mm_vision_tower": "openai/clip-vit-large-patch14-336", 26 | "model_type": "llava", 27 | "num_attention_heads": 40, 28 | "num_hidden_layers": 40, 29 | "num_key_value_heads": 40, 30 | "pad_token_id": 0, 31 | "pretraining_tp": 1, 32 | "rms_norm_eps": 1e-05, 33 | "rope_scaling": null, 34 | "tie_word_embeddings": false, 35 | "tokenizer_model_max_length": 4096, 36 | "tokenizer_padding_side": "right", 37 | "torch_dtype": "float16", 38 | "transformers_version": "4.31.0", 39 | "tune_mm_mlp_adapter": false, 40 | "tune_mm_vision_resampler": false, 41 | "unfreeze_mm_vision_tower": false, 42 | "use_cache": true, 43 | "use_mm_proj": true, 44 | "vocab_size": 32000 45 | } 46 | -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/connectivity/llava-v1.5-13b-lora(128, 256)-unfreeze_vit-False-epoch-1/non_lora_trainables.bin: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:0d5914fcb619abb930abd9edf33d3c559f6600b054d0f41625c638d64ed3b6b2 3 | size 62936807 4 | -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/connectivity/llava-v1.5-7b-lora(128, 256)-unfreeze_vit-False-epoch-1/README.md: -------------------------------------------------------------------------------- 1 | --- 2 | library_name: peft 3 | --- 4 | ## Training procedure 5 | 6 | ### Framework versions 7 | 8 | 9 | - PEFT 0.4.0 10 | -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/connectivity/llava-v1.5-7b-lora(128, 256)-unfreeze_vit-False-epoch-1/adapter_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "auto_mapping": null, 3 | "base_model_name_or_path": "/mnt/sdb1/local_llm/llava-v1.5-7b", 4 | "bias": "none", 5 | "fan_in_fan_out": false, 6 | "inference_mode": true, 7 | "init_lora_weights": true, 8 | "layers_pattern": null, 9 | "layers_to_transform": null, 10 | "lora_alpha": 256, 11 | "lora_dropout": 0.05, 12 | "modules_to_save": null, 13 | "peft_type": "LORA", 14 | "r": 128, 15 | "revision": null, 16 | "target_modules": [ 17 | "o_proj", 18 | "k_proj", 19 | "up_proj", 20 | "gate_proj", 21 | "q_proj", 22 | "v_proj", 23 | "down_proj" 24 | ], 25 | "task_type": "CAUSAL_LM" 26 | } -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/connectivity/llava-v1.5-7b-lora(128, 256)-unfreeze_vit-False-epoch-1/adapter_model.bin: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:ce8f4ddcfae5b784f4eeacf4d4597d0fd5191b125229c3d686ec740f1e388578 3 | size 639786637 4 | -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/connectivity/llava-v1.5-7b-lora(128, 256)-unfreeze_vit-False-epoch-1/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "_name_or_path": "/mnt/sdb1/local_llm/llava-v1.5-7b", 3 | "architectures": [ 4 | "LlavaLlamaForCausalLM" 5 | ], 6 | "bos_token_id": 1, 7 | "eos_token_id": 2, 8 | "freeze_mm_mlp_adapter": false, 9 | "freeze_mm_vision_resampler": false, 10 | "hidden_act": "silu", 11 | "hidden_size": 4096, 12 | "image_aspect_ratio": "pad", 13 | "initializer_range": 0.02, 14 | "intermediate_size": 11008, 15 | "max_length": 4096, 16 | "max_position_embeddings": 4096, 17 | "mm_hidden_size": 1024, 18 | "mm_projector_lr": 2e-05, 19 | "mm_projector_type": "mlp2x_gelu", 20 | "mm_resampler_type": null, 21 | "mm_use_im_patch_token": false, 22 | "mm_use_im_start_end": false, 23 | "mm_vision_select_feature": "patch", 24 | "mm_vision_select_layer": -2, 25 | "mm_vision_tower": "openai/clip-vit-large-patch14-336", 26 | "model_type": "llava", 27 | "num_attention_heads": 32, 28 | "num_hidden_layers": 32, 29 | "num_key_value_heads": 32, 30 | "pad_token_id": 0, 31 | "pretraining_tp": 1, 32 | "rms_norm_eps": 1e-05, 33 | "rope_scaling": null, 34 | "tie_word_embeddings": false, 35 | "tokenizer_model_max_length": 4096, 36 | "tokenizer_padding_side": "right", 37 | "torch_dtype": "float16", 38 | "transformers_version": "4.31.0", 39 | "tune_mm_mlp_adapter": false, 40 | "tune_mm_vision_resampler": false, 41 | "unfreeze_mm_vision_tower": false, 42 | "use_cache": true, 43 | "use_mm_proj": true, 44 | "vocab_size": 32000 45 | } 46 | -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/connectivity/llava-v1.5-7b-lora(128, 256)-unfreeze_vit-False-epoch-1/non_lora_trainables.bin: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:704d673281648ba78ef8f965fa580a4ef39d7ecc4c41458ce8c44b1cc103e18a 3 | size 41961191 4 | -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/cycle/llava-v1.5-13b-lora(128, 256)-unfreeze_vit-False-epoch-10/README.md: -------------------------------------------------------------------------------- 1 | --- 2 | library_name: peft 3 | --- 4 | ## Training procedure 5 | 6 | ### Framework versions 7 | 8 | 9 | - PEFT 0.4.0 10 | -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/cycle/llava-v1.5-13b-lora(128, 256)-unfreeze_vit-False-epoch-10/adapter_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "auto_mapping": null, 3 | "base_model_name_or_path": "/mnt/sdb1/local_llm/llava-v1.5-13b", 4 | "bias": "none", 5 | "fan_in_fan_out": false, 6 | "inference_mode": true, 7 | "init_lora_weights": true, 8 | "layers_pattern": null, 9 | "layers_to_transform": null, 10 | "lora_alpha": 256, 11 | "lora_dropout": 0.05, 12 | "modules_to_save": null, 13 | "peft_type": "LORA", 14 | "r": 128, 15 | "revision": null, 16 | "target_modules": [ 17 | "gate_proj", 18 | "v_proj", 19 | "k_proj", 20 | "down_proj", 21 | "q_proj", 22 | "up_proj", 23 | "o_proj" 24 | ], 25 | "task_type": "CAUSAL_LM" 26 | } -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/cycle/llava-v1.5-13b-lora(128, 256)-unfreeze_vit-False-epoch-10/adapter_model.bin: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:7ef3c10b602d166a7090780e890485d92bc0ae5a657d42a6af840fe4168796e2 3 | size 1001584333 4 | -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/cycle/llava-v1.5-13b-lora(128, 256)-unfreeze_vit-False-epoch-10/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "_name_or_path": "/mnt/sdb1/local_llm/llava-v1.5-13b", 3 | "architectures": [ 4 | "LlavaLlamaForCausalLM" 5 | ], 6 | "bos_token_id": 1, 7 | "eos_token_id": 2, 8 | "freeze_mm_mlp_adapter": false, 9 | "freeze_mm_vision_resampler": false, 10 | "hidden_act": "silu", 11 | "hidden_size": 5120, 12 | "image_aspect_ratio": "pad", 13 | "initializer_range": 0.02, 14 | "intermediate_size": 13824, 15 | "max_length": 4096, 16 | "max_position_embeddings": 4096, 17 | "mm_hidden_size": 1024, 18 | "mm_projector_lr": 2e-05, 19 | "mm_projector_type": "mlp2x_gelu", 20 | "mm_resampler_type": null, 21 | "mm_use_im_patch_token": false, 22 | "mm_use_im_start_end": false, 23 | "mm_vision_select_feature": "patch", 24 | "mm_vision_select_layer": -2, 25 | "mm_vision_tower": "openai/clip-vit-large-patch14-336", 26 | "model_type": "llava", 27 | "num_attention_heads": 40, 28 | "num_hidden_layers": 40, 29 | "num_key_value_heads": 40, 30 | "pad_token_id": 0, 31 | "pretraining_tp": 1, 32 | "rms_norm_eps": 1e-05, 33 | "rope_scaling": null, 34 | "tie_word_embeddings": false, 35 | "tokenizer_model_max_length": 2400, 36 | "tokenizer_padding_side": "right", 37 | "torch_dtype": "float16", 38 | "transformers_version": "4.31.0", 39 | "tune_mm_mlp_adapter": false, 40 | "tune_mm_vision_resampler": false, 41 | "unfreeze_mm_vision_tower": false, 42 | "use_cache": true, 43 | "use_mm_proj": true, 44 | "vocab_size": 32000 45 | } 46 | -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/cycle/llava-v1.5-13b-lora(128, 256)-unfreeze_vit-False-epoch-10/non_lora_trainables.bin: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:69d1569fe02146d8a5c241d48afee2afd13bbb59a275ae50c5d7e388d4deeb54 3 | size 62936807 4 | -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/cycle/llava-v1.5-7b-lora(128, 256)-unfreeze_vit-False-epoch-20/README.md: -------------------------------------------------------------------------------- 1 | --- 2 | library_name: peft 3 | --- 4 | ## Training procedure 5 | 6 | ### Framework versions 7 | 8 | 9 | - PEFT 0.4.0 10 | -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/cycle/llava-v1.5-7b-lora(128, 256)-unfreeze_vit-False-epoch-20/adapter_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "auto_mapping": null, 3 | "base_model_name_or_path": "/mnt/sdb1/local_llm/llava-v1.5-7b", 4 | "bias": "none", 5 | "fan_in_fan_out": false, 6 | "inference_mode": true, 7 | "init_lora_weights": true, 8 | "layers_pattern": null, 9 | "layers_to_transform": null, 10 | "lora_alpha": 256, 11 | "lora_dropout": 0.05, 12 | "modules_to_save": null, 13 | "peft_type": "LORA", 14 | "r": 128, 15 | "revision": null, 16 | "target_modules": [ 17 | "down_proj", 18 | "k_proj", 19 | "q_proj", 20 | "o_proj", 21 | "up_proj", 22 | "v_proj", 23 | "gate_proj" 24 | ], 25 | "task_type": "CAUSAL_LM" 26 | } -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/cycle/llava-v1.5-7b-lora(128, 256)-unfreeze_vit-False-epoch-20/adapter_model.bin: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:7b4897dc9caa4ab6f5aeda1c508a9dbdbd2d753808f88e4726f594492b29f418 3 | size 639786637 4 | -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/cycle/llava-v1.5-7b-lora(128, 256)-unfreeze_vit-False-epoch-20/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "_name_or_path": "/mnt/sdb1/local_llm/llava-v1.5-7b", 3 | "architectures": [ 4 | "LlavaLlamaForCausalLM" 5 | ], 6 | "bos_token_id": 1, 7 | "eos_token_id": 2, 8 | "freeze_mm_mlp_adapter": false, 9 | "freeze_mm_vision_resampler": false, 10 | "hidden_act": "silu", 11 | "hidden_size": 4096, 12 | "image_aspect_ratio": "pad", 13 | "initializer_range": 0.02, 14 | "intermediate_size": 11008, 15 | "max_length": 4096, 16 | "max_position_embeddings": 4096, 17 | "mm_hidden_size": 1024, 18 | "mm_projector_lr": 2e-05, 19 | "mm_projector_type": "mlp2x_gelu", 20 | "mm_resampler_type": null, 21 | "mm_use_im_patch_token": false, 22 | "mm_use_im_start_end": false, 23 | "mm_vision_select_feature": "patch", 24 | "mm_vision_select_layer": -2, 25 | "mm_vision_tower": "openai/clip-vit-large-patch14-336", 26 | "model_type": "llava", 27 | "num_attention_heads": 32, 28 | "num_hidden_layers": 32, 29 | "num_key_value_heads": 32, 30 | "pad_token_id": 0, 31 | "pretraining_tp": 1, 32 | "rms_norm_eps": 1e-05, 33 | "rope_scaling": null, 34 | "tie_word_embeddings": false, 35 | "tokenizer_model_max_length": 2400, 36 | "tokenizer_padding_side": "right", 37 | "torch_dtype": "float16", 38 | "transformers_version": "4.31.0", 39 | "tune_mm_mlp_adapter": false, 40 | "tune_mm_vision_resampler": false, 41 | "unfreeze_mm_vision_tower": false, 42 | "use_cache": true, 43 | "use_mm_proj": true, 44 | "vocab_size": 32000 45 | } 46 | -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/cycle/llava-v1.5-7b-lora(128, 256)-unfreeze_vit-False-epoch-20/non_lora_trainables.bin: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:5f0bed17df06b534f69c42a7669a60cb481ca6f3f0e246be66b6fb6dbf11916f 3 | size 41961191 4 | -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/flow/llava-v1.5-13b-lora(128, 256)-unfreeze_vit-False-epoch-10/README.md: -------------------------------------------------------------------------------- 1 | --- 2 | library_name: peft 3 | --- 4 | ## Training procedure 5 | 6 | ### Framework versions 7 | 8 | 9 | - PEFT 0.4.0 10 | -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/flow/llava-v1.5-13b-lora(128, 256)-unfreeze_vit-False-epoch-10/adapter_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "auto_mapping": null, 3 | "base_model_name_or_path": "/mnt/sdb1/local_llm/llava-v1.5-13b", 4 | "bias": "none", 5 | "fan_in_fan_out": false, 6 | "inference_mode": true, 7 | "init_lora_weights": true, 8 | "layers_pattern": null, 9 | "layers_to_transform": null, 10 | "lora_alpha": 256, 11 | "lora_dropout": 0.05, 12 | "modules_to_save": null, 13 | "peft_type": "LORA", 14 | "r": 128, 15 | "revision": null, 16 | "target_modules": [ 17 | "gate_proj", 18 | "v_proj", 19 | "o_proj", 20 | "k_proj", 21 | "q_proj", 22 | "up_proj", 23 | "down_proj" 24 | ], 25 | "task_type": "CAUSAL_LM" 26 | } -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/flow/llava-v1.5-13b-lora(128, 256)-unfreeze_vit-False-epoch-10/adapter_model.bin: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:dd4304e86c8b19cd2187c5d6deef9af524bd683d6ed5b29b47411cf8bfac2542 3 | size 1001584333 4 | -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/flow/llava-v1.5-13b-lora(128, 256)-unfreeze_vit-False-epoch-10/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "_name_or_path": "/mnt/sdb1/local_llm/llava-v1.5-13b", 3 | "architectures": [ 4 | "LlavaLlamaForCausalLM" 5 | ], 6 | "bos_token_id": 1, 7 | "eos_token_id": 2, 8 | "freeze_mm_mlp_adapter": false, 9 | "freeze_mm_vision_resampler": false, 10 | "hidden_act": "silu", 11 | "hidden_size": 5120, 12 | "image_aspect_ratio": "pad", 13 | "initializer_range": 0.02, 14 | "intermediate_size": 13824, 15 | "max_length": 4096, 16 | "max_position_embeddings": 4096, 17 | "mm_hidden_size": 1024, 18 | "mm_projector_lr": 2e-05, 19 | "mm_projector_type": "mlp2x_gelu", 20 | "mm_resampler_type": null, 21 | "mm_use_im_patch_token": false, 22 | "mm_use_im_start_end": false, 23 | "mm_vision_select_feature": "patch", 24 | "mm_vision_select_layer": -2, 25 | "mm_vision_tower": "openai/clip-vit-large-patch14-336", 26 | "model_type": "llava", 27 | "num_attention_heads": 40, 28 | "num_hidden_layers": 40, 29 | "num_key_value_heads": 40, 30 | "pad_token_id": 0, 31 | "pretraining_tp": 1, 32 | "rms_norm_eps": 1e-05, 33 | "rope_scaling": null, 34 | "tie_word_embeddings": false, 35 | "tokenizer_model_max_length": 2400, 36 | "tokenizer_padding_side": "right", 37 | "torch_dtype": "float16", 38 | "transformers_version": "4.31.0", 39 | "tune_mm_mlp_adapter": false, 40 | "tune_mm_vision_resampler": false, 41 | "unfreeze_mm_vision_tower": false, 42 | "use_cache": true, 43 | "use_mm_proj": true, 44 | "vocab_size": 32000 45 | } 46 | -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/flow/llava-v1.5-13b-lora(128, 256)-unfreeze_vit-False-epoch-10/non_lora_trainables.bin: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:af884afc6c30619022349a2dcb2f6f0d1e72bfc8de89730030a6d624cee4335e 3 | size 62936807 4 | -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/flow/llava-v1.5-7b-lora(128, 256)-unfreeze_vit-False-epoch-20/README.md: -------------------------------------------------------------------------------- 1 | --- 2 | library_name: peft 3 | --- 4 | ## Training procedure 5 | 6 | ### Framework versions 7 | 8 | 9 | - PEFT 0.4.0 10 | -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/flow/llava-v1.5-7b-lora(128, 256)-unfreeze_vit-False-epoch-20/adapter_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "auto_mapping": null, 3 | "base_model_name_or_path": "/mnt/sdb1/local_llm/llava-v1.5-7b", 4 | "bias": "none", 5 | "fan_in_fan_out": false, 6 | "inference_mode": true, 7 | "init_lora_weights": true, 8 | "layers_pattern": null, 9 | "layers_to_transform": null, 10 | "lora_alpha": 256, 11 | "lora_dropout": 0.05, 12 | "modules_to_save": null, 13 | "peft_type": "LORA", 14 | "r": 128, 15 | "revision": null, 16 | "target_modules": [ 17 | "k_proj", 18 | "gate_proj", 19 | "v_proj", 20 | "down_proj", 21 | "q_proj", 22 | "up_proj", 23 | "o_proj" 24 | ], 25 | "task_type": "CAUSAL_LM" 26 | } -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/flow/llava-v1.5-7b-lora(128, 256)-unfreeze_vit-False-epoch-20/adapter_model.bin: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:1910ec548947e972a00a181763f56d467402231962439f0aa8ca4dd24609a678 3 | size 639786637 4 | -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/flow/llava-v1.5-7b-lora(128, 256)-unfreeze_vit-False-epoch-20/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "_name_or_path": "/mnt/sdb1/local_llm/llava-v1.5-7b", 3 | "architectures": [ 4 | "LlavaLlamaForCausalLM" 5 | ], 6 | "bos_token_id": 1, 7 | "eos_token_id": 2, 8 | "freeze_mm_mlp_adapter": false, 9 | "freeze_mm_vision_resampler": false, 10 | "hidden_act": "silu", 11 | "hidden_size": 4096, 12 | "image_aspect_ratio": "pad", 13 | "initializer_range": 0.02, 14 | "intermediate_size": 11008, 15 | "max_length": 4096, 16 | "max_position_embeddings": 4096, 17 | "mm_hidden_size": 1024, 18 | "mm_projector_lr": 2e-05, 19 | "mm_projector_type": "mlp2x_gelu", 20 | "mm_resampler_type": null, 21 | "mm_use_im_patch_token": false, 22 | "mm_use_im_start_end": false, 23 | "mm_vision_select_feature": "patch", 24 | "mm_vision_select_layer": -2, 25 | "mm_vision_tower": "openai/clip-vit-large-patch14-336", 26 | "model_type": "llava", 27 | "num_attention_heads": 32, 28 | "num_hidden_layers": 32, 29 | "num_key_value_heads": 32, 30 | "pad_token_id": 0, 31 | "pretraining_tp": 1, 32 | "rms_norm_eps": 1e-05, 33 | "rope_scaling": null, 34 | "tie_word_embeddings": false, 35 | "tokenizer_model_max_length": 2400, 36 | "tokenizer_padding_side": "right", 37 | "torch_dtype": "float16", 38 | "transformers_version": "4.31.0", 39 | "tune_mm_mlp_adapter": false, 40 | "tune_mm_vision_resampler": false, 41 | "unfreeze_mm_vision_tower": false, 42 | "use_cache": true, 43 | "use_mm_proj": true, 44 | "vocab_size": 32000 45 | } 46 | -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/flow/llava-v1.5-7b-lora(128, 256)-unfreeze_vit-False-epoch-20/non_lora_trainables.bin: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:4cc97d643847651c21eb15dce89ee5b78c1261c473eec3e57687ddc80d0925c2 3 | size 41961191 4 | -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/hamilton/llava-v1.5-13b-lora(128, 256)-unfreeze_vit-False-epoch-30/README.md: -------------------------------------------------------------------------------- 1 | --- 2 | library_name: peft 3 | --- 4 | ## Training procedure 5 | 6 | ### Framework versions 7 | 8 | 9 | - PEFT 0.4.0 10 | -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/hamilton/llava-v1.5-13b-lora(128, 256)-unfreeze_vit-False-epoch-30/adapter_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "auto_mapping": null, 3 | "base_model_name_or_path": "/mnt/sdb1/local_llm/llava-v1.5-13b", 4 | "bias": "none", 5 | "fan_in_fan_out": false, 6 | "inference_mode": true, 7 | "init_lora_weights": true, 8 | "layers_pattern": null, 9 | "layers_to_transform": null, 10 | "lora_alpha": 256, 11 | "lora_dropout": 0.05, 12 | "modules_to_save": null, 13 | "peft_type": "LORA", 14 | "r": 128, 15 | "revision": null, 16 | "target_modules": [ 17 | "gate_proj", 18 | "up_proj", 19 | "v_proj", 20 | "o_proj", 21 | "q_proj", 22 | "down_proj", 23 | "k_proj" 24 | ], 25 | "task_type": "CAUSAL_LM" 26 | } -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/hamilton/llava-v1.5-13b-lora(128, 256)-unfreeze_vit-False-epoch-30/adapter_model.bin: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:490efb7622367579674900c674139f79b87ce47d4be887e9dd3c98783aac4335 3 | size 1001584333 4 | -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/hamilton/llava-v1.5-13b-lora(128, 256)-unfreeze_vit-False-epoch-30/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "_name_or_path": "/mnt/sdb1/local_llm/llava-v1.5-13b", 3 | "architectures": [ 4 | "LlavaLlamaForCausalLM" 5 | ], 6 | "bos_token_id": 1, 7 | "eos_token_id": 2, 8 | "freeze_mm_mlp_adapter": false, 9 | "freeze_mm_vision_resampler": false, 10 | "hidden_act": "silu", 11 | "hidden_size": 5120, 12 | "image_aspect_ratio": "pad", 13 | "initializer_range": 0.02, 14 | "intermediate_size": 13824, 15 | "max_length": 4096, 16 | "max_position_embeddings": 4096, 17 | "mm_hidden_size": 1024, 18 | "mm_projector_lr": 2e-05, 19 | "mm_projector_type": "mlp2x_gelu", 20 | "mm_resampler_type": null, 21 | "mm_use_im_patch_token": false, 22 | "mm_use_im_start_end": false, 23 | "mm_vision_select_feature": "patch", 24 | "mm_vision_select_layer": -2, 25 | "mm_vision_tower": "openai/clip-vit-large-patch14-336", 26 | "model_type": "llava", 27 | "num_attention_heads": 40, 28 | "num_hidden_layers": 40, 29 | "num_key_value_heads": 40, 30 | "pad_token_id": 0, 31 | "pretraining_tp": 1, 32 | "rms_norm_eps": 1e-05, 33 | "rope_scaling": null, 34 | "tie_word_embeddings": false, 35 | "tokenizer_model_max_length": 2400, 36 | "tokenizer_padding_side": "right", 37 | "torch_dtype": "float16", 38 | "transformers_version": "4.31.0", 39 | "tune_mm_mlp_adapter": false, 40 | "tune_mm_vision_resampler": false, 41 | "unfreeze_mm_vision_tower": false, 42 | "use_cache": true, 43 | "use_mm_proj": true, 44 | "vocab_size": 32000 45 | } 46 | -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/hamilton/llava-v1.5-13b-lora(128, 256)-unfreeze_vit-False-epoch-30/non_lora_trainables.bin: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:94577e569adfe85ee9b5954b7597e76b30ead0a047c275ccf7cf38334e9f1a16 3 | size 62936807 4 | -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/hamilton/llava-v1.5-7b-lora(128, 256)-unfreeze_vit-False-epoch-30/README.md: -------------------------------------------------------------------------------- 1 | --- 2 | library_name: peft 3 | --- 4 | ## Training procedure 5 | 6 | ### Framework versions 7 | 8 | 9 | - PEFT 0.4.0 10 | -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/hamilton/llava-v1.5-7b-lora(128, 256)-unfreeze_vit-False-epoch-30/adapter_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "auto_mapping": null, 3 | "base_model_name_or_path": "/mnt/sdb1/local_llm/llava-v1.5-7b", 4 | "bias": "none", 5 | "fan_in_fan_out": false, 6 | "inference_mode": true, 7 | "init_lora_weights": true, 8 | "layers_pattern": null, 9 | "layers_to_transform": null, 10 | "lora_alpha": 256, 11 | "lora_dropout": 0.05, 12 | "modules_to_save": null, 13 | "peft_type": "LORA", 14 | "r": 128, 15 | "revision": null, 16 | "target_modules": [ 17 | "gate_proj", 18 | "k_proj", 19 | "down_proj", 20 | "q_proj", 21 | "v_proj", 22 | "o_proj", 23 | "up_proj" 24 | ], 25 | "task_type": "CAUSAL_LM" 26 | } -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/hamilton/llava-v1.5-7b-lora(128, 256)-unfreeze_vit-False-epoch-30/adapter_model.bin: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:f1a90a8b50ad85c25a90e17da5907c4d7ce41cb56f6277c6b0228eb4d98d8f62 3 | size 639786637 4 | -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/hamilton/llava-v1.5-7b-lora(128, 256)-unfreeze_vit-False-epoch-30/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "_name_or_path": "/mnt/sdb1/local_llm/llava-v1.5-7b", 3 | "architectures": [ 4 | "LlavaLlamaForCausalLM" 5 | ], 6 | "bos_token_id": 1, 7 | "eos_token_id": 2, 8 | "freeze_mm_mlp_adapter": false, 9 | "freeze_mm_vision_resampler": false, 10 | "hidden_act": "silu", 11 | "hidden_size": 4096, 12 | "image_aspect_ratio": "pad", 13 | "initializer_range": 0.02, 14 | "intermediate_size": 11008, 15 | "max_length": 4096, 16 | "max_position_embeddings": 4096, 17 | "mm_hidden_size": 1024, 18 | "mm_projector_lr": 2e-05, 19 | "mm_projector_type": "mlp2x_gelu", 20 | "mm_resampler_type": null, 21 | "mm_use_im_patch_token": false, 22 | "mm_use_im_start_end": false, 23 | "mm_vision_select_feature": "patch", 24 | "mm_vision_select_layer": -2, 25 | "mm_vision_tower": "openai/clip-vit-large-patch14-336", 26 | "model_type": "llava", 27 | "num_attention_heads": 32, 28 | "num_hidden_layers": 32, 29 | "num_key_value_heads": 32, 30 | "pad_token_id": 0, 31 | "pretraining_tp": 1, 32 | "rms_norm_eps": 1e-05, 33 | "rope_scaling": null, 34 | "tie_word_embeddings": false, 35 | "tokenizer_model_max_length": 2400, 36 | "tokenizer_padding_side": "right", 37 | "torch_dtype": "float16", 38 | "transformers_version": "4.31.0", 39 | "tune_mm_mlp_adapter": false, 40 | "tune_mm_vision_resampler": false, 41 | "unfreeze_mm_vision_tower": false, 42 | "use_cache": true, 43 | "use_mm_proj": true, 44 | "vocab_size": 32000 45 | } 46 | -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/hamilton/llava-v1.5-7b-lora(128, 256)-unfreeze_vit-False-epoch-30/non_lora_trainables.bin: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:34391e9c7fff5d4da1f90fe3938b08f7b41fc3723cc14769258c909eb2fcf6b7 3 | size 41961191 4 | -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/matching/llava-v1.5-13b-lora(128, 256)-unfreeze_vit-False-epoch-50/README.md: -------------------------------------------------------------------------------- 1 | --- 2 | library_name: peft 3 | --- 4 | ## Training procedure 5 | 6 | ### Framework versions 7 | 8 | 9 | - PEFT 0.4.0 10 | -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/matching/llava-v1.5-13b-lora(128, 256)-unfreeze_vit-False-epoch-50/adapter_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "auto_mapping": null, 3 | "base_model_name_or_path": "/mnt/sdb1/local_llm/llava-v1.5-13b", 4 | "bias": "none", 5 | "fan_in_fan_out": false, 6 | "inference_mode": true, 7 | "init_lora_weights": true, 8 | "layers_pattern": null, 9 | "layers_to_transform": null, 10 | "lora_alpha": 256, 11 | "lora_dropout": 0.05, 12 | "modules_to_save": null, 13 | "peft_type": "LORA", 14 | "r": 128, 15 | "revision": null, 16 | "target_modules": [ 17 | "o_proj", 18 | "gate_proj", 19 | "q_proj", 20 | "up_proj", 21 | "down_proj", 22 | "k_proj", 23 | "v_proj" 24 | ], 25 | "task_type": "CAUSAL_LM" 26 | } -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/matching/llava-v1.5-13b-lora(128, 256)-unfreeze_vit-False-epoch-50/adapter_model.bin: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:1ebbbba10d61f97d9dffc878db0b73d3e86216c8d0765a75db0abce7e0c82458 3 | size 1001584333 4 | -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/matching/llava-v1.5-13b-lora(128, 256)-unfreeze_vit-False-epoch-50/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "_name_or_path": "/mnt/sdb1/local_llm/llava-v1.5-13b", 3 | "architectures": [ 4 | "LlavaLlamaForCausalLM" 5 | ], 6 | "bos_token_id": 1, 7 | "eos_token_id": 2, 8 | "freeze_mm_mlp_adapter": false, 9 | "freeze_mm_vision_resampler": false, 10 | "hidden_act": "silu", 11 | "hidden_size": 5120, 12 | "image_aspect_ratio": "pad", 13 | "initializer_range": 0.02, 14 | "intermediate_size": 13824, 15 | "max_length": 4096, 16 | "max_position_embeddings": 4096, 17 | "mm_hidden_size": 1024, 18 | "mm_projector_lr": 2e-05, 19 | "mm_projector_type": "mlp2x_gelu", 20 | "mm_resampler_type": null, 21 | "mm_use_im_patch_token": false, 22 | "mm_use_im_start_end": false, 23 | "mm_vision_select_feature": "patch", 24 | "mm_vision_select_layer": -2, 25 | "mm_vision_tower": "openai/clip-vit-large-patch14-336", 26 | "model_type": "llava", 27 | "num_attention_heads": 40, 28 | "num_hidden_layers": 40, 29 | "num_key_value_heads": 40, 30 | "pad_token_id": 0, 31 | "pretraining_tp": 1, 32 | "rms_norm_eps": 1e-05, 33 | "rope_scaling": null, 34 | "tie_word_embeddings": false, 35 | "tokenizer_model_max_length": 2400, 36 | "tokenizer_padding_side": "right", 37 | "torch_dtype": "float16", 38 | "transformers_version": "4.31.0", 39 | "tune_mm_mlp_adapter": false, 40 | "tune_mm_vision_resampler": false, 41 | "unfreeze_mm_vision_tower": false, 42 | "use_cache": true, 43 | "use_mm_proj": true, 44 | "vocab_size": 32000 45 | } 46 | -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/matching/llava-v1.5-13b-lora(128, 256)-unfreeze_vit-False-epoch-50/non_lora_trainables.bin: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:98ac5c7ac3444703b8fe02d358654bde2c7097045f2b293cd4a536ea270499a9 3 | size 62936807 4 | -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/matching/llava-v1.5-7b-lora(128, 256)-unfreeze_vit-False-epoch-5/README.md: -------------------------------------------------------------------------------- 1 | --- 2 | library_name: peft 3 | --- 4 | ## Training procedure 5 | 6 | ### Framework versions 7 | 8 | 9 | - PEFT 0.4.0 10 | -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/matching/llava-v1.5-7b-lora(128, 256)-unfreeze_vit-False-epoch-5/adapter_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "auto_mapping": null, 3 | "base_model_name_or_path": "/mnt/sdb1/local_llm/llava-v1.5-7b", 4 | "bias": "none", 5 | "fan_in_fan_out": false, 6 | "inference_mode": true, 7 | "init_lora_weights": true, 8 | "layers_pattern": null, 9 | "layers_to_transform": null, 10 | "lora_alpha": 256, 11 | "lora_dropout": 0.05, 12 | "modules_to_save": null, 13 | "peft_type": "LORA", 14 | "r": 128, 15 | "revision": null, 16 | "target_modules": [ 17 | "v_proj", 18 | "o_proj", 19 | "down_proj", 20 | "q_proj", 21 | "k_proj", 22 | "gate_proj", 23 | "up_proj" 24 | ], 25 | "task_type": "CAUSAL_LM" 26 | } -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/matching/llava-v1.5-7b-lora(128, 256)-unfreeze_vit-False-epoch-5/adapter_model.bin: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:3ef5d2b584bec4062d6a9ffa8c81a60d4e8e4a6b8ff4c75d4db1386c12c0c330 3 | size 639786637 4 | -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/matching/llava-v1.5-7b-lora(128, 256)-unfreeze_vit-False-epoch-5/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "_name_or_path": "/mnt/sdb1/local_llm/llava-v1.5-7b", 3 | "architectures": [ 4 | "LlavaLlamaForCausalLM" 5 | ], 6 | "bos_token_id": 1, 7 | "eos_token_id": 2, 8 | "freeze_mm_mlp_adapter": false, 9 | "freeze_mm_vision_resampler": false, 10 | "hidden_act": "silu", 11 | "hidden_size": 4096, 12 | "image_aspect_ratio": "pad", 13 | "initializer_range": 0.02, 14 | "intermediate_size": 11008, 15 | "max_length": 4096, 16 | "max_position_embeddings": 4096, 17 | "mm_hidden_size": 1024, 18 | "mm_projector_lr": 2e-05, 19 | "mm_projector_type": "mlp2x_gelu", 20 | "mm_resampler_type": null, 21 | "mm_use_im_patch_token": false, 22 | "mm_use_im_start_end": false, 23 | "mm_vision_select_feature": "patch", 24 | "mm_vision_select_layer": -2, 25 | "mm_vision_tower": "openai/clip-vit-large-patch14-336", 26 | "model_type": "llava", 27 | "num_attention_heads": 32, 28 | "num_hidden_layers": 32, 29 | "num_key_value_heads": 32, 30 | "pad_token_id": 0, 31 | "pretraining_tp": 1, 32 | "rms_norm_eps": 1e-05, 33 | "rope_scaling": null, 34 | "tie_word_embeddings": false, 35 | "tokenizer_model_max_length": 2400, 36 | "tokenizer_padding_side": "right", 37 | "torch_dtype": "float16", 38 | "transformers_version": "4.31.0", 39 | "tune_mm_mlp_adapter": false, 40 | "tune_mm_vision_resampler": false, 41 | "unfreeze_mm_vision_tower": false, 42 | "use_cache": true, 43 | "use_mm_proj": true, 44 | "vocab_size": 32000 45 | } 46 | -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/matching/llava-v1.5-7b-lora(128, 256)-unfreeze_vit-False-epoch-5/non_lora_trainables.bin: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:5f42b7a7ead07a09f16c8efc3bc34df6a74cf16632428659bae0c84d550a225e 3 | size 41961191 4 | -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/shortest_path/llava-v1.5-13b-lora(128, 256)-unfreeze_vit-False-epoch-10/README.md: -------------------------------------------------------------------------------- 1 | --- 2 | library_name: peft 3 | --- 4 | ## Training procedure 5 | 6 | ### Framework versions 7 | 8 | 9 | - PEFT 0.4.0 10 | -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/shortest_path/llava-v1.5-13b-lora(128, 256)-unfreeze_vit-False-epoch-10/adapter_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "auto_mapping": null, 3 | "base_model_name_or_path": "/mnt/sdb1/local_llm/llava-v1.5-13b", 4 | "bias": "none", 5 | "fan_in_fan_out": false, 6 | "inference_mode": true, 7 | "init_lora_weights": true, 8 | "layers_pattern": null, 9 | "layers_to_transform": null, 10 | "lora_alpha": 256, 11 | "lora_dropout": 0.05, 12 | "modules_to_save": null, 13 | "peft_type": "LORA", 14 | "r": 128, 15 | "revision": null, 16 | "target_modules": [ 17 | "up_proj", 18 | "down_proj", 19 | "v_proj", 20 | "o_proj", 21 | "gate_proj", 22 | "q_proj", 23 | "k_proj" 24 | ], 25 | "task_type": "CAUSAL_LM" 26 | } -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/shortest_path/llava-v1.5-13b-lora(128, 256)-unfreeze_vit-False-epoch-10/adapter_model.bin: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:1f39328c6324ae4428ab35d07ce8f707d6754604a7d9717384db60c1f7611131 3 | size 1001584333 4 | -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/shortest_path/llava-v1.5-13b-lora(128, 256)-unfreeze_vit-False-epoch-10/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "_name_or_path": "/mnt/sdb1/local_llm/llava-v1.5-13b", 3 | "architectures": [ 4 | "LlavaLlamaForCausalLM" 5 | ], 6 | "bos_token_id": 1, 7 | "eos_token_id": 2, 8 | "freeze_mm_mlp_adapter": false, 9 | "freeze_mm_vision_resampler": false, 10 | "hidden_act": "silu", 11 | "hidden_size": 5120, 12 | "image_aspect_ratio": "pad", 13 | "initializer_range": 0.02, 14 | "intermediate_size": 13824, 15 | "max_length": 4096, 16 | "max_position_embeddings": 4096, 17 | "mm_hidden_size": 1024, 18 | "mm_projector_lr": 2e-05, 19 | "mm_projector_type": "mlp2x_gelu", 20 | "mm_resampler_type": null, 21 | "mm_use_im_patch_token": false, 22 | "mm_use_im_start_end": false, 23 | "mm_vision_select_feature": "patch", 24 | "mm_vision_select_layer": -2, 25 | "mm_vision_tower": "openai/clip-vit-large-patch14-336", 26 | "model_type": "llava", 27 | "num_attention_heads": 40, 28 | "num_hidden_layers": 40, 29 | "num_key_value_heads": 40, 30 | "pad_token_id": 0, 31 | "pretraining_tp": 1, 32 | "rms_norm_eps": 1e-05, 33 | "rope_scaling": null, 34 | "tie_word_embeddings": false, 35 | "tokenizer_model_max_length": 2400, 36 | "tokenizer_padding_side": "right", 37 | "torch_dtype": "float16", 38 | "transformers_version": "4.31.0", 39 | "tune_mm_mlp_adapter": false, 40 | "tune_mm_vision_resampler": false, 41 | "unfreeze_mm_vision_tower": false, 42 | "use_cache": true, 43 | "use_mm_proj": true, 44 | "vocab_size": 32000 45 | } 46 | -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/shortest_path/llava-v1.5-13b-lora(128, 256)-unfreeze_vit-False-epoch-10/non_lora_trainables.bin: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:00a0ff56ab44c6ddc4931485674002472d4501ef99bf3bd8d2d565ef3101c514 3 | size 62936807 4 | -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/shortest_path/llava-v1.5-7b-lora(128, 256)-unfreeze_vit-False-epoch-10/README.md: -------------------------------------------------------------------------------- 1 | --- 2 | library_name: peft 3 | --- 4 | ## Training procedure 5 | 6 | ### Framework versions 7 | 8 | 9 | - PEFT 0.4.0 10 | -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/shortest_path/llava-v1.5-7b-lora(128, 256)-unfreeze_vit-False-epoch-10/adapter_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "auto_mapping": null, 3 | "base_model_name_or_path": "/mnt/sdb1/local_llm/llava-v1.5-7b", 4 | "bias": "none", 5 | "fan_in_fan_out": false, 6 | "inference_mode": true, 7 | "init_lora_weights": true, 8 | "layers_pattern": null, 9 | "layers_to_transform": null, 10 | "lora_alpha": 256, 11 | "lora_dropout": 0.05, 12 | "modules_to_save": null, 13 | "peft_type": "LORA", 14 | "r": 128, 15 | "revision": null, 16 | "target_modules": [ 17 | "down_proj", 18 | "up_proj", 19 | "q_proj", 20 | "v_proj", 21 | "o_proj", 22 | "gate_proj", 23 | "k_proj" 24 | ], 25 | "task_type": "CAUSAL_LM" 26 | } -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/shortest_path/llava-v1.5-7b-lora(128, 256)-unfreeze_vit-False-epoch-10/adapter_model.bin: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:ee7a2750bbd059d4d64ce62d0ec837e34d2d8acb9e84f54e77d2ac39c3525aaa 3 | size 639786637 4 | -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/shortest_path/llava-v1.5-7b-lora(128, 256)-unfreeze_vit-False-epoch-10/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "_name_or_path": "/mnt/sdb1/local_llm/llava-v1.5-7b", 3 | "architectures": [ 4 | "LlavaLlamaForCausalLM" 5 | ], 6 | "bos_token_id": 1, 7 | "eos_token_id": 2, 8 | "freeze_mm_mlp_adapter": false, 9 | "freeze_mm_vision_resampler": false, 10 | "hidden_act": "silu", 11 | "hidden_size": 4096, 12 | "image_aspect_ratio": "pad", 13 | "initializer_range": 0.02, 14 | "intermediate_size": 11008, 15 | "max_length": 4096, 16 | "max_position_embeddings": 4096, 17 | "mm_hidden_size": 1024, 18 | "mm_projector_lr": 2e-05, 19 | "mm_projector_type": "mlp2x_gelu", 20 | "mm_resampler_type": null, 21 | "mm_use_im_patch_token": false, 22 | "mm_use_im_start_end": false, 23 | "mm_vision_select_feature": "patch", 24 | "mm_vision_select_layer": -2, 25 | "mm_vision_tower": "openai/clip-vit-large-patch14-336", 26 | "model_type": "llava", 27 | "num_attention_heads": 32, 28 | "num_hidden_layers": 32, 29 | "num_key_value_heads": 32, 30 | "pad_token_id": 0, 31 | "pretraining_tp": 1, 32 | "rms_norm_eps": 1e-05, 33 | "rope_scaling": null, 34 | "tie_word_embeddings": false, 35 | "tokenizer_model_max_length": 2400, 36 | "tokenizer_padding_side": "right", 37 | "torch_dtype": "float16", 38 | "transformers_version": "4.31.0", 39 | "tune_mm_mlp_adapter": false, 40 | "tune_mm_vision_resampler": false, 41 | "unfreeze_mm_vision_tower": false, 42 | "use_cache": true, 43 | "use_mm_proj": true, 44 | "vocab_size": 32000 45 | } 46 | -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/shortest_path/llava-v1.5-7b-lora(128, 256)-unfreeze_vit-False-epoch-10/non_lora_trainables.bin: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:e7efe7855dd84b3c40c8d44ae6408edd774163528aecde200551795088db05a4 3 | size 41961191 4 | -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/topology/llava-v1.5-13b-lora(128, 256)-unfreeze_vit-False-epoch-10/README.md: -------------------------------------------------------------------------------- 1 | --- 2 | library_name: peft 3 | --- 4 | ## Training procedure 5 | 6 | ### Framework versions 7 | 8 | 9 | - PEFT 0.4.0 10 | -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/topology/llava-v1.5-13b-lora(128, 256)-unfreeze_vit-False-epoch-10/adapter_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "auto_mapping": null, 3 | "base_model_name_or_path": "/mnt/sdb1/local_llm/llava-v1.5-13b", 4 | "bias": "none", 5 | "fan_in_fan_out": false, 6 | "inference_mode": true, 7 | "init_lora_weights": true, 8 | "layers_pattern": null, 9 | "layers_to_transform": null, 10 | "lora_alpha": 256, 11 | "lora_dropout": 0.05, 12 | "modules_to_save": null, 13 | "peft_type": "LORA", 14 | "r": 128, 15 | "revision": null, 16 | "target_modules": [ 17 | "k_proj", 18 | "q_proj", 19 | "v_proj", 20 | "down_proj", 21 | "gate_proj", 22 | "o_proj", 23 | "up_proj" 24 | ], 25 | "task_type": "CAUSAL_LM" 26 | } -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/topology/llava-v1.5-13b-lora(128, 256)-unfreeze_vit-False-epoch-10/adapter_model.bin: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:6f77510707288aca26d1d6941f51d0a47873438fc76c679204c2287e59d36b14 3 | size 1001584333 4 | -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/topology/llava-v1.5-13b-lora(128, 256)-unfreeze_vit-False-epoch-10/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "_name_or_path": "/mnt/sdb1/local_llm/llava-v1.5-13b", 3 | "architectures": [ 4 | "LlavaLlamaForCausalLM" 5 | ], 6 | "bos_token_id": 1, 7 | "eos_token_id": 2, 8 | "freeze_mm_mlp_adapter": false, 9 | "freeze_mm_vision_resampler": false, 10 | "hidden_act": "silu", 11 | "hidden_size": 5120, 12 | "image_aspect_ratio": "pad", 13 | "initializer_range": 0.02, 14 | "intermediate_size": 13824, 15 | "max_length": 4096, 16 | "max_position_embeddings": 4096, 17 | "mm_hidden_size": 1024, 18 | "mm_projector_lr": 2e-05, 19 | "mm_projector_type": "mlp2x_gelu", 20 | "mm_resampler_type": null, 21 | "mm_use_im_patch_token": false, 22 | "mm_use_im_start_end": false, 23 | "mm_vision_select_feature": "patch", 24 | "mm_vision_select_layer": -2, 25 | "mm_vision_tower": "openai/clip-vit-large-patch14-336", 26 | "model_type": "llava", 27 | "num_attention_heads": 40, 28 | "num_hidden_layers": 40, 29 | "num_key_value_heads": 40, 30 | "pad_token_id": 0, 31 | "pretraining_tp": 1, 32 | "rms_norm_eps": 1e-05, 33 | "rope_scaling": null, 34 | "tie_word_embeddings": false, 35 | "tokenizer_model_max_length": 2400, 36 | "tokenizer_padding_side": "right", 37 | "torch_dtype": "float16", 38 | "transformers_version": "4.31.0", 39 | "tune_mm_mlp_adapter": false, 40 | "tune_mm_vision_resampler": false, 41 | "unfreeze_mm_vision_tower": false, 42 | "use_cache": true, 43 | "use_mm_proj": true, 44 | "vocab_size": 32000 45 | } 46 | -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/topology/llava-v1.5-13b-lora(128, 256)-unfreeze_vit-False-epoch-10/non_lora_trainables.bin: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:bce5425a9a12daf21a8af16418222c1b6d4d4132c83db3c12e49e7157c31b41c 3 | size 62936807 4 | -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/topology/llava-v1.5-7b-lora(128, 256)-unfreeze_vit-False-epoch-10/README.md: -------------------------------------------------------------------------------- 1 | --- 2 | library_name: peft 3 | --- 4 | ## Training procedure 5 | 6 | ### Framework versions 7 | 8 | 9 | - PEFT 0.4.0 10 | -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/topology/llava-v1.5-7b-lora(128, 256)-unfreeze_vit-False-epoch-10/adapter_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "auto_mapping": null, 3 | "base_model_name_or_path": "/mnt/sdb1/local_llm/llava-v1.5-7b", 4 | "bias": "none", 5 | "fan_in_fan_out": false, 6 | "inference_mode": true, 7 | "init_lora_weights": true, 8 | "layers_pattern": null, 9 | "layers_to_transform": null, 10 | "lora_alpha": 256, 11 | "lora_dropout": 0.05, 12 | "modules_to_save": null, 13 | "peft_type": "LORA", 14 | "r": 128, 15 | "revision": null, 16 | "target_modules": [ 17 | "k_proj", 18 | "gate_proj", 19 | "q_proj", 20 | "o_proj", 21 | "v_proj", 22 | "up_proj", 23 | "down_proj" 24 | ], 25 | "task_type": "CAUSAL_LM" 26 | } -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/topology/llava-v1.5-7b-lora(128, 256)-unfreeze_vit-False-epoch-10/adapter_model.bin: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:0c769757d96cc15035de8ef336700a988e49d03925008de3d90e650b2aecd8ee 3 | size 639786637 4 | -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/topology/llava-v1.5-7b-lora(128, 256)-unfreeze_vit-False-epoch-10/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "_name_or_path": "/mnt/sdb1/local_llm/llava-v1.5-7b", 3 | "architectures": [ 4 | "LlavaLlamaForCausalLM" 5 | ], 6 | "bos_token_id": 1, 7 | "eos_token_id": 2, 8 | "freeze_mm_mlp_adapter": false, 9 | "freeze_mm_vision_resampler": false, 10 | "hidden_act": "silu", 11 | "hidden_size": 4096, 12 | "image_aspect_ratio": "pad", 13 | "initializer_range": 0.02, 14 | "intermediate_size": 11008, 15 | "max_length": 4096, 16 | "max_position_embeddings": 4096, 17 | "mm_hidden_size": 1024, 18 | "mm_projector_lr": 2e-05, 19 | "mm_projector_type": "mlp2x_gelu", 20 | "mm_resampler_type": null, 21 | "mm_use_im_patch_token": false, 22 | "mm_use_im_start_end": false, 23 | "mm_vision_select_feature": "patch", 24 | "mm_vision_select_layer": -2, 25 | "mm_vision_tower": "openai/clip-vit-large-patch14-336", 26 | "model_type": "llava", 27 | "num_attention_heads": 32, 28 | "num_hidden_layers": 32, 29 | "num_key_value_heads": 32, 30 | "pad_token_id": 0, 31 | "pretraining_tp": 1, 32 | "rms_norm_eps": 1e-05, 33 | "rope_scaling": null, 34 | "tie_word_embeddings": false, 35 | "tokenizer_model_max_length": 2400, 36 | "tokenizer_padding_side": "right", 37 | "torch_dtype": "float16", 38 | "transformers_version": "4.31.0", 39 | "tune_mm_mlp_adapter": false, 40 | "tune_mm_vision_resampler": false, 41 | "unfreeze_mm_vision_tower": false, 42 | "use_cache": true, 43 | "use_mm_proj": true, 44 | "vocab_size": 32000 45 | } 46 | -------------------------------------------------------------------------------- /checkpoints/Vision_Text/GVLQA_BASE/topology/llava-v1.5-7b-lora(128, 256)-unfreeze_vit-False-epoch-10/non_lora_trainables.bin: -------------------------------------------------------------------------------- 1 | version https://git-lfs.github.com/spec/v1 2 | oid sha256:00e57fb82a142ed5b8c1baafe736fbd04b2cd286dd0f2bf2e42d7ed0a8bce8b6 3 | size 41961191 4 | -------------------------------------------------------------------------------- /fastchat/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = "0.2.34" 2 | -------------------------------------------------------------------------------- /fastchat/__pycache__/__init__.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/fastchat/__pycache__/__init__.cpython-310.pyc -------------------------------------------------------------------------------- /fastchat/__pycache__/constants.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/fastchat/__pycache__/constants.cpython-310.pyc -------------------------------------------------------------------------------- /fastchat/__pycache__/conversation.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/fastchat/__pycache__/conversation.cpython-310.pyc -------------------------------------------------------------------------------- /fastchat/__pycache__/utils.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/fastchat/__pycache__/utils.cpython-310.pyc -------------------------------------------------------------------------------- /fastchat/constants.py: -------------------------------------------------------------------------------- 1 | """ 2 | Global constants. 3 | """ 4 | 5 | from enum import IntEnum 6 | import os 7 | 8 | REPO_PATH = os.path.dirname(os.path.dirname(__file__)) 9 | 10 | ##### For the gradio web server 11 | SERVER_ERROR_MSG = ( 12 | "**NETWORK ERROR DUE TO HIGH TRAFFIC. PLEASE REGENERATE OR REFRESH THIS PAGE.**" 13 | ) 14 | MODERATION_MSG = "$MODERATION$ YOUR INPUT VIOLATES OUR CONTENT MODERATION GUIDELINES." 15 | CONVERSATION_LIMIT_MSG = "YOU HAVE REACHED THE CONVERSATION LENGTH LIMIT. PLEASE CLEAR HISTORY AND START A NEW CONVERSATION." 16 | INACTIVE_MSG = "THIS SESSION HAS BEEN INACTIVE FOR TOO LONG. PLEASE REFRESH THIS PAGE." 17 | SLOW_MODEL_MSG = "⚠️ Both models will show the responses all at once. Please stay patient as it may take over 30 seconds." 18 | RATE_LIMIT_MSG = "**RATE LIMIT OF THIS MODEL IS REACHED. PLEASE COME BACK LATER OR TRY OTHER MODELS.**" 19 | # Maximum input length 20 | INPUT_CHAR_LEN_LIMIT = int(os.getenv("FASTCHAT_INPUT_CHAR_LEN_LIMIT", 12000)) 21 | # Maximum conversation turns 22 | CONVERSATION_TURN_LIMIT = 50 23 | # Session expiration time 24 | SESSION_EXPIRATION_TIME = 3600 25 | # The output dir of log files 26 | LOGDIR = os.getenv("LOGDIR", ".") 27 | # CPU Instruction Set Architecture 28 | CPU_ISA = os.getenv("CPU_ISA") 29 | 30 | 31 | ##### For the controller and workers (could be overwritten through ENV variables.) 32 | CONTROLLER_HEART_BEAT_EXPIRATION = int( 33 | os.getenv("FASTCHAT_CONTROLLER_HEART_BEAT_EXPIRATION", 90) 34 | ) 35 | WORKER_HEART_BEAT_INTERVAL = int(os.getenv("FASTCHAT_WORKER_HEART_BEAT_INTERVAL", 45)) 36 | WORKER_API_TIMEOUT = int(os.getenv("FASTCHAT_WORKER_API_TIMEOUT", 100)) 37 | WORKER_API_EMBEDDING_BATCH_SIZE = int( 38 | os.getenv("FASTCHAT_WORKER_API_EMBEDDING_BATCH_SIZE", 4) 39 | ) 40 | 41 | 42 | class ErrorCode(IntEnum): 43 | """ 44 | https://platform.openai.com/docs/guides/error-codes/api-errors 45 | """ 46 | 47 | VALIDATION_TYPE_ERROR = 40001 48 | 49 | INVALID_AUTH_KEY = 40101 50 | INCORRECT_AUTH_KEY = 40102 51 | NO_PERMISSION = 40103 52 | 53 | INVALID_MODEL = 40301 54 | PARAM_OUT_OF_RANGE = 40302 55 | CONTEXT_OVERFLOW = 40303 56 | 57 | RATE_LIMIT = 42901 58 | QUOTA_EXCEEDED = 42902 59 | ENGINE_OVERLOADED = 42903 60 | 61 | INTERNAL_ERROR = 50001 62 | CUDA_OUT_OF_MEMORY = 50002 63 | GRADIO_REQUEST_ERROR = 50003 64 | GRADIO_STREAM_UNKNOWN_ERROR = 50004 65 | CONTROLLER_NO_WORKER = 50005 66 | CONTROLLER_WORKER_TIMEOUT = 50006 67 | -------------------------------------------------------------------------------- /fastchat/custom_eval/__pycache__/eval.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/fastchat/custom_eval/__pycache__/eval.cpython-310.pyc -------------------------------------------------------------------------------- /fastchat/data/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/fastchat/data/__init__.py -------------------------------------------------------------------------------- /fastchat/data/convert_alpaca.py: -------------------------------------------------------------------------------- 1 | """ 2 | Convert alpaca dataset into sharegpt format. 3 | 4 | Usage: python3 -m fastchat.data.convert_alpaca --in alpaca_data.json 5 | """ 6 | 7 | import argparse 8 | import json 9 | 10 | from transformers import AutoTokenizer, AutoModelForCausalLM 11 | import numpy as np 12 | 13 | 14 | if __name__ == "__main__": 15 | parser = argparse.ArgumentParser() 16 | parser.add_argument("--in-file", type=str) 17 | parser.add_argument("--out-file", type=str) 18 | args = parser.parse_args() 19 | 20 | content = json.load(open(args.in_file, "r")) 21 | new_content = [] 22 | for i, c in enumerate(content): 23 | if len(c["input"].strip()) > 1: 24 | q, a = c["instruction"] + "\nInput:\n" + c["input"], c["output"] 25 | else: 26 | q, a = c["instruction"], c["output"] 27 | new_content.append( 28 | { 29 | "id": f"alpaca_{i}", 30 | "conversations": [ 31 | {"from": "human", "value": q}, 32 | {"from": "gpt", "value": a}, 33 | ], 34 | } 35 | ) 36 | 37 | print(f"#out: {len(new_content)}") 38 | json.dump(new_content, open(args.out_file, "w"), indent=2, ensure_ascii=False) 39 | -------------------------------------------------------------------------------- /fastchat/data/extract_gpt4_only.py: -------------------------------------------------------------------------------- 1 | """ 2 | Extract the conversations generated by GPT-4 only. 3 | 4 | Usage: python3 -m fastchat.data.extract_gpt4_only --in sharegpt.json 5 | """ 6 | import argparse 7 | import json 8 | 9 | 10 | if __name__ == "__main__": 11 | parser = argparse.ArgumentParser() 12 | parser.add_argument("--in-file", type=str, required=True) 13 | parser.add_argument("--out-file", type=str) 14 | parser.add_argument("--begin", type=int) 15 | parser.add_argument("--end", type=int) 16 | args = parser.parse_args() 17 | 18 | content = json.load(open(args.in_file, "r")) 19 | content = content[args.begin : args.end] 20 | new_content = [] 21 | for c in content: 22 | model = c.get("model", None) 23 | if model == "gpt4" or model is None: 24 | new_content.append(c) 25 | 26 | if args.out_file: 27 | out_file = args.out_file 28 | else: 29 | out_file = args.in_file.replace(".json", "_gpt4.json") 30 | 31 | print(f"#in: {len(content)}, #out: {len(new_content)}") 32 | json.dump(new_content, open(out_file, "w"), indent=2, ensure_ascii=False) 33 | -------------------------------------------------------------------------------- /fastchat/data/extract_single_round.py: -------------------------------------------------------------------------------- 1 | """ 2 | Extract the first round of the conversations. 3 | 4 | Usage: python3 -m fastchat.data.extract_single_round --in sharegpt.json 5 | """ 6 | import argparse 7 | import json 8 | 9 | 10 | if __name__ == "__main__": 11 | parser = argparse.ArgumentParser() 12 | parser.add_argument("--in-file", type=str, required=True) 13 | parser.add_argument("--out-file", type=str) 14 | parser.add_argument("--begin", type=int) 15 | parser.add_argument("--end", type=int) 16 | args = parser.parse_args() 17 | 18 | content = json.load(open(args.in_file, "r")) 19 | content = content[args.begin : args.end] 20 | for c in content: 21 | c["conversations"] = c["conversations"][:2] 22 | 23 | if args.out_file: 24 | out_file = args.out_file 25 | else: 26 | out_file = args.in_file.replace(".json", "_single.json") 27 | 28 | print(f"#in: {len(content)}, #out: {len(content)}") 29 | json.dump(content, open(out_file, "w"), indent=2, ensure_ascii=False) 30 | -------------------------------------------------------------------------------- /fastchat/data/filter_wrong_format.py: -------------------------------------------------------------------------------- 1 | """ 2 | Filter conversations with wrong formats. 3 | 4 | Usage: 5 | python3 -m fastchat.data.filter_wrong_format --in input.json --out output.json 6 | 7 | """ 8 | import argparse 9 | import json 10 | import re 11 | 12 | from tqdm import tqdm 13 | 14 | wrong_indices_pattern = re.compile("\n1\. [^2]*\n1\. ") 15 | 16 | 17 | def should_skip(conv): 18 | # Filter wrong list indices like https://sharegpt.com/c/1pREAGO 19 | for sentence in conv["conversations"]: 20 | val = sentence["value"] 21 | sub = re.search(wrong_indices_pattern, val) 22 | if sub is not None: 23 | return True 24 | 25 | return False 26 | 27 | 28 | if __name__ == "__main__": 29 | parser = argparse.ArgumentParser() 30 | parser.add_argument("--in-file", type=str, required=True) 31 | parser.add_argument("--out-file", type=str, required=True) 32 | args = parser.parse_args() 33 | 34 | content = json.load(open(args.in_file, "r")) 35 | 36 | new_content = [] 37 | for conv in tqdm(content): 38 | if should_skip(conv): 39 | print(f"{conv['id']} contains a wrong format.") 40 | else: 41 | new_content.append(conv) 42 | 43 | print(f"#in: {len(content)}, #out: {len(new_content)}") 44 | json.dump(new_content, open(args.out_file, "w"), indent=2, ensure_ascii=False) 45 | -------------------------------------------------------------------------------- /fastchat/data/get_stats.py: -------------------------------------------------------------------------------- 1 | """ 2 | Get stats of a dataset. 3 | 4 | Usage: python3 -m fastchat.data.get_stats --in sharegpt.json 5 | """ 6 | 7 | import argparse 8 | from concurrent.futures import ProcessPoolExecutor 9 | import json 10 | 11 | import numpy as np 12 | from tqdm import tqdm 13 | from transformers import AutoTokenizer, AutoModelForCausalLM 14 | 15 | K = 1e3 16 | M = 1e6 17 | 18 | 19 | def tokenize_one_sample(c): 20 | for i in range(len(c["conversations"])): 21 | v = c["conversations"][i]["value"] 22 | c["conversations"][i]["value"] = tokenizer.tokenize(v) 23 | return c 24 | 25 | 26 | def tokenize_dataset(content): 27 | processed = [] 28 | with ProcessPoolExecutor() as executor: 29 | for result in tqdm( 30 | executor.map(tokenize_one_sample, content), total=len(content) 31 | ): 32 | processed.append(result) 33 | 34 | return processed 35 | 36 | 37 | def compute_stats(content): 38 | sample_lens = [] 39 | sample_turns = [] 40 | prompt_lens = [] 41 | res_lens = [] 42 | 43 | for c in content: 44 | sample_len = 0 45 | sample_turns.append(len(c["conversations"]) // 2) 46 | for i in range(len(c["conversations"]) // 2): 47 | p = c["conversations"][i * 2]["value"] 48 | r = c["conversations"][i * 2 + 1]["value"] 49 | 50 | turn_len = len(p) + len(r) 51 | sample_len += turn_len 52 | prompt_lens.append(len(p)) 53 | res_lens.append(len(r)) 54 | sample_lens.append(sample_len) 55 | 56 | return sample_lens, sample_turns, prompt_lens, res_lens 57 | 58 | 59 | if __name__ == "__main__": 60 | parser = argparse.ArgumentParser() 61 | parser.add_argument("--in-file", type=str) 62 | parser.add_argument( 63 | "--model-name-or-path", type=str, default="meta-llama/Llama-2-7b-chat-hf" 64 | ) 65 | args = parser.parse_args() 66 | 67 | content = json.load(open(args.in_file, "r")) 68 | tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path, use_fast=False) 69 | content = tokenize_dataset(content) 70 | 71 | sample_lens, sample_turns, prompt_lens, res_lens = compute_stats(content) 72 | print(f"#sequence: {len(content)/K:.2f} K") 73 | print(f"#tokens: {np.sum(sample_lens)/M:.2f} M") 74 | print(f"avg. turns: {np.mean(sample_turns):.2f}") 75 | print(f"avg. prompt length: {np.mean(prompt_lens):.2f}") 76 | print(f"avg. response length: {np.mean(res_lens):.2f}") 77 | 78 | print("\n- Histogram -") 79 | bin_edges = [0, 1024, 2048, 4096, 8192, 16384, 32768] 80 | hist = np.histogram(sample_lens, bins=bin_edges)[0] 81 | for i in range(len(hist)): 82 | print(f"L{bin_edges[i]} - {bin_edges[i+1]}: {hist[i]}") 83 | -------------------------------------------------------------------------------- /fastchat/data/inspect_data.py: -------------------------------------------------------------------------------- 1 | """ 2 | Usage: 3 | python3 -m fastchat.data.inspect_data --in sharegpt_20230322_clean_lang_split.json 4 | """ 5 | import argparse 6 | import json 7 | import random 8 | 9 | 10 | if __name__ == "__main__": 11 | parser = argparse.ArgumentParser() 12 | parser.add_argument("--in-file", type=str, required=True) 13 | parser.add_argument("--begin", type=int) 14 | parser.add_argument("--random-n", type=int) 15 | args = parser.parse_args() 16 | 17 | content = json.load(open(args.in_file, "r")) 18 | 19 | if args.random_n: 20 | indices = [random.randint(0, len(content) - 1) for _ in range(args.random_n)] 21 | elif args.begin: 22 | indices = range(args.begin, len(content)) 23 | else: 24 | indices = range(0, len(content)) 25 | 26 | for idx in indices: 27 | sample = content[idx] 28 | print("=" * 40) 29 | print(f"no: {idx}, id: {sample['id']}") 30 | for conv in sample["conversations"]: 31 | print(conv["from"] + ": ") 32 | print(conv["value"]) 33 | input() 34 | -------------------------------------------------------------------------------- /fastchat/data/merge.py: -------------------------------------------------------------------------------- 1 | """ 2 | Merge two conversation files into one 3 | 4 | Usage: python3 -m fastchat.data.merge --in file1.json file2.json --out merged.json 5 | """ 6 | 7 | import argparse 8 | import json 9 | 10 | 11 | if __name__ == "__main__": 12 | parser = argparse.ArgumentParser() 13 | parser.add_argument("--in-file", type=str, required=True, nargs="+") 14 | parser.add_argument("--out-file", type=str, default="merged.json") 15 | args = parser.parse_args() 16 | 17 | new_content = [] 18 | for in_file in args.in_file: 19 | content = json.load(open(in_file, "r")) 20 | new_content.extend(content) 21 | 22 | print(f"#out: {len(new_content)}") 23 | json.dump(new_content, open(args.out_file, "w"), indent=2, ensure_ascii=False) 24 | -------------------------------------------------------------------------------- /fastchat/data/optional_clean.py: -------------------------------------------------------------------------------- 1 | """ 2 | Do optional cleaning (e.g., remove some languages). 3 | 4 | Usage: 5 | python3 -m fastchat.data.optional_clean --in input.json --out output.json --keep-lang en 6 | python3 -m fastchat.data.optional_clean --in input.json --out output.json --skip-lang en 7 | 8 | Requirement: 9 | pip3 install polyglot pyicu pycld2 10 | """ 11 | import argparse 12 | import json 13 | import re 14 | 15 | import polyglot 16 | from polyglot.detect import Detector 17 | import pycld2 18 | from tqdm import tqdm 19 | 20 | 21 | def skip(conv, args): 22 | # Remove certain languages 23 | if args.keep_lang != "all" or args.skip_lang is not None: 24 | text = "\n".join([x["value"] for x in conv["conversations"]]) 25 | try: 26 | lang_code = Detector(text).language.code 27 | except (pycld2.error, polyglot.detect.base.UnknownLanguage): 28 | lang_code = "unknown" 29 | 30 | if args.keep_lang != "all" and lang_code != args.keep_lang: 31 | return True 32 | 33 | if lang_code == args.skip_lang: 34 | return True 35 | 36 | # Remove repetitive numbers 37 | if args.reduce_rep: 38 | for sentence in conv["conversations"]: 39 | val = sentence["value"] 40 | sub = re.search(r"(\d)\1{8}", val) 41 | if sub is not None: 42 | return True 43 | 44 | return False 45 | 46 | 47 | if __name__ == "__main__": 48 | parser = argparse.ArgumentParser() 49 | parser.add_argument("--in-file", type=str, required=True) 50 | parser.add_argument("--out-file", type=str) 51 | parser.add_argument( 52 | "--keep-lang", 53 | type=str, 54 | default="all", 55 | choices=["all", "en"], 56 | help="Only keep certain langauges.", 57 | ) 58 | parser.add_argument("--skip-lang", type=str, help="Skip a specific language.") 59 | # NOTE: Be careful about reduce_rep which may remove some good data. 60 | # For example, addresses could have long consecutive 0's 61 | parser.add_argument("--reduce-rep", action="store_true") 62 | args = parser.parse_args() 63 | 64 | in_file = args.in_file 65 | out_file = args.out_file 66 | keep_lang = args.keep_lang 67 | skip_lang = args.skip_lang 68 | reduce_rep = args.reduce_rep 69 | assert keep_lang == "all" or skip_lang is None 70 | 71 | if out_file is None: 72 | out_file = "sharegpt_clean" 73 | if keep_lang != "all": 74 | out_file += "_" + keep_lang 75 | if skip_lang is not None: 76 | out_file += "_skip_" + skip_lang 77 | if reduce_rep: 78 | out_file += "_reduce_rep" 79 | out_file += ".json" 80 | 81 | content = json.load(open(in_file, "r")) 82 | num_conv = len(content) 83 | 84 | new_content = [] 85 | for conv in tqdm(content): 86 | if not skip(conv, args): 87 | new_content.append(conv) 88 | 89 | print(f"#in: {len(content)}, #out: {len(new_content)}") 90 | json.dump(new_content, open(out_file, "w"), indent=2, ensure_ascii=False) 91 | -------------------------------------------------------------------------------- /fastchat/data/optional_replace.py: -------------------------------------------------------------------------------- 1 | """ 2 | Do optional replace of bos/eos/pad/unk. 3 | 4 | Usage: 5 | python3 -m fastchat.data.optional_replace --in input.json --out output.json --model-name-or-path 6 | 7 | Requirement: 8 | pip3 install transformers tqdm 9 | """ 10 | import argparse 11 | import json 12 | import traceback 13 | 14 | import transformers 15 | from tqdm import tqdm 16 | 17 | 18 | def replace_special_tokens( 19 | tokenizer: transformers.PreTrainedTokenizer, text: str 20 | ) -> str: 21 | if not text: 22 | return text 23 | 24 | def _insert_vline(token: str) -> str: 25 | if len(token) < 2: 26 | return " " 27 | elif len(token) == 2: 28 | return f"{token[0]}|{token[1]}" 29 | else: 30 | return f"{token[:1]}|{token[1:-1]}|{token[-1:]}" 31 | 32 | if tokenizer.bos_token: 33 | text = text.replace(tokenizer.bos_token, _insert_vline(tokenizer.bos_token)) 34 | if tokenizer.eos_token: 35 | text = text.replace(tokenizer.eos_token, _insert_vline(tokenizer.eos_token)) 36 | if tokenizer.pad_token: 37 | text = text.replace(tokenizer.pad_token, _insert_vline(tokenizer.pad_token)) 38 | if tokenizer.unk_token: 39 | text = text.replace(tokenizer.unk_token, _insert_vline(tokenizer.unk_token)) 40 | return text 41 | 42 | 43 | def replace(conv, tokenizer): 44 | # Replace bos/eos/pad/unk tokens 45 | if tokenizer: 46 | try: 47 | for sentence in conv["conversations"]: 48 | sentence["value"] = replace_special_tokens(tokenizer, sentence["value"]) 49 | except Exception as e: 50 | traceback.print_exc() 51 | 52 | 53 | if __name__ == "__main__": 54 | parser = argparse.ArgumentParser() 55 | parser.add_argument("--in-file", type=str, required=True) 56 | parser.add_argument("--out-file", type=str) 57 | parser.add_argument( 58 | "--model-name-or-path", 59 | type=str, 60 | help="The directory or address where the model token is stored.", 61 | ) 62 | args = parser.parse_args() 63 | 64 | in_file = args.in_file 65 | out_file = args.out_file 66 | tokenizer = None 67 | if args.model_name_or_path: 68 | tokenizer = transformers.AutoTokenizer.from_pretrained( 69 | args.model_name_or_path, 70 | trust_remote_code=True, 71 | use_fast=False, 72 | ) 73 | 74 | if out_file is None: 75 | out_file = f"{in_file}_replace.json" 76 | 77 | content = json.load(open(in_file, "r")) 78 | 79 | for conv in tqdm(content): 80 | replace(conv, tokenizer) 81 | 82 | json.dump(content, open(out_file, "w"), indent=2, ensure_ascii=False) 83 | -------------------------------------------------------------------------------- /fastchat/data/prepare_all.py: -------------------------------------------------------------------------------- 1 | """Prepare all datasets.""" 2 | 3 | import argparse 4 | import os 5 | 6 | from fastchat.utils import run_cmd 7 | 8 | 9 | if __name__ == "__main__": 10 | parser = argparse.ArgumentParser() 11 | parser.add_argument("--prefix", type=str, default="~/datasets/sharegpt_20230521") 12 | parser.add_argument( 13 | "--model-name-or-path", type=str, default="meta-llama/Llama-2-7b-chat-hf" 14 | ) 15 | parser.add_argument("--seq-len", type=int, default=4096) 16 | args = parser.parse_args() 17 | 18 | in_prefix = args.prefix 19 | model_path = args.model_name_or_path 20 | seq_len = args.seq_len 21 | prefix = ( 22 | f"{in_prefix}_{seq_len}".replace("4096", "4k") 23 | .replace("8192", "8k") 24 | .replace("16384", "16k") 25 | ) 26 | 27 | cmd_list = [ 28 | f"python3 -m fastchat.data.clean_sharegpt --in {in_prefix}_html.json --out {prefix}_clean.json", 29 | f"python3 -m fastchat.data.optional_clean --in {prefix}_clean.json --out {prefix}_clean_lang.json --skip-lang ko", 30 | f"python3 -m fastchat.data.split_long_conversation --in {prefix}_clean_lang.json --out {prefix}_clean_lang_split.json --model-name {model_path} --max-length {seq_len}", 31 | f"python3 -m fastchat.data.filter_wrong_format --in {prefix}_clean_lang_split.json --out {prefix}_clean_lang_split.json", 32 | f"python3 -m fastchat.data.split_train_test --in {prefix}_clean_lang_split.json --ratio 0.99", 33 | f"python3 -m fastchat.data.hardcoded_questions", 34 | f"python3 -m fastchat.data.merge --in {prefix}_clean_lang_split_train.json hardcoded.json --out {prefix}_clean_lang_split_identity.json", 35 | f"python3 -m fastchat.data.extract_gpt4_only --in {prefix}_clean_lang_split_identity.json", 36 | f"python3 -m fastchat.data.extract_single_round --in {prefix}_clean_lang_split_identity.json", 37 | ] 38 | 39 | for cmd in cmd_list: 40 | ret = run_cmd(cmd) 41 | if ret != 0: 42 | exit(ret) 43 | -------------------------------------------------------------------------------- /fastchat/data/pretty_json.py: -------------------------------------------------------------------------------- 1 | """ 2 | Usage: 3 | python3 pretty_json.py --in in.json --out out.json 4 | """ 5 | 6 | import argparse 7 | import json 8 | 9 | 10 | if __name__ == "__main__": 11 | parser = argparse.ArgumentParser() 12 | parser.add_argument("--in-file", type=str, required=True) 13 | parser.add_argument("--out-file", type=str, required=True) 14 | args = parser.parse_args() 15 | 16 | with open(args.in_file, "r") as fin: 17 | data = json.load(fin) 18 | 19 | with open(args.out_file, "w") as fout: 20 | json.dump(data, fout, indent=2, ensure_ascii=False) 21 | -------------------------------------------------------------------------------- /fastchat/data/sample.py: -------------------------------------------------------------------------------- 1 | """ 2 | Sample some conversations from a file. 3 | 4 | Usage: python3 -m fastchat.data.sample --in sharegpt.json --out sampled.json 5 | """ 6 | import argparse 7 | import json 8 | 9 | import numpy as np 10 | 11 | 12 | if __name__ == "__main__": 13 | parser = argparse.ArgumentParser() 14 | parser.add_argument("--in-file", type=str, required=True) 15 | parser.add_argument("--out-file", type=str, default="sampled.json") 16 | parser.add_argument("--begin", type=int, default=0) 17 | parser.add_argument("--end", type=int, default=100) 18 | parser.add_argument("--max-length", type=int, default=1024) 19 | parser.add_argument("--keep-order", action="store_true") 20 | args = parser.parse_args() 21 | 22 | content = json.load(open(args.in_file, "r")) 23 | if not args.keep_order: 24 | np.random.seed(42) 25 | np.random.shuffle(content) 26 | 27 | new_content = [] 28 | for i in range(args.begin, min(args.end, len(content))): 29 | sample = content[i] 30 | concat = "" 31 | for s in sample["conversations"]: 32 | concat += s["value"] 33 | 34 | if len(concat) > args.max_length: 35 | continue 36 | 37 | new_content.append(sample) 38 | 39 | print(f"#in: {len(content)}, #out: {len(new_content)}") 40 | json.dump(new_content, open(args.out_file, "w"), indent=2, ensure_ascii=False) 41 | -------------------------------------------------------------------------------- /fastchat/data/split_train_test.py: -------------------------------------------------------------------------------- 1 | """ 2 | Split the dataset into training and test set. 3 | 4 | Usage: python3 -m fastchat.data.split_train_test --in sharegpt.json 5 | """ 6 | import argparse 7 | import json 8 | 9 | import numpy as np 10 | 11 | 12 | if __name__ == "__main__": 13 | parser = argparse.ArgumentParser() 14 | parser.add_argument("--in-file", type=str, required=True) 15 | parser.add_argument("--begin", type=int, default=0) 16 | parser.add_argument("--end", type=int, default=100) 17 | parser.add_argument("--ratio", type=float, default=0.9) 18 | args = parser.parse_args() 19 | 20 | content = json.load(open(args.in_file, "r")) 21 | np.random.seed(0) 22 | 23 | perm = np.random.permutation(len(content)) 24 | content = [content[i] for i in perm] 25 | split = int(args.ratio * len(content)) 26 | 27 | train_set = content[:split] 28 | test_set = content[split:] 29 | 30 | print(f"#train: {len(train_set)}, #test: {len(test_set)}") 31 | train_name = args.in_file.replace(".json", "_train.json") 32 | test_name = args.in_file.replace(".json", "_test.json") 33 | json.dump(train_set, open(train_name, "w"), indent=2, ensure_ascii=False) 34 | json.dump(test_set, open(test_name, "w"), indent=2, ensure_ascii=False) 35 | -------------------------------------------------------------------------------- /fastchat/llm_judge/clean_judgment.py: -------------------------------------------------------------------------------- 1 | """ 2 | Clean model judgment files. 3 | """ 4 | import argparse 5 | import json 6 | 7 | selected_models = [ 8 | "alpaca-13b", 9 | "baize-v2-13b", 10 | "chatglm-6b", 11 | "claude-instant-v1", 12 | "claude-v1", 13 | "dolly-v2-12b", 14 | "falcon-40b-instruct", 15 | "fastchat-t5-3b", 16 | "gpt-3.5-turbo", 17 | "gpt-4", 18 | "gpt4all-13b-snoozy", 19 | "guanaco-33b", 20 | "guanaco-65b", 21 | "h2ogpt-oasst-open-llama-13b", 22 | "koala-13b", 23 | "llama-13b", 24 | "mpt-30b-chat", 25 | "mpt-30b-instruct", 26 | "mpt-7b-chat", 27 | "nous-hermes-13b", 28 | "oasst-sft-4-pythia-12b", 29 | "oasst-sft-7-llama-30b", 30 | "palm-2-chat-bison-001", 31 | "rwkv-4-raven-14b", 32 | "stablelm-tuned-alpha-7b", 33 | "tulu-30b", 34 | "vicuna-13b-v1.3", 35 | "vicuna-33b-v1.3", 36 | "vicuna-7b-v1.3", 37 | "wizardlm-13b", 38 | "wizardlm-30b", 39 | ] 40 | 41 | 42 | if __name__ == "__main__": 43 | parser = argparse.ArgumentParser() 44 | parser.add_argument("--infile", type=str) 45 | args = parser.parse_args() 46 | 47 | infile = args.infile 48 | outfile = infile.replace(".jsonl", "_clean.jsonl") 49 | 50 | raw_lines = open(infile).readlines() 51 | rets = [] 52 | models = set() 53 | visited = set() 54 | for line in raw_lines: 55 | obj = json.loads(line) 56 | 57 | if "model_1" in obj: # pair 58 | model = obj["model_1"] 59 | key = ( 60 | obj["model_1"], 61 | obj["model_2"], 62 | obj["question_id"], 63 | tuple(obj["judge"]), 64 | ) 65 | else: # single 66 | model = obj["model"] 67 | key = (obj["model"], obj["question_id"], tuple(obj["judge"])) 68 | 69 | if key in visited: 70 | continue 71 | visited.add(key) 72 | 73 | if model not in selected_models: 74 | continue 75 | models.add(model) 76 | rets.append(obj) 77 | 78 | models = sorted(list(models)) 79 | missing_models = [x for x in selected_models if x not in models] 80 | print(f"in models: {models}, number: {len(models)}") 81 | print(f"missing models: {missing_models}") 82 | print(f"#in: {len(raw_lines)}, #out: {len(rets)}") 83 | rets.sort( 84 | key=lambda x: ( 85 | x["model"] if "model" in x else x["model_1"], 86 | x["question_id"], 87 | x["turn"], 88 | ) 89 | ) 90 | 91 | with open(outfile, "w") as fout: 92 | for x in rets: 93 | fout.write(json.dumps(x) + "\n") 94 | -------------------------------------------------------------------------------- /fastchat/llm_judge/data/mt_bench/misc/radar.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/fastchat/llm_judge/data/mt_bench/misc/radar.png -------------------------------------------------------------------------------- /fastchat/llm_judge/download_mt_bench_pregenerated.py: -------------------------------------------------------------------------------- 1 | """ 2 | Download the pre-generated model answers and judgments for MT-bench. 3 | """ 4 | import os 5 | 6 | from fastchat.utils import run_cmd 7 | 8 | filenames = [ 9 | "data/mt_bench/model_answer/alpaca-13b.jsonl", 10 | "data/mt_bench/model_answer/baize-v2-13b.jsonl", 11 | "data/mt_bench/model_answer/chatglm-6b.jsonl", 12 | "data/mt_bench/model_answer/claude-instant-v1.jsonl", 13 | "data/mt_bench/model_answer/claude-v1.jsonl", 14 | "data/mt_bench/model_answer/dolly-v2-12b.jsonl", 15 | "data/mt_bench/model_answer/falcon-40b-instruct.jsonl", 16 | "data/mt_bench/model_answer/fastchat-t5-3b.jsonl", 17 | "data/mt_bench/model_answer/gpt-3.5-turbo.jsonl", 18 | "data/mt_bench/model_answer/gpt-4.jsonl", 19 | "data/mt_bench/model_answer/gpt4all-13b-snoozy.jsonl", 20 | "data/mt_bench/model_answer/guanaco-33b.jsonl", 21 | "data/mt_bench/model_answer/guanaco-65b.jsonl", 22 | "data/mt_bench/model_answer/h2ogpt-oasst-open-llama-13b.jsonl", 23 | "data/mt_bench/model_answer/koala-13b.jsonl", 24 | "data/mt_bench/model_answer/llama-13b.jsonl", 25 | "data/mt_bench/model_answer/mpt-30b-chat.jsonl", 26 | "data/mt_bench/model_answer/mpt-30b-instruct.jsonl", 27 | "data/mt_bench/model_answer/mpt-7b-chat.jsonl", 28 | "data/mt_bench/model_answer/nous-hermes-13b.jsonl", 29 | "data/mt_bench/model_answer/oasst-sft-4-pythia-12b.jsonl", 30 | "data/mt_bench/model_answer/oasst-sft-7-llama-30b.jsonl", 31 | "data/mt_bench/model_answer/palm-2-chat-bison-001.jsonl", 32 | "data/mt_bench/model_answer/rwkv-4-raven-14b.jsonl", 33 | "data/mt_bench/model_answer/stablelm-tuned-alpha-7b.jsonl", 34 | "data/mt_bench/model_answer/tulu-30b.jsonl", 35 | "data/mt_bench/model_answer/vicuna-13b-v1.3.jsonl", 36 | "data/mt_bench/model_answer/vicuna-33b-v1.3.jsonl", 37 | "data/mt_bench/model_answer/vicuna-7b-v1.3.jsonl", 38 | "data/mt_bench/model_answer/wizardlm-13b.jsonl", 39 | "data/mt_bench/model_answer/wizardlm-30b.jsonl", 40 | "data/mt_bench/model_judgment/gpt-4_single.jsonl", 41 | "data/mt_bench/model_judgment/gpt-4_pair.jsonl", 42 | ] 43 | 44 | 45 | if __name__ == "__main__": 46 | prefix = "https://huggingface.co/spaces/lmsys/mt-bench/resolve/main/" 47 | 48 | for name in filenames: 49 | os.makedirs(os.path.dirname(name), exist_ok=True) 50 | ret = run_cmd(f"wget -q --show-progress -O {name} {prefix + name}") 51 | assert ret == 0 52 | -------------------------------------------------------------------------------- /fastchat/model/__init__.py: -------------------------------------------------------------------------------- 1 | from fastchat.model.model_adapter import ( 2 | load_model, 3 | get_conversation_template, 4 | add_model_args, 5 | ) 6 | -------------------------------------------------------------------------------- /fastchat/model/__pycache__/__init__.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/fastchat/model/__pycache__/__init__.cpython-310.pyc -------------------------------------------------------------------------------- /fastchat/model/__pycache__/compression.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/fastchat/model/__pycache__/compression.cpython-310.pyc -------------------------------------------------------------------------------- /fastchat/model/__pycache__/llama_condense_monkey_patch.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/fastchat/model/__pycache__/llama_condense_monkey_patch.cpython-310.pyc -------------------------------------------------------------------------------- /fastchat/model/__pycache__/model_adapter.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/fastchat/model/__pycache__/model_adapter.cpython-310.pyc -------------------------------------------------------------------------------- /fastchat/model/__pycache__/model_chatglm.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/fastchat/model/__pycache__/model_chatglm.cpython-310.pyc -------------------------------------------------------------------------------- /fastchat/model/__pycache__/model_codet5p.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/fastchat/model/__pycache__/model_codet5p.cpython-310.pyc -------------------------------------------------------------------------------- /fastchat/model/__pycache__/model_exllama.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/fastchat/model/__pycache__/model_exllama.cpython-310.pyc -------------------------------------------------------------------------------- /fastchat/model/__pycache__/model_falcon.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/fastchat/model/__pycache__/model_falcon.cpython-310.pyc -------------------------------------------------------------------------------- /fastchat/model/__pycache__/model_xfastertransformer.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/fastchat/model/__pycache__/model_xfastertransformer.cpython-310.pyc -------------------------------------------------------------------------------- /fastchat/model/__pycache__/monkey_patch_non_inplace.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/fastchat/model/__pycache__/monkey_patch_non_inplace.cpython-310.pyc -------------------------------------------------------------------------------- /fastchat/model/apply_lora.py: -------------------------------------------------------------------------------- 1 | """ 2 | Apply the LoRA weights on top of a base model. 3 | 4 | Usage: 5 | python3 -m fastchat.model.apply_lora --base ~/model_weights/llama-7b --target ~/model_weights/baize-7b --lora project-baize/baize-lora-7B 6 | 7 | Dependency: 8 | pip3 install git+https://github.com/huggingface/peft.git@2822398fbe896f25d4dac5e468624dc5fd65a51b 9 | """ 10 | import argparse 11 | 12 | import torch 13 | from peft import PeftModel 14 | from transformers import AutoTokenizer, AutoModelForCausalLM 15 | 16 | 17 | def apply_lora(base_model_path, target_model_path, lora_path): 18 | print(f"Loading the base model from {base_model_path}") 19 | base = AutoModelForCausalLM.from_pretrained( 20 | base_model_path, torch_dtype=torch.float16, low_cpu_mem_usage=True 21 | ) 22 | base_tokenizer = AutoTokenizer.from_pretrained(base_model_path, use_fast=False) 23 | 24 | print(f"Loading the LoRA adapter from {lora_path}") 25 | 26 | lora_model = PeftModel.from_pretrained( 27 | base, 28 | lora_path, 29 | # torch_dtype=torch.float16 30 | ) 31 | 32 | print("Applying the LoRA") 33 | model = lora_model.merge_and_unload() 34 | 35 | print(f"Saving the target model to {target_model_path}") 36 | model.save_pretrained(target_model_path) 37 | base_tokenizer.save_pretrained(target_model_path) 38 | 39 | 40 | if __name__ == "__main__": 41 | parser = argparse.ArgumentParser() 42 | parser.add_argument("--base-model-path", type=str, required=True) 43 | parser.add_argument("--target-model-path", type=str, required=True) 44 | parser.add_argument("--lora-path", type=str, required=True) 45 | 46 | args = parser.parse_args() 47 | 48 | apply_lora(args.base_model_path, args.target_model_path, args.lora_path) 49 | -------------------------------------------------------------------------------- /fastchat/model/convert_fp16.py: -------------------------------------------------------------------------------- 1 | """ 2 | Usage: 3 | python3 -m fastchat.model.convert_fp16 --in in-folder --out out-folder 4 | """ 5 | import argparse 6 | 7 | from transformers import AutoTokenizer, AutoModelForCausalLM 8 | import torch 9 | 10 | 11 | def convert_fp16(in_checkpoint, out_checkpoint): 12 | tokenizer = AutoTokenizer.from_pretrained(in_checkpoint, use_fast=False) 13 | model = AutoModelForCausalLM.from_pretrained( 14 | in_checkpoint, torch_dtype=torch.float16, low_cpu_mem_usage=True 15 | ) 16 | model.save_pretrained(out_checkpoint) 17 | tokenizer.save_pretrained(out_checkpoint) 18 | 19 | 20 | if __name__ == "__main__": 21 | parser = argparse.ArgumentParser() 22 | parser.add_argument("--in-checkpoint", type=str, help="Path to the model") 23 | parser.add_argument("--out-checkpoint", type=str, help="Path to the output model") 24 | args = parser.parse_args() 25 | 26 | convert_fp16(args.in_checkpoint, args.out_checkpoint) 27 | -------------------------------------------------------------------------------- /fastchat/model/llama_condense_monkey_patch.py: -------------------------------------------------------------------------------- 1 | # Code adapted from https://huggingface.co/kaiokendev/superhot-13b-8k-no-rlhf-test/blob/main/llama_rope_scaled_monkey_patch.py 2 | 3 | from functools import partial 4 | 5 | import torch 6 | import transformers 7 | import transformers.models.llama.modeling_llama 8 | 9 | 10 | class CondenseRotaryEmbedding(torch.nn.Module): 11 | def __init__( 12 | self, dim, ratio, max_position_embeddings=2048, base=10000, device=None 13 | ): 14 | super().__init__() 15 | inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2).float().to(device) / dim)) 16 | self.register_buffer("inv_freq", inv_freq) 17 | 18 | # Build here to make `torch.jit.trace` work. 19 | self.ratio = ratio 20 | max_position_embeddings *= ratio 21 | self.max_seq_len_cached = max_position_embeddings 22 | # print(f"Monkey Patching condense ratio {ratio}") 23 | t = ( 24 | torch.arange( 25 | self.max_seq_len_cached, 26 | device=self.inv_freq.device, 27 | dtype=self.inv_freq.dtype, 28 | ) 29 | / ratio 30 | ) 31 | freqs = torch.einsum("i,j->ij", t, self.inv_freq) 32 | # Different from paper, but it uses a different permutation in order to obtain the same calculation 33 | emb = torch.cat((freqs, freqs), dim=-1) 34 | dtype = torch.get_default_dtype() 35 | self.register_buffer( 36 | "cos_cached", emb.cos()[None, None, :, :].to(dtype), persistent=False 37 | ) 38 | self.register_buffer( 39 | "sin_cached", emb.sin()[None, None, :, :].to(dtype), persistent=False 40 | ) 41 | 42 | def forward(self, x, seq_len=None): 43 | # x: [bs, num_attention_heads, seq_len, head_size] 44 | # This `if` block is unlikely to be run after we build sin/cos in `__init__`. Keep the logic here just in case. 45 | if seq_len > self.max_seq_len_cached: 46 | self.max_seq_len_cached = seq_len 47 | t = ( 48 | torch.arange( 49 | self.max_seq_len_cached, device=x.device, dtype=self.inv_freq.dtype 50 | ) 51 | / self.ratio 52 | ) 53 | freqs = torch.einsum("i,j->ij", t, self.inv_freq) 54 | # Different from paper, but it uses a different permutation in order to obtain the same calculation 55 | emb = torch.cat((freqs, freqs), dim=-1).to(x.device) 56 | self.register_buffer( 57 | "cos_cached", emb.cos()[None, None, :, :].to(x.dtype), persistent=False 58 | ) 59 | self.register_buffer( 60 | "sin_cached", emb.sin()[None, None, :, :].to(x.dtype), persistent=False 61 | ) 62 | return ( 63 | self.cos_cached[:, :, :seq_len, ...].to(dtype=x.dtype), 64 | self.sin_cached[:, :, :seq_len, ...].to(dtype=x.dtype), 65 | ) 66 | 67 | 68 | def replace_llama_with_condense(ratio): 69 | transformers.models.llama.modeling_llama.LlamaRotaryEmbedding = partial( 70 | CondenseRotaryEmbedding, ratio=ratio 71 | ) 72 | -------------------------------------------------------------------------------- /fastchat/model/make_delta.py: -------------------------------------------------------------------------------- 1 | """ 2 | Make the delta weights by subtracting base weights. 3 | 4 | Usage: 5 | python3 -m fastchat.model.make_delta --base ~/model_weights/llama-13b --target ~/model_weights/vicuna-13b --delta ~/model_weights/vicuna-13b-delta --hub-repo-id lmsys/vicuna-13b-delta-v1.1 6 | """ 7 | import argparse 8 | 9 | import torch 10 | from tqdm import tqdm 11 | from transformers import AutoTokenizer, AutoModelForCausalLM 12 | 13 | 14 | def make_delta(base_model_path, target_model_path, delta_path): 15 | print(f"Loading the base model from {base_model_path}") 16 | base = AutoModelForCausalLM.from_pretrained( 17 | base_model_path, torch_dtype=torch.float16, low_cpu_mem_usage=True 18 | ) 19 | 20 | print(f"Loading the target model from {target_model_path}") 21 | target = AutoModelForCausalLM.from_pretrained( 22 | target_model_path, torch_dtype=torch.float16, low_cpu_mem_usage=True 23 | ) 24 | target_tokenizer = AutoTokenizer.from_pretrained(target_model_path, use_fast=False) 25 | 26 | print("Calculating the delta") 27 | for name, param in tqdm(target.state_dict().items(), desc="Calculating delta"): 28 | assert name in base.state_dict() 29 | param.data -= base.state_dict()[name] 30 | 31 | print(f"Saving the delta to {delta_path}") 32 | if args.hub_repo_id: 33 | kwargs = {"push_to_hub": True, "repo_id": args.hub_repo_id} 34 | else: 35 | kwargs = {} 36 | target.save_pretrained(delta_path, **kwargs) 37 | target_tokenizer.save_pretrained(delta_path, **kwargs) 38 | 39 | 40 | if __name__ == "__main__": 41 | parser = argparse.ArgumentParser() 42 | parser.add_argument("--base-model-path", type=str, required=True) 43 | parser.add_argument("--target-model-path", type=str, required=True) 44 | parser.add_argument("--delta-path", type=str, required=True) 45 | parser.add_argument("--hub-repo-id", type=str) 46 | args = parser.parse_args() 47 | 48 | make_delta(args.base_model_path, args.target_model_path, args.delta_path) 49 | -------------------------------------------------------------------------------- /fastchat/model/model_exllama.py: -------------------------------------------------------------------------------- 1 | import gc 2 | import sys 3 | from typing import Dict 4 | 5 | import torch 6 | 7 | 8 | def generate_stream_exllama( 9 | model, 10 | tokenizer, 11 | params: Dict, 12 | device: str, 13 | context_len: int, 14 | stream_interval: int = 2, 15 | judge_sent_end: bool = False, 16 | ): 17 | try: 18 | from exllamav2.generator import ExLlamaV2StreamingGenerator, ExLlamaV2Sampler 19 | except ImportError as e: 20 | print(f"Error: Failed to load Exllamav2. {e}") 21 | sys.exit(-1) 22 | 23 | prompt = params["prompt"] 24 | 25 | generator = ExLlamaV2StreamingGenerator(model.model, model.cache, tokenizer) 26 | settings = ExLlamaV2Sampler.Settings() 27 | 28 | settings.temperature = float(params.get("temperature", 0.85)) 29 | settings.top_k = int(params.get("top_k", 50)) 30 | settings.top_p = float(params.get("top_p", 0.8)) 31 | settings.token_repetition_penalty = float(params.get("repetition_penalty", 1.15)) 32 | settings.disallow_tokens(generator.tokenizer, [generator.tokenizer.eos_token_id]) 33 | 34 | max_new_tokens = int(params.get("max_new_tokens", 256)) 35 | 36 | generator.set_stop_conditions(params.get("stop_token_ids", None) or []) 37 | echo = bool(params.get("echo", True)) 38 | 39 | input_ids = generator.tokenizer.encode(prompt) 40 | prompt_tokens = input_ids.shape[-1] 41 | generator.begin_stream(input_ids, settings) 42 | 43 | generated_tokens = 0 44 | if echo: 45 | output = prompt 46 | else: 47 | output = "" 48 | while True: 49 | chunk, eos, _ = generator.stream() 50 | output += chunk 51 | generated_tokens += 1 52 | if generated_tokens == max_new_tokens: 53 | finish_reason = "length" 54 | break 55 | elif eos: 56 | finish_reason = "length" 57 | break 58 | yield { 59 | "text": output, 60 | "usage": { 61 | "prompt_tokens": prompt_tokens, 62 | "completion_tokens": generated_tokens, 63 | "total_tokens": prompt_tokens + generated_tokens, 64 | }, 65 | "finish_reason": None, 66 | } 67 | 68 | yield { 69 | "text": output, 70 | "usage": { 71 | "prompt_tokens": prompt_tokens, 72 | "completion_tokens": generated_tokens, 73 | "total_tokens": prompt_tokens + generated_tokens, 74 | }, 75 | "finish_reason": finish_reason, 76 | } 77 | gc.collect() 78 | -------------------------------------------------------------------------------- /fastchat/model/model_xfastertransformer.py: -------------------------------------------------------------------------------- 1 | import gc 2 | from threading import Thread 3 | 4 | import torch 5 | from transformers import TextIteratorStreamer 6 | 7 | 8 | @torch.inference_mode() 9 | def generate_stream_xft( 10 | model, 11 | tokenizer, 12 | params, 13 | device, 14 | context_len=8192, 15 | stream_interval=2, 16 | judge_sent_end=False, 17 | ): 18 | prompt = params["prompt"] 19 | repetition_penalty = float(params.get("repetition_penalty", 1.0)) 20 | 21 | # unused now, and placehold for future. 22 | # temperature = float(params.get("temperature", 1.0)) 23 | # top_p = float(params.get("top_p", 1.0)) 24 | 25 | max_new_tokens = int(params.get("max_new_tokens", 4096)) 26 | echo = params.get("echo", True) 27 | 28 | inputs = tokenizer( 29 | prompt, return_tensors="pt", padding=model.config.padding 30 | ).input_ids 31 | input_echo_len = len(inputs[0]) 32 | max_len = max_new_tokens + input_echo_len 33 | 34 | decode_config = dict(skip_special_tokens=True, clean_up_tokenization_spaces=True) 35 | streamer = TextIteratorStreamer(tokenizer, skip_prompt=True, **decode_config) 36 | generation_kwargs = { 37 | "input_ids": inputs, 38 | "streamer": streamer, 39 | "max_length": max_len, 40 | "num_beams": model.config.beam_width, 41 | "length_penalty": repetition_penalty, 42 | "num_return_sequences": model.config.num_return_sequences, 43 | "early_stopping": model.config.early_stopping, 44 | "eos_token_id": model.config.eos_token_id, 45 | "pad_token_id": model.config.pad_token_id, 46 | } 47 | 48 | thread = Thread(target=model.model.generate, kwargs=generation_kwargs) 49 | thread.start() 50 | if echo: 51 | # means keep the prompt 52 | output = prompt 53 | else: 54 | output = "" 55 | i = 0 56 | for i, new_text in enumerate(streamer): 57 | output += new_text 58 | yield { 59 | "text": output, 60 | "usage": { 61 | "prompt_tokens": input_echo_len, 62 | "completion_tokens": i, 63 | "total_tokens": input_echo_len + i, 64 | }, 65 | "finish_reason": None, 66 | } 67 | output = output.strip() 68 | if i == max_new_tokens - 1: 69 | finish_reason = "length" 70 | else: 71 | finish_reason = "stop" 72 | yield { 73 | "text": output, 74 | "usage": { 75 | "prompt_tokens": input_echo_len, 76 | "completion_tokens": i, 77 | "total_tokens": input_echo_len + i, 78 | }, 79 | "finish_reason": finish_reason, 80 | } 81 | gc.collect() 82 | -------------------------------------------------------------------------------- /fastchat/model/rwkv_model.py: -------------------------------------------------------------------------------- 1 | import os 2 | from types import SimpleNamespace 3 | import warnings 4 | 5 | import torch 6 | 7 | os.environ["RWKV_JIT_ON"] = "1" 8 | os.environ["RWKV_CUDA_ON"] = "1" 9 | 10 | from rwkv.model import RWKV 11 | from rwkv.utils import PIPELINE, PIPELINE_ARGS 12 | 13 | 14 | class RwkvModel: 15 | def __init__(self, model_path): 16 | warnings.warn( 17 | "Experimental support. Please use ChatRWKV if you want to chat with RWKV" 18 | ) 19 | self.config = SimpleNamespace(is_encoder_decoder=False) 20 | self.model = RWKV(model=model_path, strategy="cuda fp16") 21 | # two GPUs 22 | # self.model = RWKV(model=model_path, strategy="cuda:0 fp16 *20 -> cuda:1 fp16") 23 | 24 | self.tokenizer = None 25 | self.model_path = model_path 26 | 27 | def to(self, target): 28 | assert target == "cuda" 29 | 30 | def __call__(self, input_ids, use_cache, past_key_values=None): 31 | assert use_cache == True 32 | input_ids = input_ids[0].detach().cpu().numpy() 33 | # print(input_ids) 34 | logits, state = self.model.forward(input_ids, past_key_values) 35 | # print(logits) 36 | logits = logits.unsqueeze(0).unsqueeze(0) 37 | out = SimpleNamespace(logits=logits, past_key_values=state) 38 | return out 39 | 40 | def generate( 41 | self, input_ids, do_sample, temperature, max_new_tokens, repetition_penalty=1.0 42 | ): 43 | # This function is used by fastchat.llm_judge. 44 | # Because RWKV does not support huggingface generation API, 45 | # we reuse fastchat.serve.inference.generate_stream as a workaround. 46 | from transformers import AutoTokenizer 47 | 48 | from fastchat.serve.inference import generate_stream 49 | from fastchat.conversation import get_conv_template 50 | 51 | if self.tokenizer is None: 52 | self.tokenizer = AutoTokenizer.from_pretrained( 53 | "EleutherAI/pythia-160m", use_fast=True 54 | ) 55 | prompt = self.tokenizer.decode(input_ids[0].tolist()) 56 | conv = get_conv_template("rwkv") 57 | 58 | gen_params = { 59 | "model": self.model_path, 60 | "prompt": prompt, 61 | "temperature": temperature, 62 | "repetition_penalty": repetition_penalty, 63 | "max_new_tokens": max_new_tokens, 64 | "stop": conv.stop_str, 65 | "stop_token_ids": conv.stop_token_ids, 66 | "echo": False, 67 | } 68 | res_iter = generate_stream(self, self.tokenizer, gen_params, "cuda") 69 | 70 | for res in res_iter: 71 | pass 72 | 73 | output = res["text"] 74 | output_ids = self.tokenizer.encode(output) 75 | 76 | return [input_ids[0].tolist() + output_ids] 77 | -------------------------------------------------------------------------------- /fastchat/model/upload_hub.py: -------------------------------------------------------------------------------- 1 | """ 2 | Upload weights to huggingface. 3 | 4 | Usage: 5 | python3 -m fastchat.model.upload_hub --model-path ~/model_weights/vicuna-13b --hub-repo-id lmsys/vicuna-13b-v1.3 6 | """ 7 | import argparse 8 | import tempfile 9 | 10 | import torch 11 | from transformers import AutoTokenizer, AutoModelForCausalLM 12 | 13 | 14 | def upload_hub(model_path, hub_repo_id, component, private): 15 | if component == "all": 16 | components = ["model", "tokenizer"] 17 | else: 18 | components = [component] 19 | 20 | kwargs = {"push_to_hub": True, "repo_id": hub_repo_id, "private": args.private} 21 | 22 | if "model" in components: 23 | model = AutoModelForCausalLM.from_pretrained( 24 | model_path, torch_dtype=torch.float16, low_cpu_mem_usage=True 25 | ) 26 | with tempfile.TemporaryDirectory() as tmp_path: 27 | model.save_pretrained(tmp_path, **kwargs) 28 | 29 | if "tokenizer" in components: 30 | tokenizer = AutoTokenizer.from_pretrained(model_path, use_fast=False) 31 | with tempfile.TemporaryDirectory() as tmp_path: 32 | tokenizer.save_pretrained(tmp_path, **kwargs) 33 | 34 | 35 | if __name__ == "__main__": 36 | parser = argparse.ArgumentParser() 37 | parser.add_argument("--model-path", type=str, required=True) 38 | parser.add_argument("--hub-repo-id", type=str, required=True) 39 | parser.add_argument( 40 | "--component", type=str, choices=["all", "model", "tokenizer"], default="all" 41 | ) 42 | parser.add_argument("--private", action="store_true") 43 | args = parser.parse_args() 44 | 45 | upload_hub(args.model_path, args.hub_repo_id, args.component, args.private) 46 | -------------------------------------------------------------------------------- /fastchat/modules/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/fastchat/modules/__init__.py -------------------------------------------------------------------------------- /fastchat/modules/__pycache__/__init__.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/fastchat/modules/__pycache__/__init__.cpython-310.pyc -------------------------------------------------------------------------------- /fastchat/modules/__pycache__/awq.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/fastchat/modules/__pycache__/awq.cpython-310.pyc -------------------------------------------------------------------------------- /fastchat/modules/__pycache__/exllama.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/fastchat/modules/__pycache__/exllama.cpython-310.pyc -------------------------------------------------------------------------------- /fastchat/modules/__pycache__/gptq.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/fastchat/modules/__pycache__/gptq.cpython-310.pyc -------------------------------------------------------------------------------- /fastchat/modules/__pycache__/xfastertransformer.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/fastchat/modules/__pycache__/xfastertransformer.cpython-310.pyc -------------------------------------------------------------------------------- /fastchat/modules/awq.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass, field 2 | from pathlib import Path 3 | import sys 4 | 5 | import torch 6 | from transformers import AutoTokenizer, AutoConfig, AutoModelForCausalLM, modeling_utils 7 | 8 | 9 | @dataclass 10 | class AWQConfig: 11 | ckpt: str = field( 12 | default=None, 13 | metadata={ 14 | "help": "Load quantized model. The path to the local AWQ checkpoint." 15 | }, 16 | ) 17 | wbits: int = field(default=16, metadata={"help": "#bits to use for quantization"}) 18 | groupsize: int = field( 19 | default=-1, 20 | metadata={"help": "Groupsize to use for quantization; default uses full row."}, 21 | ) 22 | 23 | 24 | def load_awq_quantized(model_name, awq_config: AWQConfig, device): 25 | print("Loading AWQ quantized model...") 26 | 27 | try: 28 | from tinychat.utils import load_quant 29 | from tinychat.modules import make_quant_norm, make_quant_attn, make_fused_mlp 30 | except ImportError as e: 31 | print(f"Error: Failed to import tinychat. {e}") 32 | print("Please double check if you have successfully installed AWQ") 33 | print("See https://github.com/lm-sys/FastChat/blob/main/docs/awq.md") 34 | sys.exit(-1) 35 | 36 | config = AutoConfig.from_pretrained(model_name, trust_remote_code=True) 37 | tokenizer = AutoTokenizer.from_pretrained( 38 | model_name, use_fast=False, trust_remote_code=True 39 | ) 40 | 41 | def skip(*args, **kwargs): 42 | pass 43 | 44 | torch.nn.init.kaiming_uniform_ = skip 45 | torch.nn.init.kaiming_normal_ = skip 46 | torch.nn.init.uniform_ = skip 47 | torch.nn.init.normal_ = skip 48 | modeling_utils._init_weights = False 49 | 50 | torch.set_default_dtype(torch.half) 51 | model = AutoModelForCausalLM.from_config(config, trust_remote_code=True) 52 | 53 | if any(name in find_awq_ckpt(awq_config) for name in ["llama", "vicuna"]): 54 | model = load_quant.load_awq_llama_fast( 55 | model, 56 | find_awq_ckpt(awq_config), 57 | awq_config.wbits, 58 | awq_config.groupsize, 59 | device, 60 | ) 61 | make_quant_attn(model, device) 62 | make_quant_norm(model) 63 | make_fused_mlp(model) 64 | else: 65 | model = load_quant.load_awq_model( 66 | model, 67 | find_awq_ckpt(awq_config), 68 | awq_config.wbits, 69 | awq_config.groupsize, 70 | device, 71 | ) 72 | return model, tokenizer 73 | 74 | 75 | def find_awq_ckpt(awq_config: AWQConfig): 76 | if Path(awq_config.ckpt).is_file(): 77 | return awq_config.ckpt 78 | 79 | for ext in ["*.pt", "*.safetensors"]: 80 | matched_result = sorted(Path(awq_config.ckpt).glob(ext)) 81 | if len(matched_result) > 0: 82 | return str(matched_result[-1]) 83 | 84 | print("Error: AWQ checkpoint not found") 85 | sys.exit(1) 86 | -------------------------------------------------------------------------------- /fastchat/modules/exllama.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass, field 2 | import sys 3 | 4 | 5 | @dataclass 6 | class ExllamaConfig: 7 | max_seq_len: int 8 | gpu_split: str = None 9 | cache_8bit: bool = False 10 | 11 | 12 | class ExllamaModel: 13 | def __init__(self, exllama_model, exllama_cache): 14 | self.model = exllama_model 15 | self.cache = exllama_cache 16 | self.config = self.model.config 17 | 18 | 19 | def load_exllama_model(model_path, exllama_config: ExllamaConfig): 20 | try: 21 | from exllamav2 import ( 22 | ExLlamaV2Config, 23 | ExLlamaV2Tokenizer, 24 | ExLlamaV2, 25 | ExLlamaV2Cache, 26 | ExLlamaV2Cache_8bit, 27 | ) 28 | except ImportError as e: 29 | print(f"Error: Failed to load Exllamav2. {e}") 30 | sys.exit(-1) 31 | 32 | exllamav2_config = ExLlamaV2Config() 33 | exllamav2_config.model_dir = model_path 34 | exllamav2_config.prepare() 35 | exllamav2_config.max_seq_len = exllama_config.max_seq_len 36 | exllamav2_config.cache_8bit = exllama_config.cache_8bit 37 | 38 | exllama_model = ExLlamaV2(exllamav2_config) 39 | tokenizer = ExLlamaV2Tokenizer(exllamav2_config) 40 | 41 | split = None 42 | if exllama_config.gpu_split: 43 | split = [float(alloc) for alloc in exllama_config.gpu_split.split(",")] 44 | exllama_model.load(split) 45 | 46 | cache_class = ExLlamaV2Cache_8bit if exllamav2_config.cache_8bit else ExLlamaV2Cache 47 | exllama_cache = cache_class(exllama_model) 48 | model = ExllamaModel(exllama_model=exllama_model, exllama_cache=exllama_cache) 49 | 50 | return model, tokenizer 51 | -------------------------------------------------------------------------------- /fastchat/modules/gptq.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass, field 2 | import os 3 | from os.path import isdir, isfile 4 | from pathlib import Path 5 | import sys 6 | 7 | from transformers import AutoTokenizer 8 | 9 | 10 | @dataclass 11 | class GptqConfig: 12 | ckpt: str = field( 13 | default=None, 14 | metadata={ 15 | "help": "Load quantized model. The path to the local GPTQ checkpoint." 16 | }, 17 | ) 18 | wbits: int = field(default=16, metadata={"help": "#bits to use for quantization"}) 19 | groupsize: int = field( 20 | default=-1, 21 | metadata={"help": "Groupsize to use for quantization; default uses full row."}, 22 | ) 23 | act_order: bool = field( 24 | default=True, 25 | metadata={"help": "Whether to apply the activation order GPTQ heuristic"}, 26 | ) 27 | 28 | 29 | def load_gptq_quantized(model_name, gptq_config: GptqConfig): 30 | print("Loading GPTQ quantized model...") 31 | 32 | try: 33 | script_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) 34 | module_path = os.path.join(script_path, "../repositories/GPTQ-for-LLaMa") 35 | 36 | sys.path.insert(0, module_path) 37 | from llama import load_quant 38 | except ImportError as e: 39 | print(f"Error: Failed to load GPTQ-for-LLaMa. {e}") 40 | print("See https://github.com/lm-sys/FastChat/blob/main/docs/gptq.md") 41 | sys.exit(-1) 42 | 43 | tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=False) 44 | # only `fastest-inference-4bit` branch cares about `act_order` 45 | if gptq_config.act_order: 46 | model = load_quant( 47 | model_name, 48 | find_gptq_ckpt(gptq_config), 49 | gptq_config.wbits, 50 | gptq_config.groupsize, 51 | act_order=gptq_config.act_order, 52 | ) 53 | else: 54 | # other branches 55 | model = load_quant( 56 | model_name, 57 | find_gptq_ckpt(gptq_config), 58 | gptq_config.wbits, 59 | gptq_config.groupsize, 60 | ) 61 | 62 | return model, tokenizer 63 | 64 | 65 | def find_gptq_ckpt(gptq_config: GptqConfig): 66 | if Path(gptq_config.ckpt).is_file(): 67 | return gptq_config.ckpt 68 | 69 | for ext in ["*.pt", "*.safetensors"]: 70 | matched_result = sorted(Path(gptq_config.ckpt).glob(ext)) 71 | if len(matched_result) > 0: 72 | return str(matched_result[-1]) 73 | 74 | print("Error: gptq checkpoint not found") 75 | sys.exit(1) 76 | -------------------------------------------------------------------------------- /fastchat/modules/xfastertransformer.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | import sys 3 | 4 | 5 | @dataclass 6 | class XftConfig: 7 | max_seq_len: int = 4096 8 | beam_width: int = 1 9 | eos_token_id: int = -1 10 | pad_token_id: int = -1 11 | num_return_sequences: int = 1 12 | is_encoder_decoder: bool = False 13 | padding: bool = True 14 | early_stopping: bool = False 15 | data_type: str = "bf16_fp16" 16 | 17 | 18 | class XftModel: 19 | def __init__(self, xft_model, xft_config): 20 | self.model = xft_model 21 | self.config = xft_config 22 | 23 | 24 | def load_xft_model(model_path, xft_config: XftConfig): 25 | try: 26 | import xfastertransformer 27 | from transformers import AutoTokenizer 28 | except ImportError as e: 29 | print(f"Error: Failed to load xFasterTransformer. {e}") 30 | sys.exit(-1) 31 | 32 | if xft_config.data_type is None or xft_config.data_type == "": 33 | data_type = "bf16_fp16" 34 | else: 35 | data_type = xft_config.data_type 36 | tokenizer = AutoTokenizer.from_pretrained( 37 | model_path, use_fast=False, padding_side="left", trust_remote_code=True 38 | ) 39 | xft_model = xfastertransformer.AutoModel.from_pretrained( 40 | model_path, dtype=data_type 41 | ) 42 | model = XftModel(xft_model=xft_model, xft_config=xft_config) 43 | if model.model.rank > 0: 44 | while True: 45 | model.model.generate() 46 | return model, tokenizer 47 | -------------------------------------------------------------------------------- /fastchat/serve/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/fastchat/serve/__init__.py -------------------------------------------------------------------------------- /fastchat/serve/__pycache__/__init__.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/fastchat/serve/__pycache__/__init__.cpython-310.pyc -------------------------------------------------------------------------------- /fastchat/serve/__pycache__/cli.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/fastchat/serve/__pycache__/cli.cpython-310.pyc -------------------------------------------------------------------------------- /fastchat/serve/__pycache__/inference.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/fastchat/serve/__pycache__/inference.cpython-310.pyc -------------------------------------------------------------------------------- /fastchat/serve/gateway/README.md: -------------------------------------------------------------------------------- 1 | # fastchat Nginx Gateway 2 | 3 | ## Purpose of the Gateway 4 | 5 | The Nginx gateway serves the following purposes: 6 | 7 | 1. Protects Gradio servers by acting as a firewall. 8 | 2. Facilitates dynamic mounting and unmounting of Gradio servers. 9 | 3. Provides load balancing for Gradio servers. 10 | 4. Offers additional security features, such as total connection limit. 11 | 5. Reduces attack surface by requiring only a single public port to be exposed for serving. 12 | 13 | ## Deployment and Updating of the Gateway 14 | 15 | ### Installing Nginx 16 | 17 | On Debian-based distributions (e.g., Ubuntu): 18 | 19 | ```bash 20 | sudo apt update 21 | sudo apt install nginx 22 | ``` 23 | On Red Hat-based distributions (e.g., CentOS, Fedora): 24 | 25 | ```bash 26 | sudo yum install epel-release 27 | sudo yum install nginx 28 | ``` 29 | 30 | ### Deployment 31 | 32 | Copy `nginx.conf` to `/etc/nginx/nginx.conf` (need sudo permission). 33 | 34 | Replace the port number 7860 in `server localhost:7860` with the port where you deploy the Gradio web server. 35 | 36 | Modify `upstream websocket` to configure Gradio servers behind the gateway. 37 | 38 | Lastly, update Nginx. 39 | 40 | 41 | ### HTTPS Deployment with a Public Domain URL 42 | 43 | Make sure you obtain the HTTPS certificate and the private key used to generate the certificate. 44 | 45 | Fill the addresses to your certificate and private key in the `[PATH_TO_SSL_CERT]` and `[PATH_TO_PRIVATE_KEY]` fields. 46 | 47 | If you have your own domain url to serve the chatbot, replace the chat.lmsys.org url with your own domain url. 48 | 49 | ### Updating 50 | 51 | Every time when `/etc/nginx/nginx.conf` is modified, you need to update the Nginx service: 52 | 53 | ```bash 54 | sudo nginx -t # check `/etc/nginx/nginx.conf` 55 | sudo systemctl reload nginx # restart Nginx service to load the new config 56 | sudo systemctl status nginx # check the status of the Nginx service. It should be active (running). 57 | ``` 58 | -------------------------------------------------------------------------------- /fastchat/serve/huggingface_api.py: -------------------------------------------------------------------------------- 1 | """ 2 | Use FastChat with Hugging Face generation APIs. 3 | 4 | Usage: 5 | python3 -m fastchat.serve.huggingface_api --model lmsys/vicuna-7b-v1.5 6 | python3 -m fastchat.serve.huggingface_api --model lmsys/fastchat-t5-3b-v1.0 7 | """ 8 | import argparse 9 | 10 | import torch 11 | 12 | from fastchat.model import load_model, get_conversation_template, add_model_args 13 | 14 | 15 | @torch.inference_mode() 16 | def main(args): 17 | # Load model 18 | model, tokenizer = load_model( 19 | args.model_path, 20 | device=args.device, 21 | num_gpus=args.num_gpus, 22 | max_gpu_memory=args.max_gpu_memory, 23 | load_8bit=args.load_8bit, 24 | cpu_offloading=args.cpu_offloading, 25 | revision=args.revision, 26 | debug=args.debug, 27 | ) 28 | 29 | # Build the prompt with a conversation template 30 | msg = args.message 31 | conv = get_conversation_template(args.model_path) 32 | conv.append_message(conv.roles[0], msg) 33 | conv.append_message(conv.roles[1], None) 34 | prompt = conv.get_prompt() 35 | 36 | # Run inference 37 | inputs = tokenizer([prompt], return_tensors="pt").to(args.device) 38 | output_ids = model.generate( 39 | **inputs, 40 | do_sample=True if args.temperature > 1e-5 else False, 41 | temperature=args.temperature, 42 | repetition_penalty=args.repetition_penalty, 43 | max_new_tokens=args.max_new_tokens, 44 | ) 45 | 46 | if model.config.is_encoder_decoder: 47 | output_ids = output_ids[0] 48 | else: 49 | output_ids = output_ids[0][len(inputs["input_ids"][0]) :] 50 | outputs = tokenizer.decode( 51 | output_ids, skip_special_tokens=True, spaces_between_special_tokens=False 52 | ) 53 | 54 | # Print results 55 | print(f"{conv.roles[0]}: {msg}") 56 | print(f"{conv.roles[1]}: {outputs}") 57 | 58 | 59 | if __name__ == "__main__": 60 | parser = argparse.ArgumentParser() 61 | add_model_args(parser) 62 | parser.add_argument("--temperature", type=float, default=0.7) 63 | parser.add_argument("--repetition_penalty", type=float, default=1.0) 64 | parser.add_argument("--max-new-tokens", type=int, default=1024) 65 | parser.add_argument("--debug", action="store_true") 66 | parser.add_argument("--message", type=str, default="Hello! Who are you?") 67 | args = parser.parse_args() 68 | 69 | # Reset default repetition penalty for T5 models. 70 | if "t5" in args.model_path and args.repetition_penalty == 1.0: 71 | args.repetition_penalty = 1.2 72 | 73 | main(args) 74 | -------------------------------------------------------------------------------- /fastchat/serve/monitor/dataset_release_scripts/arena_33k/count_unique_users.py: -------------------------------------------------------------------------------- 1 | """Count the unique users in a battle log file.""" 2 | 3 | import argparse 4 | import json 5 | 6 | 7 | if __name__ == "__main__": 8 | parser = argparse.ArgumentParser() 9 | parser.add_argument("--input", type=str) 10 | args = parser.parse_args() 11 | 12 | lines = json.load(open(args.input)) 13 | ct_anony_votes = 0 14 | all_users = set() 15 | all_models = set() 16 | for l in lines: 17 | if not l["anony"]: 18 | continue 19 | all_users.add(l["judge"]) 20 | all_models.add(l["model_a"]) 21 | all_models.add(l["model_b"]) 22 | ct_anony_votes += 1 23 | 24 | print(f"#anony_vote: {ct_anony_votes}, #user: {len(all_users)}") 25 | print(f"#model: {len(all_models)}") 26 | -------------------------------------------------------------------------------- /fastchat/serve/monitor/dataset_release_scripts/arena_33k/merge_field.py: -------------------------------------------------------------------------------- 1 | """Count the unique users in a battle log file.""" 2 | 3 | import argparse 4 | import json 5 | 6 | 7 | if __name__ == "__main__": 8 | parser = argparse.ArgumentParser() 9 | parser.add_argument("--input", type=str) 10 | parser.add_argument("--tag-file", type=str) 11 | args = parser.parse_args() 12 | 13 | # build index 14 | objs = json.load(open(args.tag_file)) 15 | new_field_dict = {} 16 | for obj in objs: 17 | new_field_dict[obj["question_id"]] = obj["toxic_chat"] 18 | 19 | objs = json.load(open(args.input)) 20 | for obj in objs: 21 | obj["toxic_chat_tag"] = new_field_dict[obj["question_id"]] 22 | 23 | output = args.input.replace(".json", "_added.json") 24 | with open(output, "w") as fout: 25 | json.dump(objs, fout, indent=2, ensure_ascii=False) 26 | -------------------------------------------------------------------------------- /fastchat/serve/monitor/dataset_release_scripts/arena_33k/sample.py: -------------------------------------------------------------------------------- 1 | """ 2 | Count the unique users in a battle log file. 3 | 4 | Usage: 5 | python3 -input in.json --number 1000 6 | """ 7 | 8 | import argparse 9 | import json 10 | import random 11 | 12 | K = 1000 13 | 14 | if __name__ == "__main__": 15 | parser = argparse.ArgumentParser() 16 | parser.add_argument("--input", type=str) 17 | parser.add_argument("--number", type=int, nargs="+") 18 | args = parser.parse_args() 19 | 20 | convs = json.load(open(args.input)) 21 | random.seed(0) 22 | random.shuffle(convs) 23 | 24 | for number in args.number: 25 | new_convs = convs[:number] 26 | 27 | output = args.input.replace(".json", f"_{number//K}k.json") 28 | with open(output, "w") as fout: 29 | json.dump(new_convs, fout, indent=2, ensure_ascii=False) 30 | 31 | print(f"#in: {len(convs)}, #out: {len(new_convs)}") 32 | print(f"Write to file: {output}") 33 | -------------------------------------------------------------------------------- /fastchat/serve/monitor/dataset_release_scripts/arena_33k/upload_hf_dataset.py: -------------------------------------------------------------------------------- 1 | """ 2 | Upload to huggingface. 3 | """ 4 | import json 5 | from datasets import Dataset, DatasetDict, load_dataset 6 | 7 | objs = json.load(open("clean_battle_conv_20230630_tagged_v3_pii_33k_added.json")) 8 | data = Dataset.from_list(objs) 9 | data.push_to_hub("lmsys/chatbot_arena_conversations", private=True) 10 | -------------------------------------------------------------------------------- /fastchat/serve/monitor/dataset_release_scripts/lmsys_chat_1m/approve_all.py: -------------------------------------------------------------------------------- 1 | import requests 2 | 3 | headers = {"authorization": "Bearer hf_XXX"} 4 | 5 | url = "https://huggingface.co/api/datasets/lmsys/lmsys-chat-1m/user-access-request/pending" 6 | a = requests.get(url, headers=headers) 7 | 8 | for u in a.json(): 9 | user = u["user"]["user"] 10 | url = "https://huggingface.co/api/datasets/lmsys/lmsys-chat-1m/user-access-request/grant" 11 | ret = requests.post(url, headers=headers, json={"user": user}) 12 | print(user, ret.status_code) 13 | assert ret.status_code == 200 14 | -------------------------------------------------------------------------------- /fastchat/serve/monitor/dataset_release_scripts/lmsys_chat_1m/compute_stats.py: -------------------------------------------------------------------------------- 1 | """ 2 | From colab: 3 | https://colab.research.google.com/drive/1oMdw_Lqgmd6DletSOLHsyD-Rc96cRShs?usp=sharing 4 | """ 5 | import argparse 6 | import datetime 7 | import json 8 | import os 9 | from pytz import timezone 10 | import time 11 | 12 | import kaleido 13 | import numpy as np 14 | import pandas as pd 15 | import plotly.express as px 16 | import plotly.graph_objects as go 17 | from tqdm import tqdm 18 | 19 | import plotly.io as pio 20 | 21 | pio.kaleido.scope.mathjax = None 22 | 23 | parser = argparse.ArgumentParser() 24 | parser.add_argument("--in-file", type=str, required=True) 25 | parser.add_argument("--scale", type=int, required=True) 26 | args = parser.parse_args() 27 | 28 | filename = args.in_file 29 | scale = args.scale 30 | convs = json.load(open(filename)) 31 | df = pd.DataFrame(convs) 32 | df 33 | 34 | print(f"#ips: {df['user_id'].nunique() * scale}") 35 | print(f"#models: {df['model'].nunique()}") 36 | print(f"#language: {df['language'].nunique()}") 37 | print(f"#turns: {df['turn'].mean()}") 38 | 39 | model_counts = df["model"].value_counts() * scale 40 | # print("model counts", model_counts) 41 | fig = px.bar(x=model_counts.index, y=model_counts) 42 | fig.update_layout( 43 | xaxis_title=None, 44 | yaxis_title="Count", 45 | height=200, 46 | width=950, 47 | margin=dict(l=0, r=0, t=0, b=0), 48 | ) 49 | fig.show() 50 | fig.write_image("model_count.pdf") 51 | 52 | 53 | model_counts = df["language"].value_counts().head(25) * scale 54 | fig = px.bar(x=model_counts.index, y=model_counts) 55 | fig.update_layout( 56 | xaxis_title=None, 57 | yaxis_title="Count", 58 | height=200, 59 | width=950, 60 | margin=dict(l=0, r=0, t=0, b=0), 61 | ) 62 | fig.show() 63 | fig.write_image("language_count.pdf") 64 | 65 | chat_dates = [ 66 | datetime.datetime.fromtimestamp(x, tz=timezone("US/Pacific")).strftime("%Y-%m-%d") 67 | for x in df["tstamp"] 68 | ] 69 | 70 | 71 | def to_remove(x): 72 | for d in ["08-09", "08-08", "08-07", "08-06", "08-05", "08-04"]: 73 | if d in x: 74 | return True 75 | return False 76 | 77 | 78 | chat_dates = [x for x in chat_dates if not to_remove(x)] 79 | 80 | chat_dates_counts = pd.value_counts(chat_dates) * scale 81 | print(f"mean #chat per day: {np.mean(chat_dates_counts):.2f}") 82 | 83 | fig = px.bar(x=chat_dates_counts.index, y=chat_dates_counts) 84 | fig.update_layout( 85 | xaxis_title="Dates", 86 | yaxis_title="Count", 87 | height=200, 88 | width=950, 89 | margin=dict(l=0, r=0, t=0, b=0), 90 | ) 91 | fig.show() 92 | fig.write_image("daily_conversation_count.pdf") 93 | 94 | import transformers 95 | 96 | tokenizer = transformers.AutoTokenizer.from_pretrained( 97 | "lmsys/vicuna-7b-v1.5", use_fast=False 98 | ) 99 | 100 | prompts = [] 101 | responses = [] 102 | for conv in df["conversation"]: 103 | for row in conv: 104 | if row["role"] == "user": 105 | prompts.append(row["content"]) 106 | else: 107 | responses.append(row["content"]) 108 | 109 | print(f"#prompts: {len(prompts)}") 110 | print(f"#responses: {len(responses)}") 111 | 112 | 113 | prompt_lens = [len(tokenizer(x).input_ids) for x in tqdm(prompts)] 114 | print() 115 | print(f"mean prompt len: {np.mean(prompt_lens):.2f}") 116 | 117 | response_lens = [len(tokenizer(x).input_ids) if x else 0 for x in tqdm(responses)] 118 | print() 119 | print(f"mean response len: {np.mean(response_lens):.2f}") 120 | -------------------------------------------------------------------------------- /fastchat/serve/monitor/dataset_release_scripts/lmsys_chat_1m/final_post_processing.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import json 3 | 4 | from tqdm import tqdm 5 | import numpy as np 6 | 7 | 8 | if __name__ == "__main__": 9 | parser = argparse.ArgumentParser() 10 | parser.add_argument("--in-file", type=str, required=True) 11 | args = parser.parse_args() 12 | 13 | # Read conversations 14 | convs = json.load(open(args.in_file)) 15 | print(f"#conv: {len(convs)}") 16 | 17 | # Delete some fileds 18 | for c in convs: 19 | del c["tstamp"] 20 | del c["user_id"] 21 | 22 | # Write 23 | print(f"#out conv: {len(convs)}") 24 | out_file = args.in_file.replace(".json", ".s2.json") 25 | print(f"Output to {out_file}") 26 | with open(out_file, "w") as fout: 27 | json.dump(convs, fout, indent=2, ensure_ascii=False) 28 | -------------------------------------------------------------------------------- /fastchat/serve/monitor/dataset_release_scripts/lmsys_chat_1m/instructions.md: -------------------------------------------------------------------------------- 1 | ``` 2 | export BASE=clean_conv_20230809_100k_pii 3 | export SCALE=10 4 | 5 | # filter words 6 | python3 filter_bad_conv.py --in $BASE.json 7 | 8 | # Clean up some fileds (e.g., timestamps) 9 | python3 final_post_processing.py --in $BASE.s1.json 10 | 11 | # upload to hf 12 | python3 upload_hf_dataset.py --in $BASE.s1.s2.json 13 | 14 | # Make another version with openai moderation tag 15 | python3 merge_oai_tag.py --in $BASE.s1.s2.json 16 | 17 | # Make visualizations 18 | python3 compute_stats.py --in $BASE.s1.json --scale $SCALE 19 | 20 | # Copy figures 21 | scp "atlas:/data/lmzheng/FastChat/fastchat/serve/monitor/dataset_release_scripts/lmsys_chat_1m/*.pdf" . 22 | ``` 23 | 24 | -------------------------------------------------------------------------------- /fastchat/serve/monitor/dataset_release_scripts/lmsys_chat_1m/merge_oai_tag.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import json 3 | import time 4 | 5 | from tqdm import tqdm 6 | 7 | 8 | if __name__ == "__main__": 9 | parser = argparse.ArgumentParser() 10 | parser.add_argument("--in-file", type=str, required=True) 11 | parser.add_argument("--sample", type=int) 12 | args = parser.parse_args() 13 | 14 | tag_file = "clean_conv_20230809_1.5M_oai_filter_v2.json" 15 | # tag_file = "clean_conv_20230809_1.5M_oai_filter_v2_100k.json" 16 | in_file = args.in_file 17 | tic = time.time() 18 | 19 | # Load tags 20 | print("Load tags...") 21 | tag_data = json.load(open(tag_file)) 22 | tag_dict = {} 23 | for c in tqdm(tag_data): 24 | tag_dict[c["conversation_id"]] = [x["oai_filter"] for x in c["conversation"]] 25 | print(f"elapsed: {time.time() - tic:.2f} s") 26 | 27 | # Append to input_file 28 | print("Load inputs...") 29 | input_data = json.load(open(in_file)) 30 | for c in tqdm(input_data): 31 | cid = c["conversation_id"] 32 | if cid in tag_dict: 33 | c["openai_moderation"] = tag_dict[cid] 34 | else: 35 | print(f"missing tag for conv {cid}") 36 | exit() 37 | print(f"elapsed: {time.time() - tic:.2f} s") 38 | 39 | # Write output 40 | print("Write outputs...") 41 | out_file = in_file.replace(".json", ".with_tag.json") 42 | print(f"Output to {out_file}") 43 | with open(out_file, "w") as fout: 44 | json.dump(input_data, fout, indent=2, ensure_ascii=False) 45 | print(f"elapsed: {time.time() - tic:.2f} s") 46 | -------------------------------------------------------------------------------- /fastchat/serve/monitor/dataset_release_scripts/lmsys_chat_1m/process_all.sh: -------------------------------------------------------------------------------- 1 | export BASE=clean_conv_20230809_1.5M_pii 2 | #export BASE=clean_conv_20230809_100k_pii 3 | export SCALE=1 4 | 5 | # Filter words 6 | python3 filter_bad_conv.py --in $BASE.json --sample 1000000 7 | 8 | # Clean up some fileds (e.g., timestamps) 9 | python3 final_post_processing.py --in $BASE.s1.json 10 | 11 | # Upload to hf 12 | python3 upload_hf_dataset.py --in $BASE.s1.s2.json 13 | 14 | # Make another version with openai moderation tag 15 | python3 merge_oai_tag.py --in $BASE.s1.s2.json 16 | 17 | # Make visualizations 18 | python3 compute_stats.py --in $BASE.s1.json --scale $SCALE 19 | -------------------------------------------------------------------------------- /fastchat/serve/monitor/dataset_release_scripts/lmsys_chat_1m/sample.py: -------------------------------------------------------------------------------- 1 | """ 2 | Count the unique users in a battle log file. 3 | 4 | Usage: 5 | python3 -input in.json --number 1000 6 | """ 7 | 8 | import argparse 9 | import json 10 | import random 11 | 12 | K = 1000 13 | 14 | if __name__ == "__main__": 15 | parser = argparse.ArgumentParser() 16 | parser.add_argument("--input", type=str) 17 | parser.add_argument("--number", type=int, nargs="+") 18 | args = parser.parse_args() 19 | 20 | convs = json.load(open(args.input)) 21 | random.seed(42) 22 | random.shuffle(convs) 23 | 24 | for number in args.number: 25 | new_convs = convs[:number] 26 | 27 | output = args.input.replace(".json", f"_{number//K}k.json") 28 | with open(output, "w") as fout: 29 | json.dump(new_convs, fout, indent=2, ensure_ascii=False) 30 | 31 | print(f"#in: {len(convs)}, #out: {len(new_convs)}") 32 | print(f"Write to file: {output}") 33 | -------------------------------------------------------------------------------- /fastchat/serve/monitor/dataset_release_scripts/lmsys_chat_1m/upload_hf_dataset.py: -------------------------------------------------------------------------------- 1 | """ 2 | Upload to huggingface. 3 | """ 4 | import argparse 5 | import json 6 | from datasets import Dataset, DatasetDict, load_dataset 7 | 8 | 9 | if __name__ == "__main__": 10 | parser = argparse.ArgumentParser() 11 | parser.add_argument("--in-file", type=str, required=True) 12 | args = parser.parse_args() 13 | 14 | objs = json.load(open(args.in_file)) 15 | print(f"#convs: {len(objs)}") 16 | data = Dataset.from_list(objs) 17 | data.push_to_hub("lmsys/lmsys-chat-1m", private=True) 18 | -------------------------------------------------------------------------------- /fastchat/serve/monitor/inspect_conv.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import code 3 | import datetime 4 | import json 5 | import os 6 | from pytz import timezone 7 | import time 8 | 9 | import pandas as pd 10 | from tqdm import tqdm 11 | 12 | 13 | def get_log_files(max_num_files=None): 14 | dates = [] 15 | for month in [4, 5]: 16 | for day in range(1, 32): 17 | dates.append(f"2023-{month:02d}-{day:02d}") 18 | 19 | num_servers = 14 20 | filenames = [] 21 | for d in dates: 22 | for i in range(num_servers): 23 | name = os.path.expanduser(f"~/fastchat_logs/server{i}/{d}-conv.json") 24 | if os.path.exists(name): 25 | filenames.append(name) 26 | max_num_files = max_num_files or len(filenames) 27 | filenames = filenames[-max_num_files:] 28 | return filenames 29 | 30 | 31 | def pretty_print_conversation(messages): 32 | for role, msg in messages: 33 | print(f"[[{role}]]: {msg}") 34 | 35 | 36 | def inspect_convs(log_files): 37 | data = [] 38 | for filename in tqdm(log_files, desc="read files"): 39 | for retry in range(5): 40 | try: 41 | lines = open(filename).readlines() 42 | break 43 | except FileNotFoundError: 44 | time.sleep(2) 45 | 46 | for l in lines: 47 | row = json.loads(l) 48 | 49 | if "states" not in row: 50 | continue 51 | if row["type"] not in ["leftvote", "rightvote", "bothbad_vote"]: 52 | continue 53 | 54 | model_names = row["states"][0]["model_name"], row["states"][1]["model_name"] 55 | if row["type"] == "leftvote": 56 | winner, loser = model_names[0], model_names[1] 57 | winner_conv, loser_conv = row["states"][0], row["states"][1] 58 | elif row["type"] == "rightvote": 59 | loser, winner = model_names[0], model_names[1] 60 | loser_conv, winner_conv = row["states"][0], row["states"][1] 61 | 62 | if loser == "bard" and winner == "vicuna-13b": 63 | print("=" * 20) 64 | print(f"Winner: {winner}") 65 | pretty_print_conversation(winner_conv["messages"]) 66 | print(f"Loser: {loser}") 67 | pretty_print_conversation(loser_conv["messages"]) 68 | print("=" * 20) 69 | input() 70 | 71 | # if row["type"] == "bothbad_vote" and "gpt-4" in model_names: 72 | # print("=" * 20) 73 | # print(f"Model A: {model_names[0]}") 74 | # pretty_print_conversation(row["states"][0]["messages"]) 75 | # print(f"Model B: {model_names[1]}") 76 | # pretty_print_conversation(row["states"][1]["messages"]) 77 | # print("=" * 20) 78 | # input() 79 | 80 | 81 | if __name__ == "__main__": 82 | parser = argparse.ArgumentParser() 83 | parser.add_argument("--max-num-files", type=int) 84 | args = parser.parse_args() 85 | 86 | log_files = get_log_files(args.max_num_files) 87 | inspect_convs(log_files) 88 | -------------------------------------------------------------------------------- /fastchat/serve/monitor/intersect_conv_file.py: -------------------------------------------------------------------------------- 1 | """ 2 | Take the intersection of two conversation files. 3 | 4 | Usage: python3 -m fastchat.data.merge --input input.json --conv-id conv_id_file.json --out intersect.json 5 | """ 6 | 7 | import argparse 8 | import json 9 | 10 | 11 | if __name__ == "__main__": 12 | parser = argparse.ArgumentParser() 13 | parser.add_argument("--input", type=str, required=True) 14 | parser.add_argument("--conv-id", type=str, required=True) 15 | parser.add_argument("--out-file", type=str, default="intersect.json") 16 | args = parser.parse_args() 17 | 18 | conv_id_objs = json.load(open(args.conv_id, "r")) 19 | conv_ids = set(x["conversation_id"] for x in conv_id_objs) 20 | 21 | objs = json.load(open(args.input, "r")) 22 | after_objs = [x for x in objs if x["conversation_id"] in conv_ids] 23 | 24 | print(f"#in: {len(objs)}, #out: {len(after_objs)}") 25 | json.dump(after_objs, open(args.out_file, "w"), indent=2, ensure_ascii=False) 26 | -------------------------------------------------------------------------------- /fastchat/serve/monitor/leaderboard_csv_to_html.py: -------------------------------------------------------------------------------- 1 | """ 2 | Convert a leaderboard csv file to html table used in the blog. 3 | 4 | Usage: 5 | python3 leaderboard_csv_to_html.py --in leaderboard_table_20230619.csv 6 | """ 7 | import argparse 8 | 9 | import numpy as np 10 | 11 | from fastchat.serve.monitor.monitor import load_leaderboard_table_csv 12 | 13 | 14 | def model_hyperlink(model_name, link): 15 | return f' {model_name} ' 16 | 17 | 18 | if __name__ == "__main__": 19 | parser = argparse.ArgumentParser() 20 | parser.add_argument("--input", type=str, required=True) 21 | args = parser.parse_args() 22 | 23 | data = load_leaderboard_table_csv(args.input, add_hyperlink=False) 24 | headers = [ 25 | "Model", 26 | "MT-bench (score)", 27 | "Arena Elo rating", 28 | "MMLU", 29 | "License", 30 | ] 31 | values = [] 32 | for item in data: 33 | row = [] 34 | for key in headers: 35 | value = item[key] 36 | row.append(value) 37 | row[0] = model_hyperlink(item["Model"], item["Link"]) 38 | values.append(row) 39 | values.sort(key=lambda x: -x[1] if not np.isnan(x[1]) else 1e9) 40 | 41 | for value in values: 42 | row = "" 43 | for x in value: 44 | try: 45 | if np.isnan(x): 46 | x = "-" 47 | except TypeError: 48 | pass 49 | row += f" {x} " 50 | row += "" 51 | print(row) 52 | -------------------------------------------------------------------------------- /fastchat/serve/monitor/summarize_cluster.py: -------------------------------------------------------------------------------- 1 | """ 2 | Usage: 3 | python3 summarize_cluster.py --in results_c20_kmeans_cluster.pkl --model gpt-4 --num-prompts 100 4 | python3 summarize_cluster.py --in results_c20_kmeans_cluster.pkl --model azure-gpt-4-32k --num-prompts 200 5 | """ 6 | import argparse 7 | import pickle 8 | 9 | from fastchat.llm_judge.common import ( 10 | chat_completion_openai, 11 | chat_completion_openai_azure, 12 | chat_completion_anthropic, 13 | ) 14 | from fastchat.conversation import get_conv_template 15 | 16 | 17 | def truncate_string(s, l): 18 | half = int(l // 2) 19 | return s[:half] + s[-half:] if len(s) > l else s 20 | 21 | 22 | if __name__ == "__main__": 23 | parser = argparse.ArgumentParser() 24 | parser.add_argument("--input-file", type=str, required=True) 25 | parser.add_argument("--model", type=str, default="gpt-3.5-turbo") 26 | parser.add_argument("--num-prompts", type=int, default=100) 27 | args = parser.parse_args() 28 | 29 | model = args.model 30 | 31 | cluster_infos = pickle.load(open(args.input_file, "rb")) 32 | num_total_prompts = sum([x[0] for x in cluster_infos]) 33 | 34 | topics = [] 35 | percentages = [] 36 | for i, info in enumerate(cluster_infos): 37 | num_samples, topk_prompts, random_prompts = info 38 | percentage = num_samples / num_total_prompts 39 | print( 40 | f"cluster {i}, #prompts {num_samples}, percentage: {percentage * 100:.2f}%" 41 | ) 42 | instruct = "Given a list of user messages, use less than 8 words to summarize a central topic for all messages in English. Your output should only include a single line. Try to be specific." 43 | split = int(args.num_prompts * 0.8) 44 | prompt = "\n".join( 45 | [truncate_string(x, l=200) for x in topk_prompts[:split]] 46 | + [ 47 | truncate_string(x, l=200) 48 | for x in random_prompts[: args.num_prompts - split] 49 | ] 50 | ) 51 | prompt = "BEGIN OF THE MESSAGE LIST\n" + prompt + "\nEND OF THE MESSAGE LIST." 52 | 53 | if "azure-" in model: 54 | template_name = "chatgpt" 55 | completion_func = chat_completion_openai_azure 56 | elif "gpt" in model: 57 | template_name = "chatgpt" 58 | completion_func = chat_completion_openai 59 | elif "claude" in model: 60 | template_name = "claude" 61 | completion_func = chat_completion_anthropic 62 | 63 | conv = get_conv_template(template_name) 64 | conv.set_system_message(instruct) 65 | conv.append_message(conv.roles[0], prompt) 66 | conv.append_message(conv.roles[1], None) 67 | 68 | topic = completion_func(model, conv, temperature=0, max_tokens=256) 69 | print(topic) 70 | 71 | topics.append(topic) 72 | percentages.append(round(percentage, 6)) 73 | 74 | print() 75 | print(f"topics: {topics}") 76 | print(f"percentages: {percentages}") 77 | -------------------------------------------------------------------------------- /fastchat/serve/monitor/tag_openai_moderation.py: -------------------------------------------------------------------------------- 1 | """ 2 | Add OpenAI moderation API results to all conversations. 3 | """ 4 | import argparse 5 | from concurrent.futures import ThreadPoolExecutor 6 | import json 7 | import os 8 | import time 9 | 10 | import openai 11 | import requests 12 | from tqdm import tqdm 13 | 14 | 15 | API_MAX_RETRY = 16 16 | API_RETRY_SLEEP = 10 17 | API_ERROR_OUTPUT = "$ERROR$" 18 | 19 | 20 | def tag_moderation(text): 21 | result = API_ERROR_OUTPUT 22 | for _ in range(API_MAX_RETRY): 23 | try: 24 | result = openai.Moderation.create(input=text)["results"][0] 25 | break 26 | except openai.error.OpenAIError as e: 27 | print(type(e), e) 28 | time.sleep(API_RETRY_SLEEP) 29 | 30 | return result 31 | 32 | 33 | def tag_openai_moderation(x): 34 | conv = x["conversation_a"] 35 | user_prompts = "\n".join([x["content"] for x in conv if x["role"] == "user"]) 36 | result = tag_moderation(user_prompts) 37 | x["openai_moderation"] = result 38 | 39 | 40 | if __name__ == "__main__": 41 | parser = argparse.ArgumentParser() 42 | parser.add_argument("--input", type=str, required=True) 43 | parser.add_argument( 44 | "--parallel", type=int, default=1, help="The number of concurrent API calls." 45 | ) 46 | parser.add_argument("--first-n", type=int) 47 | args = parser.parse_args() 48 | 49 | battles = json.load(open(args.input)) 50 | 51 | if args.first_n: 52 | battles = battles[: args.first_n] 53 | 54 | with ThreadPoolExecutor(args.parallel) as executor: 55 | for line in tqdm( 56 | executor.map(tag_openai_moderation, battles), total=len(battles) 57 | ): 58 | pass 59 | 60 | output = args.input.replace(".json", "_tagged.json") 61 | with open(output, "w") as fout: 62 | json.dump(battles, fout, indent=2, ensure_ascii=False) 63 | print(f"Write cleaned data to {output}") 64 | -------------------------------------------------------------------------------- /fastchat/serve/register_worker.py: -------------------------------------------------------------------------------- 1 | """ 2 | Manually register workers. 3 | 4 | Usage: 5 | python3 -m fastchat.serve.register_worker --controller http://localhost:21001 --worker-name http://localhost:21002 6 | """ 7 | 8 | import argparse 9 | 10 | import requests 11 | 12 | if __name__ == "__main__": 13 | parser = argparse.ArgumentParser() 14 | parser.add_argument("--controller-address", type=str) 15 | parser.add_argument("--worker-name", type=str) 16 | parser.add_argument("--check-heart-beat", action="store_true") 17 | args = parser.parse_args() 18 | 19 | url = args.controller_address + "/register_worker" 20 | data = { 21 | "worker_name": args.worker_name, 22 | "check_heart_beat": args.check_heart_beat, 23 | "worker_status": None, 24 | } 25 | r = requests.post(url, json=data) 26 | assert r.status_code == 200 27 | -------------------------------------------------------------------------------- /fastchat/serve/shutdown_serve.py: -------------------------------------------------------------------------------- 1 | """ 2 | Usage： 3 | python shutdown_serve.py --down all 4 | options: "all","controller","model_worker","openai_api_server"， `all` means to stop all related servers 5 | """ 6 | 7 | import argparse 8 | import os 9 | import subprocess 10 | 11 | parser = argparse.ArgumentParser() 12 | parser.add_argument( 13 | "--down", choices=["all", "controller", "model_worker", "openai_api_server"] 14 | ) 15 | args = parser.parse_args() 16 | base_shell = "ps -eo user,pid,cmd|grep fastchat.serve{}|grep -v grep|awk '{{print $2}}'|xargs kill -9" 17 | if args.down == "all": 18 | shell_script = base_shell.format("") 19 | else: 20 | serve = f".{args.down}" 21 | shell_script = base_shell.format(serve) 22 | print(f"execute shell cmd: {shell_script}") 23 | subprocess.run(shell_script, shell=True, check=True) 24 | print(f"{args.down} has been shutdown!") 25 | -------------------------------------------------------------------------------- /fastchat/serve/test_message.py: -------------------------------------------------------------------------------- 1 | """Send a test message.""" 2 | import argparse 3 | import json 4 | 5 | import requests 6 | 7 | from fastchat.model.model_adapter import get_conversation_template 8 | 9 | 10 | def main(): 11 | model_name = args.model_name 12 | 13 | if args.worker_address: 14 | worker_addr = args.worker_address 15 | else: 16 | controller_addr = args.controller_address 17 | ret = requests.post(controller_addr + "/refresh_all_workers") 18 | ret = requests.post(controller_addr + "/list_models") 19 | models = ret.json()["models"] 20 | models.sort() 21 | print(f"Models: {models}") 22 | 23 | ret = requests.post( 24 | controller_addr + "/get_worker_address", json={"model": model_name} 25 | ) 26 | worker_addr = ret.json()["address"] 27 | print(f"worker_addr: {worker_addr}") 28 | 29 | if worker_addr == "": 30 | print(f"No available workers for {model_name}") 31 | return 32 | 33 | conv = get_conversation_template(model_name) 34 | conv.append_message(conv.roles[0], args.message) 35 | conv.append_message(conv.roles[1], None) 36 | prompt = conv.get_prompt() 37 | 38 | headers = {"User-Agent": "FastChat Client"} 39 | gen_params = { 40 | "model": model_name, 41 | "prompt": prompt, 42 | "temperature": args.temperature, 43 | "max_new_tokens": args.max_new_tokens, 44 | "stop": conv.stop_str, 45 | "stop_token_ids": conv.stop_token_ids, 46 | "echo": False, 47 | } 48 | response = requests.post( 49 | worker_addr + "/worker_generate_stream", 50 | headers=headers, 51 | json=gen_params, 52 | stream=True, 53 | ) 54 | 55 | print(f"{conv.roles[0]}: {args.message}") 56 | print(f"{conv.roles[1]}: ", end="") 57 | prev = 0 58 | for chunk in response.iter_lines(decode_unicode=False, delimiter=b"\0"): 59 | if chunk: 60 | data = json.loads(chunk.decode()) 61 | output = data["text"].strip() 62 | print(output[prev:], end="", flush=True) 63 | prev = len(output) 64 | print("") 65 | 66 | 67 | if __name__ == "__main__": 68 | parser = argparse.ArgumentParser() 69 | parser.add_argument( 70 | "--controller-address", type=str, default="http://localhost:21001" 71 | ) 72 | parser.add_argument("--worker-address", type=str) 73 | parser.add_argument("--model-name", type=str, required=True) 74 | parser.add_argument("--temperature", type=float, default=0.0) 75 | parser.add_argument("--max-new-tokens", type=int, default=32) 76 | parser.add_argument( 77 | "--message", type=str, default="Tell me a story with more than 1000 words." 78 | ) 79 | args = parser.parse_args() 80 | 81 | main() 82 | -------------------------------------------------------------------------------- /fastchat/train/__pycache__/llama2_flash_attn_monkey_patch.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/fastchat/train/__pycache__/llama2_flash_attn_monkey_patch.cpython-310.pyc -------------------------------------------------------------------------------- /fastchat/train/__pycache__/llama_flash_attn_monkey_patch.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/fastchat/train/__pycache__/llama_flash_attn_monkey_patch.cpython-310.pyc -------------------------------------------------------------------------------- /fastchat/train/__pycache__/train.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/fastchat/train/__pycache__/train.cpython-310.pyc -------------------------------------------------------------------------------- /fastchat/train/train_mem.py: -------------------------------------------------------------------------------- 1 | # Make it more memory efficient by monkey patching the LLaMA model with FlashAttn. 2 | 3 | # Need to call this before importing transformers. 4 | from fastchat.train.llama2_flash_attn_monkey_patch import ( 5 | replace_llama_attn_with_flash_attn, 6 | ) 7 | 8 | replace_llama_attn_with_flash_attn() 9 | 10 | from fastchat.train.train import train 11 | 12 | if __name__ == "__main__": 13 | train() 14 | -------------------------------------------------------------------------------- /fastchat/train/train_xformers.py: -------------------------------------------------------------------------------- 1 | # Make it more memory efficient by monkey patching the LLaMA model with xformers attention. 2 | 3 | # Need to call this before importing transformers. 4 | from fastchat.train.llama_xformers_attn_monkey_patch import ( 5 | replace_llama_attn_with_xformers_attn, 6 | ) 7 | 8 | replace_llama_attn_with_xformers_attn() 9 | 10 | from fastchat.train.train import train 11 | 12 | if __name__ == "__main__": 13 | train() 14 | -------------------------------------------------------------------------------- /llava/__init__.py: -------------------------------------------------------------------------------- 1 | from .model import LlavaLlamaForCausalLM 2 | -------------------------------------------------------------------------------- /llava/__pycache__/__init__.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/llava/__pycache__/__init__.cpython-310.pyc -------------------------------------------------------------------------------- /llava/__pycache__/__init__.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/llava/__pycache__/__init__.cpython-311.pyc -------------------------------------------------------------------------------- /llava/__pycache__/constants.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/llava/__pycache__/constants.cpython-310.pyc -------------------------------------------------------------------------------- /llava/__pycache__/conversation.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/llava/__pycache__/conversation.cpython-310.pyc -------------------------------------------------------------------------------- /llava/__pycache__/mm_utils.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/llava/__pycache__/mm_utils.cpython-310.pyc -------------------------------------------------------------------------------- /llava/__pycache__/utils.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/llava/__pycache__/utils.cpython-310.pyc -------------------------------------------------------------------------------- /llava/constants.py: -------------------------------------------------------------------------------- 1 | CONTROLLER_HEART_BEAT_EXPIRATION = 30 2 | WORKER_HEART_BEAT_INTERVAL = 15 3 | 4 | LOGDIR = "." 5 | 6 | # Model Constants 7 | IGNORE_INDEX = -100 8 | IMAGE_TOKEN_INDEX = -200 9 | DEFAULT_IMAGE_TOKEN = "" 10 | DEFAULT_IMAGE_PATCH_TOKEN = "" 11 | DEFAULT_IM_START_TOKEN = "" 12 | DEFAULT_IM_END_TOKEN = "" 13 | IMAGE_PLACEHOLDER = "" 14 | -------------------------------------------------------------------------------- /llava/custom_eval/__pycache__/eval.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/llava/custom_eval/__pycache__/eval.cpython-310.pyc -------------------------------------------------------------------------------- /llava/eval/eval_pope.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | import argparse 4 | 5 | def eval_pope(answers, label_file): 6 | label_list = [json.loads(q)['label'] for q in open(label_file, 'r')] 7 | 8 | for answer in answers: 9 | text = answer['text'] 10 | 11 | # Only keep the first sentence 12 | if text.find('.') != -1: 13 | text = text.split('.')[0] 14 | 15 | text = text.replace(',', '') 16 | words = text.split(' ') 17 | if 'No' in words or 'not' in words or 'no' in words: 18 | answer['text'] = 'no' 19 | else: 20 | answer['text'] = 'yes' 21 | 22 | for i in range(len(label_list)): 23 | if label_list[i] == 'no': 24 | label_list[i] = 0 25 | else: 26 | label_list[i] = 1 27 | 28 | pred_list = [] 29 | for answer in answers: 30 | if answer['text'] == 'no': 31 | pred_list.append(0) 32 | else: 33 | pred_list.append(1) 34 | 35 | pos = 1 36 | neg = 0 37 | yes_ratio = pred_list.count(1) / len(pred_list) 38 | 39 | TP, TN, FP, FN = 0, 0, 0, 0 40 | for pred, label in zip(pred_list, label_list): 41 | if pred == pos and label == pos: 42 | TP += 1 43 | elif pred == pos and label == neg: 44 | FP += 1 45 | elif pred == neg and label == neg: 46 | TN += 1 47 | elif pred == neg and label == pos: 48 | FN += 1 49 | 50 | print('TP\tFP\tTN\tFN\t') 51 | print('{}\t{}\t{}\t{}'.format(TP, FP, TN, FN)) 52 | 53 | precision = float(TP) / float(TP + FP) 54 | recall = float(TP) / float(TP + FN) 55 | f1 = 2*precision*recall / (precision + recall) 56 | acc = (TP + TN) / (TP + TN + FP + FN) 57 | print('Accuracy: {}'.format(acc)) 58 | print('Precision: {}'.format(precision)) 59 | print('Recall: {}'.format(recall)) 60 | print('F1 score: {}'.format(f1)) 61 | print('Yes ratio: {}'.format(yes_ratio)) 62 | print('%.3f, %.3f, %.3f, %.3f, %.3f' % (f1, acc, precision, recall, yes_ratio) ) 63 | 64 | if __name__ == "__main__": 65 | parser = argparse.ArgumentParser() 66 | parser.add_argument("--annotation-dir", type=str) 67 | parser.add_argument("--question-file", type=str) 68 | parser.add_argument("--result-file", type=str) 69 | args = parser.parse_args() 70 | 71 | questions = [json.loads(line) for line in open(args.question_file)] 72 | questions = {question['question_id']: question for question in questions} 73 | answers = [json.loads(q) for q in open(args.result_file)] 74 | for file in os.listdir(args.annotation_dir): 75 | assert file.startswith('coco_pope_') 76 | assert file.endswith('.json') 77 | category = file[10:-5] 78 | cur_answers = [x for x in answers if questions[x['question_id']]['category'] == category] 79 | print('Category: {}, # samples: {}'.format(category, len(cur_answers))) 80 | eval_pope(cur_answers, os.path.join(args.annotation_dir, file)) 81 | print("====================================") 82 | -------------------------------------------------------------------------------- /llava/eval/eval_textvqa.py: -------------------------------------------------------------------------------- 1 | import os 2 | import argparse 3 | import json 4 | import re 5 | 6 | from llava.eval.m4c_evaluator import TextVQAAccuracyEvaluator 7 | 8 | 9 | def get_args(): 10 | parser = argparse.ArgumentParser() 11 | parser.add_argument('--annotation-file', type=str) 12 | parser.add_argument('--result-file', type=str) 13 | parser.add_argument('--result-dir', type=str) 14 | return parser.parse_args() 15 | 16 | 17 | def prompt_processor(prompt): 18 | if prompt.startswith('OCR tokens: '): 19 | pattern = r"Question: (.*?) Short answer:" 20 | match = re.search(pattern, prompt, re.DOTALL) 21 | question = match.group(1) 22 | elif 'Reference OCR token: ' in prompt and len(prompt.split('\n')) == 3: 23 | if prompt.startswith('Reference OCR token:'): 24 | question = prompt.split('\n')[1] 25 | else: 26 | question = prompt.split('\n')[0] 27 | elif len(prompt.split('\n')) == 2: 28 | question = prompt.split('\n')[0] 29 | else: 30 | assert False 31 | 32 | return question.lower() 33 | 34 | 35 | def eval_single(annotation_file, result_file): 36 | experiment_name = os.path.splitext(os.path.basename(result_file))[0] 37 | print(experiment_name) 38 | annotations = json.load(open(annotation_file))['data'] 39 | annotations = {(annotation['image_id'], annotation['question'].lower()): annotation for annotation in annotations} 40 | results = [json.loads(line) for line in open(result_file)] 41 | 42 | pred_list = [] 43 | for result in results: 44 | annotation = annotations[(result['question_id'], prompt_processor(result['prompt']))] 45 | pred_list.append({ 46 | "pred_answer": result['text'], 47 | "gt_answers": annotation['answers'], 48 | }) 49 | 50 | evaluator = TextVQAAccuracyEvaluator() 51 | print('Samples: {}\nAccuracy: {:.2f}%\n'.format(len(pred_list), 100. * evaluator.eval_pred_list(pred_list))) 52 | 53 | 54 | if __name__ == "__main__": 55 | args = get_args() 56 | 57 | if args.result_file is not None: 58 | eval_single(args.annotation_file, args.result_file) 59 | 60 | if args.result_dir is not None: 61 | for result_file in sorted(os.listdir(args.result_dir)): 62 | if not result_file.endswith('.jsonl'): 63 | print(f'Skipping {result_file}') 64 | continue 65 | eval_single(args.annotation_file, os.path.join(args.result_dir, result_file)) 66 | -------------------------------------------------------------------------------- /llava/eval/model_qa.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | from transformers import AutoTokenizer, AutoModelForCausalLM, StoppingCriteria 3 | import torch 4 | import os 5 | import json 6 | from tqdm import tqdm 7 | import shortuuid 8 | 9 | from llava.conversation import default_conversation 10 | from llava.utils import disable_torch_init 11 | 12 | 13 | # new stopping implementation 14 | class KeywordsStoppingCriteria(StoppingCriteria): 15 | def __init__(self, keywords, tokenizer, input_ids): 16 | self.keywords = keywords 17 | self.tokenizer = tokenizer 18 | self.start_len = None 19 | self.input_ids = input_ids 20 | 21 | def __call__(self, output_ids: torch.LongTensor, scores: torch.FloatTensor, **kwargs) -> bool: 22 | if self.start_len is None: 23 | self.start_len = self.input_ids.shape[1] 24 | else: 25 | outputs = self.tokenizer.batch_decode(output_ids[:, self.start_len:], skip_special_tokens=True)[0] 26 | for keyword in self.keywords: 27 | if keyword in outputs: 28 | return True 29 | return False 30 | 31 | 32 | @torch.inference_mode() 33 | def eval_model(model_name, questions_file, answers_file): 34 | # Model 35 | disable_torch_init() 36 | model_name = os.path.expanduser(model_name) 37 | tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=False) 38 | model = AutoModelForCausalLM.from_pretrained(model_name, 39 | torch_dtype=torch.float16).cuda() 40 | 41 | 42 | ques_file = open(os.path.expanduser(questions_file), "r") 43 | ans_file = open(os.path.expanduser(answers_file), "w") 44 | for i, line in enumerate(tqdm(ques_file)): 45 | idx = json.loads(line)["question_id"] 46 | qs = json.loads(line)["text"] 47 | cat = json.loads(line)["category"] 48 | conv = default_conversation.copy() 49 | conv.append_message(conv.roles[0], qs) 50 | prompt = conv.get_prompt() 51 | inputs = tokenizer([prompt]) 52 | input_ids = torch.as_tensor(inputs.input_ids).cuda() 53 | stopping_criteria = KeywordsStoppingCriteria([conv.sep], tokenizer, input_ids) 54 | output_ids = model.generate( 55 | input_ids, 56 | do_sample=True, 57 | use_cache=True, 58 | temperature=0.7, 59 | max_new_tokens=1024, 60 | stopping_criteria=[stopping_criteria]) 61 | outputs = tokenizer.batch_decode(output_ids, skip_special_tokens=True)[0] 62 | try: 63 | index = outputs.index(conv.sep, len(prompt)) 64 | except ValueError: 65 | outputs += conv.sep 66 | index = outputs.index(conv.sep, len(prompt)) 67 | 68 | outputs = outputs[len(prompt) + len(conv.roles[1]) + 2:index].strip() 69 | ans_id = shortuuid.uuid() 70 | ans_file.write(json.dumps({"question_id": idx, 71 | "text": outputs, 72 | "answer_id": ans_id, 73 | "model_id": model_name, 74 | "metadata": {}}) + "\n") 75 | ans_file.flush() 76 | ans_file.close() 77 | 78 | if __name__ == "__main__": 79 | parser = argparse.ArgumentParser() 80 | parser.add_argument("--model-name", type=str, default="facebook/opt-350m") 81 | parser.add_argument("--question-file", type=str, default="tables/question.jsonl") 82 | parser.add_argument("--answers-file", type=str, default="answer.jsonl") 83 | args = parser.parse_args() 84 | 85 | eval_model(args.model_name, args.question_file, args.answers_file) 86 | -------------------------------------------------------------------------------- /llava/eval/qa_baseline_gpt35.py: -------------------------------------------------------------------------------- 1 | """Generate answers with GPT-3.5""" 2 | # Note: you need to be using OpenAI Python v0.27.0 for the code below to work 3 | import argparse 4 | import json 5 | import os 6 | import time 7 | import concurrent.futures 8 | 9 | import openai 10 | import tqdm 11 | import shortuuid 12 | 13 | MODEL = 'gpt-3.5-turbo' 14 | MODEL_ID = 'gpt-3.5-turbo:20230327' 15 | 16 | def get_answer(question_id: int, question: str, max_tokens: int): 17 | ans = { 18 | 'answer_id': shortuuid.uuid(), 19 | 'question_id': question_id, 20 | 'model_id': MODEL_ID, 21 | } 22 | for _ in range(3): 23 | try: 24 | response = openai.ChatCompletion.create( 25 | model=MODEL, 26 | messages=[{ 27 | 'role': 'system', 28 | 'content': 'You are a helpful assistant.' 29 | }, { 30 | 'role': 'user', 31 | 'content': question, 32 | }], 33 | max_tokens=max_tokens, 34 | ) 35 | ans['text'] = response['choices'][0]['message']['content'] 36 | return ans 37 | except Exception as e: 38 | print('[ERROR]', e) 39 | ans['text'] = '#ERROR#' 40 | time.sleep(1) 41 | return ans 42 | 43 | 44 | if __name__ == '__main__': 45 | parser = argparse.ArgumentParser(description='ChatGPT answer generation.') 46 | parser.add_argument('-q', '--question') 47 | parser.add_argument('-o', '--output') 48 | parser.add_argument('--max-tokens', type=int, default=1024, help='maximum number of tokens produced in the output') 49 | args = parser.parse_args() 50 | 51 | questions_dict = {} 52 | with open(os.path.expanduser(args.question)) as f: 53 | for line in f: 54 | if not line: 55 | continue 56 | q = json.loads(line) 57 | questions_dict[q['question_id']] = q['text'] 58 | 59 | answers = [] 60 | 61 | with concurrent.futures.ThreadPoolExecutor(max_workers=32) as executor: 62 | futures = [] 63 | for qid, question in questions_dict.items(): 64 | future = executor.submit(get_answer, qid, question, args.max_tokens) 65 | futures.append(future) 66 | 67 | for future in tqdm.tqdm(concurrent.futures.as_completed(futures), total=len(futures)): 68 | answers.append(future.result()) 69 | 70 | answers.sort(key=lambda x: x['question_id']) 71 | 72 | with open(os.path.expanduser(args.output), 'w') as f: 73 | table = [json.dumps(ans) for ans in answers] 74 | f.write('\n'.join(table)) 75 | -------------------------------------------------------------------------------- /llava/eval/summarize_gpt_review.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | from collections import defaultdict 4 | 5 | import numpy as np 6 | 7 | import argparse 8 | 9 | def parse_args(): 10 | parser = argparse.ArgumentParser(description='ChatGPT-based QA evaluation.') 11 | parser.add_argument('-d', '--dir', default=None) 12 | parser.add_argument('-v', '--version', default=None) 13 | parser.add_argument('-s', '--select', nargs='*', default=None) 14 | parser.add_argument('-f', '--files', nargs='*', default=[]) 15 | parser.add_argument('-i', '--ignore', nargs='*', default=[]) 16 | return parser.parse_args() 17 | 18 | 19 | if __name__ == '__main__': 20 | args = parse_args() 21 | 22 | if args.ignore is not None: 23 | args.ignore = [int(x) for x in args.ignore] 24 | 25 | if len(args.files) > 0: 26 | review_files = args.files 27 | else: 28 | review_files = [x for x in os.listdir(args.dir) if x.endswith('.jsonl') and (x.startswith('gpt4_text') or x.startswith('reviews_') or x.startswith('review_') or 'review' in args.dir)] 29 | 30 | for review_file in sorted(review_files): 31 | config = os.path.basename(review_file).replace('gpt4_text_', '').replace('.jsonl', '') 32 | if args.select is not None and any(x not in config for x in args.select): 33 | continue 34 | if '0613' in config: 35 | version = '0613' 36 | else: 37 | version = '0314' 38 | if args.version is not None and args.version != version: 39 | continue 40 | scores = defaultdict(list) 41 | print(config) 42 | with open(os.path.join(args.dir, review_file) if args.dir is not None else review_file) as f: 43 | for review_str in f: 44 | review = json.loads(review_str) 45 | if review['question_id'] in args.ignore: 46 | continue 47 | if 'category' in review: 48 | scores[review['category']].append(review['tuple']) 49 | scores['all'].append(review['tuple']) 50 | else: 51 | if 'tuple' in review: 52 | scores['all'].append(review['tuple']) 53 | else: 54 | scores['all'].append(review['score']) 55 | for k, v in sorted(scores.items()): 56 | stats = np.asarray(v).mean(0).tolist() 57 | stats = [round(x, 3) for x in stats] 58 | # print(k, stats, round(stats[1]/stats[0]*100, 1)) 59 | print(k, round(stats[1]/stats[0]*100, 1), round(stats[0] * 10, 1), round(stats[1] * 10, 1)) 60 | print('=================================') 61 | -------------------------------------------------------------------------------- /llava/eval/table/model.jsonl: -------------------------------------------------------------------------------- 1 | {"model_id": "vicuna-13b:20230322-clean-lang", "model_name": "vicuna-13b", "model_version": "20230322-clean-lang", "model_metadata": "vicuna-13b-20230322-clean-lang"} 2 | {"model_id": "alpaca-13b:v1", "model_name": "alpaca-13b", "model_version": "v1", "model_metadata": "alpaca-13b"} 3 | {"model_id": "llama-13b:v1", "model_name": "llama-13b", "model_version": "v1", "model_metadata": "hf-llama-13b"} 4 | {"model_id": "bard:20230327", "model_name": "bard", "model_version": "20230327", "model_metadata": "Google Bard 20230327"} 5 | {"model_id": "gpt-3.5-turbo:20230327", "model_name": "gpt-3.5-turbo", "model_version": "20230327", "model_metadata": "OpenAI ChatGPT gpt-3.5-turbo Chat Completion"} 6 | -------------------------------------------------------------------------------- /llava/eval/table/reviewer.jsonl: -------------------------------------------------------------------------------- 1 | {"reviewer_id": "gpt-4-0328-default", "prompt_id": 1, "metadata": {"temperature": 0.2, "max_tokens": 1024}, "description": "GPT-4 for general questions"} 2 | {"reviewer_id": "gpt-4-0328-coding", "prompt_id": 2, "metadata": {"temperature": 0.2, "max_tokens": 1024}, "description": "GPT-4 for coding questions"} 3 | {"reviewer_id": "gpt-4-0328-math", "prompt_id": 3, "metadata": {"temperature": 0.2, "max_tokens": 1024}, "description": "GPT-4 for math questions"} 4 | {"reviewer_id": "gpt-4-0417-visual", "prompt_id": 4, "metadata": {"temperature": 0.2, "max_tokens": 1024}, "description": "GPT-4 for math questions"} 5 | -------------------------------------------------------------------------------- /llava/eval/webpage/figures/alpaca.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/llava/eval/webpage/figures/alpaca.png -------------------------------------------------------------------------------- /llava/eval/webpage/figures/bard.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/llava/eval/webpage/figures/bard.jpg -------------------------------------------------------------------------------- /llava/eval/webpage/figures/chatgpt.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /llava/eval/webpage/figures/llama.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/llava/eval/webpage/figures/llama.jpg -------------------------------------------------------------------------------- /llava/eval/webpage/figures/swords_FILL0_wght300_GRAD0_opsz48.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /llava/eval/webpage/figures/vicuna.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/llava/eval/webpage/figures/vicuna.jpeg -------------------------------------------------------------------------------- /llava/eval/webpage/styles.css: -------------------------------------------------------------------------------- 1 | body { 2 | font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif; 3 | background-color: #f8f9fa; 4 | } 5 | 6 | .navbar-dark .navbar-nav .nav-link { 7 | color: #f1cf68; 8 | font-size: 1.1rem; 9 | padding: 0.5rem 0.6rem; 10 | } 11 | 12 | .card-header { 13 | font-weight: bold; 14 | } 15 | 16 | .card { 17 | box-shadow: 0 4px 8px rgba(0, 0, 0, 0.1); 18 | transition: 0.3s; 19 | } 20 | 21 | .card:hover { 22 | box-shadow: 0 8px 16px rgba(0, 0, 0, 0.2); 23 | } 24 | 25 | button { 26 | transition: background-color 0.3s; 27 | } 28 | 29 | button:hover { 30 | background-color: #007bff; 31 | } 32 | 33 | @media (max-width: 767px) { 34 | .form-row .form-group { 35 | margin-bottom: 10px; 36 | } 37 | } 38 | 39 | /* Extra styles */ 40 | 41 | .expandable-card .card-text-container { 42 | max-height: 200px; 43 | overflow-y: hidden; 44 | position: relative; 45 | } 46 | 47 | .expandable-card.expanded .card-text-container { 48 | max-height: none; 49 | } 50 | 51 | .expand-btn { 52 | position: relative; 53 | display: none; 54 | background-color: rgba(255, 255, 255, 0.8); 55 | color: #510c75; 56 | border-color: transparent; 57 | } 58 | 59 | .expand-btn:hover { 60 | background-color: rgba(200, 200, 200, 0.8); 61 | text-decoration: none; 62 | border-color: transparent; 63 | color: #510c75; 64 | } 65 | 66 | .expand-btn:focus { 67 | outline: none; 68 | text-decoration: none; 69 | } 70 | 71 | .expandable-card:not(.expanded) .card-text-container:after { 72 | content: ""; 73 | position: absolute; 74 | bottom: 0; 75 | left: 0; 76 | width: 100%; 77 | height: 90px; 78 | background: linear-gradient(rgba(255, 255, 255, 0.2), rgba(255, 255, 255, 1)); 79 | } 80 | 81 | .expandable-card:not(.expanded) .expand-btn { 82 | margin-top: -40px; 83 | } 84 | 85 | .card-body { 86 | padding-bottom: 5px; 87 | } 88 | 89 | .vertical-flex-layout { 90 | justify-content: center; 91 | align-items: center; 92 | height: 100%; 93 | display: flex; 94 | flex-direction: column; 95 | gap: 5px; 96 | } 97 | 98 | .figure-img { 99 | max-width: 100%; 100 | height: auto; 101 | } 102 | 103 | .adjustable-font-size { 104 | font-size: calc(0.5rem + 2vw); 105 | } 106 | -------------------------------------------------------------------------------- /llava/model/__init__.py: -------------------------------------------------------------------------------- 1 | from .language_model.llava_llama import LlavaLlamaForCausalLM, LlavaConfig 2 | from .language_model.llava_mpt import LlavaMPTForCausalLM, LlavaMPTConfig 3 | -------------------------------------------------------------------------------- /llava/model/__pycache__/__init__.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/llava/model/__pycache__/__init__.cpython-310.pyc -------------------------------------------------------------------------------- /llava/model/__pycache__/__init__.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/llava/model/__pycache__/__init__.cpython-311.pyc -------------------------------------------------------------------------------- /llava/model/__pycache__/builder.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/llava/model/__pycache__/builder.cpython-310.pyc -------------------------------------------------------------------------------- /llava/model/__pycache__/llava_arch.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/llava/model/__pycache__/llava_arch.cpython-310.pyc -------------------------------------------------------------------------------- /llava/model/apply_delta.py: -------------------------------------------------------------------------------- 1 | """ 2 | Usage: 3 | python3 -m fastchat.model.apply_delta --base ~/model_weights/llama-7b --target ~/model_weights/vicuna-7b --delta lmsys/vicuna-7b-delta 4 | """ 5 | import argparse 6 | 7 | import torch 8 | from tqdm import tqdm 9 | from transformers import AutoTokenizer, AutoModelForCausalLM 10 | from llava import LlavaLlamaForCausalLM 11 | 12 | 13 | def apply_delta(base_model_path, target_model_path, delta_path): 14 | print("Loading base model") 15 | base = AutoModelForCausalLM.from_pretrained( 16 | base_model_path, torch_dtype=torch.float16, low_cpu_mem_usage=True) 17 | 18 | print("Loading delta") 19 | delta = LlavaLlamaForCausalLM.from_pretrained(delta_path, torch_dtype=torch.float16, low_cpu_mem_usage=True) 20 | delta_tokenizer = AutoTokenizer.from_pretrained(delta_path) 21 | 22 | print("Applying delta") 23 | for name, param in tqdm(delta.state_dict().items(), desc="Applying delta"): 24 | if name not in base.state_dict(): 25 | assert name in ['model.mm_projector.weight', 'model.mm_projector.bias'], f'{name} not in base model' 26 | continue 27 | if param.data.shape == base.state_dict()[name].shape: 28 | param.data += base.state_dict()[name] 29 | else: 30 | assert name in ['model.embed_tokens.weight', 'lm_head.weight'], \ 31 | f'{name} dimension mismatch: {param.data.shape} vs {base.state_dict()[name].shape}' 32 | bparam = base.state_dict()[name] 33 | param.data[:bparam.shape[0], :bparam.shape[1]] += bparam 34 | 35 | print("Saving target model") 36 | delta.save_pretrained(target_model_path) 37 | delta_tokenizer.save_pretrained(target_model_path) 38 | 39 | 40 | if __name__ == "__main__": 41 | parser = argparse.ArgumentParser() 42 | parser.add_argument("--base-model-path", type=str, required=True) 43 | parser.add_argument("--target-model-path", type=str, required=True) 44 | parser.add_argument("--delta-path", type=str, required=True) 45 | 46 | args = parser.parse_args() 47 | 48 | apply_delta(args.base_model_path, args.target_model_path, args.delta_path) 49 | -------------------------------------------------------------------------------- /llava/model/consolidate.py: -------------------------------------------------------------------------------- 1 | """ 2 | Usage: 3 | python3 -m llava.model.consolidate --src ~/model_weights/llava-7b --dst ~/model_weights/llava-7b_consolidate 4 | """ 5 | import argparse 6 | 7 | import torch 8 | from transformers import AutoTokenizer, AutoModelForCausalLM 9 | from llava.model import * 10 | from llava.model.utils import auto_upgrade 11 | 12 | 13 | def consolidate_ckpt(src_path, dst_path): 14 | print("Loading model") 15 | auto_upgrade(src_path) 16 | src_model = AutoModelForCausalLM.from_pretrained(src_path, torch_dtype=torch.float16, low_cpu_mem_usage=True) 17 | src_tokenizer = AutoTokenizer.from_pretrained(src_path, use_fast=False) 18 | src_model.save_pretrained(dst_path) 19 | src_tokenizer.save_pretrained(dst_path) 20 | 21 | 22 | if __name__ == "__main__": 23 | parser = argparse.ArgumentParser() 24 | parser.add_argument("--src", type=str, required=True) 25 | parser.add_argument("--dst", type=str, required=True) 26 | 27 | args = parser.parse_args() 28 | 29 | consolidate_ckpt(args.src, args.dst) 30 | -------------------------------------------------------------------------------- /llava/model/language_model/__pycache__/llava_llama.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/llava/model/language_model/__pycache__/llava_llama.cpython-310.pyc -------------------------------------------------------------------------------- /llava/model/language_model/__pycache__/llava_llama.cpython-311.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/llava/model/language_model/__pycache__/llava_llama.cpython-311.pyc -------------------------------------------------------------------------------- /llava/model/language_model/__pycache__/llava_mpt.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/llava/model/language_model/__pycache__/llava_mpt.cpython-310.pyc -------------------------------------------------------------------------------- /llava/model/language_model/mpt/__pycache__/adapt_tokenizer.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/llava/model/language_model/mpt/__pycache__/adapt_tokenizer.cpython-310.pyc -------------------------------------------------------------------------------- /llava/model/language_model/mpt/__pycache__/attention.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/llava/model/language_model/mpt/__pycache__/attention.cpython-310.pyc -------------------------------------------------------------------------------- /llava/model/language_model/mpt/__pycache__/blocks.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/llava/model/language_model/mpt/__pycache__/blocks.cpython-310.pyc -------------------------------------------------------------------------------- /llava/model/language_model/mpt/__pycache__/configuration_mpt.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/llava/model/language_model/mpt/__pycache__/configuration_mpt.cpython-310.pyc -------------------------------------------------------------------------------- /llava/model/language_model/mpt/__pycache__/custom_embedding.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/llava/model/language_model/mpt/__pycache__/custom_embedding.cpython-310.pyc -------------------------------------------------------------------------------- /llava/model/language_model/mpt/__pycache__/flash_attn_triton.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/llava/model/language_model/mpt/__pycache__/flash_attn_triton.cpython-310.pyc -------------------------------------------------------------------------------- /llava/model/language_model/mpt/__pycache__/hf_prefixlm_converter.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/llava/model/language_model/mpt/__pycache__/hf_prefixlm_converter.cpython-310.pyc -------------------------------------------------------------------------------- /llava/model/language_model/mpt/__pycache__/meta_init_context.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/llava/model/language_model/mpt/__pycache__/meta_init_context.cpython-310.pyc -------------------------------------------------------------------------------- /llava/model/language_model/mpt/__pycache__/modeling_mpt.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/llava/model/language_model/mpt/__pycache__/modeling_mpt.cpython-310.pyc -------------------------------------------------------------------------------- /llava/model/language_model/mpt/__pycache__/norm.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/llava/model/language_model/mpt/__pycache__/norm.cpython-310.pyc -------------------------------------------------------------------------------- /llava/model/language_model/mpt/__pycache__/param_init_fns.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/llava/model/language_model/mpt/__pycache__/param_init_fns.cpython-310.pyc -------------------------------------------------------------------------------- /llava/model/language_model/mpt/adapt_tokenizer.py: -------------------------------------------------------------------------------- 1 | from typing import Union 2 | from transformers import AutoTokenizer, PreTrainedTokenizer, PreTrainedTokenizerFast 3 | Tokenizer = Union[PreTrainedTokenizer, PreTrainedTokenizerFast] 4 | NUM_SENTINEL_TOKENS: int = 100 5 | 6 | def adapt_tokenizer_for_denoising(tokenizer: Tokenizer): 7 | """Adds sentinel tokens and padding token (if missing). 8 | 9 | Expands the tokenizer vocabulary to include sentinel tokens 10 | used in mixture-of-denoiser tasks as well as a padding token. 11 | 12 | All added tokens are added as special tokens. No tokens are 13 | added if sentinel tokens and padding token already exist. 14 | """ 15 | sentinels_to_add = [f'' for i in range(NUM_SENTINEL_TOKENS)] 16 | tokenizer.add_tokens(sentinels_to_add, special_tokens=True) 17 | if tokenizer.pad_token is None: 18 | tokenizer.add_tokens('', special_tokens=True) 19 | tokenizer.pad_token = '' 20 | assert tokenizer.pad_token_id is not None 21 | sentinels = ''.join([f'' for i in range(NUM_SENTINEL_TOKENS)]) 22 | _sentinel_token_ids = tokenizer(sentinels, add_special_tokens=False).input_ids 23 | tokenizer.sentinel_token_ids = _sentinel_token_ids 24 | 25 | class AutoTokenizerForMOD(AutoTokenizer): 26 | """AutoTokenizer + Adaptation for MOD. 27 | 28 | A simple wrapper around AutoTokenizer to make instantiating 29 | an MOD-adapted tokenizer a bit easier. 30 | 31 | MOD-adapted tokenizers have sentinel tokens (e.g., ), 32 | a padding token, and a property to get the token ids of the 33 | sentinel tokens. 34 | """ 35 | 36 | @classmethod 37 | def from_pretrained(cls, *args, **kwargs): 38 | """See `AutoTokenizer.from_pretrained` docstring.""" 39 | tokenizer = super().from_pretrained(*args, **kwargs) 40 | adapt_tokenizer_for_denoising(tokenizer) 41 | return tokenizer -------------------------------------------------------------------------------- /llava/model/language_model/mpt/blocks.py: -------------------------------------------------------------------------------- 1 | """GPT Blocks used for the GPT Model.""" 2 | from typing import Dict, Optional, Tuple 3 | import torch 4 | import torch.nn as nn 5 | from .attention import ATTN_CLASS_REGISTRY 6 | from .norm import NORM_CLASS_REGISTRY 7 | 8 | class MPTMLP(nn.Module): 9 | 10 | def __init__(self, d_model: int, expansion_ratio: int, device: Optional[str]=None): 11 | super().__init__() 12 | self.up_proj = nn.Linear(d_model, expansion_ratio * d_model, device=device) 13 | self.act = nn.GELU(approximate='none') 14 | self.down_proj = nn.Linear(expansion_ratio * d_model, d_model, device=device) 15 | self.down_proj._is_residual = True 16 | 17 | def forward(self, x): 18 | return self.down_proj(self.act(self.up_proj(x))) 19 | 20 | class MPTBlock(nn.Module): 21 | 22 | def __init__(self, d_model: int, n_heads: int, expansion_ratio: int, attn_config: Dict={'attn_type': 'multihead_attention', 'attn_pdrop': 0.0, 'attn_impl': 'triton', 'qk_ln': False, 'clip_qkv': None, 'softmax_scale': None, 'prefix_lm': False, 'attn_uses_sequence_id': False, 'alibi': False, 'alibi_bias_max': 8}, resid_pdrop: float=0.0, norm_type: str='low_precision_layernorm', verbose: int=0, device: Optional[str]=None, **kwargs): 23 | del kwargs 24 | super().__init__() 25 | norm_class = NORM_CLASS_REGISTRY[norm_type.lower()] 26 | attn_class = ATTN_CLASS_REGISTRY[attn_config['attn_type']] 27 | self.norm_1 = norm_class(d_model, device=device) 28 | self.attn = attn_class(attn_impl=attn_config['attn_impl'], clip_qkv=attn_config['clip_qkv'], qk_ln=attn_config['qk_ln'], softmax_scale=attn_config['softmax_scale'], attn_pdrop=attn_config['attn_pdrop'], d_model=d_model, n_heads=n_heads, verbose=verbose, device=device) 29 | self.norm_2 = norm_class(d_model, device=device) 30 | self.ffn = MPTMLP(d_model=d_model, expansion_ratio=expansion_ratio, device=device) 31 | self.resid_attn_dropout = nn.Dropout(resid_pdrop) 32 | self.resid_ffn_dropout = nn.Dropout(resid_pdrop) 33 | 34 | def forward(self, x: torch.Tensor, past_key_value: Optional[Tuple[torch.Tensor]]=None, attn_bias: Optional[torch.Tensor]=None, attention_mask: Optional[torch.ByteTensor]=None, is_causal: bool=True) -> Tuple[torch.Tensor, Optional[Tuple[torch.Tensor]]]: 35 | a = self.norm_1(x) 36 | (b, attn_weights, past_key_value) = self.attn(a, past_key_value=past_key_value, attn_bias=attn_bias, attention_mask=attention_mask, is_causal=is_causal) 37 | x = x + self.resid_attn_dropout(b) 38 | m = self.norm_2(x) 39 | n = self.ffn(m) 40 | x = x + self.resid_ffn_dropout(n) 41 | return (x, attn_weights, past_key_value) -------------------------------------------------------------------------------- /llava/model/language_model/mpt/custom_embedding.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import torch.nn.functional as F 4 | from torch import Tensor 5 | 6 | class SharedEmbedding(nn.Embedding): 7 | 8 | def forward(self, input: Tensor, unembed: bool=False) -> Tensor: 9 | if unembed: 10 | return F.linear(input, self.weight) 11 | return super().forward(input) -------------------------------------------------------------------------------- /llava/model/language_model/mpt/norm.py: -------------------------------------------------------------------------------- 1 | import torch 2 | 3 | def _cast_if_autocast_enabled(tensor): 4 | if torch.is_autocast_enabled(): 5 | if tensor.device.type == 'cuda': 6 | dtype = torch.get_autocast_gpu_dtype() 7 | elif tensor.device.type == 'cpu': 8 | dtype = torch.get_autocast_cpu_dtype() 9 | else: 10 | raise NotImplementedError() 11 | return tensor.to(dtype=dtype) 12 | return tensor 13 | 14 | class LPLayerNorm(torch.nn.LayerNorm): 15 | 16 | def __init__(self, normalized_shape, eps=1e-05, elementwise_affine=True, device=None, dtype=None): 17 | super().__init__(normalized_shape=normalized_shape, eps=eps, elementwise_affine=elementwise_affine, device=device, dtype=dtype) 18 | 19 | def forward(self, x): 20 | module_device = x.device 21 | downcast_x = _cast_if_autocast_enabled(x) 22 | downcast_weight = _cast_if_autocast_enabled(self.weight) if self.weight is not None else self.weight 23 | downcast_bias = _cast_if_autocast_enabled(self.bias) if self.bias is not None else self.bias 24 | with torch.autocast(enabled=False, device_type=module_device.type): 25 | return torch.nn.functional.layer_norm(downcast_x, self.normalized_shape, downcast_weight, downcast_bias, self.eps) 26 | 27 | def rms_norm(x, weight=None, eps=1e-05): 28 | output = x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + eps) 29 | if weight is not None: 30 | return output * weight 31 | return output 32 | 33 | class RMSNorm(torch.nn.Module): 34 | 35 | def __init__(self, normalized_shape, eps=1e-05, weight=True, dtype=None, device=None): 36 | super().__init__() 37 | self.eps = eps 38 | if weight: 39 | self.weight = torch.nn.Parameter(torch.ones(normalized_shape, dtype=dtype, device=device)) 40 | else: 41 | self.register_parameter('weight', None) 42 | 43 | def forward(self, x): 44 | return rms_norm(x.float(), self.weight, self.eps).to(dtype=x.dtype) 45 | 46 | class LPRMSNorm(RMSNorm): 47 | 48 | def __init__(self, normalized_shape, eps=1e-05, weight=True, dtype=None, device=None): 49 | super().__init__(normalized_shape=normalized_shape, eps=eps, weight=weight, dtype=dtype, device=device) 50 | 51 | def forward(self, x): 52 | downcast_x = _cast_if_autocast_enabled(x) 53 | downcast_weight = _cast_if_autocast_enabled(self.weight) if self.weight is not None else self.weight 54 | with torch.autocast(enabled=False, device_type=x.device.type): 55 | return rms_norm(downcast_x, downcast_weight, self.eps).to(dtype=x.dtype) 56 | NORM_CLASS_REGISTRY = {'layernorm': torch.nn.LayerNorm, 'low_precision_layernorm': LPLayerNorm, 'rmsnorm': RMSNorm, 'low_precision_rmsnorm': LPRMSNorm} -------------------------------------------------------------------------------- /llava/model/make_delta.py: -------------------------------------------------------------------------------- 1 | """ 2 | Usage: 3 | python3 -m llava.model.make_delta --base ~/model_weights/llama-7b --target ~/model_weights/llava-7b --delta ~/model_weights/llava-7b-delta --hub-repo-id liuhaotian/llava-7b-delta 4 | """ 5 | import argparse 6 | 7 | import torch 8 | from tqdm import tqdm 9 | from transformers import AutoTokenizer, AutoModelForCausalLM 10 | from llava.model.utils import auto_upgrade 11 | 12 | 13 | def make_delta(base_model_path, target_model_path, delta_path, hub_repo_id): 14 | print("Loading base model") 15 | base = AutoModelForCausalLM.from_pretrained( 16 | base_model_path, torch_dtype=torch.float16, low_cpu_mem_usage=True) 17 | 18 | print("Loading target model") 19 | auto_upgrade(target_model_path) 20 | target = AutoModelForCausalLM.from_pretrained(target_model_path, torch_dtype=torch.float16, low_cpu_mem_usage=True) 21 | 22 | print("Calculating delta") 23 | for name, param in tqdm(target.state_dict().items(), desc="Calculating delta"): 24 | if name not in base.state_dict(): 25 | assert name in ['model.mm_projector.weight', 'model.mm_projector.bias'], f'{name} not in base model' 26 | continue 27 | if param.data.shape == base.state_dict()[name].shape: 28 | param.data -= base.state_dict()[name] 29 | else: 30 | assert name in ['model.embed_tokens.weight', 'lm_head.weight'], f'{name} dimension mismatch: {param.data.shape} vs {base.state_dict()[name].shape}' 31 | bparam = base.state_dict()[name] 32 | param.data[:bparam.shape[0], :bparam.shape[1]] -= bparam 33 | 34 | print("Saving delta") 35 | if hub_repo_id: 36 | kwargs = {"push_to_hub": True, "repo_id": hub_repo_id} 37 | else: 38 | kwargs = {} 39 | target.save_pretrained(delta_path, **kwargs) 40 | target_tokenizer = AutoTokenizer.from_pretrained(target_model_path) 41 | target_tokenizer.save_pretrained(delta_path, **kwargs) 42 | 43 | 44 | if __name__ == "__main__": 45 | parser = argparse.ArgumentParser() 46 | parser.add_argument("--base-model-path", type=str, required=True) 47 | parser.add_argument("--target-model-path", type=str, required=True) 48 | parser.add_argument("--delta-path", type=str, required=True) 49 | parser.add_argument("--hub-repo-id", type=str, default=None) 50 | args = parser.parse_args() 51 | 52 | make_delta(args.base_model_path, args.target_model_path, args.delta_path, args.hub_repo_id) 53 | -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/__pycache__/builder.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/llava/model/multimodal_encoder/__pycache__/builder.cpython-310.pyc -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/__pycache__/clip_encoder.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/llava/model/multimodal_encoder/__pycache__/clip_encoder.cpython-310.pyc -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/builder.py: -------------------------------------------------------------------------------- 1 | import os 2 | from .clip_encoder import CLIPVisionTower 3 | 4 | 5 | def build_vision_tower(vision_tower_cfg, **kwargs): 6 | vision_tower = getattr(vision_tower_cfg, 'mm_vision_tower', getattr(vision_tower_cfg, 'vision_tower', None)) 7 | is_absolute_path_exists = os.path.exists(vision_tower) 8 | if is_absolute_path_exists or vision_tower.startswith("openai") or vision_tower.startswith("laion"): 9 | return CLIPVisionTower(vision_tower, args=vision_tower_cfg, **kwargs) 10 | 11 | raise ValueError(f'Unknown vision tower: {vision_tower}') 12 | -------------------------------------------------------------------------------- /llava/model/multimodal_encoder/clip_encoder.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | 4 | from transformers import CLIPVisionModel, CLIPImageProcessor, CLIPVisionConfig 5 | 6 | 7 | class CLIPVisionTower(nn.Module): 8 | def __init__(self, vision_tower, args, delay_load=False): 9 | super().__init__() 10 | 11 | self.is_loaded = False 12 | 13 | self.vision_tower_name = vision_tower 14 | self.select_layer = args.mm_vision_select_layer 15 | self.select_feature = getattr(args, 'mm_vision_select_feature', 'patch') 16 | 17 | if not delay_load: 18 | self.load_model() 19 | else: 20 | self.cfg_only = CLIPVisionConfig.from_pretrained(self.vision_tower_name) 21 | 22 | def load_model(self): 23 | self.image_processor = CLIPImageProcessor.from_pretrained(self.vision_tower_name) 24 | self.vision_tower = CLIPVisionModel.from_pretrained(self.vision_tower_name) 25 | self.vision_tower.requires_grad_(False) 26 | 27 | self.is_loaded = True 28 | 29 | def feature_select(self, image_forward_outs): 30 | image_features = image_forward_outs.hidden_states[self.select_layer] 31 | if self.select_feature == 'patch': 32 | image_features = image_features[:, 1:] 33 | elif self.select_feature == 'cls_patch': 34 | image_features = image_features 35 | else: 36 | raise ValueError(f'Unexpected select feature: {self.select_feature}') 37 | return image_features 38 | 39 | @torch.no_grad() 40 | def forward(self, images): 41 | if type(images) is list: 42 | image_features = [] 43 | for image in images: 44 | image_forward_out = self.vision_tower(image.to(device=self.device, dtype=self.dtype).unsqueeze(0), output_hidden_states=True) 45 | image_feature = self.feature_select(image_forward_out).to(image.dtype) 46 | image_features.append(image_feature) 47 | else: 48 | image_forward_outs = self.vision_tower(images.to(device=self.device, dtype=self.dtype), output_hidden_states=True) 49 | image_features = self.feature_select(image_forward_outs).to(images.dtype) 50 | 51 | return image_features 52 | 53 | @property 54 | def dummy_feature(self): 55 | return torch.zeros(1, self.hidden_size, device=self.device, dtype=self.dtype) 56 | 57 | @property 58 | def dtype(self): 59 | return self.vision_tower.dtype 60 | 61 | @property 62 | def device(self): 63 | return self.vision_tower.device 64 | 65 | @property 66 | def config(self): 67 | if self.is_loaded: 68 | return self.vision_tower.config 69 | else: 70 | return self.cfg_only 71 | 72 | @property 73 | def hidden_size(self): 74 | return self.config.hidden_size 75 | 76 | @property 77 | def num_patches(self): 78 | return (self.config.image_size // self.config.patch_size) ** 2 79 | -------------------------------------------------------------------------------- /llava/model/multimodal_projector/__pycache__/builder.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/llava/model/multimodal_projector/__pycache__/builder.cpython-310.pyc -------------------------------------------------------------------------------- /llava/model/multimodal_projector/builder.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn as nn 3 | import re 4 | 5 | 6 | class IdentityMap(nn.Module): 7 | def __init__(self): 8 | super().__init__() 9 | 10 | def forward(self, x, *args, **kwargs): 11 | return x 12 | 13 | @property 14 | def config(self): 15 | return {"mm_projector_type": 'identity'} 16 | 17 | 18 | class SimpleResBlock(nn.Module): 19 | def __init__(self, channels): 20 | super().__init__() 21 | self.pre_norm = nn.LayerNorm(channels) 22 | 23 | self.proj = nn.Sequential( 24 | nn.Linear(channels, channels), 25 | nn.GELU(), 26 | nn.Linear(channels, channels) 27 | ) 28 | def forward(self, x): 29 | x = self.pre_norm(x) 30 | return x + self.proj(x) 31 | 32 | 33 | def build_vision_projector(config, delay_load=False, **kwargs): 34 | projector_type = getattr(config, 'mm_projector_type', 'linear') 35 | 36 | if projector_type == 'linear': 37 | return nn.Linear(config.mm_hidden_size, config.hidden_size) 38 | 39 | mlp_gelu_match = re.match(r'^mlp(\d+)x_gelu$', projector_type) 40 | if mlp_gelu_match: 41 | mlp_depth = int(mlp_gelu_match.group(1)) 42 | modules = [nn.Linear(config.mm_hidden_size, config.hidden_size)] 43 | for _ in range(1, mlp_depth): 44 | modules.append(nn.GELU()) 45 | modules.append(nn.Linear(config.hidden_size, config.hidden_size)) 46 | return nn.Sequential(*modules) 47 | 48 | if projector_type == 'identity': 49 | return IdentityMap() 50 | 51 | raise ValueError(f'Unknown projector type: {projector_type}') 52 | -------------------------------------------------------------------------------- /llava/model/utils.py: -------------------------------------------------------------------------------- 1 | from transformers import AutoConfig 2 | 3 | 4 | def auto_upgrade(config): 5 | cfg = AutoConfig.from_pretrained(config) 6 | if 'llava' in config and 'llava' not in cfg.model_type: 7 | assert cfg.model_type == 'llama' 8 | print("You are using newer LLaVA code base, while the checkpoint of v0 is from older code base.") 9 | print("You must upgrade the checkpoint to the new code base (this can be done automatically).") 10 | confirm = input("Please confirm that you want to upgrade the checkpoint. [Y/N]") 11 | if confirm.lower() in ["y", "yes"]: 12 | print("Upgrading checkpoint...") 13 | assert len(cfg.architectures) == 1 14 | setattr(cfg.__class__, "model_type", "llava") 15 | cfg.architectures[0] = 'LlavaLlamaForCausalLM' 16 | cfg.save_pretrained(config) 17 | print("Checkpoint upgraded.") 18 | else: 19 | print("Checkpoint upgrade aborted.") 20 | exit(1) 21 | -------------------------------------------------------------------------------- /llava/serve/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/llava/serve/__init__.py -------------------------------------------------------------------------------- /llava/serve/examples/extreme_ironing.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/llava/serve/examples/extreme_ironing.jpg -------------------------------------------------------------------------------- /llava/serve/examples/waterview.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/llava/serve/examples/waterview.jpg -------------------------------------------------------------------------------- /llava/serve/register_worker.py: -------------------------------------------------------------------------------- 1 | """ 2 | Manually register workers. 3 | 4 | Usage: 5 | python3 -m fastchat.serve.register_worker --controller http://localhost:21001 --worker-name http://localhost:21002 6 | """ 7 | 8 | import argparse 9 | 10 | import requests 11 | 12 | if __name__ == "__main__": 13 | parser = argparse.ArgumentParser() 14 | parser.add_argument("--controller-address", type=str) 15 | parser.add_argument("--worker-name", type=str) 16 | parser.add_argument("--check-heart-beat", action="store_true") 17 | args = parser.parse_args() 18 | 19 | url = args.controller_address + "/register_worker" 20 | data = { 21 | "worker_name": args.worker_name, 22 | "check_heart_beat": args.check_heart_beat, 23 | "worker_status": None, 24 | } 25 | r = requests.post(url, json=data) 26 | assert r.status_code == 200 27 | -------------------------------------------------------------------------------- /llava/serve/test_message.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import json 3 | 4 | import requests 5 | 6 | from llava.conversation import default_conversation 7 | 8 | 9 | def main(): 10 | if args.worker_address: 11 | worker_addr = args.worker_address 12 | else: 13 | controller_addr = args.controller_address 14 | ret = requests.post(controller_addr + "/refresh_all_workers") 15 | ret = requests.post(controller_addr + "/list_models") 16 | models = ret.json()["models"] 17 | models.sort() 18 | print(f"Models: {models}") 19 | 20 | ret = requests.post(controller_addr + "/get_worker_address", 21 | json={"model": args.model_name}) 22 | worker_addr = ret.json()["address"] 23 | print(f"worker_addr: {worker_addr}") 24 | 25 | if worker_addr == "": 26 | return 27 | 28 | conv = default_conversation.copy() 29 | conv.append_message(conv.roles[0], args.message) 30 | prompt = conv.get_prompt() 31 | 32 | headers = {"User-Agent": "LLaVA Client"} 33 | pload = { 34 | "model": args.model_name, 35 | "prompt": prompt, 36 | "max_new_tokens": args.max_new_tokens, 37 | "temperature": 0.7, 38 | "stop": conv.sep, 39 | } 40 | response = requests.post(worker_addr + "/worker_generate_stream", headers=headers, 41 | json=pload, stream=True) 42 | 43 | print(prompt.replace(conv.sep, "\n"), end="") 44 | for chunk in response.iter_lines(chunk_size=8192, decode_unicode=False, delimiter=b"\0"): 45 | if chunk: 46 | data = json.loads(chunk.decode("utf-8")) 47 | output = data["text"].split(conv.sep)[-1] 48 | print(output, end="\r") 49 | print("") 50 | 51 | 52 | if __name__ == "__main__": 53 | parser = argparse.ArgumentParser() 54 | parser.add_argument("--controller-address", type=str, default="http://localhost:21001") 55 | parser.add_argument("--worker-address", type=str) 56 | parser.add_argument("--model-name", type=str, default="facebook/opt-350m") 57 | parser.add_argument("--max-new-tokens", type=int, default=32) 58 | parser.add_argument("--message", type=str, default= 59 | "Tell me a story with more than 1000 words.") 60 | args = parser.parse_args() 61 | 62 | main() 63 | -------------------------------------------------------------------------------- /llava/test_topo.py: -------------------------------------------------------------------------------- 1 | 2 | import networkx as nx 3 | 4 | 5 | # 检查哈密顿路径是否可行 6 | def is_hamiltonian_path(G, path): 7 | # 检查路径是否包含每个节点 8 | if set(path) != set(G.nodes): 9 | return False 10 | # 检查路径是否连续 11 | for i in range(len(path) - 1): 12 | if not G.has_edge(path[i], path[i + 1]): 13 | return False 14 | return True 15 | outputs = "0->2->4->1->5->3." 16 | image_file = "/mnt/sdb1/NLGraph/NLGraph/hamilton/image/easy/standard/graph8_dot_ellipse_1.0_white_filled.png" 17 | candidate_path = outputs.split(".")[0].split('->') 18 | candidate_path = list(map(int, candidate_path)) 19 | graph_file = image_file.replace("image", "graph").split("_")[0] + '.txt' 20 | 21 | 22 | 23 | # 创建无向图 24 | G = nx.Graph() 25 | with open(graph_file,"r") as f: 26 | n, m = [int(x) for x in next(f).split()] 27 | array = [] 28 | for line in f: # read rest of lines 29 | array.append([int(x) for x in line.split()]) 30 | edges = array[:m] 31 | assert len(edges) == m 32 | G.add_nodes_from(range(n)) 33 | for edge in edges: 34 | G.add_edge(edge[0], edge[1]) 35 | print(G, candidate_path) 36 | if is_hamiltonian_path(G=G, path=candidate_path): print('true') 37 | else: print('false') -------------------------------------------------------------------------------- /llava/train/__pycache__/linkpred_builder.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/llava/train/__pycache__/linkpred_builder.cpython-310.pyc -------------------------------------------------------------------------------- /llava/train/__pycache__/llama_flash_attn_monkey_patch.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/llava/train/__pycache__/llama_flash_attn_monkey_patch.cpython-310.pyc -------------------------------------------------------------------------------- /llava/train/__pycache__/llava_trainer.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/llava/train/__pycache__/llava_trainer.cpython-310.pyc -------------------------------------------------------------------------------- /llava/train/__pycache__/nodecls_builder.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/llava/train/__pycache__/nodecls_builder.cpython-310.pyc -------------------------------------------------------------------------------- /llava/train/__pycache__/train.cpython-310.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/WEIYanbin1999/GITA/67eac669068fc5646bebf26b743a7f57c77c3c19/llava/train/__pycache__/train.cpython-310.pyc -------------------------------------------------------------------------------- /llava/train/train_mem.py: -------------------------------------------------------------------------------- 1 | # Adopted from https://github.com/lm-sys/FastChat. Below is the original copyright: 2 | # Adopted from tatsu-lab@stanford_alpaca. Below is the original copyright: 3 | # Make it more memory efficient by monkey patching the LLaMA model with FlashAttn. 4 | 5 | # Need to call this before importing transformers. 6 | from llava.train.llama_flash_attn_monkey_patch import replace_llama_attn_with_flash_attn 7 | from llava.train.train import train 8 | 9 | replace_llama_attn_with_flash_attn() 10 | 11 | if __name__ == "__main__": 12 | train() 13 | -------------------------------------------------------------------------------- /llava/train/train_xformers.py: -------------------------------------------------------------------------------- 1 | # Make it more memory efficient by monkey patching the LLaMA model with xformers attention. 2 | 3 | # Need to call this before importing transformers. 4 | from llava.train.llama_xformers_attn_monkey_patch import ( 5 | replace_llama_attn_with_xformers_attn, 6 | ) 7 | 8 | replace_llama_attn_with_xformers_attn() 9 | 10 | from llava.train.train import train 11 | 12 | if __name__ == "__main__": 13 | train() 14 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools>=61.0"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [project] 6 | name = "gita" 7 | version = "1.1.2" 8 | description = "A language and visual assistant for graph reasoning." 9 | readme = "README.md" 10 | requires-python = ">=3.8" 11 | classifiers = [ 12 | "Programming Language :: Python :: 3", 13 | "License :: OSI Approved :: Apache Software License", 14 | ] 15 | dependencies = [ 16 | # please make sure the compability between torch version and cuda version 17 | "torch==2.0.1", "torchvision==0.15.2", "torchviz", 18 | "transformers==4.31.0", "tokenizers==0.13.3", "sentencepiece==0.1.99", "shortuuid", 19 | "accelerate==0.21.0", "peft", "bitsandbytes", 20 | "pydantic", "markdown2[all]", "numpy", "scikit-learn==1.2.2", 21 | "gradio==4.16.0", "gradio-client==0.8.1", 22 | "requests", "httpx==0.24.0", "uvicorn", "fastapi", 23 | "einops==0.6.1", "einops-exts==0.0.4", "timm==0.6.13" 24 | ] 25 | 26 | [project.optional-dependencies] 27 | train = ["deepspeed==0.12.6", "ninja", "wandb"] 28 | build = ["build", "twine"] 29 | 30 | [project.urls] 31 | "Homepage" = "https://v-graph.github.io/" 32 | "Bug Tracker" = "https://github.com/WEIYanbin1999/GIT/issues" 33 | 34 | [tool.setuptools.packages.find] 35 | exclude = ["assets*", "benchmark*", "docs", "dist*", "playground*", "scripts*", "tests*"] 36 | 37 | [tool.wheel] 38 | exclude = ["assets*", "benchmark*", "docs", "dist*", "playground*", "scripts*", "tests*"] 39 | -------------------------------------------------------------------------------- /scripts/eval/eval_loop.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | gpu_ids=( 4 | "6" 5 | ) 6 | 7 | declare -a hyper_1=( 8 | "cycle" 9 | "flow" 10 | "hamilton" 11 | "matching" 12 | "shortest_path" 13 | "topology" 14 | "connectivity" 15 | 16 | # "CiteSeer" 17 | # "Cora" 18 | # "email-Eu-core" 19 | # "PolBlogs" 20 | 21 | # "ca-GrQc" 22 | # "ca-HepTh" 23 | ) 24 | 25 | declare -a hyper_2=( 26 | "7b none none none zero-shot Text_Only GVLQA-BASE none none" 27 | "13b none none none zero-shot Text_Only GVLQA-BASE none none" 28 | 29 | # "7b none none none zero-shot Vision_Text GVLQA-BASE none none" 30 | # "13b none none none zero-shot Vision_Text GVLQA-BASE none none" 31 | 32 | # "7b none none none zero-shot Vision_Only GVLQA-BASE none none" 33 | # "13b none none none zero-shot Vision_Only GVLQA-BASE none none" 34 | 35 | # "7b 5 64 16 fine-tuned Vision_Text NODECLS True False" 36 | # "7b 10 64 16 fine-tuned Vision_Text NODECLS True False" 37 | # "7b 5 64 16 fine-tuned Vision_Text NODECLS True True" 38 | # "7b 10 64 16 fine-tuned Vision_Text NODECLS True True" 39 | # "7b 5 64 16 fine-tuned Vision_Text NODECLS False False" 40 | # "7b 10 64 16 fine-tuned Vision_Text NODECLS False False" 41 | # "7b 5 64 16 fine-tuned Vision_Text NODECLS False True" 42 | # "7b 10 64 16 fine-tuned Vision_Text NODECLS False True" 43 | ) 44 | 45 | declare -a params=() 46 | 47 | for h1 in "${hyper_1[@]}"; do 48 | for h2 in "${hyper_2[@]}"; do 49 | params+=("${h1} ${h2}") 50 | done 51 | done 52 | 53 | for gpu_index in "${!gpu_ids[@]}"; do 54 | gpu_id=${gpu_ids[$gpu_index]} 55 | start_index=$(("$gpu_index" * ${#params[@]} / ${#gpu_ids[@]})) 56 | end_index=$((("$gpu_index" + 1) * ${#params[@]} / ${#gpu_ids[@]})) 57 | 58 | for task_index in $(seq $start_index $((end_index - 1))); do 59 | bash ./scripts/eval/eval.sh $gpu_id ${params[$task_index]} & 60 | wait $! 61 | done & 62 | done 63 | 64 | wait 65 | echo "Evaluation process has completed!!!" 66 | -------------------------------------------------------------------------------- /scripts/train/finetune_lora_loop.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | gpu_ids=( 4 | "0,1,2,3,4,5,6,7" 5 | ) 6 | 7 | declare -a hyper_1=( 8 | "hamilton" 9 | "cycle" 10 | "flow" 11 | "matching" 12 | "shortest_path" 13 | "topology" 14 | "connectivity" 15 | 16 | # "CiteSeer" 17 | # "Cora" 18 | # "email-Eu-core" 19 | # "PolBlogs" 20 | 21 | # "ca-GrQc" 22 | # "ca-HepTh" 23 | ) 24 | 25 | declare -a hyper_2=( 26 | "7b 5 16 64 16 Vision_Only GVLQA-AUGET True False" 27 | "7b 10 16 64 16 Vision_Only GVLQA-BASE True False" 28 | 29 | "7b 5 32 64 16 Vision_Text GVLQA-BASE False False" 30 | "7b 10 32 64 16 Vision_Text GVLQA-BASE False False" 31 | ) 32 | 33 | declare -a params=() 34 | 35 | for h1 in "${hyper_1[@]}"; do 36 | for h2 in "${hyper_2[@]}"; do 37 | params+=("${h1} ${h2}") 38 | done 39 | done 40 | 41 | for gpu_index in "${!gpu_ids[@]}"; do 42 | gpu_id=${gpu_ids[$gpu_index]} 43 | start_index=$(("$gpu_index" * ${#params[@]} / ${#gpu_ids[@]})) 44 | end_index=$((("$gpu_index" + 1) * ${#params[@]} / ${#gpu_ids[@]})) 45 | 46 | for task_index in $(seq $start_index $((end_index - 1))); do 47 | random_port=$(shuf -i 10000-50000 -n 1) 48 | bash ./scripts/train/finetune_lora.sh $gpu_id $random_port ${params[$task_index]} & 49 | wait $! 50 | done & 51 | done 52 | 53 | wait 54 | echo "Training process has completed!!!" 55 | -------------------------------------------------------------------------------- /scripts/zero2.json: -------------------------------------------------------------------------------- 1 | { 2 | "fp16": { 3 | "enabled": "auto", 4 | "loss_scale": 0, 5 | "loss_scale_window": 1000, 6 | "initial_scale_power": 16, 7 | "hysteresis": 2, 8 | "min_loss_scale": 1 9 | }, 10 | "bf16": { 11 | "enabled": "auto" 12 | }, 13 | "train_micro_batch_size_per_gpu": "auto", 14 | "train_batch_size": "auto", 15 | "gradient_accumulation_steps": "auto", 16 | "zero_optimization": { 17 | "stage": 2, 18 | "overlap_comm": true, 19 | "contiguous_gradients": true, 20 | "sub_group_size": 1e9, 21 | "reduce_bucket_size": "auto" 22 | } 23 | } -------------------------------------------------------------------------------- /scripts/zero3.json: -------------------------------------------------------------------------------- 1 | { 2 | "fp16": { 3 | "enabled": "auto", 4 | "loss_scale": 0, 5 | "loss_scale_window": 1000, 6 | "initial_scale_power": 16, 7 | "hysteresis": 2, 8 | "min_loss_scale": 1 9 | }, 10 | "bf16": { 11 | "enabled": "auto" 12 | }, 13 | "train_micro_batch_size_per_gpu": "auto", 14 | "train_batch_size": "auto", 15 | "gradient_accumulation_steps": "auto", 16 | "zero_optimization": { 17 | "stage": 3, 18 | "overlap_comm": true, 19 | "contiguous_gradients": true, 20 | "sub_group_size": 1e9, 21 | "reduce_bucket_size": "auto", 22 | "stage3_prefetch_bucket_size": "auto", 23 | "stage3_param_persistence_threshold": "auto", 24 | "stage3_max_live_parameters": 1e9, 25 | "stage3_max_reuse_distance": 1e9, 26 | "stage3_gather_16bit_weights_on_model_save": true 27 | } 28 | } -------------------------------------------------------------------------------- /scripts/zero3_offload.json: -------------------------------------------------------------------------------- 1 | { 2 | "fp16": { 3 | "enabled": "auto", 4 | "loss_scale": 0, 5 | "loss_scale_window": 1000, 6 | "initial_scale_power": 16, 7 | "hysteresis": 2, 8 | "min_loss_scale": 1 9 | }, 10 | 11 | "bf16": { 12 | "enabled": "auto" 13 | }, 14 | 15 | "optimizer": { 16 | "type": "AdamW", 17 | "params": { 18 | "lr": "auto", 19 | "betas": "auto", 20 | "eps": "auto", 21 | "weight_decay": "auto" 22 | } 23 | }, 24 | 25 | "scheduler": { 26 | "type": "WarmupLR", 27 | "params": { 28 | "warmup_min_lr": "auto", 29 | "warmup_max_lr": "auto", 30 | "warmup_num_steps": "auto" 31 | } 32 | }, 33 | 34 | "zero_optimization": { 35 | "stage": 3, 36 | "offload_optimizer": { 37 | "device": "cpu", 38 | "pin_memory": true 39 | }, 40 | "offload_param": { 41 | "device": "cpu", 42 | "pin_memory": true 43 | }, 44 | "overlap_comm": true, 45 | "contiguous_gradients": true, 46 | "sub_group_size": 1e9, 47 | "reduce_bucket_size": "auto", 48 | "stage3_prefetch_bucket_size": "auto", 49 | "stage3_param_persistence_threshold": "auto", 50 | "stage3_max_live_parameters": 1e9, 51 | "stage3_max_reuse_distance": 1e9, 52 | "gather_16bit_weights_on_model_save": true 53 | }, 54 | "gradient_accumulation_steps": "auto", 55 | "gradient_clipping": "auto", 56 | "train_batch_size": "auto", 57 | "train_micro_batch_size_per_gpu": "auto", 58 | "steps_per_print": 1e5, 59 | "wall_clock_breakdown": false 60 | } --------------------------------------------------------------------------------