├── .gitignore ├── A_exps └── lv1.5_7b.yml ├── B_scripts └── launch.sh ├── C_datasets ├── .gitkeep └── YOUR_DATASET │ ├── .gitkeep │ ├── Images │ └── .gitkeep │ └── Questions │ └── .gitkeep ├── D_answers └── .gitkeep ├── LICENSE ├── README.md ├── env_conda.txt ├── env_pip.txt └── src ├── __init__.py ├── constants.py ├── conversation.py ├── eval ├── __init__.py ├── eval_and_save_pope.py ├── eval_gpt_review.py ├── eval_gpt_review_bench.py ├── eval_gpt_review_visual.py ├── eval_science_qa.py ├── eval_science_qa_gpt4.py ├── eval_science_qa_gpt4_requery.py ├── eval_textvqa.py ├── generate_webpage_data_from_table.py ├── m4c_evaluator.py ├── model_qa.py ├── model_vqa.py ├── model_vqa_loader.py ├── model_vqa_mmbench.py ├── model_vqa_science.py ├── qa_baseline_gpt35.py ├── quick_eval_pope.py ├── run_llava.py └── summarize_gpt_review.py ├── inference.py ├── llm_modeling ├── __init__.py ├── modeling_llama.py └── utils.py ├── logic ├── __init__.py ├── constants.py └── logic.py ├── mm_utils.py ├── model ├── __init__.py ├── builder.py ├── configuration_llama.py ├── llava.py ├── llava_arch.py ├── multimodal_encoder │ ├── builder.py │ └── clip_encoder.py ├── multimodal_projector │ └── builder.py └── utils.py ├── stash.py └── utils.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seilk/VisAttnSink/HEAD/.gitignore -------------------------------------------------------------------------------- /A_exps/lv1.5_7b.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seilk/VisAttnSink/HEAD/A_exps/lv1.5_7b.yml -------------------------------------------------------------------------------- /B_scripts/launch.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seilk/VisAttnSink/HEAD/B_scripts/launch.sh -------------------------------------------------------------------------------- /C_datasets/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /C_datasets/YOUR_DATASET/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /C_datasets/YOUR_DATASET/Images/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /C_datasets/YOUR_DATASET/Questions/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /D_answers/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seilk/VisAttnSink/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seilk/VisAttnSink/HEAD/README.md -------------------------------------------------------------------------------- /env_conda.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seilk/VisAttnSink/HEAD/env_conda.txt -------------------------------------------------------------------------------- /env_pip.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seilk/VisAttnSink/HEAD/env_pip.txt -------------------------------------------------------------------------------- /src/__init__.py: -------------------------------------------------------------------------------- 1 | from .model.llava import TunedLlamaForCausalLM 2 | -------------------------------------------------------------------------------- /src/constants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seilk/VisAttnSink/HEAD/src/constants.py -------------------------------------------------------------------------------- /src/conversation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seilk/VisAttnSink/HEAD/src/conversation.py -------------------------------------------------------------------------------- /src/eval/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/eval/eval_and_save_pope.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seilk/VisAttnSink/HEAD/src/eval/eval_and_save_pope.py -------------------------------------------------------------------------------- /src/eval/eval_gpt_review.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seilk/VisAttnSink/HEAD/src/eval/eval_gpt_review.py -------------------------------------------------------------------------------- /src/eval/eval_gpt_review_bench.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seilk/VisAttnSink/HEAD/src/eval/eval_gpt_review_bench.py -------------------------------------------------------------------------------- /src/eval/eval_gpt_review_visual.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seilk/VisAttnSink/HEAD/src/eval/eval_gpt_review_visual.py -------------------------------------------------------------------------------- /src/eval/eval_science_qa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seilk/VisAttnSink/HEAD/src/eval/eval_science_qa.py -------------------------------------------------------------------------------- /src/eval/eval_science_qa_gpt4.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seilk/VisAttnSink/HEAD/src/eval/eval_science_qa_gpt4.py -------------------------------------------------------------------------------- /src/eval/eval_science_qa_gpt4_requery.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seilk/VisAttnSink/HEAD/src/eval/eval_science_qa_gpt4_requery.py -------------------------------------------------------------------------------- /src/eval/eval_textvqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seilk/VisAttnSink/HEAD/src/eval/eval_textvqa.py -------------------------------------------------------------------------------- /src/eval/generate_webpage_data_from_table.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seilk/VisAttnSink/HEAD/src/eval/generate_webpage_data_from_table.py -------------------------------------------------------------------------------- /src/eval/m4c_evaluator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seilk/VisAttnSink/HEAD/src/eval/m4c_evaluator.py -------------------------------------------------------------------------------- /src/eval/model_qa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seilk/VisAttnSink/HEAD/src/eval/model_qa.py -------------------------------------------------------------------------------- /src/eval/model_vqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seilk/VisAttnSink/HEAD/src/eval/model_vqa.py -------------------------------------------------------------------------------- /src/eval/model_vqa_loader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seilk/VisAttnSink/HEAD/src/eval/model_vqa_loader.py -------------------------------------------------------------------------------- /src/eval/model_vqa_mmbench.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seilk/VisAttnSink/HEAD/src/eval/model_vqa_mmbench.py -------------------------------------------------------------------------------- /src/eval/model_vqa_science.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seilk/VisAttnSink/HEAD/src/eval/model_vqa_science.py -------------------------------------------------------------------------------- /src/eval/qa_baseline_gpt35.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seilk/VisAttnSink/HEAD/src/eval/qa_baseline_gpt35.py -------------------------------------------------------------------------------- /src/eval/quick_eval_pope.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seilk/VisAttnSink/HEAD/src/eval/quick_eval_pope.py -------------------------------------------------------------------------------- /src/eval/run_llava.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seilk/VisAttnSink/HEAD/src/eval/run_llava.py -------------------------------------------------------------------------------- /src/eval/summarize_gpt_review.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seilk/VisAttnSink/HEAD/src/eval/summarize_gpt_review.py -------------------------------------------------------------------------------- /src/inference.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seilk/VisAttnSink/HEAD/src/inference.py -------------------------------------------------------------------------------- /src/llm_modeling/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seilk/VisAttnSink/HEAD/src/llm_modeling/__init__.py -------------------------------------------------------------------------------- /src/llm_modeling/modeling_llama.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seilk/VisAttnSink/HEAD/src/llm_modeling/modeling_llama.py -------------------------------------------------------------------------------- /src/llm_modeling/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seilk/VisAttnSink/HEAD/src/llm_modeling/utils.py -------------------------------------------------------------------------------- /src/logic/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seilk/VisAttnSink/HEAD/src/logic/__init__.py -------------------------------------------------------------------------------- /src/logic/constants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seilk/VisAttnSink/HEAD/src/logic/constants.py -------------------------------------------------------------------------------- /src/logic/logic.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seilk/VisAttnSink/HEAD/src/logic/logic.py -------------------------------------------------------------------------------- /src/mm_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seilk/VisAttnSink/HEAD/src/mm_utils.py -------------------------------------------------------------------------------- /src/model/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/model/builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seilk/VisAttnSink/HEAD/src/model/builder.py -------------------------------------------------------------------------------- /src/model/configuration_llama.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seilk/VisAttnSink/HEAD/src/model/configuration_llama.py -------------------------------------------------------------------------------- /src/model/llava.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seilk/VisAttnSink/HEAD/src/model/llava.py -------------------------------------------------------------------------------- /src/model/llava_arch.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seilk/VisAttnSink/HEAD/src/model/llava_arch.py -------------------------------------------------------------------------------- /src/model/multimodal_encoder/builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seilk/VisAttnSink/HEAD/src/model/multimodal_encoder/builder.py -------------------------------------------------------------------------------- /src/model/multimodal_encoder/clip_encoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seilk/VisAttnSink/HEAD/src/model/multimodal_encoder/clip_encoder.py -------------------------------------------------------------------------------- /src/model/multimodal_projector/builder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seilk/VisAttnSink/HEAD/src/model/multimodal_projector/builder.py -------------------------------------------------------------------------------- /src/model/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seilk/VisAttnSink/HEAD/src/model/utils.py -------------------------------------------------------------------------------- /src/stash.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seilk/VisAttnSink/HEAD/src/stash.py -------------------------------------------------------------------------------- /src/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/seilk/VisAttnSink/HEAD/src/utils.py --------------------------------------------------------------------------------