├── README.md ├── assets ├── data.png ├── pipeline.png └── teaser.png ├── eval ├── mmstar │ ├── answers │ │ └── ScaleCap-VLM-7B-LLM-72B.jsonl │ ├── captions │ │ └── ScaleCap-VLM-7B-LLM-72B.jsonl │ └── results │ │ └── ScaleCap-VLM-7B-LLM-72B_gpt-4-turbo-grade-1runs.json └── mmvet │ ├── answers │ └── ScaleCap-VLM-7B-LLM-72B.jsonl │ ├── bard_set.json │ ├── captions │ └── ScaleCap-VLM-7B-LLM-72B.jsonl │ ├── convert_answers.py │ ├── evaluate_7b.sh │ ├── evaluator.py │ ├── llava-mm-vet.jsonl │ ├── mm-vet.json │ └── results │ ├── ScaleCap-VLM-7B-LLM-72B.json │ ├── ScaleCap-VLM-7B-LLM-72B_gpt-4-turbo-cap-int-score-1runs.csv │ ├── ScaleCap-VLM-7B-LLM-72B_gpt-4-turbo-cap-score-1runs.csv │ └── ScaleCap-VLM-7B-LLM-72B_gpt-4-turbo-grade-1runs.json ├── example.py ├── grocery_file ├── initiallize_vlm_3b.ipynb └── initiallize_vlm_7b.ipynb ├── prism_benchmark ├── chartqa.py ├── mmstar.py ├── mmvet.py └── textvqa.py ├── pyproject.toml ├── scripts ├── bench_scripts │ └── launch_mmvet.sh └── launch_example.sh ├── setup.sh └── utils ├── __init__.py ├── prompts.py └── scalecap_utils.py /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cooperx521/ScaleCap/HEAD/README.md -------------------------------------------------------------------------------- /assets/data.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cooperx521/ScaleCap/HEAD/assets/data.png -------------------------------------------------------------------------------- /assets/pipeline.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cooperx521/ScaleCap/HEAD/assets/pipeline.png -------------------------------------------------------------------------------- /assets/teaser.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cooperx521/ScaleCap/HEAD/assets/teaser.png -------------------------------------------------------------------------------- /eval/mmstar/answers/ScaleCap-VLM-7B-LLM-72B.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cooperx521/ScaleCap/HEAD/eval/mmstar/answers/ScaleCap-VLM-7B-LLM-72B.jsonl -------------------------------------------------------------------------------- /eval/mmstar/captions/ScaleCap-VLM-7B-LLM-72B.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cooperx521/ScaleCap/HEAD/eval/mmstar/captions/ScaleCap-VLM-7B-LLM-72B.jsonl -------------------------------------------------------------------------------- /eval/mmstar/results/ScaleCap-VLM-7B-LLM-72B_gpt-4-turbo-grade-1runs.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cooperx521/ScaleCap/HEAD/eval/mmstar/results/ScaleCap-VLM-7B-LLM-72B_gpt-4-turbo-grade-1runs.json -------------------------------------------------------------------------------- /eval/mmvet/answers/ScaleCap-VLM-7B-LLM-72B.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cooperx521/ScaleCap/HEAD/eval/mmvet/answers/ScaleCap-VLM-7B-LLM-72B.jsonl -------------------------------------------------------------------------------- /eval/mmvet/bard_set.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cooperx521/ScaleCap/HEAD/eval/mmvet/bard_set.json -------------------------------------------------------------------------------- /eval/mmvet/captions/ScaleCap-VLM-7B-LLM-72B.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cooperx521/ScaleCap/HEAD/eval/mmvet/captions/ScaleCap-VLM-7B-LLM-72B.jsonl -------------------------------------------------------------------------------- /eval/mmvet/convert_answers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cooperx521/ScaleCap/HEAD/eval/mmvet/convert_answers.py -------------------------------------------------------------------------------- /eval/mmvet/evaluate_7b.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cooperx521/ScaleCap/HEAD/eval/mmvet/evaluate_7b.sh -------------------------------------------------------------------------------- /eval/mmvet/evaluator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cooperx521/ScaleCap/HEAD/eval/mmvet/evaluator.py -------------------------------------------------------------------------------- /eval/mmvet/llava-mm-vet.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cooperx521/ScaleCap/HEAD/eval/mmvet/llava-mm-vet.jsonl -------------------------------------------------------------------------------- /eval/mmvet/mm-vet.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cooperx521/ScaleCap/HEAD/eval/mmvet/mm-vet.json -------------------------------------------------------------------------------- /eval/mmvet/results/ScaleCap-VLM-7B-LLM-72B.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cooperx521/ScaleCap/HEAD/eval/mmvet/results/ScaleCap-VLM-7B-LLM-72B.json -------------------------------------------------------------------------------- /eval/mmvet/results/ScaleCap-VLM-7B-LLM-72B_gpt-4-turbo-cap-int-score-1runs.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cooperx521/ScaleCap/HEAD/eval/mmvet/results/ScaleCap-VLM-7B-LLM-72B_gpt-4-turbo-cap-int-score-1runs.csv -------------------------------------------------------------------------------- /eval/mmvet/results/ScaleCap-VLM-7B-LLM-72B_gpt-4-turbo-cap-score-1runs.csv: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cooperx521/ScaleCap/HEAD/eval/mmvet/results/ScaleCap-VLM-7B-LLM-72B_gpt-4-turbo-cap-score-1runs.csv -------------------------------------------------------------------------------- /eval/mmvet/results/ScaleCap-VLM-7B-LLM-72B_gpt-4-turbo-grade-1runs.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cooperx521/ScaleCap/HEAD/eval/mmvet/results/ScaleCap-VLM-7B-LLM-72B_gpt-4-turbo-grade-1runs.json -------------------------------------------------------------------------------- /example.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cooperx521/ScaleCap/HEAD/example.py -------------------------------------------------------------------------------- /grocery_file/initiallize_vlm_3b.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cooperx521/ScaleCap/HEAD/grocery_file/initiallize_vlm_3b.ipynb -------------------------------------------------------------------------------- /grocery_file/initiallize_vlm_7b.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cooperx521/ScaleCap/HEAD/grocery_file/initiallize_vlm_7b.ipynb -------------------------------------------------------------------------------- /prism_benchmark/chartqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cooperx521/ScaleCap/HEAD/prism_benchmark/chartqa.py -------------------------------------------------------------------------------- /prism_benchmark/mmstar.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cooperx521/ScaleCap/HEAD/prism_benchmark/mmstar.py -------------------------------------------------------------------------------- /prism_benchmark/mmvet.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cooperx521/ScaleCap/HEAD/prism_benchmark/mmvet.py -------------------------------------------------------------------------------- /prism_benchmark/textvqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cooperx521/ScaleCap/HEAD/prism_benchmark/textvqa.py -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cooperx521/ScaleCap/HEAD/pyproject.toml -------------------------------------------------------------------------------- /scripts/bench_scripts/launch_mmvet.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cooperx521/ScaleCap/HEAD/scripts/bench_scripts/launch_mmvet.sh -------------------------------------------------------------------------------- /scripts/launch_example.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cooperx521/ScaleCap/HEAD/scripts/launch_example.sh -------------------------------------------------------------------------------- /setup.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cooperx521/ScaleCap/HEAD/setup.sh -------------------------------------------------------------------------------- /utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /utils/prompts.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cooperx521/ScaleCap/HEAD/utils/prompts.py -------------------------------------------------------------------------------- /utils/scalecap_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Cooperx521/ScaleCap/HEAD/utils/scalecap_utils.py --------------------------------------------------------------------------------