├── .gitignore ├── LICENSE ├── README.md ├── mmmu-pro.png ├── mmmu-pro ├── README.md ├── evaluate.py ├── infer │ ├── infer_gemini.py │ ├── infer_gpt.py │ ├── infer_lmdeploy.py │ └── infer_transformers.py ├── output │ ├── gpt-4o_standard(10 options)_cot.jsonl │ ├── gpt-4o_standard(10 options)_direct.jsonl │ ├── gpt-4o_vision_cot.jsonl │ └── gpt-4o_vision_direct.jsonl ├── prompts.yaml └── tool │ ├── README.md │ ├── data.jsonl │ ├── index.html │ ├── output │ ├── page_1.png │ ├── page_10.png │ ├── page_11.png │ ├── page_12.png │ ├── page_13.png │ ├── page_14.png │ ├── page_15.png │ ├── page_16.png │ ├── page_17.png │ ├── page_18.png │ ├── page_19.png │ ├── page_2.png │ ├── page_20.png │ ├── page_21.png │ ├── page_3.png │ ├── page_4.png │ ├── page_5.png │ ├── page_6.png │ ├── page_7.png │ ├── page_8.png │ └── page_9.png │ ├── screenshot_generator.py │ └── static │ ├── background_images │ ├── background1.jpg │ ├── background2.jpg │ ├── background3.jpg │ ├── background4.jpg │ ├── background5.jpg │ ├── background6.jpg │ ├── background7.jpg │ ├── background8.jpg │ └── background9.jpg │ └── images │ ├── .bashrc │ ├── .profile │ ├── .zcompdump-xcs-research-share-96ztq-master-0-5.8.1.zwc │ ├── ai2d_00000053.png │ ├── ai2d_00000443.png │ ├── ai2d_00000539.png │ ├── ai2d_00000667.png │ ├── ai2d_00000670.png │ ├── ai2d_00001162.png │ ├── ai2d_00001264.png │ ├── ai2d_00001317.png │ ├── ai2d_00002168.png │ ├── ai2d_00002293.png │ ├── test_Agriculture_195_1.png │ ├── test_Art_113_1.png │ ├── test_Clinical_Medicine_118_1.png │ ├── test_Clinical_Medicine_69_1.png │ ├── test_Energy_and_Power_276_1.png │ ├── test_History_134_1.png │ ├── test_History_1_1.png │ ├── test_Music_251_1.png │ ├── test_Music_251_2.png │ ├── test_Music_251_3.png │ ├── test_Music_251_4.png │ ├── test_Music_251_5.png │ ├── validation_Biology_1_1.png │ ├── validation_Biology_20_1.png │ ├── validation_Clinical_Medicine_23_1.png │ ├── validation_Clinical_Medicine_2_1.png │ ├── validation_Design_19_1.png │ ├── validation_Energy_and_Power_10_1.png │ ├── validation_Energy_and_Power_24_1.png │ ├── validation_Energy_and_Power_2_1.png │ └── validation_Music_11_1.png ├── mmmu.png └── mmmu ├── README.md ├── answer_dict_val.json ├── configs └── llava1.5.yaml ├── example_outputs ├── llava1.5_13b │ ├── Accounting │ │ └── output.json │ ├── Agriculture │ │ └── output.json │ ├── Architecture_and_Engineering │ │ └── output.json │ ├── Art │ │ └── output.json │ ├── Art_Theory │ │ └── output.json │ ├── Basic_Medical_Science │ │ └── output.json │ ├── Biology │ │ └── output.json │ ├── Chemistry │ │ └── output.json │ ├── Clinical_Medicine │ │ └── output.json │ ├── Computer_Science │ │ └── output.json │ ├── Design │ │ └── output.json │ ├── Diagnostics_and_Laboratory_Medicine │ │ └── output.json │ ├── Economics │ │ └── output.json │ ├── Electronics │ │ └── output.json │ ├── Energy_and_Power │ │ └── output.json │ ├── Finance │ │ └── output.json │ ├── Geography │ │ └── output.json │ ├── History │ │ └── output.json │ ├── Literature │ │ └── output.json │ ├── Manage │ │ └── output.json │ ├── Marketing │ │ └── output.json │ ├── Materials │ │ └── output.json │ ├── Math │ │ └── output.json │ ├── Mechanical_Engineering │ │ └── output.json │ ├── Music │ │ └── output.json │ ├── Pharmacy │ │ └── output.json │ ├── Physics │ │ └── output.json │ ├── Psychology │ │ └── output.json │ ├── Public_Health │ │ └── output.json │ ├── Sociology │ │ └── output.json │ └── total_val_output.json ├── llava1.5_13b_val.json └── qwen_vl │ ├── Accounting │ └── output.json │ ├── Agriculture │ └── output.json │ ├── Architecture_and_Engineering │ └── output.json │ ├── Art │ └── output.json │ ├── Art_Theory │ └── output.json │ ├── Basic_Medical_Science │ └── output.json │ ├── Biology │ └── output.json │ ├── Chemistry │ └── output.json │ ├── Clinical_Medicine │ └── output.json │ ├── Computer_Science │ └── output.json │ ├── Design │ └── output.json │ ├── Diagnostics_and_Laboratory_Medicine │ └── output.json │ ├── Economics │ └── output.json │ ├── Electronics │ └── output.json │ ├── Energy_and_Power │ └── output.json │ ├── Finance │ └── output.json │ ├── Geography │ └── output.json │ ├── History │ └── output.json │ ├── Literature │ └── output.json │ ├── Manage │ └── output.json │ ├── Marketing │ └── output.json │ ├── Materials │ └── output.json │ ├── Math │ └── output.json │ ├── Mechanical_Engineering │ └── output.json │ ├── Music │ └── output.json │ ├── Pharmacy │ └── output.json │ ├── Physics │ └── output.json │ ├── Psychology │ └── output.json │ ├── Public_Health │ └── output.json │ ├── Sociology │ └── output.json │ └── total_val_output.json ├── main_eval_only.py ├── main_parse_and_eval.py ├── print_results.py ├── run_llava.py └── utils ├── data_utils.py ├── eval_utils.py └── model_utils.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/README.md -------------------------------------------------------------------------------- /mmmu-pro.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro.png -------------------------------------------------------------------------------- /mmmu-pro/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/README.md -------------------------------------------------------------------------------- /mmmu-pro/evaluate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/evaluate.py -------------------------------------------------------------------------------- /mmmu-pro/infer/infer_gemini.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/infer/infer_gemini.py -------------------------------------------------------------------------------- /mmmu-pro/infer/infer_gpt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/infer/infer_gpt.py -------------------------------------------------------------------------------- /mmmu-pro/infer/infer_lmdeploy.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/infer/infer_lmdeploy.py -------------------------------------------------------------------------------- /mmmu-pro/infer/infer_transformers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/infer/infer_transformers.py -------------------------------------------------------------------------------- /mmmu-pro/output/gpt-4o_standard(10 options)_cot.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/output/gpt-4o_standard(10 options)_cot.jsonl -------------------------------------------------------------------------------- /mmmu-pro/output/gpt-4o_standard(10 options)_direct.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/output/gpt-4o_standard(10 options)_direct.jsonl -------------------------------------------------------------------------------- /mmmu-pro/output/gpt-4o_vision_cot.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/output/gpt-4o_vision_cot.jsonl -------------------------------------------------------------------------------- /mmmu-pro/output/gpt-4o_vision_direct.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/output/gpt-4o_vision_direct.jsonl -------------------------------------------------------------------------------- /mmmu-pro/prompts.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/prompts.yaml -------------------------------------------------------------------------------- /mmmu-pro/tool/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/README.md -------------------------------------------------------------------------------- /mmmu-pro/tool/data.jsonl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/data.jsonl -------------------------------------------------------------------------------- /mmmu-pro/tool/index.html: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/index.html -------------------------------------------------------------------------------- /mmmu-pro/tool/output/page_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/output/page_1.png -------------------------------------------------------------------------------- /mmmu-pro/tool/output/page_10.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/output/page_10.png -------------------------------------------------------------------------------- /mmmu-pro/tool/output/page_11.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/output/page_11.png -------------------------------------------------------------------------------- /mmmu-pro/tool/output/page_12.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/output/page_12.png -------------------------------------------------------------------------------- /mmmu-pro/tool/output/page_13.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/output/page_13.png -------------------------------------------------------------------------------- /mmmu-pro/tool/output/page_14.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/output/page_14.png -------------------------------------------------------------------------------- /mmmu-pro/tool/output/page_15.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/output/page_15.png -------------------------------------------------------------------------------- /mmmu-pro/tool/output/page_16.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/output/page_16.png -------------------------------------------------------------------------------- /mmmu-pro/tool/output/page_17.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/output/page_17.png -------------------------------------------------------------------------------- /mmmu-pro/tool/output/page_18.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/output/page_18.png -------------------------------------------------------------------------------- /mmmu-pro/tool/output/page_19.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/output/page_19.png -------------------------------------------------------------------------------- /mmmu-pro/tool/output/page_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/output/page_2.png -------------------------------------------------------------------------------- /mmmu-pro/tool/output/page_20.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/output/page_20.png -------------------------------------------------------------------------------- /mmmu-pro/tool/output/page_21.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/output/page_21.png -------------------------------------------------------------------------------- /mmmu-pro/tool/output/page_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/output/page_3.png -------------------------------------------------------------------------------- /mmmu-pro/tool/output/page_4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/output/page_4.png -------------------------------------------------------------------------------- /mmmu-pro/tool/output/page_5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/output/page_5.png -------------------------------------------------------------------------------- /mmmu-pro/tool/output/page_6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/output/page_6.png -------------------------------------------------------------------------------- /mmmu-pro/tool/output/page_7.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/output/page_7.png -------------------------------------------------------------------------------- /mmmu-pro/tool/output/page_8.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/output/page_8.png -------------------------------------------------------------------------------- /mmmu-pro/tool/output/page_9.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/output/page_9.png -------------------------------------------------------------------------------- /mmmu-pro/tool/screenshot_generator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/screenshot_generator.py -------------------------------------------------------------------------------- /mmmu-pro/tool/static/background_images/background1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/static/background_images/background1.jpg -------------------------------------------------------------------------------- /mmmu-pro/tool/static/background_images/background2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/static/background_images/background2.jpg -------------------------------------------------------------------------------- /mmmu-pro/tool/static/background_images/background3.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/static/background_images/background3.jpg -------------------------------------------------------------------------------- /mmmu-pro/tool/static/background_images/background4.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/static/background_images/background4.jpg -------------------------------------------------------------------------------- /mmmu-pro/tool/static/background_images/background5.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/static/background_images/background5.jpg -------------------------------------------------------------------------------- /mmmu-pro/tool/static/background_images/background6.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/static/background_images/background6.jpg -------------------------------------------------------------------------------- /mmmu-pro/tool/static/background_images/background7.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/static/background_images/background7.jpg -------------------------------------------------------------------------------- /mmmu-pro/tool/static/background_images/background8.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/static/background_images/background8.jpg -------------------------------------------------------------------------------- /mmmu-pro/tool/static/background_images/background9.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/static/background_images/background9.jpg -------------------------------------------------------------------------------- /mmmu-pro/tool/static/images/.bashrc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/static/images/.bashrc -------------------------------------------------------------------------------- /mmmu-pro/tool/static/images/.profile: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/static/images/.profile -------------------------------------------------------------------------------- /mmmu-pro/tool/static/images/.zcompdump-xcs-research-share-96ztq-master-0-5.8.1.zwc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/static/images/.zcompdump-xcs-research-share-96ztq-master-0-5.8.1.zwc -------------------------------------------------------------------------------- /mmmu-pro/tool/static/images/ai2d_00000053.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/static/images/ai2d_00000053.png -------------------------------------------------------------------------------- /mmmu-pro/tool/static/images/ai2d_00000443.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/static/images/ai2d_00000443.png -------------------------------------------------------------------------------- /mmmu-pro/tool/static/images/ai2d_00000539.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/static/images/ai2d_00000539.png -------------------------------------------------------------------------------- /mmmu-pro/tool/static/images/ai2d_00000667.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/static/images/ai2d_00000667.png -------------------------------------------------------------------------------- /mmmu-pro/tool/static/images/ai2d_00000670.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/static/images/ai2d_00000670.png -------------------------------------------------------------------------------- /mmmu-pro/tool/static/images/ai2d_00001162.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/static/images/ai2d_00001162.png -------------------------------------------------------------------------------- /mmmu-pro/tool/static/images/ai2d_00001264.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/static/images/ai2d_00001264.png -------------------------------------------------------------------------------- /mmmu-pro/tool/static/images/ai2d_00001317.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/static/images/ai2d_00001317.png -------------------------------------------------------------------------------- /mmmu-pro/tool/static/images/ai2d_00002168.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/static/images/ai2d_00002168.png -------------------------------------------------------------------------------- /mmmu-pro/tool/static/images/ai2d_00002293.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/static/images/ai2d_00002293.png -------------------------------------------------------------------------------- /mmmu-pro/tool/static/images/test_Agriculture_195_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/static/images/test_Agriculture_195_1.png -------------------------------------------------------------------------------- /mmmu-pro/tool/static/images/test_Art_113_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/static/images/test_Art_113_1.png -------------------------------------------------------------------------------- /mmmu-pro/tool/static/images/test_Clinical_Medicine_118_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/static/images/test_Clinical_Medicine_118_1.png -------------------------------------------------------------------------------- /mmmu-pro/tool/static/images/test_Clinical_Medicine_69_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/static/images/test_Clinical_Medicine_69_1.png -------------------------------------------------------------------------------- /mmmu-pro/tool/static/images/test_Energy_and_Power_276_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/static/images/test_Energy_and_Power_276_1.png -------------------------------------------------------------------------------- /mmmu-pro/tool/static/images/test_History_134_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/static/images/test_History_134_1.png -------------------------------------------------------------------------------- /mmmu-pro/tool/static/images/test_History_1_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/static/images/test_History_1_1.png -------------------------------------------------------------------------------- /mmmu-pro/tool/static/images/test_Music_251_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/static/images/test_Music_251_1.png -------------------------------------------------------------------------------- /mmmu-pro/tool/static/images/test_Music_251_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/static/images/test_Music_251_2.png -------------------------------------------------------------------------------- /mmmu-pro/tool/static/images/test_Music_251_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/static/images/test_Music_251_3.png -------------------------------------------------------------------------------- /mmmu-pro/tool/static/images/test_Music_251_4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/static/images/test_Music_251_4.png -------------------------------------------------------------------------------- /mmmu-pro/tool/static/images/test_Music_251_5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/static/images/test_Music_251_5.png -------------------------------------------------------------------------------- /mmmu-pro/tool/static/images/validation_Biology_1_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/static/images/validation_Biology_1_1.png -------------------------------------------------------------------------------- /mmmu-pro/tool/static/images/validation_Biology_20_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/static/images/validation_Biology_20_1.png -------------------------------------------------------------------------------- /mmmu-pro/tool/static/images/validation_Clinical_Medicine_23_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/static/images/validation_Clinical_Medicine_23_1.png -------------------------------------------------------------------------------- /mmmu-pro/tool/static/images/validation_Clinical_Medicine_2_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/static/images/validation_Clinical_Medicine_2_1.png -------------------------------------------------------------------------------- /mmmu-pro/tool/static/images/validation_Design_19_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/static/images/validation_Design_19_1.png -------------------------------------------------------------------------------- /mmmu-pro/tool/static/images/validation_Energy_and_Power_10_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/static/images/validation_Energy_and_Power_10_1.png -------------------------------------------------------------------------------- /mmmu-pro/tool/static/images/validation_Energy_and_Power_24_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/static/images/validation_Energy_and_Power_24_1.png -------------------------------------------------------------------------------- /mmmu-pro/tool/static/images/validation_Energy_and_Power_2_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/static/images/validation_Energy_and_Power_2_1.png -------------------------------------------------------------------------------- /mmmu-pro/tool/static/images/validation_Music_11_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu-pro/tool/static/images/validation_Music_11_1.png -------------------------------------------------------------------------------- /mmmu.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu.png -------------------------------------------------------------------------------- /mmmu/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/README.md -------------------------------------------------------------------------------- /mmmu/answer_dict_val.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/answer_dict_val.json -------------------------------------------------------------------------------- /mmmu/configs/llava1.5.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/configs/llava1.5.yaml -------------------------------------------------------------------------------- /mmmu/example_outputs/llava1.5_13b/Accounting/output.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/example_outputs/llava1.5_13b/Accounting/output.json -------------------------------------------------------------------------------- /mmmu/example_outputs/llava1.5_13b/Agriculture/output.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/example_outputs/llava1.5_13b/Agriculture/output.json -------------------------------------------------------------------------------- /mmmu/example_outputs/llava1.5_13b/Architecture_and_Engineering/output.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/example_outputs/llava1.5_13b/Architecture_and_Engineering/output.json -------------------------------------------------------------------------------- /mmmu/example_outputs/llava1.5_13b/Art/output.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/example_outputs/llava1.5_13b/Art/output.json -------------------------------------------------------------------------------- /mmmu/example_outputs/llava1.5_13b/Art_Theory/output.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/example_outputs/llava1.5_13b/Art_Theory/output.json -------------------------------------------------------------------------------- /mmmu/example_outputs/llava1.5_13b/Basic_Medical_Science/output.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/example_outputs/llava1.5_13b/Basic_Medical_Science/output.json -------------------------------------------------------------------------------- /mmmu/example_outputs/llava1.5_13b/Biology/output.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/example_outputs/llava1.5_13b/Biology/output.json -------------------------------------------------------------------------------- /mmmu/example_outputs/llava1.5_13b/Chemistry/output.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/example_outputs/llava1.5_13b/Chemistry/output.json -------------------------------------------------------------------------------- /mmmu/example_outputs/llava1.5_13b/Clinical_Medicine/output.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/example_outputs/llava1.5_13b/Clinical_Medicine/output.json -------------------------------------------------------------------------------- /mmmu/example_outputs/llava1.5_13b/Computer_Science/output.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/example_outputs/llava1.5_13b/Computer_Science/output.json -------------------------------------------------------------------------------- /mmmu/example_outputs/llava1.5_13b/Design/output.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/example_outputs/llava1.5_13b/Design/output.json -------------------------------------------------------------------------------- /mmmu/example_outputs/llava1.5_13b/Diagnostics_and_Laboratory_Medicine/output.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/example_outputs/llava1.5_13b/Diagnostics_and_Laboratory_Medicine/output.json -------------------------------------------------------------------------------- /mmmu/example_outputs/llava1.5_13b/Economics/output.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/example_outputs/llava1.5_13b/Economics/output.json -------------------------------------------------------------------------------- /mmmu/example_outputs/llava1.5_13b/Electronics/output.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/example_outputs/llava1.5_13b/Electronics/output.json -------------------------------------------------------------------------------- /mmmu/example_outputs/llava1.5_13b/Energy_and_Power/output.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/example_outputs/llava1.5_13b/Energy_and_Power/output.json -------------------------------------------------------------------------------- /mmmu/example_outputs/llava1.5_13b/Finance/output.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/example_outputs/llava1.5_13b/Finance/output.json -------------------------------------------------------------------------------- /mmmu/example_outputs/llava1.5_13b/Geography/output.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/example_outputs/llava1.5_13b/Geography/output.json -------------------------------------------------------------------------------- /mmmu/example_outputs/llava1.5_13b/History/output.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/example_outputs/llava1.5_13b/History/output.json -------------------------------------------------------------------------------- /mmmu/example_outputs/llava1.5_13b/Literature/output.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/example_outputs/llava1.5_13b/Literature/output.json -------------------------------------------------------------------------------- /mmmu/example_outputs/llava1.5_13b/Manage/output.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/example_outputs/llava1.5_13b/Manage/output.json -------------------------------------------------------------------------------- /mmmu/example_outputs/llava1.5_13b/Marketing/output.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/example_outputs/llava1.5_13b/Marketing/output.json -------------------------------------------------------------------------------- /mmmu/example_outputs/llava1.5_13b/Materials/output.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/example_outputs/llava1.5_13b/Materials/output.json -------------------------------------------------------------------------------- /mmmu/example_outputs/llava1.5_13b/Math/output.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/example_outputs/llava1.5_13b/Math/output.json -------------------------------------------------------------------------------- /mmmu/example_outputs/llava1.5_13b/Mechanical_Engineering/output.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/example_outputs/llava1.5_13b/Mechanical_Engineering/output.json -------------------------------------------------------------------------------- /mmmu/example_outputs/llava1.5_13b/Music/output.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/example_outputs/llava1.5_13b/Music/output.json -------------------------------------------------------------------------------- /mmmu/example_outputs/llava1.5_13b/Pharmacy/output.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/example_outputs/llava1.5_13b/Pharmacy/output.json -------------------------------------------------------------------------------- /mmmu/example_outputs/llava1.5_13b/Physics/output.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/example_outputs/llava1.5_13b/Physics/output.json -------------------------------------------------------------------------------- /mmmu/example_outputs/llava1.5_13b/Psychology/output.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/example_outputs/llava1.5_13b/Psychology/output.json -------------------------------------------------------------------------------- /mmmu/example_outputs/llava1.5_13b/Public_Health/output.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/example_outputs/llava1.5_13b/Public_Health/output.json -------------------------------------------------------------------------------- /mmmu/example_outputs/llava1.5_13b/Sociology/output.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/example_outputs/llava1.5_13b/Sociology/output.json -------------------------------------------------------------------------------- /mmmu/example_outputs/llava1.5_13b/total_val_output.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/example_outputs/llava1.5_13b/total_val_output.json -------------------------------------------------------------------------------- /mmmu/example_outputs/llava1.5_13b_val.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/example_outputs/llava1.5_13b_val.json -------------------------------------------------------------------------------- /mmmu/example_outputs/qwen_vl/Accounting/output.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/example_outputs/qwen_vl/Accounting/output.json -------------------------------------------------------------------------------- /mmmu/example_outputs/qwen_vl/Agriculture/output.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/example_outputs/qwen_vl/Agriculture/output.json -------------------------------------------------------------------------------- /mmmu/example_outputs/qwen_vl/Architecture_and_Engineering/output.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/example_outputs/qwen_vl/Architecture_and_Engineering/output.json -------------------------------------------------------------------------------- /mmmu/example_outputs/qwen_vl/Art/output.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/example_outputs/qwen_vl/Art/output.json -------------------------------------------------------------------------------- /mmmu/example_outputs/qwen_vl/Art_Theory/output.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/example_outputs/qwen_vl/Art_Theory/output.json -------------------------------------------------------------------------------- /mmmu/example_outputs/qwen_vl/Basic_Medical_Science/output.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/example_outputs/qwen_vl/Basic_Medical_Science/output.json -------------------------------------------------------------------------------- /mmmu/example_outputs/qwen_vl/Biology/output.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/example_outputs/qwen_vl/Biology/output.json -------------------------------------------------------------------------------- /mmmu/example_outputs/qwen_vl/Chemistry/output.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/example_outputs/qwen_vl/Chemistry/output.json -------------------------------------------------------------------------------- /mmmu/example_outputs/qwen_vl/Clinical_Medicine/output.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/example_outputs/qwen_vl/Clinical_Medicine/output.json -------------------------------------------------------------------------------- /mmmu/example_outputs/qwen_vl/Computer_Science/output.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/example_outputs/qwen_vl/Computer_Science/output.json -------------------------------------------------------------------------------- /mmmu/example_outputs/qwen_vl/Design/output.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/example_outputs/qwen_vl/Design/output.json -------------------------------------------------------------------------------- /mmmu/example_outputs/qwen_vl/Diagnostics_and_Laboratory_Medicine/output.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/example_outputs/qwen_vl/Diagnostics_and_Laboratory_Medicine/output.json -------------------------------------------------------------------------------- /mmmu/example_outputs/qwen_vl/Economics/output.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/example_outputs/qwen_vl/Economics/output.json -------------------------------------------------------------------------------- /mmmu/example_outputs/qwen_vl/Electronics/output.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/example_outputs/qwen_vl/Electronics/output.json -------------------------------------------------------------------------------- /mmmu/example_outputs/qwen_vl/Energy_and_Power/output.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/example_outputs/qwen_vl/Energy_and_Power/output.json -------------------------------------------------------------------------------- /mmmu/example_outputs/qwen_vl/Finance/output.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/example_outputs/qwen_vl/Finance/output.json -------------------------------------------------------------------------------- /mmmu/example_outputs/qwen_vl/Geography/output.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/example_outputs/qwen_vl/Geography/output.json -------------------------------------------------------------------------------- /mmmu/example_outputs/qwen_vl/History/output.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/example_outputs/qwen_vl/History/output.json -------------------------------------------------------------------------------- /mmmu/example_outputs/qwen_vl/Literature/output.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/example_outputs/qwen_vl/Literature/output.json -------------------------------------------------------------------------------- /mmmu/example_outputs/qwen_vl/Manage/output.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/example_outputs/qwen_vl/Manage/output.json -------------------------------------------------------------------------------- /mmmu/example_outputs/qwen_vl/Marketing/output.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/example_outputs/qwen_vl/Marketing/output.json -------------------------------------------------------------------------------- /mmmu/example_outputs/qwen_vl/Materials/output.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/example_outputs/qwen_vl/Materials/output.json -------------------------------------------------------------------------------- /mmmu/example_outputs/qwen_vl/Math/output.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/example_outputs/qwen_vl/Math/output.json -------------------------------------------------------------------------------- /mmmu/example_outputs/qwen_vl/Mechanical_Engineering/output.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/example_outputs/qwen_vl/Mechanical_Engineering/output.json -------------------------------------------------------------------------------- /mmmu/example_outputs/qwen_vl/Music/output.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/example_outputs/qwen_vl/Music/output.json -------------------------------------------------------------------------------- /mmmu/example_outputs/qwen_vl/Pharmacy/output.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/example_outputs/qwen_vl/Pharmacy/output.json -------------------------------------------------------------------------------- /mmmu/example_outputs/qwen_vl/Physics/output.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/example_outputs/qwen_vl/Physics/output.json -------------------------------------------------------------------------------- /mmmu/example_outputs/qwen_vl/Psychology/output.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/example_outputs/qwen_vl/Psychology/output.json -------------------------------------------------------------------------------- /mmmu/example_outputs/qwen_vl/Public_Health/output.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/example_outputs/qwen_vl/Public_Health/output.json -------------------------------------------------------------------------------- /mmmu/example_outputs/qwen_vl/Sociology/output.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/example_outputs/qwen_vl/Sociology/output.json -------------------------------------------------------------------------------- /mmmu/example_outputs/qwen_vl/total_val_output.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/example_outputs/qwen_vl/total_val_output.json -------------------------------------------------------------------------------- /mmmu/main_eval_only.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/main_eval_only.py -------------------------------------------------------------------------------- /mmmu/main_parse_and_eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/main_parse_and_eval.py -------------------------------------------------------------------------------- /mmmu/print_results.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/print_results.py -------------------------------------------------------------------------------- /mmmu/run_llava.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/run_llava.py -------------------------------------------------------------------------------- /mmmu/utils/data_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/utils/data_utils.py -------------------------------------------------------------------------------- /mmmu/utils/eval_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/utils/eval_utils.py -------------------------------------------------------------------------------- /mmmu/utils/model_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MMMU-Benchmark/MMMU/HEAD/mmmu/utils/model_utils.py --------------------------------------------------------------------------------