├── .github ├── issue_template.md ├── pull_request_template.md └── workflows │ └── black.yml ├── .gitignore ├── .pre-commit-config.yaml ├── LICENSE ├── README.md ├── assets ├── comparison_plot.png ├── evaluation_results.png ├── tasks-min.png ├── tasks.pdf └── tasks.png ├── docs ├── README.md ├── commands.md ├── current_tasks.md ├── model_guide.md └── task_guide.md ├── lmms_eval ├── __init__.py ├── __main__.py ├── api │ ├── __init__.py │ ├── filter.py │ ├── instance.py │ ├── metrics.py │ ├── model.py │ ├── registry.py │ ├── samplers.py │ └── task.py ├── evaluator.py ├── filters │ ├── __init__.py │ ├── decontamination.py │ ├── extraction.py │ ├── selection.py │ └── transformation.py ├── logging_utils.py ├── models │ ├── __init__.py │ ├── batch_gpt4.py │ ├── claude.py │ ├── from_log.py │ ├── fuyu.py │ ├── gemini_api.py │ ├── gpt4v.py │ ├── idefics2.py │ ├── instructblip.py │ ├── internvl.py │ ├── internvl2.py │ ├── llama_vid.py │ ├── llava.py │ ├── llava_hf.py │ ├── llava_sglang.py │ ├── llava_vid.py │ ├── longva.py │ ├── mantis.py │ ├── minicpm_v.py │ ├── model_utils │ │ ├── __init__.py │ │ ├── load_video.py │ │ └── qwen │ │ │ └── qwen_generate_utils.py │ ├── mplug_owl_video.py │ ├── mplug_owl_video │ │ ├── __init__.py │ │ ├── configuration_mplug_owl.py │ │ ├── modeling_mplug_owl.py │ │ ├── processing_mplug_owl.py │ │ └── tokenization_mplug_owl.py │ ├── phi3v.py │ ├── qwen_vl.py │ ├── qwen_vl_api.py │ ├── reka.py │ ├── tinyllava.py │ ├── video_chatgpt.py │ ├── video_chatgpt │ │ ├── __init__.py │ │ ├── constants.py │ │ ├── eval │ │ │ ├── __init__.py │ │ │ └── model_utils.py │ │ ├── inference.py │ │ ├── model │ │ │ ├── __init__.py │ │ │ ├── consolidate.py │ │ │ ├── make_delta.py │ │ │ ├── utils.py │ │ │ └── video_chatgpt.py │ │ ├── single_video_inference.py │ │ ├── utils.py │ │ └── video_conversation.py │ ├── video_llava.py │ ├── vila.py │ ├── xcomposer2_4KHD.py │ └── xcomposer2_4khd.py ├── tasks │ ├── __init__.py │ ├── _task_utils │ │ ├── file_utils.py │ │ ├── gpt_eval_utils.py │ │ ├── video_loader.py │ │ └── vqa_eval_metric.py │ ├── mmmu │ │ ├── arial.ttf │ │ ├── mmmu.yaml │ │ ├── mmmu_group_img.yaml │ │ ├── mmmu_group_img_test.yaml │ │ ├── mmmu_group_img_val.yaml │ │ ├── mmmu_test.yaml │ │ ├── mmmu_val.yaml │ │ ├── utils.py │ │ └── utils_group_img.py │ └── urbench │ │ ├── arial.ttf │ │ ├── camera_localization_group_img_test.yaml │ │ ├── camera_localization_group_img_val.yaml │ │ ├── camera_localization_test.yaml │ │ ├── camera_localization_val.yaml │ │ ├── city_retrieval_group_img_test.yaml │ │ ├── city_retrieval_group_img_val.yaml │ │ ├── city_retrieval_test.yaml │ │ ├── city_retrieval_val.yaml │ │ ├── counting_test.yaml │ │ ├── counting_val.yaml │ │ ├── image_retrieval_group_img_test.yaml │ │ ├── image_retrieval_group_img_val.yaml │ │ ├── image_retrieval_test.yaml │ │ ├── image_retrieval_val.yaml │ │ ├── object_attribute_recognition_group_img_test.yaml │ │ ├── object_attribute_recognition_group_img_val.yaml │ │ ├── object_attribute_recognition_test.yaml │ │ ├── object_attribute_recognition_val.yaml │ │ ├── object_grounding_test.yaml │ │ ├── object_grounding_val.yaml │ │ ├── object_matching_group_img_test.yaml │ │ ├── object_matching_group_img_val.yaml │ │ ├── object_matching_test.yaml │ │ ├── object_matching_val.yaml │ │ ├── orientation_group_img_test.yaml │ │ ├── orientation_group_img_val.yaml │ │ ├── orientation_test.yaml │ │ ├── orientation_val.yaml │ │ ├── road_understanding_group_img_test.yaml │ │ ├── road_understanding_group_img_val.yaml │ │ ├── road_understanding_test.yaml │ │ ├── road_understanding_val.yaml │ │ ├── role_based_reasoing_val.yaml │ │ ├── role_based_reasoning_group_img_val.yaml │ │ ├── role_based_reasoning_test.yaml │ │ ├── scene_comparison_group_img_test.yaml │ │ ├── scene_comparison_group_img_val.yaml │ │ ├── scene_comparison_test.yaml │ │ ├── scene_comparison_val.yaml │ │ ├── scene_recognition_group_img_test.yaml │ │ ├── scene_recognition_group_img_val.yaml │ │ ├── scene_recognition_test.yaml │ │ ├── scene_recognition_val.yaml │ │ ├── traffic_sign_reasoning_group_img_val.yaml │ │ ├── traffic_sign_reasoning_test.yaml │ │ ├── traffic_sign_reasoning_val.yaml │ │ ├── urbench.yaml │ │ ├── urbench_group_img_test_all.yaml │ │ ├── urbench_group_img_val_all.yaml │ │ ├── urbench_test_all.yaml │ │ ├── urbench_val_all.yaml │ │ ├── utils.py │ │ ├── utils_group_img.py │ │ ├── visual_prompt_reasoning_test.yaml │ │ └── visual_prompt_reasoning_val.yaml └── utils.py ├── miscs ├── example_eval.yaml ├── llava_repr_requirements.txt ├── llava_result_check.md ├── llava_sglang_result_check.md ├── repr_scripts.sh ├── repr_torch_envs.txt ├── scienceqa_id.txt ├── script.sh ├── test_llava.py ├── test_scienceqa.py ├── tinyllava_repr_requirements.txt └── tinyllava_repr_scripts.sh ├── pyproject.toml ├── setup.py ├── static └── appendix.pdf └── tools ├── get_video_avg_time.py ├── make_image_hf_dataset.ipynb ├── make_video_hf_dataset.ipynb └── makecvrr.ipynb /.github/issue_template.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/.github/issue_template.md -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/.github/pull_request_template.md -------------------------------------------------------------------------------- /.github/workflows/black.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/.github/workflows/black.yml -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/.gitignore -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/.pre-commit-config.yaml -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/README.md -------------------------------------------------------------------------------- /assets/comparison_plot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/assets/comparison_plot.png -------------------------------------------------------------------------------- /assets/evaluation_results.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/assets/evaluation_results.png -------------------------------------------------------------------------------- /assets/tasks-min.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/assets/tasks-min.png -------------------------------------------------------------------------------- /assets/tasks.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/assets/tasks.pdf -------------------------------------------------------------------------------- /assets/tasks.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/assets/tasks.png -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/docs/README.md -------------------------------------------------------------------------------- /docs/commands.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/docs/commands.md -------------------------------------------------------------------------------- /docs/current_tasks.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/docs/current_tasks.md -------------------------------------------------------------------------------- /docs/model_guide.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/docs/model_guide.md -------------------------------------------------------------------------------- /docs/task_guide.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/docs/task_guide.md -------------------------------------------------------------------------------- /lmms_eval/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lmms_eval/__main__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/__main__.py -------------------------------------------------------------------------------- /lmms_eval/api/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lmms_eval/api/filter.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/api/filter.py -------------------------------------------------------------------------------- /lmms_eval/api/instance.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/api/instance.py -------------------------------------------------------------------------------- /lmms_eval/api/metrics.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/api/metrics.py -------------------------------------------------------------------------------- /lmms_eval/api/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/api/model.py -------------------------------------------------------------------------------- /lmms_eval/api/registry.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/api/registry.py -------------------------------------------------------------------------------- /lmms_eval/api/samplers.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/api/samplers.py -------------------------------------------------------------------------------- /lmms_eval/api/task.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/api/task.py -------------------------------------------------------------------------------- /lmms_eval/evaluator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/evaluator.py -------------------------------------------------------------------------------- /lmms_eval/filters/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/filters/__init__.py -------------------------------------------------------------------------------- /lmms_eval/filters/decontamination.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/filters/decontamination.py -------------------------------------------------------------------------------- /lmms_eval/filters/extraction.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/filters/extraction.py -------------------------------------------------------------------------------- /lmms_eval/filters/selection.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/filters/selection.py -------------------------------------------------------------------------------- /lmms_eval/filters/transformation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/filters/transformation.py -------------------------------------------------------------------------------- /lmms_eval/logging_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/logging_utils.py -------------------------------------------------------------------------------- /lmms_eval/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/models/__init__.py -------------------------------------------------------------------------------- /lmms_eval/models/batch_gpt4.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/models/batch_gpt4.py -------------------------------------------------------------------------------- /lmms_eval/models/claude.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/models/claude.py -------------------------------------------------------------------------------- /lmms_eval/models/from_log.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/models/from_log.py -------------------------------------------------------------------------------- /lmms_eval/models/fuyu.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/models/fuyu.py -------------------------------------------------------------------------------- /lmms_eval/models/gemini_api.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/models/gemini_api.py -------------------------------------------------------------------------------- /lmms_eval/models/gpt4v.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/models/gpt4v.py -------------------------------------------------------------------------------- /lmms_eval/models/idefics2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/models/idefics2.py -------------------------------------------------------------------------------- /lmms_eval/models/instructblip.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/models/instructblip.py -------------------------------------------------------------------------------- /lmms_eval/models/internvl.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/models/internvl.py -------------------------------------------------------------------------------- /lmms_eval/models/internvl2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/models/internvl2.py -------------------------------------------------------------------------------- /lmms_eval/models/llama_vid.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/models/llama_vid.py -------------------------------------------------------------------------------- /lmms_eval/models/llava.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/models/llava.py -------------------------------------------------------------------------------- /lmms_eval/models/llava_hf.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/models/llava_hf.py -------------------------------------------------------------------------------- /lmms_eval/models/llava_sglang.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/models/llava_sglang.py -------------------------------------------------------------------------------- /lmms_eval/models/llava_vid.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/models/llava_vid.py -------------------------------------------------------------------------------- /lmms_eval/models/longva.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/models/longva.py -------------------------------------------------------------------------------- /lmms_eval/models/mantis.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/models/mantis.py -------------------------------------------------------------------------------- /lmms_eval/models/minicpm_v.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/models/minicpm_v.py -------------------------------------------------------------------------------- /lmms_eval/models/model_utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lmms_eval/models/model_utils/load_video.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/models/model_utils/load_video.py -------------------------------------------------------------------------------- /lmms_eval/models/model_utils/qwen/qwen_generate_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/models/model_utils/qwen/qwen_generate_utils.py -------------------------------------------------------------------------------- /lmms_eval/models/mplug_owl_video.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/models/mplug_owl_video.py -------------------------------------------------------------------------------- /lmms_eval/models/mplug_owl_video/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/models/mplug_owl_video/__init__.py -------------------------------------------------------------------------------- /lmms_eval/models/mplug_owl_video/configuration_mplug_owl.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/models/mplug_owl_video/configuration_mplug_owl.py -------------------------------------------------------------------------------- /lmms_eval/models/mplug_owl_video/modeling_mplug_owl.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/models/mplug_owl_video/modeling_mplug_owl.py -------------------------------------------------------------------------------- /lmms_eval/models/mplug_owl_video/processing_mplug_owl.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/models/mplug_owl_video/processing_mplug_owl.py -------------------------------------------------------------------------------- /lmms_eval/models/mplug_owl_video/tokenization_mplug_owl.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/models/mplug_owl_video/tokenization_mplug_owl.py -------------------------------------------------------------------------------- /lmms_eval/models/phi3v.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/models/phi3v.py -------------------------------------------------------------------------------- /lmms_eval/models/qwen_vl.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/models/qwen_vl.py -------------------------------------------------------------------------------- /lmms_eval/models/qwen_vl_api.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/models/qwen_vl_api.py -------------------------------------------------------------------------------- /lmms_eval/models/reka.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/models/reka.py -------------------------------------------------------------------------------- /lmms_eval/models/tinyllava.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/models/tinyllava.py -------------------------------------------------------------------------------- /lmms_eval/models/video_chatgpt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/models/video_chatgpt.py -------------------------------------------------------------------------------- /lmms_eval/models/video_chatgpt/__init__.py: -------------------------------------------------------------------------------- 1 | from .model import VideoChatGPTLlamaForCausalLM 2 | -------------------------------------------------------------------------------- /lmms_eval/models/video_chatgpt/constants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/models/video_chatgpt/constants.py -------------------------------------------------------------------------------- /lmms_eval/models/video_chatgpt/eval/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lmms_eval/models/video_chatgpt/eval/model_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/models/video_chatgpt/eval/model_utils.py -------------------------------------------------------------------------------- /lmms_eval/models/video_chatgpt/inference.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/models/video_chatgpt/inference.py -------------------------------------------------------------------------------- /lmms_eval/models/video_chatgpt/model/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/models/video_chatgpt/model/__init__.py -------------------------------------------------------------------------------- /lmms_eval/models/video_chatgpt/model/consolidate.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/models/video_chatgpt/model/consolidate.py -------------------------------------------------------------------------------- /lmms_eval/models/video_chatgpt/model/make_delta.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/models/video_chatgpt/model/make_delta.py -------------------------------------------------------------------------------- /lmms_eval/models/video_chatgpt/model/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/models/video_chatgpt/model/utils.py -------------------------------------------------------------------------------- /lmms_eval/models/video_chatgpt/model/video_chatgpt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/models/video_chatgpt/model/video_chatgpt.py -------------------------------------------------------------------------------- /lmms_eval/models/video_chatgpt/single_video_inference.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/models/video_chatgpt/single_video_inference.py -------------------------------------------------------------------------------- /lmms_eval/models/video_chatgpt/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/models/video_chatgpt/utils.py -------------------------------------------------------------------------------- /lmms_eval/models/video_chatgpt/video_conversation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/models/video_chatgpt/video_conversation.py -------------------------------------------------------------------------------- /lmms_eval/models/video_llava.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/models/video_llava.py -------------------------------------------------------------------------------- /lmms_eval/models/vila.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/models/vila.py -------------------------------------------------------------------------------- /lmms_eval/models/xcomposer2_4KHD.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/models/xcomposer2_4KHD.py -------------------------------------------------------------------------------- /lmms_eval/models/xcomposer2_4khd.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/models/xcomposer2_4khd.py -------------------------------------------------------------------------------- /lmms_eval/tasks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/__init__.py -------------------------------------------------------------------------------- /lmms_eval/tasks/_task_utils/file_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/_task_utils/file_utils.py -------------------------------------------------------------------------------- /lmms_eval/tasks/_task_utils/gpt_eval_utils.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lmms_eval/tasks/_task_utils/video_loader.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/_task_utils/video_loader.py -------------------------------------------------------------------------------- /lmms_eval/tasks/_task_utils/vqa_eval_metric.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/_task_utils/vqa_eval_metric.py -------------------------------------------------------------------------------- /lmms_eval/tasks/mmmu/arial.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/mmmu/arial.ttf -------------------------------------------------------------------------------- /lmms_eval/tasks/mmmu/mmmu.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/mmmu/mmmu.yaml -------------------------------------------------------------------------------- /lmms_eval/tasks/mmmu/mmmu_group_img.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/mmmu/mmmu_group_img.yaml -------------------------------------------------------------------------------- /lmms_eval/tasks/mmmu/mmmu_group_img_test.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/mmmu/mmmu_group_img_test.yaml -------------------------------------------------------------------------------- /lmms_eval/tasks/mmmu/mmmu_group_img_val.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/mmmu/mmmu_group_img_val.yaml -------------------------------------------------------------------------------- /lmms_eval/tasks/mmmu/mmmu_test.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/mmmu/mmmu_test.yaml -------------------------------------------------------------------------------- /lmms_eval/tasks/mmmu/mmmu_val.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/mmmu/mmmu_val.yaml -------------------------------------------------------------------------------- /lmms_eval/tasks/mmmu/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/mmmu/utils.py -------------------------------------------------------------------------------- /lmms_eval/tasks/mmmu/utils_group_img.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/mmmu/utils_group_img.py -------------------------------------------------------------------------------- /lmms_eval/tasks/urbench/arial.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/urbench/arial.ttf -------------------------------------------------------------------------------- /lmms_eval/tasks/urbench/camera_localization_group_img_test.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/urbench/camera_localization_group_img_test.yaml -------------------------------------------------------------------------------- /lmms_eval/tasks/urbench/camera_localization_group_img_val.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/urbench/camera_localization_group_img_val.yaml -------------------------------------------------------------------------------- /lmms_eval/tasks/urbench/camera_localization_test.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/urbench/camera_localization_test.yaml -------------------------------------------------------------------------------- /lmms_eval/tasks/urbench/camera_localization_val.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/urbench/camera_localization_val.yaml -------------------------------------------------------------------------------- /lmms_eval/tasks/urbench/city_retrieval_group_img_test.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/urbench/city_retrieval_group_img_test.yaml -------------------------------------------------------------------------------- /lmms_eval/tasks/urbench/city_retrieval_group_img_val.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/urbench/city_retrieval_group_img_val.yaml -------------------------------------------------------------------------------- /lmms_eval/tasks/urbench/city_retrieval_test.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/urbench/city_retrieval_test.yaml -------------------------------------------------------------------------------- /lmms_eval/tasks/urbench/city_retrieval_val.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/urbench/city_retrieval_val.yaml -------------------------------------------------------------------------------- /lmms_eval/tasks/urbench/counting_test.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/urbench/counting_test.yaml -------------------------------------------------------------------------------- /lmms_eval/tasks/urbench/counting_val.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/urbench/counting_val.yaml -------------------------------------------------------------------------------- /lmms_eval/tasks/urbench/image_retrieval_group_img_test.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/urbench/image_retrieval_group_img_test.yaml -------------------------------------------------------------------------------- /lmms_eval/tasks/urbench/image_retrieval_group_img_val.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/urbench/image_retrieval_group_img_val.yaml -------------------------------------------------------------------------------- /lmms_eval/tasks/urbench/image_retrieval_test.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/urbench/image_retrieval_test.yaml -------------------------------------------------------------------------------- /lmms_eval/tasks/urbench/image_retrieval_val.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/urbench/image_retrieval_val.yaml -------------------------------------------------------------------------------- /lmms_eval/tasks/urbench/object_attribute_recognition_group_img_test.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/urbench/object_attribute_recognition_group_img_test.yaml -------------------------------------------------------------------------------- /lmms_eval/tasks/urbench/object_attribute_recognition_group_img_val.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/urbench/object_attribute_recognition_group_img_val.yaml -------------------------------------------------------------------------------- /lmms_eval/tasks/urbench/object_attribute_recognition_test.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/urbench/object_attribute_recognition_test.yaml -------------------------------------------------------------------------------- /lmms_eval/tasks/urbench/object_attribute_recognition_val.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/urbench/object_attribute_recognition_val.yaml -------------------------------------------------------------------------------- /lmms_eval/tasks/urbench/object_grounding_test.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/urbench/object_grounding_test.yaml -------------------------------------------------------------------------------- /lmms_eval/tasks/urbench/object_grounding_val.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/urbench/object_grounding_val.yaml -------------------------------------------------------------------------------- /lmms_eval/tasks/urbench/object_matching_group_img_test.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/urbench/object_matching_group_img_test.yaml -------------------------------------------------------------------------------- /lmms_eval/tasks/urbench/object_matching_group_img_val.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/urbench/object_matching_group_img_val.yaml -------------------------------------------------------------------------------- /lmms_eval/tasks/urbench/object_matching_test.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/urbench/object_matching_test.yaml -------------------------------------------------------------------------------- /lmms_eval/tasks/urbench/object_matching_val.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/urbench/object_matching_val.yaml -------------------------------------------------------------------------------- /lmms_eval/tasks/urbench/orientation_group_img_test.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/urbench/orientation_group_img_test.yaml -------------------------------------------------------------------------------- /lmms_eval/tasks/urbench/orientation_group_img_val.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/urbench/orientation_group_img_val.yaml -------------------------------------------------------------------------------- /lmms_eval/tasks/urbench/orientation_test.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/urbench/orientation_test.yaml -------------------------------------------------------------------------------- /lmms_eval/tasks/urbench/orientation_val.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/urbench/orientation_val.yaml -------------------------------------------------------------------------------- /lmms_eval/tasks/urbench/road_understanding_group_img_test.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/urbench/road_understanding_group_img_test.yaml -------------------------------------------------------------------------------- /lmms_eval/tasks/urbench/road_understanding_group_img_val.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/urbench/road_understanding_group_img_val.yaml -------------------------------------------------------------------------------- /lmms_eval/tasks/urbench/road_understanding_test.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/urbench/road_understanding_test.yaml -------------------------------------------------------------------------------- /lmms_eval/tasks/urbench/road_understanding_val.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/urbench/road_understanding_val.yaml -------------------------------------------------------------------------------- /lmms_eval/tasks/urbench/role_based_reasoing_val.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/urbench/role_based_reasoing_val.yaml -------------------------------------------------------------------------------- /lmms_eval/tasks/urbench/role_based_reasoning_group_img_val.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/urbench/role_based_reasoning_group_img_val.yaml -------------------------------------------------------------------------------- /lmms_eval/tasks/urbench/role_based_reasoning_test.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/urbench/role_based_reasoning_test.yaml -------------------------------------------------------------------------------- /lmms_eval/tasks/urbench/scene_comparison_group_img_test.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/urbench/scene_comparison_group_img_test.yaml -------------------------------------------------------------------------------- /lmms_eval/tasks/urbench/scene_comparison_group_img_val.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/urbench/scene_comparison_group_img_val.yaml -------------------------------------------------------------------------------- /lmms_eval/tasks/urbench/scene_comparison_test.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/urbench/scene_comparison_test.yaml -------------------------------------------------------------------------------- /lmms_eval/tasks/urbench/scene_comparison_val.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/urbench/scene_comparison_val.yaml -------------------------------------------------------------------------------- /lmms_eval/tasks/urbench/scene_recognition_group_img_test.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/urbench/scene_recognition_group_img_test.yaml -------------------------------------------------------------------------------- /lmms_eval/tasks/urbench/scene_recognition_group_img_val.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/urbench/scene_recognition_group_img_val.yaml -------------------------------------------------------------------------------- /lmms_eval/tasks/urbench/scene_recognition_test.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/urbench/scene_recognition_test.yaml -------------------------------------------------------------------------------- /lmms_eval/tasks/urbench/scene_recognition_val.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/urbench/scene_recognition_val.yaml -------------------------------------------------------------------------------- /lmms_eval/tasks/urbench/traffic_sign_reasoning_group_img_val.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/urbench/traffic_sign_reasoning_group_img_val.yaml -------------------------------------------------------------------------------- /lmms_eval/tasks/urbench/traffic_sign_reasoning_test.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/urbench/traffic_sign_reasoning_test.yaml -------------------------------------------------------------------------------- /lmms_eval/tasks/urbench/traffic_sign_reasoning_val.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/urbench/traffic_sign_reasoning_val.yaml -------------------------------------------------------------------------------- /lmms_eval/tasks/urbench/urbench.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/urbench/urbench.yaml -------------------------------------------------------------------------------- /lmms_eval/tasks/urbench/urbench_group_img_test_all.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/urbench/urbench_group_img_test_all.yaml -------------------------------------------------------------------------------- /lmms_eval/tasks/urbench/urbench_group_img_val_all.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/urbench/urbench_group_img_val_all.yaml -------------------------------------------------------------------------------- /lmms_eval/tasks/urbench/urbench_test_all.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/urbench/urbench_test_all.yaml -------------------------------------------------------------------------------- /lmms_eval/tasks/urbench/urbench_val_all.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/urbench/urbench_val_all.yaml -------------------------------------------------------------------------------- /lmms_eval/tasks/urbench/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/urbench/utils.py -------------------------------------------------------------------------------- /lmms_eval/tasks/urbench/utils_group_img.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/urbench/utils_group_img.py -------------------------------------------------------------------------------- /lmms_eval/tasks/urbench/visual_prompt_reasoning_test.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/urbench/visual_prompt_reasoning_test.yaml -------------------------------------------------------------------------------- /lmms_eval/tasks/urbench/visual_prompt_reasoning_val.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/tasks/urbench/visual_prompt_reasoning_val.yaml -------------------------------------------------------------------------------- /lmms_eval/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/lmms_eval/utils.py -------------------------------------------------------------------------------- /miscs/example_eval.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/miscs/example_eval.yaml -------------------------------------------------------------------------------- /miscs/llava_repr_requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/miscs/llava_repr_requirements.txt -------------------------------------------------------------------------------- /miscs/llava_result_check.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /miscs/llava_sglang_result_check.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /miscs/repr_scripts.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/miscs/repr_scripts.sh -------------------------------------------------------------------------------- /miscs/repr_torch_envs.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/miscs/repr_torch_envs.txt -------------------------------------------------------------------------------- /miscs/scienceqa_id.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/miscs/scienceqa_id.txt -------------------------------------------------------------------------------- /miscs/script.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/miscs/script.sh -------------------------------------------------------------------------------- /miscs/test_llava.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/miscs/test_llava.py -------------------------------------------------------------------------------- /miscs/test_scienceqa.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/miscs/test_scienceqa.py -------------------------------------------------------------------------------- /miscs/tinyllava_repr_requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/miscs/tinyllava_repr_requirements.txt -------------------------------------------------------------------------------- /miscs/tinyllava_repr_scripts.sh: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/miscs/tinyllava_repr_scripts.sh -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/pyproject.toml -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/setup.py -------------------------------------------------------------------------------- /static/appendix.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/static/appendix.pdf -------------------------------------------------------------------------------- /tools/get_video_avg_time.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/tools/get_video_avg_time.py -------------------------------------------------------------------------------- /tools/make_image_hf_dataset.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/tools/make_image_hf_dataset.ipynb -------------------------------------------------------------------------------- /tools/make_video_hf_dataset.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/tools/make_video_hf_dataset.ipynb -------------------------------------------------------------------------------- /tools/makecvrr.ipynb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatalab/UrBench/HEAD/tools/makecvrr.ipynb --------------------------------------------------------------------------------