├── .gitignore ├── LICENSE ├── README.md ├── README_CN.md ├── adb_client.py ├── agent ├── __init__.py ├── llm │ ├── glm4.py │ ├── llama.py │ └── qwen_llm_model.py ├── mllm │ ├── claude_model.py │ └── qwen_model.py ├── model.py └── utils.py ├── assets ├── avd-subgoal-subcates.png ├── before-after-sft.png ├── data-and-train-1010-with-figure.png ├── leaderboard.png └── main-picture.png ├── configs ├── glm-4-plus-mac-XML.yaml ├── gpt-4o-linux-XML.yaml ├── gpt-4o-mac-SoM.yaml └── xml-sft-model.yaml ├── docs ├── instruction_tuning.md ├── modify_androidlab.md ├── prepare_for_linux.md └── prepare_for_mac.md ├── eval.py ├── evaluation ├── __init__.py ├── auto_test.py ├── config │ ├── bluecoins.yaml │ ├── calendar.yaml │ ├── cantook.yaml │ ├── clock.yaml │ ├── contacts.yaml │ ├── map.yaml │ ├── pimusic.yaml │ ├── setting.yaml │ └── zoom.yaml ├── configs.py ├── definition.py ├── docker_utils.py ├── evaluation.py ├── parallel.py ├── task.py ├── tasks │ ├── __init__.py │ ├── bluecoins │ │ ├── __init__.py │ │ └── bluecoins.py │ ├── calendar │ │ ├── __init__.py │ │ └── calendar.py │ ├── cantook │ │ ├── __init__.py │ │ └── cantook.py │ ├── clock │ │ ├── __init__.py │ │ └── clock.py │ ├── contacts │ │ ├── __init__.py │ │ └── contacts.py │ ├── human_ground_turth │ │ └── ground_truth_length.json │ ├── map_me │ │ ├── __init__.py │ │ └── map.py │ ├── pimusic │ │ ├── __init__.py │ │ └── pimusic.py │ ├── setting │ │ ├── __init__.py │ │ └── setting.py │ └── zoom │ │ ├── __init__.py │ │ └── zoom.py └── utils.py ├── generate_result.py ├── page_executor ├── __init__.py ├── simple_vision_executor.py ├── text_executor.py └── utils.py ├── recorder ├── __init__.py └── json_recoder.py ├── requirements.txt ├── templates ├── __init__.py ├── android_screenshot_template.py ├── packages.py ├── seeact_screenshot_prompts.py ├── seeact_xml_prompts.py └── text_only_mobile.py ├── tools ├── check_result_multiprocess.py ├── delete_blank.py ├── modify_mobile_to_avd.py └── modify_mobile_to_docker.py └── utils_mobile ├── and_controller.py ├── specialCheck.py ├── utils.py └── xml_tool.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/README.md -------------------------------------------------------------------------------- /README_CN.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/README_CN.md -------------------------------------------------------------------------------- /adb_client.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/adb_client.py -------------------------------------------------------------------------------- /agent/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/agent/__init__.py -------------------------------------------------------------------------------- /agent/llm/glm4.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/agent/llm/glm4.py -------------------------------------------------------------------------------- /agent/llm/llama.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/agent/llm/llama.py -------------------------------------------------------------------------------- /agent/llm/qwen_llm_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/agent/llm/qwen_llm_model.py -------------------------------------------------------------------------------- /agent/mllm/claude_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/agent/mllm/claude_model.py -------------------------------------------------------------------------------- /agent/mllm/qwen_model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/agent/mllm/qwen_model.py -------------------------------------------------------------------------------- /agent/model.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/agent/model.py -------------------------------------------------------------------------------- /agent/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/agent/utils.py -------------------------------------------------------------------------------- /assets/avd-subgoal-subcates.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/assets/avd-subgoal-subcates.png -------------------------------------------------------------------------------- /assets/before-after-sft.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/assets/before-after-sft.png -------------------------------------------------------------------------------- /assets/data-and-train-1010-with-figure.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/assets/data-and-train-1010-with-figure.png -------------------------------------------------------------------------------- /assets/leaderboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/assets/leaderboard.png -------------------------------------------------------------------------------- /assets/main-picture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/assets/main-picture.png -------------------------------------------------------------------------------- /configs/glm-4-plus-mac-XML.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/configs/glm-4-plus-mac-XML.yaml -------------------------------------------------------------------------------- /configs/gpt-4o-linux-XML.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/configs/gpt-4o-linux-XML.yaml -------------------------------------------------------------------------------- /configs/gpt-4o-mac-SoM.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/configs/gpt-4o-mac-SoM.yaml -------------------------------------------------------------------------------- /configs/xml-sft-model.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/configs/xml-sft-model.yaml -------------------------------------------------------------------------------- /docs/instruction_tuning.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/docs/instruction_tuning.md -------------------------------------------------------------------------------- /docs/modify_androidlab.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/docs/modify_androidlab.md -------------------------------------------------------------------------------- /docs/prepare_for_linux.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/docs/prepare_for_linux.md -------------------------------------------------------------------------------- /docs/prepare_for_mac.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/docs/prepare_for_mac.md -------------------------------------------------------------------------------- /eval.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/eval.py -------------------------------------------------------------------------------- /evaluation/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /evaluation/auto_test.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/evaluation/auto_test.py -------------------------------------------------------------------------------- /evaluation/config/bluecoins.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/evaluation/config/bluecoins.yaml -------------------------------------------------------------------------------- /evaluation/config/calendar.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/evaluation/config/calendar.yaml -------------------------------------------------------------------------------- /evaluation/config/cantook.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/evaluation/config/cantook.yaml -------------------------------------------------------------------------------- /evaluation/config/clock.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/evaluation/config/clock.yaml -------------------------------------------------------------------------------- /evaluation/config/contacts.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/evaluation/config/contacts.yaml -------------------------------------------------------------------------------- /evaluation/config/map.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/evaluation/config/map.yaml -------------------------------------------------------------------------------- /evaluation/config/pimusic.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/evaluation/config/pimusic.yaml -------------------------------------------------------------------------------- /evaluation/config/setting.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/evaluation/config/setting.yaml -------------------------------------------------------------------------------- /evaluation/config/zoom.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/evaluation/config/zoom.yaml -------------------------------------------------------------------------------- /evaluation/configs.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/evaluation/configs.py -------------------------------------------------------------------------------- /evaluation/definition.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/evaluation/definition.py -------------------------------------------------------------------------------- /evaluation/docker_utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/evaluation/docker_utils.py -------------------------------------------------------------------------------- /evaluation/evaluation.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/evaluation/evaluation.py -------------------------------------------------------------------------------- /evaluation/parallel.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/evaluation/parallel.py -------------------------------------------------------------------------------- /evaluation/task.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/evaluation/task.py -------------------------------------------------------------------------------- /evaluation/tasks/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /evaluation/tasks/bluecoins/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/evaluation/tasks/bluecoins/__init__.py -------------------------------------------------------------------------------- /evaluation/tasks/bluecoins/bluecoins.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/evaluation/tasks/bluecoins/bluecoins.py -------------------------------------------------------------------------------- /evaluation/tasks/calendar/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/evaluation/tasks/calendar/__init__.py -------------------------------------------------------------------------------- /evaluation/tasks/calendar/calendar.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/evaluation/tasks/calendar/calendar.py -------------------------------------------------------------------------------- /evaluation/tasks/cantook/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/evaluation/tasks/cantook/__init__.py -------------------------------------------------------------------------------- /evaluation/tasks/cantook/cantook.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/evaluation/tasks/cantook/cantook.py -------------------------------------------------------------------------------- /evaluation/tasks/clock/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/evaluation/tasks/clock/__init__.py -------------------------------------------------------------------------------- /evaluation/tasks/clock/clock.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/evaluation/tasks/clock/clock.py -------------------------------------------------------------------------------- /evaluation/tasks/contacts/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/evaluation/tasks/contacts/__init__.py -------------------------------------------------------------------------------- /evaluation/tasks/contacts/contacts.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/evaluation/tasks/contacts/contacts.py -------------------------------------------------------------------------------- /evaluation/tasks/human_ground_turth/ground_truth_length.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/evaluation/tasks/human_ground_turth/ground_truth_length.json -------------------------------------------------------------------------------- /evaluation/tasks/map_me/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/evaluation/tasks/map_me/__init__.py -------------------------------------------------------------------------------- /evaluation/tasks/map_me/map.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/evaluation/tasks/map_me/map.py -------------------------------------------------------------------------------- /evaluation/tasks/pimusic/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/evaluation/tasks/pimusic/__init__.py -------------------------------------------------------------------------------- /evaluation/tasks/pimusic/pimusic.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/evaluation/tasks/pimusic/pimusic.py -------------------------------------------------------------------------------- /evaluation/tasks/setting/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/evaluation/tasks/setting/__init__.py -------------------------------------------------------------------------------- /evaluation/tasks/setting/setting.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/evaluation/tasks/setting/setting.py -------------------------------------------------------------------------------- /evaluation/tasks/zoom/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/evaluation/tasks/zoom/__init__.py -------------------------------------------------------------------------------- /evaluation/tasks/zoom/zoom.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/evaluation/tasks/zoom/zoom.py -------------------------------------------------------------------------------- /evaluation/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/evaluation/utils.py -------------------------------------------------------------------------------- /generate_result.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/generate_result.py -------------------------------------------------------------------------------- /page_executor/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/page_executor/__init__.py -------------------------------------------------------------------------------- /page_executor/simple_vision_executor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/page_executor/simple_vision_executor.py -------------------------------------------------------------------------------- /page_executor/text_executor.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/page_executor/text_executor.py -------------------------------------------------------------------------------- /page_executor/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/page_executor/utils.py -------------------------------------------------------------------------------- /recorder/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/recorder/__init__.py -------------------------------------------------------------------------------- /recorder/json_recoder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/recorder/json_recoder.py -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/requirements.txt -------------------------------------------------------------------------------- /templates/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/templates/__init__.py -------------------------------------------------------------------------------- /templates/android_screenshot_template.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/templates/android_screenshot_template.py -------------------------------------------------------------------------------- /templates/packages.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/templates/packages.py -------------------------------------------------------------------------------- /templates/seeact_screenshot_prompts.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/templates/seeact_screenshot_prompts.py -------------------------------------------------------------------------------- /templates/seeact_xml_prompts.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/templates/seeact_xml_prompts.py -------------------------------------------------------------------------------- /templates/text_only_mobile.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/templates/text_only_mobile.py -------------------------------------------------------------------------------- /tools/check_result_multiprocess.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/tools/check_result_multiprocess.py -------------------------------------------------------------------------------- /tools/delete_blank.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/tools/delete_blank.py -------------------------------------------------------------------------------- /tools/modify_mobile_to_avd.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/tools/modify_mobile_to_avd.py -------------------------------------------------------------------------------- /tools/modify_mobile_to_docker.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/tools/modify_mobile_to_docker.py -------------------------------------------------------------------------------- /utils_mobile/and_controller.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/utils_mobile/and_controller.py -------------------------------------------------------------------------------- /utils_mobile/specialCheck.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/utils_mobile/specialCheck.py -------------------------------------------------------------------------------- /utils_mobile/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/utils_mobile/utils.py -------------------------------------------------------------------------------- /utils_mobile/xml_tool.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/THUDM/Android-Lab/HEAD/utils_mobile/xml_tool.py --------------------------------------------------------------------------------