├── src └── mobile_agent_benchmark │ ├── __init__.py │ ├── assets │ ├── music1.mp3 │ ├── music2.mp3 │ └── wallpaper.png │ ├── tasks │ ├── note_open_about.py │ ├── calendar_daily_view.py │ ├── calendar_open_about.py │ ├── contacts_about.py │ ├── calendar_snooze_time.py │ ├── calendar_next_month.py │ ├── calendar_search.py │ ├── messager_show_archived.py │ ├── recorder_bit_rate.py │ ├── music_player_about_FAQ.py │ ├── filemanager_check_storage.py │ ├── music_player_open_settings.py │ ├── calendar_start_week.py │ ├── note_add.py │ ├── recorder_recycle_bin.py │ ├── launcher_check_contributor.py │ ├── launcher_open_about_FAQ.py │ ├── messager_change_font_size.py │ ├── recorder_extension.py │ ├── recorder_empty_trash.py │ ├── gallery_remember_playback_position.py │ ├── calculator_length.py │ ├── gallery_set_favorite.py │ ├── gallery_show_hidden_items.py │ ├── music_player_config_equalizer.py │ ├── recorder_recycle_settings.py │ ├── contacts_search.py │ ├── gallery_play_videos_automatically.py │ ├── launcher_setting_close_app_when_launching.py │ ├── contacts_remove_dialog.py │ ├── filemanager_search.py │ ├── note_new_checklist_items.py │ ├── launcher_remove_app.py │ ├── launcher_search_app.py │ ├── launcher_hide_app_name.py │ ├── calculator_add.py │ ├── calculator_minus.py │ ├── gallery_group_by_file_type.py │ ├── calculator_cube.py │ ├── calculator_mixed.py │ ├── gallery_use_24_hour_time_format.py │ ├── calculator_divide.py │ ├── calculator_square.py │ ├── calendar_create_and_search.py │ ├── calculator_multiply.py │ ├── calculator_point.py │ ├── filemanager_check_file_properties.py │ ├── contacts_delete.py │ ├── recorder_theme.py │ ├── launcher_add_apps.py │ ├── filemanager_delete_file.py │ ├── note_new_checklist.py │ ├── launcher_sort_by_custom.py │ ├── filemanager_delete_txt.py │ ├── gallery_sort_by_size_asc.py │ ├── contacts_filter.py │ ├── music_player_create_playlist_and_search.py │ ├── messager_create_conversation_and_check_message_properties.py │ ├── music_player_album_sort_by_year.py │ ├── launcher_sort_by_title_desc.py │ ├── messager_create_conversation_and_search.py │ ├── gallery_list_view_type.py │ ├── recorder_delete.py │ ├── recorder_rename.py │ ├── messager_start_a_conversation.py │ ├── contacts_modify.py │ ├── music_player_playlist_sort_desc.py │ ├── recorder_delete_all.py │ ├── filemanager_create_new_file.py │ ├── recorder_rename_all.py │ ├── filemanager_rename_file.py │ ├── music_player_create_playlist_and_sort_desc.py │ ├── music_player_create_playlist.py │ ├── filemanager_delete_videos.py │ ├── note_delete.py │ ├── music_player_rescan_media.py │ ├── filemanager_sort_folder_by_size_desc.py │ ├── note_rename.py │ ├── filemanager_hide_folder.py │ ├── calculator_recalculate.py │ ├── calendar_delete_tasks.py │ ├── note_open.py │ ├── contacts_favorite.py │ ├── contacts_hide_email.py │ ├── launcher_rename_app.py │ ├── gallery_filter_by_images_and_videos.py │ ├── music_player_search_playlist.py │ ├── note_search.py │ ├── contacts_create.py │ ├── messager_add_block_numbers.py │ ├── messager_check_message_properties.py │ ├── gallery_set_wallpaper.py │ ├── calendar_new_task.py │ ├── messager_search_message.py │ ├── calendar_laundry.py │ ├── messager_search_contacts.py │ ├── __init__.py │ ├── note_item_checked.py │ ├── messager_make_conversation_archived.py │ ├── note_lock.py │ ├── task_utils.py │ └── contacts_sort.py │ ├── dummy_agent.py │ ├── bench_task.py │ ├── server.py │ ├── log_manager.py │ └── task_orchestrator.py ├── .gitignore ├── pyproject.toml ├── environment.yml └── README.md /src/mobile_agent_benchmark/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__ 2 | dist/ 3 | logs/ 4 | log/ 5 | .vscode/ 6 | .idea 7 | build/ 8 | mobile_agent_benchmark.egg-info/ 9 | .DS_Store -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/assets/music1.mp3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MobileAgentBench/mobile-agent-bench/HEAD/src/mobile_agent_benchmark/assets/music1.mp3 -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/assets/music2.mp3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MobileAgentBench/mobile-agent-bench/HEAD/src/mobile_agent_benchmark/assets/music2.mp3 -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/assets/wallpaper.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/MobileAgentBench/mobile-agent-bench/HEAD/src/mobile_agent_benchmark/assets/wallpaper.png -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/note_open_about.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from com.dtmilano.android.viewclient import ViewClient 3 | 4 | class OpenAboutTask(Task): 5 | def __init__(self): 6 | super().__init__( 7 | task_name="note_open_about", 8 | prompt="open about page", 9 | min_steps=2, 10 | package="com.simplemobiletools.notes.pro", 11 | max_steps=4, 12 | stop_after_finish=False, 13 | permissions=[] 14 | ) 15 | def check_finish(self, view_client, app_event) -> bool: 16 | activity_name = self.get_top_activity_name() 17 | if 'AboutActivity' in activity_name: 18 | return True 19 | return False -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools>=61.0"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [project] 6 | name = "mobile_agent_benchmark" 7 | version = "0.0.1" 8 | authors = [ 9 | { name="Author", email="mail@example.com" }, 10 | ] 11 | description = "An automated benchmark framework for mobile agents" 12 | readme = "README.md" 13 | requires-python = ">=3.8" 14 | classifiers = [ 15 | "Programming Language :: Python :: 3", 16 | ] 17 | dependencies = [ 18 | "numpy", 19 | "opencv-python", 20 | "androidviewclient==23.3.0" 21 | ] 22 | 23 | [tool.setuptools.packages.find] 24 | where = ["src"] 25 | 26 | [tool.setuptools.package-data] 27 | "mobile_agent_benchmark.configs" = ["*.json"] 28 | "mobile_agent_benchmark.assets" = ["*.jpg", "*.png"] -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/calendar_daily_view.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from com.dtmilano.android.viewclient import ViewClient 3 | 4 | class CalendarOpenAboutTask(Task): 5 | def __init__(self, task_name="calendar_daily_view", 6 | prompt="switch to daily view", 7 | min_steps=2, 8 | package="com.simplemobiletools.calendar.pro", 9 | max_steps=4, 10 | stop_after_finish=False, 11 | permissions=["android.permission.POST_NOTIFICATIONS"]): 12 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish,permissions) 13 | 14 | def check_finish(self, view_client, app_event) -> bool: 15 | if view_client.findViewById('com.simplemobiletools.calendar.pro:id/day_events') is not None: 16 | return True 17 | return False 18 | -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/calendar_open_about.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from com.dtmilano.android.viewclient import ViewClient 3 | 4 | class CalendarOpenAboutTask(Task): 5 | def __init__(self, task_name="calendar_open_about", 6 | prompt="open about page", 7 | min_steps=2, 8 | package="com.simplemobiletools.calendar.pro", 9 | max_steps=4, 10 | stop_after_finish=False, 11 | permissions=["android.permission.POST_NOTIFICATIONS"]): 12 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish,permissions) 13 | 14 | def check_finish(self, view_client, app_event) -> bool: 15 | activity_name = view_client.device.getTopActivityName() 16 | if 'AboutActivity' in activity_name: 17 | return True 18 | return False 19 | -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/contacts_about.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | 4 | class ContactsAbout(Task): 5 | def __init__(self, task_name="contacts_about", 6 | prompt="open About View", 7 | min_steps=2, 8 | package="com.simplemobiletools.contacts.pro", 9 | max_steps=4, 10 | stop_after_finish=False, 11 | permissions=["android.permission.READ_CONTACTS"]): 12 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish,permissions) 13 | 14 | self.found = False 15 | 16 | def check_finish(self, view_client, app_event: AppEvent) -> bool: 17 | title_views = view_client.findViewWithText('Frequently asked questions') 18 | if title_views is not None: 19 | return True 20 | return False -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/calendar_snooze_time.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from com.dtmilano.android.viewclient import ViewClient 3 | 4 | class CalendarSnoozeTime(Task): 5 | def __init__(self, task_name="calendar_snooze_time", 6 | prompt="change snooze time to 1 minute", 7 | min_steps=4, 8 | package="com.simplemobiletools.calendar.pro", 9 | max_steps=8, 10 | stop_after_finish=False, 11 | permissions=["android.permission.POST_NOTIFICATIONS"]): 12 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish,permissions) 13 | 14 | def check_finish(self, view_client, app_event) -> bool: 15 | view = view_client.findViewById('com.simplemobiletools.calendar.pro:id/settings_snooze_time') 16 | if view is not None and view.text() == '1 minute': 17 | return True 18 | return False 19 | -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/calendar_next_month.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | from com.dtmilano.android.viewclient import ViewClient 4 | 5 | class CalendarNextMonth(Task): 6 | def __init__(self, task_name="calendar_next_month", 7 | prompt="show events of next month", 8 | min_steps=1, 9 | package="com.simplemobiletools.calendar.pro", 10 | max_steps=2, 11 | stop_after_finish=False, 12 | permissions=["android.permission.POST_NOTIFICATIONS"]): 13 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish, permissions) 14 | 15 | def check_finish(self, view_client, app_event) -> bool: 16 | if app_event is not None: 17 | if app_event.type == AppEventType.Click and app_event.id_str == "com.simplemobiletools.calendar.pro:id/top_right_arrow": 18 | return True 19 | return False 20 | -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/calendar_search.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | from com.dtmilano.android.viewclient import ViewClient 4 | 5 | class CalendarSearch(Task): 6 | def __init__(self, task_name="calendar_search", 7 | prompt="search event 'laundry'", 8 | min_steps=2, 9 | package="com.simplemobiletools.calendar.pro", 10 | max_steps=4, 11 | stop_after_finish=False, 12 | permissions=["android.permission.POST_NOTIFICATIONS"]): 13 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish,permissions) 14 | 15 | def check_finish(self, view_client, app_event) -> bool: 16 | search_bar = view_client.findViewById('com.simplemobiletools.calendar.pro:id/top_toolbar_search') 17 | if search_bar is not None: 18 | text = search_bar.text() 19 | if text.lower() == 'laundry': 20 | return True 21 | return False -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/messager_show_archived.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | import subprocess 4 | 5 | class MessagerShowArchivedTask(Task): 6 | def __init__(self, task_name="messager_show_archived_messages", 7 | prompt="show me the archived conversations", 8 | min_steps=2, 9 | package="com.simplemobiletools.smsmessenger", 10 | max_steps=4, 11 | stop_after_finish=False, 12 | permissions=[]): 13 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish,permissions) 14 | 15 | def setup(self, view_client): 16 | subprocess.run(["adb","shell","pm", "disable-user", "--user", "0", "com.google.android.apps.messaging"]) 17 | return 18 | 19 | def check_finish(self, view_client, app_event: AppEvent) -> bool: 20 | archivedView = view_client.findViewWithText('Archive') 21 | if archivedView is not None: 22 | return True 23 | return False -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/recorder_bit_rate.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | from com.dtmilano.android.viewclient import ViewClient 4 | import time 5 | from . import task_utils 6 | 7 | class RecorderBitRate(Task): 8 | def __init__(self, task_name="recorder_bit_rate", 9 | prompt="change bitrate to 32 kbps", 10 | min_steps=3, 11 | package="com.simplemobiletools.voicerecorder", 12 | max_steps=6, 13 | stop_after_finish=False, 14 | permissions=["android.permission.RECORD_AUDIO", "android.permission.POST_NOTIFICATIONS"]): 15 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish, permissions) 16 | 17 | 18 | def check_finish(self, view_client, app_event) -> bool: 19 | bitrate = view_client.findViewWithText("Bitrate") 20 | bps = view_client.findViewWithText("32 kbps") 21 | if bitrate is not None and bps is not None: 22 | return True 23 | return False -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/music_player_about_FAQ.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from com.dtmilano.android.viewclient import ViewClient 3 | from . import task_utils 4 | class MusicPlayerOpenFAQTask(Task): 5 | def __init__(self, task_name="music_player_open_faq", 6 | prompt="open faq page", 7 | min_steps=3, 8 | package="com.simplemobiletools.musicplayer", 9 | max_steps=6, 10 | stop_after_finish=False, 11 | permissions = ["android.permission.READ_MEDIA_AUDIO", "android.permission.POST_NOTIFICATIONS"]): 12 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish, permissions) 13 | def setup(self, view_client): 14 | task_utils.music_player_permissions_for_old_android_version(view_client) 15 | return 16 | def check_finish(self, view_client, app_event) -> bool: 17 | activity_name = view_client.device.getTopActivityName() 18 | if 'FAQActivity' in activity_name: 19 | return True 20 | return False 21 | -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/filemanager_check_storage.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | from .task_utils import * 4 | 5 | class FileManagerCheckStorage(Task): 6 | def __init__(self, task_name="filemanager_check_storage", 7 | prompt="check the storage page", 8 | min_steps=1, 9 | package="com.simplemobiletools.filemanager.pro", 10 | max_steps=2, 11 | stop_after_finish=False, 12 | permissions=["android.permission.WRITE_EXTERNAL_STORAGE","android.permission.POST_NOTIFICATIONS"]): 13 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish, permissions) 14 | 15 | def setup(self, view_client): 16 | filemanager_permissions(view_client) 17 | 18 | def check_finish(self, view_client, app_event) -> bool: 19 | view_dict = view_client.getViewsById() 20 | for each_k in view_dict.keys(): 21 | if 'Total storage' in view_dict[each_k].text(): 22 | return True 23 | return False -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/dummy_agent.py: -------------------------------------------------------------------------------- 1 | # start emulator 2 | # /Users/user/Library/Android/sdk/emulator/emulator @Pixel_3a_API_34_extension_level_7_arm64-v8a 3 | # start adb: 4 | # /Users/user/Library/Android/sdk/platform-tools/adb -e shell 5 | # download apks 6 | # https://github.com/SimpleMobileTools 7 | # download free music (no copyright issue) 8 | # https://pixabay.com/music/ 9 | 10 | 11 | import time 12 | from mobile_agent_benchmark.task_orchestrator import TaskOrchestrator 13 | 14 | orchestrator = TaskOrchestrator(log_dir='/Users/user/Documents/research/benchmark/mobile_agent_benchmark/logs') 15 | 16 | 17 | def dummy_agent(prompt): 18 | print("Agent running task:", prompt) 19 | for i in range(10): 20 | print("step", i, "Please execute your action") 21 | orchestrator.before_one_action() 22 | time.sleep(3) 23 | print("Please wait!") 24 | should_stop = orchestrator.after_one_action("agent step") 25 | if should_stop: 26 | break 27 | print("agent done") 28 | 29 | 30 | if __name__ == '__main__': 31 | orchestrator.run(dummy_agent) 32 | 33 | -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/music_player_open_settings.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from com.dtmilano.android.viewclient import ViewClient 3 | from . import task_utils 4 | 5 | class MusicPlayerOpenSettingsTask(Task): 6 | def __init__(self, task_name="music_player_open_settings", 7 | prompt="open setting page", 8 | min_steps=2, 9 | package="com.simplemobiletools.musicplayer", 10 | max_steps=4, 11 | stop_after_finish=False, 12 | permissions = ["android.permission.READ_MEDIA_AUDIO", "android.permission.POST_NOTIFICATIONS"]): 13 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish, permissions) 14 | 15 | def setup(self, view_client): 16 | task_utils.music_player_permissions_for_old_android_version(view_client) 17 | pass 18 | def check_finish(self, view_client, app_event) -> bool: 19 | activity_name = view_client.device.getTopActivityName() 20 | if 'SettingsActivity' in activity_name: 21 | return True 22 | return False -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/calendar_start_week.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | from com.dtmilano.android.viewclient import ViewClient 4 | 5 | class CalendarStartWeek(Task): 6 | def __init__(self, task_name="calendar_start_week", 7 | prompt="go to settings and make weeks start on Monday", 8 | min_steps=3, 9 | package="com.simplemobiletools.calendar.pro", 10 | max_steps=6, 11 | stop_after_finish=False, 12 | permissions=["android.permission.POST_NOTIFICATIONS"]): 13 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish, permissions) 14 | 15 | self.text_filled = False 16 | self.last_event = None 17 | 18 | 19 | def check_finish(self, view_client, app_event: AppEvent) -> bool: 20 | view = view_client.findViewById("com.simplemobiletools.calendar.pro:id/settings_start_week_on") 21 | if view is not None: 22 | if view.text() == "Monday": 23 | return True 24 | return False 25 | -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/note_add.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from com.dtmilano.android.viewclient import ViewClient 3 | from ..server import AppEvent, AppEventType 4 | 5 | 6 | class OpenNoteTask(Task): 7 | def __init__(self): 8 | super().__init__( 9 | task_name="note_add", 10 | prompt="add a new note named 'TODO List'", 11 | min_steps=3, 12 | package="com.simplemobiletools.notes.pro", 13 | max_steps=6, 14 | stop_after_finish=False, 15 | permissions=[] 16 | ) 17 | self.last_event = None 18 | 19 | def check_finish(self, view_client, app_event) -> bool: 20 | """ 21 | Check if a note named 'to_do_list' is currently open by identifying the presence of the EditText for note text and verifying the text content. 22 | """ 23 | title = view_client.findViewWithText("TODO List") 24 | if title is not None: 25 | if title.map['class'] == "android.widget.TextView": 26 | # if typed elsewhere, it will be EditTextView 27 | return True 28 | return False 29 | -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/recorder_recycle_bin.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | from com.dtmilano.android.viewclient import ViewClient 4 | from . import task_utils 5 | 6 | class RecorderRecycleBin(Task): 7 | def __init__(self, task_name="recorder_recycle_bin", 8 | prompt="go to recycle bin page", 9 | min_steps=1, 10 | package="com.simplemobiletools.voicerecorder", 11 | max_steps=2, 12 | stop_after_finish=False, 13 | permissions=["android.permission.RECORD_AUDIO", "android.permission.POST_NOTIFICATIONS"]): 14 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish, permissions) 15 | 16 | def setup(self, view_client): 17 | task_utils.recorder_permissions_for_old_android_version(view_client) 18 | pass 19 | 20 | def check_finish(self, view_client, app_event) -> bool: 21 | if app_event is not None and app_event.type == AppEventType.Click and 'Recycle Bin' in app_event.text_list: 22 | return True 23 | return False 24 | -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/launcher_check_contributor.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | from . import task_utils 4 | 5 | 6 | class LauncherCheckContributor(Task): 7 | def __init__( 8 | self, 9 | task_name="launcher_check_contributor", 10 | prompt="Check who is the contributor of the app", 11 | min_steps=3, 12 | package="com.simplemobiletools.applauncher", 13 | max_steps=6, 14 | stop_after_finish=False, 15 | permissions=[], 16 | ): 17 | super().__init__( 18 | task_name, 19 | prompt, 20 | min_steps, 21 | package, 22 | max_steps, 23 | stop_after_finish, 24 | permissions, 25 | ) 26 | 27 | def setup(self, view_client): 28 | pass 29 | 30 | def check_finish(self, view_client, app_event) -> bool: 31 | if ( 32 | app_event is not None 33 | and app_event.type == AppEventType.WindowStateChange 34 | and "Contributors" in app_event.text_list 35 | ): 36 | return True 37 | return False 38 | -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/launcher_open_about_FAQ.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | from . import task_utils 4 | 5 | 6 | class LauncherOpenAboutFAQ(Task): 7 | def __init__( 8 | self, 9 | task_name="launcher_open_about_FAQ", 10 | prompt="Open About page and go Frequently Asked Questions", 11 | min_steps=3, 12 | package="com.simplemobiletools.applauncher", 13 | max_steps=6, 14 | stop_after_finish=False, 15 | permissions=[], 16 | ): 17 | super().__init__( 18 | task_name, 19 | prompt, 20 | min_steps, 21 | package, 22 | max_steps, 23 | stop_after_finish, 24 | permissions, 25 | ) 26 | 27 | def setup(self, view_client): 28 | pass 29 | 30 | def check_finish(self, view_client, app_event) -> bool: 31 | if ( 32 | app_event is not None 33 | and app_event.type == AppEventType.WindowStateChange 34 | and "Frequently asked questions" in app_event.text_list 35 | ): 36 | return True 37 | return False 38 | -------------------------------------------------------------------------------- /environment.yml: -------------------------------------------------------------------------------- 1 | name: android 2 | channels: 3 | - defaults 4 | dependencies: 5 | - ca-certificates=2024.3.11 6 | - libcxx=14.0.6 7 | - libffi=3.4.4 8 | - ncurses=6.4 9 | - openssl=3.0.13 10 | - pip=23.3.1 11 | - python=3.9.19 12 | - readline=8.2 13 | - setuptools=68.2.2 14 | - sqlite=3.41.2 15 | - tk=8.6.12 16 | - tzdata=2024a 17 | - wheel=0.41.2 18 | - xz=5.4.6 19 | - zlib=1.2.13 20 | - pip: 21 | - androidviewclient==23.3.0 22 | - build==1.2.1 23 | - certifi==2024.2.2 24 | - charset-normalizer==3.3.2 25 | - contourpy==1.2.1 26 | - culebratester-client==2.0.70 27 | - cycler==0.12.1 28 | - fonttools==4.51.0 29 | - idna==3.7 30 | - importlib-metadata==7.1.0 31 | - importlib-resources==6.4.0 32 | - kiwisolver==1.4.5 33 | - matplotlib==3.8.4 34 | - numpy==1.26.4 35 | - opencv-python==4.9.0.80 36 | - packaging==24.0 37 | - pillow==10.3.0 38 | - pyparsing==3.1.2 39 | - pyproject-hooks==1.0.0 40 | - python-dateutil==2.9.0.post0 41 | - requests==2.31.0 42 | - six==1.16.0 43 | - tomli==2.0.1 44 | - urllib3==2.2.1 45 | - zipp==3.18.1 46 | prefix: /opt/homebrew/anaconda3/envs/android 47 | -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/messager_change_font_size.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | import subprocess 4 | 5 | class MessagerFontChangeTask(Task): 6 | def __init__(self, task_name="messager_change_font_size", 7 | prompt="Change the Font size to 'Large' in the settings interface", 8 | min_steps=3, 9 | package="com.simplemobiletools.smsmessenger", 10 | max_steps=6, 11 | stop_after_finish=False, 12 | permissions=[]): 13 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish,permissions) 14 | 15 | def setup(self, view_client): 16 | subprocess.run(["adb","shell","pm", "disable-user", "--user", "0", "com.google.android.apps.messaging"]) 17 | return 18 | 19 | def check_finish(self, view_client, app_event: AppEvent) -> bool: 20 | fontSizeView = view_client.findViewById('com.simplemobiletools.smsmessenger:id/settings_font_size') 21 | if fontSizeView is not None: 22 | text = fontSizeView.text() 23 | if text.lower() == 'large': 24 | return True 25 | return False -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/recorder_extension.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | from com.dtmilano.android.viewclient import ViewClient 4 | import time 5 | from . import task_utils 6 | 7 | class RecorderExtension(Task): 8 | def __init__(self, task_name="recorder_extension", 9 | prompt="use mp3 as the format for new recordings", 10 | min_steps=3, 11 | package="com.simplemobiletools.voicerecorder", 12 | max_steps=6, 13 | stop_after_finish=False, 14 | permissions=["android.permission.RECORD_AUDIO", "android.permission.POST_NOTIFICATIONS"]): 15 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish, permissions) 16 | 17 | def setup(self, view_client): 18 | task_utils.recorder_permissions_for_old_android_version(view_client) 19 | pass 20 | 21 | def check_finish(self, view_client, app_event) -> bool: 22 | extension = view_client.findViewWithText("Extension") 23 | mp3 = view_client.findViewWithText("mp3") 24 | if extension is not None and mp3 is not None: 25 | return True 26 | return False -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/recorder_empty_trash.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | from com.dtmilano.android.viewclient import ViewClient 4 | import time 5 | from . import task_utils 6 | 7 | class RecorderEmptyTrash(Task): 8 | def __init__(self, task_name="recorder_empty_trash", 9 | prompt="go to settings and empty the recycle bin", 10 | min_steps=3, 11 | package="com.simplemobiletools.voicerecorder", 12 | max_steps=6, 13 | stop_after_finish=False, 14 | permissions=["android.permission.RECORD_AUDIO", "android.permission.POST_NOTIFICATIONS"]): 15 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish, permissions) 16 | 17 | def setup(self, view_client): 18 | task_utils.recorder_permissions_for_old_android_version(view_client) 19 | pass 20 | 21 | def check_finish(self, view_client, app_event) -> bool: 22 | if app_event is not None: 23 | if app_event.type == AppEventType.Click and app_event.id_str == 'com.simplemobiletools.voicerecorder:id/settings_empty_recycle_bin_holder': 24 | return True 25 | return False -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/gallery_remember_playback_position.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | from com.dtmilano.android.viewclient import ViewClient 4 | from . import task_utils 5 | 6 | class GalleryRememberPlaybackPosition(Task): 7 | def __init__(self, task_name="gallery_remember_playback_position", 8 | prompt="Enable remember the last video playback position in settings", 9 | min_steps=4, 10 | package="com.simplemobiletools.gallery.pro", 11 | max_steps=8, 12 | stop_after_finish=False, 13 | permissions=["android.permission.READ_EXTERNAL_STORAGE", "android.permission.READ_MEDIA_IMAGES", "android.permission.READ_MEDIA_VIDEO", "android.permission.READ_MEDIA_VISUAL_USER_SELECTED", "android.permission.ACCESS_MEDIA_LOCATION"]): 14 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish, permissions) 15 | 16 | def check_finish(self, view_client: ViewClient, app_event: AppEvent) -> bool: 17 | if task_utils.is_box_checked(view_client, "com.simplemobiletools.gallery.pro:id/settings_remember_last_video_position"): 18 | return True 19 | return False 20 | -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/calculator_length.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | 4 | 5 | class CalculatorConvertLengthTask(Task): 6 | def __init__(self, task_name="calculator_convert_length", 7 | prompt="Use Unit converter function to calculate how many kilometers 1mile is equal to", 8 | min_steps=6, 9 | package="com.simplemobiletools.calculator", 10 | max_steps=12, 11 | stop_after_finish=False, 12 | permissions=[]): 13 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish, permissions) 14 | 15 | self.mile_detected = False 16 | self.km_detected = False 17 | 18 | def check_finish(self, view_client, app_event: AppEvent) -> bool: 19 | mile = view_client.findViewWithText('Mile') 20 | kilometer = view_client.findViewWithText('Kilometer') 21 | mile_number = view_client.findViewWithText('1') 22 | kilometer_number = view_client.findViewWithText('1.609344') 23 | if mile is not None and kilometer is not None: 24 | if mile_number is not None and kilometer_number is not None: 25 | return True 26 | return False -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/gallery_set_favorite.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | from com.dtmilano.android.viewclient import ViewClient 4 | 5 | class GallerySortBySizeAsc(Task): 6 | def __init__(self, task_name="gallery_set_favorite", 7 | prompt="Go to Downloads Folder and set the first image as favorite", 8 | min_steps=3, 9 | package="com.simplemobiletools.gallery.pro", 10 | max_steps=6, 11 | stop_after_finish=False, 12 | permissions=["android.permission.READ_EXTERNAL_STORAGE", "android.permission.READ_MEDIA_IMAGES", "android.permission.READ_MEDIA_VIDEO", "android.permission.READ_MEDIA_VISUAL_USER_SELECTED", "android.permission.ACCESS_MEDIA_LOCATION"]): 13 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish, permissions) 14 | 15 | def check_finish(self, view_client, app_event: AppEvent) -> bool: 16 | if app_event is not None: 17 | if app_event.type == AppEventType.Click and app_event.id_str == "com.simplemobiletools.gallery.pro:id/bottom_favorite" and "Toggle favorite" in app_event.text_list: 18 | return True 19 | return False 20 | -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/gallery_show_hidden_items.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | from com.dtmilano.android.viewclient import ViewClient 4 | 5 | class GallerySortBySizeAsc(Task): 6 | def __init__(self, task_name="gallery_show_hidden_items", 7 | prompt="show hidden items in the gallery in settings", 8 | min_steps=3, 9 | package="com.simplemobiletools.gallery.pro", 10 | max_steps=6, 11 | stop_after_finish=False, 12 | permissions=["android.permission.READ_EXTERNAL_STORAGE", "android.permission.READ_MEDIA_IMAGES", "android.permission.READ_MEDIA_VIDEO", "android.permission.READ_MEDIA_VISUAL_USER_SELECTED", "android.permission.ACCESS_MEDIA_LOCATION"]): 13 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish, permissions) 14 | 15 | def check_finish(self, view_client, app_event: AppEvent) -> bool: 16 | if app_event is not None: 17 | if app_event.type == AppEventType.Click and app_event.id_str == "com.simplemobiletools.gallery.pro:id/settings_show_hidden_items_holder" and "Show hidden items" in app_event.text_list: 18 | return True 19 | return False 20 | -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/music_player_config_equalizer.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from com.dtmilano.android.viewclient import ViewClient 3 | from . import task_utils 4 | class MusicPlayerConfigEqualizer(Task): 5 | def __init__(self, task_name="music_player_config_equalizer", 6 | prompt="config equalizer to Heavy Metal", 7 | min_steps=3, 8 | package="com.simplemobiletools.musicplayer", 9 | max_steps=6, 10 | stop_after_finish=False, 11 | permissions = ["android.permission.READ_MEDIA_AUDIO", "android.permission.POST_NOTIFICATIONS"]): 12 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish, permissions) 13 | self.equalizer_changed = "" 14 | def setup(self, view_client): 15 | task_utils.music_player_permissions_for_old_android_version(view_client) 16 | pass 17 | def check_finish(self, view_client, app_event) -> bool: 18 | view = view_client.findViewById('com.simplemobiletools.musicplayer:id/equalizer_preset') 19 | if view is not None: 20 | self.equalizer_changed = view.text() 21 | if self.equalizer_changed == "Heavy Metal": 22 | return True 23 | return False -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/recorder_recycle_settings.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | from com.dtmilano.android.viewclient import ViewClient 4 | import time 5 | from . import task_utils 6 | 7 | class RecorderRecycleSettings(Task): 8 | def __init__(self, task_name="recorder_recycle_settings", 9 | prompt="change settings, so that the deleted items will not go to recycle bin", 10 | min_steps=3, 11 | package="com.simplemobiletools.voicerecorder", 12 | max_steps=6, 13 | stop_after_finish=False, 14 | permissions=["android.permission.RECORD_AUDIO", "android.permission.POST_NOTIFICATIONS"]): 15 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish, permissions) 16 | 17 | def setup(self, view_client): 18 | task_utils.recorder_permissions_for_old_android_version(view_client) 19 | pass 20 | 21 | def check_finish(self, view_client, app_event) -> bool: 22 | view = view_client.findViewById('com.simplemobiletools.voicerecorder:id/settings_use_recycle_bin') 23 | if view is not None: 24 | if not view.checked(): 25 | return True 26 | return False 27 | -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/contacts_search.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | from .contacts_favorite import ContactsFavorite 4 | 5 | class ContactsSearch(Task): 6 | def __init__(self, task_name="contacts_search", 7 | prompt="Search contact Yuzai", 8 | min_steps=2, 9 | package="com.simplemobiletools.contacts.pro", 10 | max_steps=4, 11 | stop_after_finish=False, 12 | permissions=["android.permission.READ_CONTACTS"]): 13 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish,permissions) 14 | 15 | self.found = False 16 | self.prepare = ContactsFavorite() 17 | 18 | # create contact 19 | def setup(self, view_client): 20 | self.prepare.setup(view_client) 21 | 22 | # delete contact Yuzai 23 | def teardown(self, view_client): 24 | self.prepare.teardown(view_client) 25 | 26 | def check_finish(self, view_client, app_event: AppEvent) -> bool: 27 | search = view_client.findViewById("com.simplemobiletools.contacts.pro:id/top_toolbar_search") 28 | if search is not None: 29 | if search.text().lower() == "yuzai": 30 | return True 31 | return False -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/gallery_play_videos_automatically.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | from com.dtmilano.android.viewclient import ViewClient 4 | 5 | class GalleryPlayVideosAutomatically(Task): 6 | def __init__(self, task_name="gallery_play_videos_automatically", 7 | prompt="Go to Gallery settings and enable play videos automatically", 8 | min_steps=4, 9 | package="com.simplemobiletools.gallery.pro", 10 | max_steps=8, 11 | stop_after_finish=False, 12 | permissions=["android.permission.READ_EXTERNAL_STORAGE", "android.permission.READ_MEDIA_IMAGES", "android.permission.READ_MEDIA_VIDEO", "android.permission.READ_MEDIA_VISUAL_USER_SELECTED", "android.permission.ACCESS_MEDIA_LOCATION"]): 13 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish, permissions) 14 | 15 | def check_finish(self, view_client, app_event: AppEvent) -> bool: 16 | if app_event is not None: 17 | if app_event.type == AppEventType.Click and app_event.id_str == "com.simplemobiletools.gallery.pro:id/settings_autoplay_videos_holder" and "Play videos automatically" in app_event.text_list: 18 | return True 19 | return False 20 | -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/launcher_setting_close_app_when_launching.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | from . import task_utils 4 | 5 | 6 | class LauncherSettingCloseAppWhenLaunching(Task): 7 | def __init__( 8 | self, 9 | task_name="launcher_setting_close_app_when_launching", 10 | prompt="Change Setting Close this app at launching a different one to false", 11 | min_steps=3, 12 | package="com.simplemobiletools.applauncher", 13 | max_steps=6, 14 | stop_after_finish=False, 15 | permissions=[], 16 | ): 17 | super().__init__( 18 | task_name, 19 | prompt, 20 | min_steps, 21 | package, 22 | max_steps, 23 | stop_after_finish, 24 | permissions, 25 | ) 26 | 27 | def setup(self, view_client): 28 | pass 29 | 30 | def check_finish(self, view_client, app_event) -> bool: 31 | if view_client.findViewById( 32 | "com.simplemobiletools.applauncher:id/settings_close_app" 33 | ) is not None and not task_utils.is_box_checked( 34 | view_client=view_client, 35 | id_str="com.simplemobiletools.applauncher:id/settings_close_app", 36 | ): 37 | return True 38 | return False 39 | -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/contacts_remove_dialog.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | 4 | class ContactsRemoveDialog(Task): 5 | def __init__(self, task_name="contacts_remove_dialog", 6 | prompt="Remove the dialog button, and then return to the main view", 7 | min_steps=4, 8 | package="com.simplemobiletools.contacts.pro", 9 | max_steps=8, 10 | stop_after_finish=False, 11 | permissions=["android.permission.READ_CONTACTS"]): 12 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish,permissions) 13 | 14 | self.dialog = False 15 | self.add = False 16 | 17 | def check_finish(self, view_client, app_event: AppEvent) -> bool: 18 | # make sure can found add button on main view 19 | add_button = view_client.findViewById('com.simplemobiletools.contacts.pro:id/fragment_fab') 20 | if add_button is not None: 21 | self.add = True 22 | # make sure cannot found dialog button on main view 23 | dialog_buuton = view_client.findViewById('com.simplemobiletools.contacts.pro:id/main_dialpad_button') 24 | if dialog_buuton is None: 25 | self.dialog = True 26 | return self.dialog and self.add -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/filemanager_search.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | from .task_utils import * 4 | 5 | 6 | class FileManagerSearchFile(Task): 7 | def __init__(self, task_name="filemanager_search_file", 8 | prompt="Search a file named 'testfile.txt'", 9 | min_steps=2, 10 | package="com.simplemobiletools.filemanager.pro", 11 | max_steps=4, 12 | stop_after_finish=False, 13 | permissions=["android.permission.WRITE_EXTERNAL_STORAGE", "android.permission.POST_NOTIFICATIONS"]): 14 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish, permissions) 15 | 16 | self.file_icon_clicked = False 17 | 18 | def setup(self, view_client): 19 | filemanager_permissions(view_client) 20 | filemanager_create_file_under_download(view_client) 21 | 22 | def teardown(self, view_client): 23 | filemanager_delete_all_files_under_folder(view_client) 24 | 25 | def check_finish(self, view_client, app_event: AppEvent) -> bool: 26 | search = view_client.findViewById("com.simplemobiletools.filemanager.pro:id/top_toolbar_search") 27 | if search is not None: 28 | if search.text().lower() == "testfile.txt": 29 | return True 30 | return False -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/note_new_checklist_items.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from com.dtmilano.android.viewclient import ViewClient 3 | from ..server import AppEvent, AppEventType 4 | 5 | class ChecklistItemTask(Task): 6 | def __init__(self): 7 | super().__init__( 8 | task_name="note_new_checklist_items", 9 | prompt="create a checklist named 'Shopping list' and add an item named 'Milk'", 10 | min_steps=7, 11 | package="com.simplemobiletools.notes.pro", 12 | max_steps=14, 13 | stop_after_finish=False, 14 | permissions=[] 15 | ) 16 | self.last_event = None 17 | self.checklist = False 18 | 19 | 20 | def check_finish(self, view_client, app_event): 21 | """ 22 | Verify that specific notes have been deleted by checking their absence. 23 | """ 24 | 25 | title_view = view_client.findViewWithText('Shopping list') 26 | text_view = view_client.findViewWithText('Milk') 27 | 28 | if title_view is not None and text_view is not None: 29 | if title_view.map['class'] == 'android.widget.TextView' \ 30 | and text_view.map['resource-id'] == 'com.simplemobiletools.notes.pro:id/checklist_title': 31 | return True 32 | return False 33 | 34 | 35 | 36 | 37 | 38 | -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/launcher_remove_app.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | from .launcher_rename_app import LauncherRenameApp 4 | from . import task_utils 5 | 6 | 7 | class LauncherRemoveApp(Task): 8 | def __init__( 9 | self, 10 | task_name="launcher_remove_app", 11 | prompt="Remove Chrome from Launcher", 12 | min_steps=3, 13 | package="com.simplemobiletools.applauncher", 14 | max_steps=6, 15 | stop_after_finish=False, 16 | permissions=[], 17 | ): 18 | super().__init__( 19 | task_name, 20 | prompt, 21 | min_steps, 22 | package, 23 | max_steps, 24 | stop_after_finish, 25 | permissions, 26 | ) 27 | 28 | self.create_task = LauncherRenameApp() 29 | 30 | # Add Apps 31 | def setup(self, view_client): 32 | self.create_task.setup(view_client) 33 | 34 | pass 35 | 36 | def check_finish(self, view_client, app_event) -> bool: 37 | if ( 38 | view_client.findViewById( 39 | "com.simplemobiletools.applauncher:id/coordinator_layout" 40 | ) 41 | is not None 42 | and view_client.findViewWithText("Camera") is not None 43 | and view_client.findViewWithText("Chrome") is None 44 | ): 45 | return True 46 | 47 | return False 48 | -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/launcher_search_app.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | from .launcher_rename_app import LauncherRenameApp 4 | from . import task_utils 5 | 6 | 7 | class LauncherSearchApp(Task): 8 | def __init__( 9 | self, 10 | task_name="launcher_search_app", 11 | prompt="Search for Chrome in Launcher", 12 | min_steps=2, 13 | package="com.simplemobiletools.applauncher", 14 | max_steps=4, 15 | stop_after_finish=False, 16 | permissions=[], 17 | ): 18 | super().__init__( 19 | task_name, 20 | prompt, 21 | min_steps, 22 | package, 23 | max_steps, 24 | stop_after_finish, 25 | permissions, 26 | ) 27 | 28 | self.create_task = LauncherRenameApp() 29 | 30 | # Add Apps 31 | def setup(self, view_client): 32 | self.create_task.setup(view_client) 33 | 34 | pass 35 | 36 | def check_finish(self, view_client, app_event) -> bool: 37 | if ( 38 | view_client.findViewById( 39 | "com.simplemobiletools.applauncher:id/coordinator_layout" 40 | ) 41 | is not None 42 | and view_client.findViewWithText("Camera") is None 43 | and view_client.findViewWithText("Chrome") is not None 44 | ): 45 | return True 46 | 47 | return False 48 | -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/launcher_hide_app_name.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | from .launcher_rename_app import LauncherRenameApp 4 | from . import task_utils 5 | 6 | 7 | class LauncherHideAppName(Task): 8 | def __init__( 9 | self, 10 | task_name="launcher_hide_app_name", 11 | prompt="Hide app name in Launcher", 12 | min_steps=1, 13 | package="com.simplemobiletools.applauncher", 14 | max_steps=2, 15 | stop_after_finish=False, 16 | permissions=[], 17 | ): 18 | super().__init__( 19 | task_name, 20 | prompt, 21 | min_steps, 22 | package, 23 | max_steps, 24 | stop_after_finish, 25 | permissions, 26 | ) 27 | self.create_task = LauncherRenameApp() 28 | 29 | # Add Apps 30 | def setup(self, view_client): 31 | self.create_task.setup(view_client) 32 | pass 33 | 34 | def check_finish(self, view_client, app_event) -> bool: 35 | if ( 36 | app_event is not None 37 | and app_event.type == AppEventType.Click 38 | and app_event.id_str == "com.simplemobiletools.applauncher:id/toggle_app_name" 39 | ): 40 | self.hide_clicked = True 41 | 42 | if view_client.findViewWithText("Chrome") is None and self.hide_clicked: 43 | return True 44 | return False 45 | -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/calculator_add.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | 4 | class CalculatorAddTask(Task): 5 | def __init__(self, task_name="calculator_add", 6 | prompt="Calculate the result of '3 + 5'", 7 | min_steps=4, 8 | package="com.simplemobiletools.calculator", 9 | max_steps=8, 10 | stop_after_finish=False, 11 | permissions=[]): 12 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish,permissions) 13 | 14 | self.text_filled = False 15 | self.last_event = None 16 | 17 | def check_finish(self, view_client, app_event: AppEvent) -> bool: 18 | result = view_client.findViewById('com.simplemobiletools.calculator:id/result') 19 | if result is not None and result.text().strip() == '8': 20 | self.text_filled = True 21 | 22 | if app_event is not None and app_event.package == self.package: 23 | # last step is = 24 | if app_event.type == AppEventType.Click and len(app_event.text_list) == 1 and \ 25 | app_event.text_list[0] == '=': 26 | if self.last_event is not None and self.last_event.type == AppEventType.Click and len( 27 | self.last_event.text_list) == 1: 28 | return self.text_filled 29 | self.last_event = app_event 30 | return False 31 | 32 | -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/calculator_minus.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | 4 | class CalculatorMinusTask(Task): 5 | def __init__(self, task_name="calculator_minus", 6 | prompt="Calculate the result of '12 - 4'", 7 | min_steps=5, 8 | package="com.simplemobiletools.calculator", 9 | max_steps=10, 10 | stop_after_finish=False, 11 | permissions=[]): 12 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish,permissions) 13 | 14 | self.text_filled = False 15 | self.last_event = None 16 | 17 | def check_finish(self, view_client, app_event: AppEvent) -> bool: 18 | result = view_client.findViewById('com.simplemobiletools.calculator:id/result') 19 | if result is not None and result.text().strip() == '8': 20 | self.text_filled = True 21 | 22 | if app_event is not None and app_event.package == self.package: 23 | # last step is = 24 | if app_event.type == AppEventType.Click and len(app_event.text_list) == 1 and \ 25 | app_event.text_list[0] == '=': 26 | if self.last_event is not None and self.last_event.type == AppEventType.Click and len( 27 | self.last_event.text_list) == 1: 28 | return self.text_filled 29 | self.last_event = app_event 30 | return False 31 | -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/gallery_group_by_file_type.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | from com.dtmilano.android.viewclient import ViewClient 4 | from . import task_utils 5 | 6 | class GalleryGroupByFileType(Task): 7 | def __init__(self, task_name="gallery_group_by_file_type", 8 | prompt="Go to the downloads folder, group the images by file type", 9 | min_steps=5, 10 | package="com.simplemobiletools.gallery.pro", 11 | max_steps=10, 12 | stop_after_finish=False, 13 | permissions=["android.permission.READ_EXTERNAL_STORAGE", "android.permission.READ_MEDIA_IMAGES", "android.permission.READ_MEDIA_VIDEO", "android.permission.READ_MEDIA_VISUAL_USER_SELECTED", "android.permission.ACCESS_MEDIA_LOCATION"]): 14 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish, permissions) 15 | 16 | self.group_by_file_type_checked = False 17 | 18 | def check_finish(self, view_client: ViewClient, app_event: AppEvent) -> bool: 19 | if task_utils.is_box_checked(view_client, "com.simplemobiletools.gallery.pro:id/grouping_dialog_radio_file_type"): 20 | self.group_by_file_type_checked = True 21 | if app_event is not None: 22 | if app_event.type == AppEventType.Click and "OK" in app_event.text_list and self.group_by_file_type_checked: 23 | return True 24 | return False 25 | -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/calculator_cube.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | 4 | class CalculatorCubeTask(Task): 5 | def __init__(self, task_name="calculator_cube", 6 | prompt="Calculate the result of '3 ^ 3'", 7 | min_steps=4, 8 | package="com.simplemobiletools.calculator", 9 | max_steps=8, 10 | stop_after_finish=False, 11 | permissions=[]): 12 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish,permissions) 13 | 14 | self.text_filled = False 15 | self.last_event = None 16 | 17 | def check_finish(self, view_client, app_event: AppEvent) -> bool: 18 | result = view_client.findViewById('com.simplemobiletools.calculator:id/result') 19 | if result is not None and result.text().strip() == '27': 20 | self.text_filled = True 21 | 22 | if app_event is not None and app_event.package == self.package: 23 | # last step is = 24 | if app_event.type == AppEventType.Click and len(app_event.text_list) == 1 and \ 25 | app_event.text_list[0] == '=': 26 | if self.last_event is not None and self.last_event.type == AppEventType.Click and len( 27 | self.last_event.text_list) == 1: 28 | return self.text_filled 29 | self.last_event = app_event 30 | return False 31 | 32 | -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/calculator_mixed.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | 4 | class CalculatorMixedTask(Task): 5 | def __init__(self, task_name="calculator_mixed", 6 | prompt="Calculate the result of '18+(24×3)-(9÷3)'", 7 | min_steps=17, 8 | package="com.simplemobiletools.calculator", 9 | max_steps=34, 10 | stop_after_finish=False, 11 | permissions=[]): 12 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish,permissions) 13 | 14 | self.text_filled = False 15 | self.last_event = None 16 | 17 | def check_finish(self, view_client, app_event: AppEvent) -> bool: 18 | result = view_client.findViewById('com.simplemobiletools.calculator:id/result') 19 | if result is not None and result.text().strip() == '87': 20 | self.text_filled = True 21 | 22 | if app_event is not None and app_event.package == self.package: 23 | # last step is = 24 | if app_event.type == AppEventType.Click and len(app_event.text_list) == 1 and \ 25 | app_event.text_list[0] == '=': 26 | if self.last_event is not None and self.last_event.type == AppEventType.Click and len( 27 | self.last_event.text_list) == 1: 28 | return self.text_filled 29 | self.last_event = app_event 30 | return False -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/gallery_use_24_hour_time_format.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | from com.dtmilano.android.viewclient import ViewClient 4 | from . import task_utils 5 | 6 | class GalleryUse24HourTimeFormat(Task): 7 | def __init__(self, task_name="gallery_use_24_hour_time_format", 8 | prompt="Change the date and time format to 24-hour format in gallery settings", 9 | min_steps=5, 10 | package="com.simplemobiletools.gallery.pro", 11 | max_steps=10, 12 | stop_after_finish=False, 13 | permissions=["android.permission.READ_EXTERNAL_STORAGE", "android.permission.READ_MEDIA_IMAGES", "android.permission.READ_MEDIA_VIDEO", "android.permission.READ_MEDIA_VISUAL_USER_SELECTED", "android.permission.ACCESS_MEDIA_LOCATION"]): 14 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish, permissions) 15 | 16 | self.box_24_hour_checked = False 17 | 18 | def check_finish(self, view_client: ViewClient, app_event: AppEvent) -> bool: 19 | if task_utils.is_box_checked(view_client, "com.simplemobiletools.gallery.pro:id/change_date_time_dialog_24_hour"): 20 | self.box_24_hour_checked = True 21 | if app_event is not None: 22 | if app_event.type == AppEventType.Click and "OK" in app_event.text_list and self.box_24_hour_checked: 23 | return True 24 | return False 25 | -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/calculator_divide.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | 4 | class CalculatorDivideTask(Task): 5 | def __init__(self, task_name="calculator_divide", 6 | prompt="Calculate the result of '12 ÷ 3'", 7 | min_steps=5, 8 | package="com.simplemobiletools.calculator", 9 | max_steps=10, 10 | stop_after_finish=False, 11 | permissions=[]): 12 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish,permissions) 13 | 14 | self.text_filled = False 15 | self.last_event = None 16 | 17 | def check_finish(self, view_client, app_event: AppEvent) -> bool: 18 | result = view_client.findViewById('com.simplemobiletools.calculator:id/result') 19 | if result is not None and result.text().strip() == '4': 20 | self.text_filled = True 21 | 22 | if app_event is not None and app_event.package == self.package: 23 | # last step is = 24 | if app_event.type == AppEventType.Click and len(app_event.text_list) == 1 and \ 25 | app_event.text_list[0] == '=': 26 | if self.last_event is not None and self.last_event.type == AppEventType.Click and len( 27 | self.last_event.text_list) == 1: 28 | return self.text_filled 29 | self.last_event = app_event 30 | return False 31 | 32 | -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/calculator_square.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | 4 | class CalculatorSquareTask(Task): 5 | def __init__(self, task_name="calculator_square", 6 | prompt="Calculate the result of '√16 + 3'", 7 | min_steps=6, 8 | package="com.simplemobiletools.calculator", 9 | max_steps=12, 10 | stop_after_finish=True, 11 | permissions=[]): 12 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish,permissions) 13 | 14 | self.text_filled = False 15 | self.last_event = None 16 | 17 | def check_finish(self, view_client, app_event: AppEvent) -> bool: 18 | result = view_client.findViewById('com.simplemobiletools.calculator:id/result') 19 | if result is not None and result.text().strip() == '7': 20 | self.text_filled = True 21 | 22 | if app_event is not None and app_event.package == self.package: 23 | # last step is = 24 | if app_event.type == AppEventType.Click and len(app_event.text_list) == 1 and \ 25 | app_event.text_list[0] == '=': 26 | if self.last_event is not None and self.last_event.type == AppEventType.Click and len( 27 | self.last_event.text_list) == 1: 28 | return self.text_filled 29 | self.last_event = app_event 30 | return False 31 | 32 | -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/calendar_create_and_search.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | from com.dtmilano.android.viewclient import ViewClient 4 | 5 | from .calendar_laundry import CalendarLaundryTask 6 | from .calendar_search import CalendarSearch 7 | 8 | class CalendarCreateAndSearch(Task): 9 | def __init__(self, task_name="calendar_create_and_search", 10 | prompt="Create a new event 'laundry' and then search for it", 11 | min_steps=6, 12 | package="com.simplemobiletools.calendar.pro", 13 | max_steps=12, 14 | stop_after_finish=False, 15 | permissions=["android.permission.POST_NOTIFICATIONS"]): 16 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish, permissions) 17 | 18 | self.create_task = CalendarLaundryTask() 19 | self.search_task = CalendarSearch() 20 | 21 | self.create_task_finished = False 22 | 23 | def setup(self, view_client): 24 | self.create_task.setup(view_client) 25 | 26 | def check_finish(self, view_client, app_event: AppEvent) -> bool: 27 | if not self.create_task_finished: 28 | self.create_task_finished = self.create_task.check_finish(view_client, app_event) 29 | else: 30 | print("create finished, checking search") 31 | return self.search_task.check_finish(view_client, app_event) 32 | 33 | return False -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/calculator_multiply.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | 4 | class CalculatorMultiplyTask(Task): 5 | def __init__(self, task_name="calculator_multiply", 6 | prompt="Calculate the result of '12*4'", 7 | min_steps=5, 8 | package="com.simplemobiletools.calculator", 9 | max_steps=10, 10 | stop_after_finish=False, 11 | permissions=[]): 12 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish, permissions) 13 | 14 | self.text_filled = False 15 | self.last_event = None 16 | 17 | def check_finish(self, view_client, app_event: AppEvent) -> bool: 18 | result = view_client.findViewById('com.simplemobiletools.calculator:id/result') 19 | if result is not None and result.text().strip() == '48': 20 | self.text_filled = True 21 | 22 | if app_event is not None and app_event.package == self.package: 23 | # last step is = 24 | if app_event.type == AppEventType.Click and len(app_event.text_list) == 1 and \ 25 | app_event.text_list[0] == '=': 26 | if self.last_event is not None and self.last_event.type == AppEventType.Click and len( 27 | self.last_event.text_list) == 1: 28 | return self.text_filled 29 | self.last_event = app_event 30 | return False 31 | 32 | -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/calculator_point.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | 4 | class CalculatorPointTask(Task): 5 | def __init__(self, task_name="calculator_point", 6 | prompt="Calculate the result of '19.7 - 81.3'", 7 | min_steps=10, 8 | package="com.simplemobiletools.calculator", 9 | max_steps=20, 10 | stop_after_finish=False, 11 | permissions=[]): 12 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish,permissions) 13 | 14 | self.text_filled = False 15 | self.last_event = None 16 | 17 | def check_finish(self, view_client, app_event: AppEvent) -> bool: 18 | result = view_client.findViewById('com.simplemobiletools.calculator:id/result') 19 | if result is not None and result.text().strip() == '-61.6': 20 | self.text_filled = True 21 | 22 | if app_event is not None and app_event.package == self.package: 23 | # last step is = 24 | if app_event.type == AppEventType.Click and len(app_event.text_list) == 1 and \ 25 | app_event.text_list[0] == '=': 26 | if self.last_event is not None and self.last_event.type == AppEventType.Click and len( 27 | self.last_event.text_list) == 1: 28 | return self.text_filled 29 | self.last_event = app_event 30 | return False 31 | 32 | -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/filemanager_check_file_properties.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | from .task_utils import * 4 | 5 | class FileManagerCheckFileProperties(Task): 6 | def __init__(self, task_name="filemanager_check_file_properties", 7 | prompt="open the folder 'Downloads' and check the properties of the file 'testfile.txt'", 8 | min_steps=2, 9 | package="com.simplemobiletools.filemanager.pro", 10 | max_steps=4, 11 | stop_after_finish=False, 12 | permissions=["android.permission.WRITE_EXTERNAL_STORAGE","android.permission.POST_NOTIFICATIONS"]): 13 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish, permissions) 14 | 15 | def setup(self, view_client): 16 | filemanager_permissions(view_client) 17 | filemanager_create_file_under_download(view_client) 18 | 19 | def teardown(self, view_client): 20 | filemanager_delete_all_files_under_folder(view_client) 21 | 22 | def check_finish(self, view_client, app_event) -> bool: 23 | if app_event is not None and app_event.package == 'com.simplemobiletools.filemanager.pro': 24 | if app_event.type == AppEventType.WindowStateChange: 25 | if len(app_event.text_list) == 12 and app_event.text_list[0] == 'Properties' and app_event.text_list[2] == 'Testfile.txt': 26 | return True 27 | return False 28 | 29 | -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/contacts_delete.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | from .contacts_favorite import ContactsFavorite 4 | class ContactsDelete(Task): 5 | def __init__(self, task_name="contacts_delete", 6 | prompt="Delete contact Yuzai", 7 | min_steps=3, 8 | package="com.simplemobiletools.contacts.pro", 9 | max_steps=6, 10 | stop_after_finish=False, 11 | permissions=["android.permission.READ_CONTACTS"]): 12 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish,permissions) 13 | 14 | self.found = False 15 | self.click = False 16 | self.prepare = ContactsFavorite() 17 | 18 | def setup(self, view_client): 19 | self.prepare.setup(view_client) 20 | 21 | def check_finish(self, view_client, app_event: AppEvent) -> bool: 22 | # Check if the 'Yes' button is clicked 23 | if app_event is not None and app_event.package == 'com.simplemobiletools.contacts.pro': 24 | if app_event.type == AppEventType.Click: 25 | if len(app_event.text_list) == 1 and app_event.text_list[0] == 'Yes': 26 | self.click = True 27 | 28 | # Check if still contains contact 'Yuzai' 29 | contact_yuzai = view_client.findViewWithText("Yuzai") 30 | if contact_yuzai is None: 31 | self.found = True 32 | 33 | return self.click and self.found 34 | 35 | -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/recorder_theme.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | from com.dtmilano.android.viewclient import ViewClient 4 | import time 5 | from . import task_utils 6 | 7 | class RecorderTheme(Task): 8 | def __init__(self, task_name="recorder_theme", 9 | prompt="change app theme to dark red", 10 | min_steps=6, 11 | package="com.simplemobiletools.voicerecorder", 12 | max_steps=12, 13 | stop_after_finish=False, 14 | permissions=["android.permission.RECORD_AUDIO", "android.permission.POST_NOTIFICATIONS"]): 15 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish, permissions) 16 | self.theme_correct = False 17 | 18 | def setup(self, view_client): 19 | task_utils.recorder_permissions_for_old_android_version(view_client) 20 | pass 21 | 22 | def check_finish(self, view_client, app_event) -> bool: 23 | theme = view_client.findViewWithText("Theme") 24 | red = view_client.findViewWithText("Dark red") 25 | if theme is not None and red is not None: 26 | self.theme_correct = True 27 | 28 | if self.theme_correct: 29 | print("theme correct, waiting for confirm") 30 | if app_event is not None: 31 | if app_event.type == AppEventType.Click and app_event.content_desc == 'Save': 32 | return True 33 | return False -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/launcher_add_apps.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | from . import task_utils 4 | 5 | 6 | class LauncherAddApps(Task): 7 | def __init__( 8 | self, 9 | task_name="launcher_add_apps", 10 | prompt="Add Chrome and Camera to launcher", 11 | min_steps=4, 12 | package="com.simplemobiletools.applauncher", 13 | max_steps=8, 14 | stop_after_finish=False, 15 | permissions=[], 16 | ): 17 | super().__init__( 18 | task_name, 19 | prompt, 20 | min_steps, 21 | package, 22 | max_steps, 23 | stop_after_finish, 24 | permissions, 25 | ) 26 | self.camera_added = False 27 | self.chrome_added = False 28 | self.ok_clicked = False 29 | 30 | def check_finish(self, view_client, app_event) -> bool: 31 | # Check if the favorite button is clicked 32 | if app_event is not None: 33 | if app_event.type == AppEventType.Click and "Camera" in app_event.text_list: 34 | self.camera_added = True 35 | if app_event.type == AppEventType.Click and "Chrome" in app_event.text_list: 36 | self.chrome_added = True 37 | if app_event.type == AppEventType.Click and "OK" in app_event.text_list: 38 | self.ok_clicked = True 39 | if self.camera_added and self.chrome_added and self.ok_clicked: 40 | return True 41 | return False 42 | -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/filemanager_delete_file.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | from .task_utils import * 4 | 5 | 6 | class FileManagerDeleteFile(Task): 7 | def __init__(self, task_name="filemanager_delete_file", 8 | prompt="Delete file named 'testfile.txt' in the 'Downloads' folder", 9 | min_steps=3, 10 | package="com.simplemobiletools.filemanager.pro", 11 | max_steps=6, 12 | stop_after_finish=False, 13 | permissions=["android.permission.WRITE_EXTERNAL_STORAGE", "android.permission.POST_NOTIFICATIONS"]): 14 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish, permissions) 15 | self.list = False 16 | 17 | def setup(self, view_client): 18 | filemanager_permissions(view_client) 19 | filemanager_create_file_under_download(view_client) 20 | 21 | def teardown(self, view_client): 22 | filemanager_delete_all_files_under_folder(view_client) 23 | 24 | def check_finish(self, view_client, app_event) -> bool: 25 | file_title_upper = view_client.findViewWithText('Testfile.txt') 26 | file_title_lower = view_client.findViewWithText('testfile.txt') 27 | correct_folder_opened = False 28 | folder = view_client.findViewWithText('> Download') 29 | # make sure in the same page 30 | if folder is not None and file_title_upper is None and file_title_lower is None: 31 | return True 32 | return False -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/note_new_checklist.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from com.dtmilano.android.viewclient import ViewClient 3 | from ..server import AppEvent, AppEventType 4 | 5 | 6 | class ChecklistTask(Task): 7 | def __init__(self): 8 | super().__init__( 9 | task_name="note_new_checklist", 10 | prompt="add a new Checklist named 'TODO List'", 11 | min_steps=4, 12 | package="com.simplemobiletools.notes.pro", 13 | max_steps=8, 14 | stop_after_finish=False, 15 | permissions=[] 16 | ) 17 | self.clicked_checklist = False 18 | self.clicked_text = False 19 | def check_finish(self, view_client, app_event) -> bool: 20 | """ 21 | Check if a note named 'to_do_list' is currently open by identifying the presence of the EditText for note text and verifying the text content. 22 | """ 23 | 24 | if app_event is not None: 25 | if app_event.type == AppEventType.Click: 26 | if 'Checklist' in app_event.text_list: 27 | self.clicked_checklist = True 28 | self.clicked_text = False 29 | elif 'Text note' in app_event.text_list: 30 | self.clicked_text = True 31 | self.clicked_checklist = False 32 | 33 | title_views = view_client.findViewWithText('TODO List') 34 | if title_views.map['class'] == "android.widget.TextView": 35 | return self.clicked_checklist and (not self.clicked_text) 36 | return False 37 | -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/launcher_sort_by_custom.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | from .launcher_rename_app import LauncherRenameApp 4 | from . import task_utils 5 | 6 | 7 | class LauncherSortByCustom(Task): 8 | def __init__( 9 | self, 10 | task_name="launcher_sort_by_custom", 11 | prompt="Sort apps by custom", 12 | min_steps=3, 13 | package="com.simplemobiletools.applauncher", 14 | max_steps=6, 15 | stop_after_finish=False, 16 | permissions=[], 17 | ): 18 | super().__init__( 19 | task_name, 20 | prompt, 21 | min_steps, 22 | package, 23 | max_steps, 24 | stop_after_finish, 25 | permissions, 26 | ) 27 | 28 | self.create_task = LauncherRenameApp() 29 | self.box_checked = False 30 | 31 | # Add Apps 32 | def setup(self, view_client): 33 | self.create_task.setup(view_client) 34 | 35 | pass 36 | 37 | def check_finish(self, view_client, app_event) -> bool: 38 | if task_utils.is_box_checked( 39 | view_client=view_client, 40 | id_str="com.simplemobiletools.applauncher:id/sorting_dialog_radio_custom", 41 | ): 42 | self.box_checked = True 43 | 44 | if ( 45 | app_event is not None 46 | and app_event.type == AppEventType.Click 47 | and "OK" in app_event.text_list 48 | and self.box_checked 49 | ): 50 | return True 51 | return False 52 | -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/filemanager_delete_txt.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | from .task_utils import * 4 | 5 | 6 | class FileManagerDeleteTxt(Task): 7 | def __init__(self, task_name="filemanager_delete_txt", 8 | prompt="Delete the txt file in Download folder", 9 | min_steps=3, 10 | package="com.simplemobiletools.filemanager.pro", 11 | max_steps=6, 12 | stop_after_finish=False, 13 | permissions=["android.permission.WRITE_EXTERNAL_STORAGE", "android.permission.POST_NOTIFICATIONS"]): 14 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish, permissions) 15 | self.list = False 16 | 17 | def setup(self, view_client): 18 | filemanager_permissions(view_client) 19 | filemanager_create_file_under_download(view_client, "voice.m4a") 20 | filemanager_create_file_under_download(view_client, "random.txt") 21 | filemanager_create_file_under_download(view_client, "video.mov") 22 | 23 | 24 | def teardown(self, view_client): 25 | filemanager_delete_all_files_under_folder(view_client) 26 | 27 | def check_finish(self, view_client, app_event) -> bool: 28 | folder = view_client.findViewWithText('> Download') 29 | voice = view_client.findViewWithText('voice.m4a') 30 | video = view_client.findViewWithText('video.mov') 31 | txt = view_client.findViewWithText('random.txt') 32 | 33 | if folder is not None and voice is not None and video is not None and txt is None: 34 | return True 35 | return False -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/gallery_sort_by_size_asc.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | from com.dtmilano.android.viewclient import ViewClient 4 | 5 | class GallerySortBySizeAsc(Task): 6 | def __init__(self, task_name="gallery_sort_by_size_asc", 7 | prompt="sort the gallery by size ascendingly", 8 | min_steps=4, 9 | package="com.simplemobiletools.gallery.pro", 10 | max_steps=8, 11 | stop_after_finish=False, 12 | permissions=["android.permission.READ_EXTERNAL_STORAGE", "android.permission.READ_MEDIA_IMAGES", "android.permission.READ_MEDIA_VIDEO", "android.permission.READ_MEDIA_VISUAL_USER_SELECTED", "android.permission.ACCESS_MEDIA_LOCATION"]): 13 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish, permissions) 14 | 15 | self.size_clicked = False 16 | self.acending_clicked = False 17 | self.last_event = None 18 | 19 | def check_finish(self, view_client, app_event: AppEvent) -> bool: 20 | if app_event is not None: 21 | if app_event.type == AppEventType.Click and app_event.id_str == "com.simplemobiletools.gallery.pro:id/sorting_dialog_radio_size": 22 | self.size_clicked = True 23 | if app_event.type == AppEventType.Click and app_event.id_str == "com.simplemobiletools.gallery.pro:id/sorting_dialog_radio_ascending": 24 | self.acending_clicked = True 25 | if app_event.type == AppEventType.Click and "OK" in app_event.text_list and self.size_clicked and self.acending_clicked: 26 | return True 27 | return False 28 | -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/contacts_filter.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | from .contacts_favorite import ContactsFavorite 4 | 5 | class ContactsFilter(Task): 6 | def __init__(self, task_name="contacts_filter", 7 | prompt="Change phone filter, which means don't show phone storage in contacts view", 8 | min_steps=3, 9 | package="com.simplemobiletools.contacts.pro", 10 | max_steps=6, 11 | stop_after_finish=False, 12 | permissions=["android.permission.READ_CONTACTS"]): 13 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish,permissions) 14 | self.prepare = ContactsFavorite() 15 | 16 | self.clicked_phone_storage = False 17 | 18 | # create contact 19 | def setup(self, view_client): 20 | self.prepare.setup(view_client) 21 | 22 | # delete contact 23 | def teardown(self, view_client): 24 | self.prepare.teardown(view_client) 25 | 26 | def check_finish(self, view_client, app_event: AppEvent) -> bool: 27 | if app_event is not None: 28 | if app_event.type == AppEventType.Click: 29 | if len(app_event.text_list): 30 | if "Phone storage" in app_event.text_list[0]: 31 | self.clicked_phone_storage = True 32 | print("phone storage clicked") 33 | if view_client.findViewWithText("Change filter") is not None: 34 | # if we filtered phone storage, we shouldn't see any contacts 35 | # and the change filter button will show up 36 | return self.clicked_phone_storage 37 | return False -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/music_player_create_playlist_and_search.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from com.dtmilano.android.viewclient import ViewClient 3 | from ..server import AppEvent, AppEventType 4 | import subprocess 5 | from .music_player_create_playlist import MusicPlayerCreatePlayList 6 | from .music_player_search_playlist import MusicPlayerSearchPlaylist 7 | from . import task_utils 8 | class MusicPlayerCreatePlayListAndSearch(Task): 9 | def __init__(self, task_name="music_player_create_playlist_and_search", 10 | prompt="create a new playlist: test, and search for it", 11 | min_steps=7, 12 | package="com.simplemobiletools.musicplayer", 13 | max_steps=14, 14 | stop_after_finish=False, 15 | permissions = ["android.permission.READ_MEDIA_AUDIO", "android.permission.POST_NOTIFICATIONS"] 16 | ): 17 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish, permissions) 18 | self.createTask = MusicPlayerCreatePlayList() 19 | self.searchTask = MusicPlayerSearchPlaylist() 20 | self.create_task_finished = False 21 | def setup(self, view_client): 22 | task_utils.music_player_permissions_for_old_android_version(view_client) 23 | task_utils.music_player_restore_playlist_functioning(view_client) 24 | pass 25 | def check_finish(self, view_client, app_event) -> bool: 26 | if not self.create_task_finished: 27 | self.create_task_finished = self.createTask.check_finish(view_client, app_event) 28 | else: 29 | print("create finished, checking search") 30 | return self.searchTask.check_finish(view_client, app_event) 31 | return False -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/messager_create_conversation_and_check_message_properties.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | import subprocess 4 | from .messager_start_a_conversation import MessagerStartConversationTask 5 | from .messager_check_message_properties import MessagerCheckMessagePropertiesTask 6 | 7 | class MessagerCreateConversationAndSawMessagePropertiesTask(Task): 8 | def __init__(self, task_name="messager_create_conversation_and_check_message_properties", 9 | prompt="start a conversation with number '123456789', send a message 'i luv u', and check for message properties ", 10 | min_steps=8, 11 | package="com.simplemobiletools.smsmessenger", 12 | max_steps=16, 13 | stop_after_finish=False, 14 | permissions=[]): 15 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish,permissions) 16 | self.createTask = MessagerStartConversationTask() 17 | self.checkTask = MessagerCheckMessagePropertiesTask() 18 | self.createTaskFinished = False 19 | 20 | def setup(self, view_client): 21 | subprocess.run(["adb","shell","pm", "disable-user", "--user", "0", "com.google.android.apps.messaging"]) 22 | return 23 | def teardown(self, view_client): 24 | self.checkTask.teardown(view_client) 25 | def check_finish(self, view_client, app_event: AppEvent) -> bool: 26 | if not self.createTaskFinished: 27 | self.createTaskFinished = self.createTask.check_finish(view_client, app_event) 28 | else: 29 | print("create finished, checking properties") 30 | return self.checkTask.check_finish(view_client, app_event) 31 | return False -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/music_player_album_sort_by_year.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | from com.dtmilano.android.viewclient import ViewClient 4 | from . import task_utils 5 | class MusicPlayerAlbumSortByYear(Task): 6 | def __init__(self, task_name="music_player_album_sort_by_year", 7 | prompt="sort the album by 'year'", 8 | min_steps=4, 9 | package="com.simplemobiletools.musicplayer", 10 | max_steps=8, 11 | stop_after_finish=False, 12 | permissions = ["android.permission.READ_MEDIA_AUDIO", "android.permission.POST_NOTIFICATIONS"]): 13 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish, permissions) 14 | self.last_event = None 15 | self.albumClicked = False 16 | def setup(self, view_client): 17 | task_utils.music_player_permissions_for_old_android_version(view_client) 18 | pass 19 | def check_finish(self, view_client, app_event) -> bool: 20 | albumView = view_client.findViewById('com.simplemobiletools.musicplayer:id/albums_list') 21 | if albumView is not None : 22 | self.albumClicked = True 23 | if app_event is not None and app_event.package == 'com.simplemobiletools.musicplayer': 24 | print('app event received') 25 | if app_event.type == AppEventType.Click: 26 | if len(app_event.text_list) == 1 and app_event.text_list[0] == 'OK': 27 | if self.last_event is not None and self.last_event.type == AppEventType.Click and self.last_event.text_list[0] == 'Year': 28 | return self.albumClicked 29 | self.last_event = app_event 30 | return False 31 | -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/launcher_sort_by_title_desc.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | from .launcher_rename_app import LauncherRenameApp 4 | from . import task_utils 5 | 6 | 7 | class LauncherSortByTitleDesc(Task): 8 | def __init__( 9 | self, 10 | task_name="launcher_sort_by_title_desc", 11 | prompt="Sort apps by title descending", 12 | min_steps=3, 13 | package="com.simplemobiletools.applauncher", 14 | max_steps=6, 15 | stop_after_finish=False, 16 | permissions=[], 17 | ): 18 | super().__init__( 19 | task_name, 20 | prompt, 21 | min_steps, 22 | package, 23 | max_steps, 24 | stop_after_finish, 25 | permissions, 26 | ) 27 | 28 | self.create_task = LauncherRenameApp() 29 | self.boxes_checked = False 30 | 31 | # Add Apps 32 | def setup(self, view_client): 33 | self.create_task.setup(view_client) 34 | 35 | pass 36 | 37 | def check_finish(self, view_client, app_event) -> bool: 38 | if task_utils.is_box_checked( 39 | view_client=view_client, 40 | id_str="com.simplemobiletools.applauncher:id/sorting_dialog_radio_title", 41 | ) and task_utils.is_box_checked( 42 | view_client=view_client, 43 | id_str="com.simplemobiletools.applauncher:id/sorting_dialog_radio_descending", 44 | ): 45 | self.boxes_checked = True 46 | 47 | if ( 48 | app_event is not None 49 | and app_event.type == AppEventType.Click 50 | and "OK" in app_event.text_list 51 | and self.boxes_checked 52 | ): 53 | return True 54 | return False 55 | -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/messager_create_conversation_and_search.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | import subprocess 4 | from .messager_start_a_conversation import MessagerStartConversationTask 5 | from .messager_search_contacts import MessagerSearchContactsTask 6 | from . import task_utils 7 | 8 | class MessagerCreateConversationAndSearchTask(Task): 9 | def __init__(self, task_name="messager_create_conversation_and_search", 10 | prompt="start a conversation with number '123456789', and send a message 'i luv u', back to the main page and search for the contact '123456789'", 11 | min_steps=8, 12 | package="com.simplemobiletools.smsmessenger", 13 | max_steps=16, 14 | stop_after_finish=False, 15 | permissions=[]): 16 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish,permissions) 17 | self.createTask = MessagerStartConversationTask() 18 | self.searchTask = MessagerSearchContactsTask() 19 | self.createTaskFinished = False 20 | 21 | def setup(self, view_client): 22 | subprocess.run(["adb","shell","pm", "disable-user", "--user", "0", "com.google.android.apps.messaging"]) 23 | return 24 | 25 | def teardown(self, view_client): 26 | task_utils.messager_delete_all(view_client) 27 | 28 | def check_finish(self, view_client, app_event: AppEvent) -> bool: 29 | if not self.createTaskFinished: 30 | self.createTaskFinished = self.createTask.check_finish(view_client, app_event) 31 | else: 32 | print("create finished, checking search") 33 | return self.searchTask.check_finish(view_client, app_event) 34 | return False -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/gallery_list_view_type.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | from com.dtmilano.android.viewclient import ViewClient 4 | from . import task_utils 5 | 6 | class GalleryListViewType(Task): 7 | def __init__(self, task_name="gallery_list_view_type", 8 | prompt="Change the view type to list view", 9 | min_steps=4, 10 | package="com.simplemobiletools.gallery.pro", 11 | max_steps=8, 12 | stop_after_finish=False, 13 | permissions=["android.permission.READ_EXTERNAL_STORAGE", "android.permission.READ_MEDIA_IMAGES", "android.permission.READ_MEDIA_VIDEO", "android.permission.READ_MEDIA_VISUAL_USER_SELECTED", "android.permission.ACCESS_MEDIA_LOCATION"]): 14 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish, permissions) 15 | 16 | self.view_type = "" 17 | 18 | def check_finish(self, view_client: ViewClient, app_event: AppEvent) -> bool: 19 | if task_utils.is_box_checked(view_client=view_client, id_str="com.simplemobiletools.gallery.pro:id/change_view_type_dialog_radio_grid"): 20 | self.view_type = "grid" 21 | elif task_utils.is_box_checked(view_client=view_client, id_str="com.simplemobiletools.gallery.pro:id/change_view_type_dialog_radio_list"): 22 | self.view_type = "list" 23 | 24 | if app_event is not None: 25 | # Sometimes the "OK" button click message can be missed, so we also check if the window state changes to "Gallery" 26 | if (app_event.type == AppEventType.Click and "OK" in app_event.text_list) or (app_event.type == AppEventType.WindowStateChange and "Gallery" in app_event.text_list) and self.view_type == "list": 27 | return True 28 | 29 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # MobileAgentBench 2 | An automated benchmark for mobile LLM agents. 3 | 4 | ## Usage 5 | 6 | ### Install AndroidStudio 7 | 8 | Install [AndroidStudio](https://developer.android.com/studio). AndroidStudio installs other debugging tools for you, such as ADB and Android emulators. 9 | 10 | You may need to setup your envrionmnet variables. 11 | 12 | ```bash 13 | export ANDROID_HOME=~/Library/Android/sdk 14 | export PATH="$PATH:$ANDROID_HOME/tools:$ANDROID_HOME/tools/bin:$ANDROID_HOME/platform-tools" 15 | ``` 16 | 17 | ### Download Benchmarking Apps 18 | 19 | The default benchmarking tasks use apps from [SimpleMobileTools](https://simplemobiletools.com). Please download and install the following apps to your testing device (Android emulator is preferred). If you're using an Android emulator, you can simply drag and drop the APK files to install. 20 | 21 | - Calculator 22 | - Calendar 23 | - Contacts 24 | - FileManager 25 | - Gallery 26 | - AppLauncher 27 | - Messager 28 | - MusicPlayer 29 | - Notes 30 | - Recorder 31 | 32 | ### Build MobileBenchMark as a Python Library 33 | 34 | Clone this repo. Run the following commands to install it as a Python library. So you can use import it in other repos. 35 | 36 | ```bash 37 | python3 -m pip install --upgrade build 38 | python3 -m build 39 | ``` 40 | 41 | You'll find the `mobile_agent_benchmark-0.0.1-py3-none-any.whl` file under the `dist` folder. Activate your agent's virtual environment, then you can run `pip install mobile_agent_benchmark-0.0.1-py3-none-any.whl` to install the library. 42 | 43 | ## Dummy Agent 44 | 45 | For testing purpose, you can run the `dummy_agent.py` file. It acts as the simplest agent. In a for loop, it does nothing but just sleep for a few seconds. You can simulate what a real agent would do to test if the benchmark can successfully detect tash completion. 46 | 47 | 48 | 49 | 50 | 51 | -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/recorder_delete.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | from com.dtmilano.android.viewclient import ViewClient 4 | import time 5 | from . import task_utils 6 | 7 | class RecorderDelete(Task): 8 | def __init__(self, task_name="recorder_delete", 9 | prompt="delete the last recorded audio", 10 | min_steps=5, 11 | package="com.simplemobiletools.voicerecorder", 12 | max_steps=10, 13 | stop_after_finish=False, 14 | permissions=["android.permission.RECORD_AUDIO", "android.permission.POST_NOTIFICATIONS"]): 15 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish, permissions) 16 | 17 | def setup(self, view_client): 18 | # To test deleting, we need to at least have something to delete 19 | # recorder only displays audio recorded by itself 20 | # so it's not easy to transfer files 21 | # we need to use UI automation to record an audio 22 | task_utils.recorder_permissions_for_old_android_version(view_client) 23 | toggle_button = view_client.findViewById("com.simplemobiletools.voicerecorder:id/toggle_recording_button") 24 | if toggle_button is None: 25 | print("setup failed") 26 | return 27 | toggle_button.touch() 28 | time.sleep(1) 29 | toggle_button.touch() 30 | pass 31 | 32 | def teardown(self, view_client): 33 | task_utils.recorder_delete_all() 34 | 35 | def check_finish(self, view_client, app_event) -> bool: 36 | view = view_client.findViewById("com.simplemobiletools.voicerecorder:id/recordings_placeholder") 37 | if view is not None: 38 | return "No recordings" in view.text() 39 | return False -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/recorder_rename.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | from com.dtmilano.android.viewclient import ViewClient 4 | import time 5 | from . import task_utils 6 | 7 | class RecorderRename(Task): 8 | def __init__(self, task_name="recorder_rename", 9 | prompt="rename the first audio to 'voice.m4a'", 10 | min_steps=6, 11 | package="com.simplemobiletools.voicerecorder", 12 | max_steps=12, 13 | stop_after_finish=False, 14 | permissions=["android.permission.RECORD_AUDIO", "android.permission.POST_NOTIFICATIONS"]): 15 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish, permissions) 16 | 17 | def setup(self, view_client): 18 | # To test deleting, we need to at least have something to delete 19 | # recorder only displays audio recorded by itself 20 | # so it's not easy to transfer files 21 | # we need to use UI automation to record an audio 22 | task_utils.recorder_permissions_for_old_android_version(view_client) 23 | toggle_button = view_client.findViewById("com.simplemobiletools.voicerecorder:id/toggle_recording_button") 24 | if toggle_button is None: 25 | print("setup failed") 26 | return 27 | toggle_button.touch() 28 | time.sleep(1) 29 | toggle_button.touch() 30 | pass 31 | 32 | def teardown(self, view_client): 33 | task_utils.recorder_delete_all() 34 | 35 | def check_finish(self, view_client, app_event) -> bool: 36 | view = view_client.findViewById('com.simplemobiletools.voicerecorder:id/recording_title') 37 | if view is not None: 38 | return view.text().lower() == 'voice.m4a' 39 | return False -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/messager_start_a_conversation.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | import subprocess 4 | from . import task_utils 5 | 6 | class MessagerStartConversationTask(Task): 7 | def __init__(self, task_name="messager_start_a_conversation", 8 | prompt="start a conversation with number '123456789', and send a message 'i luv u'", 9 | min_steps=8, 10 | package="com.simplemobiletools.smsmessenger", 11 | max_steps=16, 12 | stop_after_finish=False, 13 | permissions=[]): 14 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish,permissions) 15 | self.contactAddressCorrect = False 16 | 17 | def setup(self, view_client): 18 | subprocess.run(["adb","shell","pm", "disable-user", "--user", "0", "com.google.android.apps.messaging"]) 19 | return 20 | 21 | def teardown(self, view_client): 22 | task_utils.messager_delete_all(view_client) 23 | 24 | def check_finish(self, view_client, app_event: AppEvent) -> bool: 25 | # check if the conversation is started 26 | contactView = view_client.findViewById('com.simplemobiletools.smsmessenger:id/new_conversation_address') 27 | if contactView is not None : 28 | text = contactView.text() 29 | if text == '123456789': 30 | self.contactAddressCorrect = True 31 | print('contact address correct') 32 | messageView = view_client.findViewById('com.simplemobiletools.smsmessenger:id/thread_message_body') 33 | if messageView is not None: 34 | text = messageView.text() 35 | if text.lower() == 'i luv u': 36 | print('message correct') 37 | return self.contactAddressCorrect 38 | return False -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/contacts_modify.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | from .contacts_favorite import ContactsFavorite 4 | class ContactsModify(Task): 5 | def __init__(self, task_name="contacts_modify", 6 | prompt="Change the contact Yuzai's number to 987654321 and save it", 7 | min_steps=4, 8 | package="com.simplemobiletools.contacts.pro", 9 | max_steps=8, 10 | stop_after_finish=False, 11 | permissions=["android.permission.READ_CONTACTS"]): 12 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish,permissions) 13 | 14 | self.found = False 15 | self.name = False 16 | self.number = False 17 | self.prepare = ContactsFavorite() 18 | 19 | # create contact 20 | def setup(self, view_client): 21 | self.prepare.setup(view_client) 22 | 23 | # delete contact 24 | def teardown(self, view_client): 25 | self.prepare.teardown(view_client) 26 | 27 | def check_finish(self, view_client, app_event: AppEvent) -> bool: 28 | try: 29 | contact_name = view_client.findViewById('com.simplemobiletools.contacts.pro:id/contact_name') 30 | if contact_name is not None: 31 | text = contact_name.text() 32 | if text.lower() == 'yuzai': 33 | self.name = True 34 | 35 | phone_number = view_client.findViewById('com.simplemobiletools.contacts.pro:id/contact_number') 36 | if phone_number is not None: 37 | text = phone_number.text() 38 | if text == '987654321': 39 | self.number = True 40 | return self.name and self.number 41 | except Exception as e: 42 | print(f"Exception: {str(e)}") 43 | return False -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/music_player_playlist_sort_desc.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | from com.dtmilano.android.viewclient import ViewClient 4 | from . import task_utils 5 | 6 | class MusicPlayerSortByDesc(Task): 7 | def __init__(self, task_name="music_player_playlist_sort_desc", 8 | prompt="sort the playlist by 'desc'", 9 | min_steps=3, 10 | package="com.simplemobiletools.musicplayer", 11 | max_steps=6, 12 | stop_after_finish=False, 13 | permissions = ["android.permission.READ_MEDIA_AUDIO", "android.permission.POST_NOTIFICATIONS"]): 14 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish, permissions) 15 | self.last_event = None 16 | self.playlistClicked = False 17 | 18 | def setup(self, view_client): 19 | task_utils.music_player_permissions_for_old_android_version(view_client) 20 | pass 21 | def check_finish(self, view_client, app_event) -> bool: 22 | playlistView = view_client.findViewById('com.simplemobiletools.musicplayer:id/playlist_title') 23 | if playlistView is not None and playlistView.text().lower() == 'all tracks': 24 | self.playlistClicked = True 25 | if app_event is not None and app_event.package == 'com.simplemobiletools.musicplayer': 26 | print('app event received') 27 | if app_event.type == AppEventType.Click: 28 | if len(app_event.text_list) == 1 and app_event.text_list[0] == 'OK': 29 | if self.last_event is not None and self.last_event.type == AppEventType.Click and self.last_event.text_list[0] == 'Descending': 30 | return self.playlistClicked 31 | self.last_event = app_event 32 | return False 33 | -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/recorder_delete_all.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | from com.dtmilano.android.viewclient import ViewClient 4 | import time 5 | from .recorder_delete import RecorderDelete 6 | from . import task_utils 7 | 8 | class RecorderDeleteAll(Task): 9 | def __init__(self, task_name="recorder_delete_all", 10 | prompt="delete all recorded audio", 11 | min_steps=13, 12 | package="com.simplemobiletools.voicerecorder", 13 | max_steps=26, 14 | stop_after_finish=False, 15 | permissions=["android.permission.RECORD_AUDIO", "android.permission.POST_NOTIFICATIONS"]): 16 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish, permissions) 17 | 18 | def setup(self, view_client): 19 | # To test deleting, we need to at least have something to delete 20 | # recorder only displays audio recorded by itself 21 | # so it's not easy to transfer files 22 | # we need to use UI automation to record an audio 23 | task_utils.recorder_permissions_for_old_android_version(view_client) 24 | toggle_button = view_client.findViewById("com.simplemobiletools.voicerecorder:id/toggle_recording_button") 25 | if toggle_button is None: 26 | print("setup failed") 27 | return 28 | 29 | for i in range(3): # record 3 audio 30 | toggle_button.touch() 31 | time.sleep(1) 32 | toggle_button.touch() 33 | pass 34 | 35 | def teardown(self, view_client): 36 | task_utils.recorder_delete_all() 37 | 38 | def check_finish(self, view_client, app_event) -> bool: 39 | view = view_client.findViewById("com.simplemobiletools.voicerecorder:id/recordings_placeholder") 40 | if view is not None: 41 | return "No recordings" in view.text() 42 | return False -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/filemanager_create_new_file.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | from .task_utils import * 4 | 5 | class FileManagerCreateNewFile(Task): 6 | def __init__(self, task_name="filemanager_create_new_file", 7 | prompt="Create a new file named 'testfile.txt' in the 'Downloads' folder", 8 | min_steps=5, 9 | package="com.simplemobiletools.filemanager.pro", 10 | max_steps=10, 11 | stop_after_finish=False, 12 | permissions=["android.permission.WRITE_EXTERNAL_STORAGE","android.permission.POST_NOTIFICATIONS"]): 13 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish, permissions) 14 | 15 | self.file_icon_clicked = False 16 | 17 | def setup(self, view_client): 18 | filemanager_permissions(view_client) 19 | 20 | def teardown(self, view_client): 21 | filemanager_delete_all_files_under_folder(view_client) 22 | 23 | def check_finish(self, view_client, app_event) -> bool: 24 | if app_event is not None and app_event.package == 'com.simplemobiletools.filemanager.pro': 25 | if app_event.type == AppEventType.Click and 'File' in app_event.text_list: 26 | self.file_icon_clicked = True 27 | 28 | correct_folder_opened = False 29 | folder = view_client.findViewWithText('> Download') 30 | if folder is not None: 31 | correct_folder_opened = True 32 | 33 | view_dict = view_client.getViewsById() 34 | for k in view_dict.keys(): 35 | view = view_dict[k] 36 | if view.map['resource-id'] == 'com.simplemobiletools.filemanager.pro:id/item_name': 37 | if view.text().lower() == 'testfile.txt': 38 | # in case there are other files 39 | return correct_folder_opened and self.file_icon_clicked 40 | 41 | return False -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/recorder_rename_all.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | from com.dtmilano.android.viewclient import ViewClient 4 | import time 5 | from . import task_utils 6 | 7 | class RecorderRenameAll(Task): 8 | def __init__(self, task_name="recorder_rename_all", 9 | prompt="rename all audio to voice1.m4a, voice2.m4a, and so on", 10 | min_steps=13, 11 | package="com.simplemobiletools.voicerecorder", 12 | max_steps=26, 13 | stop_after_finish=False, 14 | permissions=["android.permission.RECORD_AUDIO", "android.permission.POST_NOTIFICATIONS"]): 15 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish, permissions) 16 | 17 | def setup(self, view_client): 18 | # To test deleting, we need to at least have something to delete 19 | # recorder only displays audio recorded by itself 20 | # so it's not easy to transfer files 21 | # we need to use UI automation to record an audio 22 | task_utils.recorder_permissions_for_old_android_version(view_client) 23 | toggle_button = view_client.findViewById("com.simplemobiletools.voicerecorder:id/toggle_recording_button") 24 | if toggle_button is None: 25 | print("setup failed") 26 | return 27 | for i in range(3): # record 3 audio 28 | toggle_button.touch() 29 | time.sleep(1) 30 | toggle_button.touch() 31 | 32 | 33 | def teardown(self, view_client): 34 | task_utils.recorder_delete_all() 35 | 36 | def check_finish(self, view_client, app_event) -> bool: 37 | view1 = view_client.findViewWithText('voice1.m4a') 38 | view2 = view_client.findViewWithText('voice2.m4a') 39 | view3 = view_client.findViewWithText('voice3.m4a') 40 | if view1 is not None and view2 is not None and view3 is not None: 41 | return True 42 | return False -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/filemanager_rename_file.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | from .task_utils import * 4 | 5 | class FileManagerRenameFile(Task): 6 | def __init__(self, task_name="filemanager_rename_file", 7 | prompt="open the folder 'Downloads' and rename the file 'Testfile.txt' to 'testfile1.txt'", 8 | min_steps=4, 9 | package="com.simplemobiletools.filemanager.pro", 10 | max_steps=8, 11 | stop_after_finish=False, 12 | permissions=["android.permission.WRITE_EXTERNAL_STORAGE","android.permission.POST_NOTIFICATIONS"]): 13 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish, permissions) 14 | self.renameFileNameCorrect = False 15 | def setup(self, view_client): 16 | filemanager_permissions(view_client) 17 | filemanager_create_file_under_download(view_client) 18 | 19 | def teardown(self, view_client): 20 | filemanager_delete_all_files_under_folder(view_client) 21 | 22 | def check_finish(self, view_client, app_event) -> bool: 23 | rename_title = view_client.findViewById('com.simplemobiletools.filemanager.pro:id/rename_item_name') 24 | if rename_title is not None: 25 | text = rename_title.text() 26 | if text.lower() == 'testfile1.txt': 27 | self.renameFileNameCorrect = True 28 | print("File name renamed correctly.") 29 | if app_event is not None and app_event.package == 'com.simplemobiletools.filemanager.pro': 30 | if app_event.type == AppEventType.WindowStateChange: 31 | if len(app_event.text_list) == 1 and app_event.text_list[0] == 'File Manager': 32 | if self.last_event is not None and self.last_event.type == AppEventType.Click and self.last_event.text_list[0] == 'OK': 33 | return self.renameFileNameCorrect 34 | self.last_event = app_event 35 | return False -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/music_player_create_playlist_and_sort_desc.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from com.dtmilano.android.viewclient import ViewClient 3 | from ..server import AppEvent, AppEventType 4 | import subprocess 5 | from .music_player_create_playlist import MusicPlayerCreatePlayList 6 | from . import task_utils 7 | class MusicPlayerCreatePlayListAndSearch(Task): 8 | def __init__(self, task_name="music_player_create_playlist_and_sort_desc", 9 | prompt="create a new playlist: test, and sort all playlist by descending order", 10 | min_steps=7, 11 | package="com.simplemobiletools.musicplayer", 12 | max_steps=14, 13 | stop_after_finish=False, 14 | permissions = ["android.permission.READ_MEDIA_AUDIO", "android.permission.POST_NOTIFICATIONS"] 15 | ): 16 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish, permissions) 17 | self.createTask = MusicPlayerCreatePlayList() 18 | self.create_task_finished = False 19 | self.last_event = None 20 | def setup(self, view_client): 21 | task_utils.music_player_permissions_for_old_android_version(view_client) 22 | pass 23 | def check_finish(self, view_client, app_event) -> bool: 24 | if not self.create_task_finished: 25 | self.create_task_finished = self.createTask.check_finish(view_client, app_event) 26 | else: 27 | print("create finished, checking sorting") 28 | if app_event is not None and app_event.package == 'com.simplemobiletools.musicplayer': 29 | if app_event.type == AppEventType.Click: 30 | if len(app_event.text_list) == 1 and app_event.text_list[0] == 'OK': 31 | if self.last_event is not None and self.last_event.type == AppEventType.Click and self.last_event.text_list[0] == 'Descending': 32 | return True 33 | self.last_event = app_event 34 | return False -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/music_player_create_playlist.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from com.dtmilano.android.viewclient import ViewClient 3 | from ..server import AppEvent, AppEventType 4 | import subprocess 5 | from . import task_utils 6 | class MusicPlayerCreatePlayList(Task): 7 | def __init__(self, task_name="music_player_create_playlist", 8 | prompt="create a new playlist:test", 9 | min_steps=5, 10 | package="com.simplemobiletools.musicplayer", 11 | max_steps=10, 12 | stop_after_finish=False, 13 | permissions = ["android.permission.READ_MEDIA_AUDIO", "android.permission.POST_NOTIFICATIONS"] 14 | ): 15 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish, permissions) 16 | self.text_filled = False 17 | self.last_event = None 18 | 19 | def setup(self, view_client): 20 | task_utils.music_player_permissions_for_old_android_version(view_client) 21 | task_utils.music_player_restore_playlist_functioning(view_client) 22 | pass 23 | def check_finish(self, view_client, app_event) -> bool: 24 | view = view_client.findViewById("com.simplemobiletools.musicplayer:id/new_playlist_title") 25 | if view is not None: 26 | view_text = view.getText() 27 | if view_text.lower() == "test": 28 | self.text_filled = True 29 | print('detected text fill') 30 | if app_event is not None and app_event.package == 'com.simplemobiletools.musicplayer': 31 | print('app event received') 32 | if app_event.type == AppEventType.WindowStateChange: 33 | if len(app_event.text_list) == 1 and app_event.text_list[0] == 'Music Player': 34 | if self.last_event is not None and self.last_event.type == AppEventType.Click and self.last_event.text_list[0] == 'OK': 35 | return self.text_filled 36 | self.last_event = app_event 37 | return False -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/filemanager_delete_videos.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | from .task_utils import * 4 | 5 | 6 | class FileManagerDeleteVideos(Task): 7 | def __init__(self, task_name="filemanager_delete_videos", 8 | prompt="Delete all videos in Download folder", 9 | min_steps=9, 10 | package="com.simplemobiletools.filemanager.pro", 11 | max_steps=18, 12 | stop_after_finish=False, 13 | permissions=["android.permission.WRITE_EXTERNAL_STORAGE", "android.permission.POST_NOTIFICATIONS"]): 14 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish, permissions) 15 | self.list = False 16 | 17 | def setup(self, view_client): 18 | filemanager_permissions(view_client) 19 | filemanager_create_file_under_download(view_client, "iccv.mp3") 20 | filemanager_create_file_under_download(view_client, "icml.mp4") 21 | filemanager_create_file_under_download(view_client, "aaai.txt") 22 | filemanager_create_file_under_download(view_client, "corl.avi") 23 | filemanager_create_file_under_download(view_client, "neurips.exe") 24 | filemanager_create_file_under_download(view_client, "cvpr.mov") 25 | 26 | 27 | def teardown(self, view_client): 28 | filemanager_delete_all_files_under_folder(view_client) 29 | 30 | def check_finish(self, view_client, app_event) -> bool: 31 | folder = view_client.findViewWithText('> Download') 32 | if folder is None: 33 | return False 34 | should_exist = ["iccv.mp3", "aaai.txt", "neurips.exe"] 35 | should_delete = ["icml.mp4", "cvpr.mov", "corl.avi"] 36 | 37 | for file in should_exist: 38 | if view_client.findViewWithText(file) is None: 39 | return False 40 | for file in should_delete: 41 | if view_client.findViewWithText(file) is not None: 42 | return False 43 | 44 | return True -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/note_delete.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from com.dtmilano.android.viewclient import ViewClient 3 | 4 | class DeleteNoteTask(Task): 5 | def __init__(self): 6 | super().__init__( 7 | task_name="note_delete", 8 | prompt="delete the 'to_do_list' and 'meeting' note", 9 | min_steps=4, 10 | package="com.simplemobiletools.notes.pro", 11 | max_steps=8, 12 | stop_after_finish=False, 13 | permissions=[] 14 | ) 15 | 16 | def setup(self, view_client): 17 | """ 18 | Prepares the environment by creating a set of notes that will be manipulated during the task. 19 | This includes interacting with the UI to input the note details and confirm their creation. 20 | """ 21 | def create_note(name): 22 | # Access the button to add a new note 23 | new_note_button = view_client.findViewById('com.simplemobiletools.notes.pro:id/new_note') 24 | new_note_button.touch() 25 | view_client.dump() 26 | 27 | title_view = view_client.findViewById('com.simplemobiletools.notes.pro:id/locked_note_title') 28 | title_view.type(name) 29 | 30 | save_button = view_client.findViewById('android:id/button1') 31 | save_button.touch() 32 | 33 | view_client.dump() 34 | 35 | # Create example notes 36 | create_note("work_list") 37 | create_note("to_do_list") 38 | create_note("meeting") 39 | pass 40 | 41 | 42 | def check_finish(self, view_client, app_event): 43 | """ 44 | Verify that specific notes have been deleted by checking their absence. 45 | """ 46 | 47 | work_list = view_client.findViewWithText("work_list") 48 | to_do_list = view_client.findViewWithText("to_do_list") 49 | meeting = view_client.findViewWithText("meeting") 50 | 51 | 52 | if work_list is not None and to_do_list is None and meeting is None: 53 | return True 54 | return False -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/music_player_rescan_media.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from com.dtmilano.android.viewclient import ViewClient 3 | from ..server import AppEvent, AppEventType 4 | import subprocess 5 | import importlib.resources as pkg_resources 6 | from . import task_utils 7 | class MusicPlayerRescanMedia(Task): 8 | def __init__(self, task_name="music_player_rescan_media", 9 | prompt="rescan media files", 10 | min_steps=2, 11 | package="com.simplemobiletools.musicplayer", 12 | max_steps=4, 13 | stop_after_finish=False, 14 | permissions = ["android.permission.READ_MEDIA_AUDIO", "android.permission.POST_NOTIFICATIONS"] 15 | ): 16 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish, permissions) 17 | self.trackNum = "" 18 | def setup(self, view_client): 19 | task_utils.music_player_permissions_for_old_android_version(view_client) 20 | pass 21 | def teardown(self, view_client): 22 | """Restore environment 23 | """ 24 | subprocess.run(["adb", "shell", "rm", "/sdcard/Music/music1.mp3"]) 25 | subprocess.run(["adb", "shell", "rm", "/sdcard/Music/music2.mp3"]) 26 | return 27 | def check_finish(self, view_client, app_event) -> bool: 28 | view = view_client.findViewById("com.simplemobiletools.musicplayer:id/folder_tracks") 29 | if view is not None: 30 | if view.getText() == "2 Tracks": 31 | return True 32 | if app_event is not None and app_event.type == AppEventType.Click: 33 | if app_event.package == 'com.simplemobiletools.musicplayer' : 34 | if app_event.text_list[0] == 'More options': 35 | subprocess.run(["adb", "push", pkg_resources.files("mobile_agent_benchmark") / "assets/music1.mp3", "/sdcard/Music/music1.mp3"]) 36 | subprocess.run(["adb", "push", pkg_resources.files("mobile_agent_benchmark") / "assets/music2.mp3", "/sdcard/Music/music2.mp3"]) 37 | return False 38 | 39 | -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/filemanager_sort_folder_by_size_desc.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | from .task_utils import * 4 | 5 | class FileManagerSortFolderBySizeDesc(Task): 6 | def __init__(self, task_name="filemanager_sort_folder_by_size_desc", 7 | prompt="In the main page, sort the folder by size in descending order", 8 | min_steps=4, 9 | package="com.simplemobiletools.filemanager.pro", 10 | max_steps=8, 11 | stop_after_finish=False, 12 | permissions=["android.permission.WRITE_EXTERNAL_STORAGE","android.permission.POST_NOTIFICATIONS"]): 13 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish, permissions) 14 | self.sizeOptionClicked = False 15 | self.descOptionClicked = False 16 | def setup(self, view_client): 17 | filemanager_permissions(view_client) 18 | 19 | def check_finish(self, view_client, app_event) -> bool: 20 | size_option = view_client.findViewById('com.simplemobiletools.filemanager.pro:id/sorting_dialog_radio_size') 21 | if size_option is not None: 22 | text = size_option.text() 23 | if text.lower() == 'size': 24 | self.sizeOptionClicked = True 25 | desc_option = view_client.findViewById('com.simplemobiletools.filemanager.pro:id/sorting_dialog_radio_descending') 26 | if desc_option is not None: 27 | text = desc_option.text() 28 | if text.lower() == 'descending': 29 | self.descOptionClicked = True 30 | if app_event is not None and app_event.package == 'com.simplemobiletools.filemanager.pro': 31 | if app_event.type == AppEventType.WindowStateChange: 32 | if len(app_event.text_list) == 1 and app_event.text_list[0] == 'File Manager': 33 | if self.last_event is not None and self.last_event.type == AppEventType.Click and self.last_event.text_list[0] == 'OK': 34 | return self.sizeOptionClicked and self.descOptionClicked 35 | self.last_event = app_event 36 | return False -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/note_rename.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from com.dtmilano.android.viewclient import ViewClient 3 | 4 | class RenameNoteTask(Task): 5 | def __init__(self): 6 | super().__init__( 7 | task_name="note_rename", 8 | prompt="rename the current note to 'finished_task'", 9 | min_steps=5, 10 | package="com.simplemobiletools.notes.pro", 11 | max_steps=10, 12 | stop_after_finish=False, 13 | permissions=[] 14 | ) 15 | self.search = False 16 | self.content = False 17 | 18 | def setup(self, view_client): 19 | """ 20 | Prepares the environment by creating a set of notes that will be manipulated during the task. 21 | This includes interacting with the UI to input the note details and confirm their creation. 22 | """ 23 | def create_note(name,text): 24 | # Access the button to add a new note 25 | new_note_button = view_client.findViewById('com.simplemobiletools.notes.pro:id/new_note') 26 | new_note_button.touch() 27 | view_client.dump() 28 | 29 | title_view = view_client.findViewById('com.simplemobiletools.notes.pro:id/locked_note_title') 30 | title_view.type(name) 31 | 32 | save_button = view_client.findViewById('android:id/button1') 33 | save_button.touch() 34 | 35 | view_client.dump() 36 | 37 | text_view = view_client.findViewById('com.simplemobiletools.notes.pro:id/text_note_view') 38 | text_view.type(text) 39 | view_client.dump() 40 | 41 | # Create example notes 42 | create_note("unfinished_task","Undo") 43 | 44 | 45 | def check_finish(self, view_client, app_event): 46 | text_view = view_client.findViewById('com.simplemobiletools.notes.pro:id/text_note_view') 47 | title_view = view_client.findViewWithText('finished_task') 48 | 49 | if text_view is not None and title_view is not None: 50 | return text_view.text() == "Undo" and title_view.map['class'] == 'android.widget.TextView' 51 | 52 | return False 53 | -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/filemanager_hide_folder.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | from .task_utils import * 4 | 5 | 6 | class FileManagerHideFolder(Task): 7 | def __init__(self, task_name="filemanager_hide_folder", 8 | prompt="Hide the folder named 'hidden' and make sure File Manager Stop showing hidden media", 9 | min_steps=3, 10 | package="com.simplemobiletools.filemanager.pro", 11 | max_steps=6, 12 | stop_after_finish=False, 13 | permissions=["android.permission.WRITE_EXTERNAL_STORAGE", "android.permission.POST_NOTIFICATIONS"]): 14 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish, permissions) 15 | self.hide_folder_name = False 16 | 17 | def setup(self, view_client): 18 | download_folder = view_client.findViewWithText('Download') 19 | download_folder.touch() 20 | view_client.dump() 21 | create_file_button = view_client.findViewById('com.simplemobiletools.filemanager.pro:id/items_fab') 22 | create_file_button.touch() 23 | view_client.dump() 24 | file_title = view_client.findViewById('com.simplemobiletools.filemanager.pro:id/item_title') 25 | file_title.setText('hidden') 26 | view_client.dump() 27 | Ok_button = view_client.findViewWithText('OK') 28 | Ok_button.touch() 29 | view_client.dump() 30 | moreOptionButton = view_client.findViewWithContentDescription('More options') 31 | moreOptionButton.touch() 32 | 33 | view_client.dump() 34 | show_hidden_folder_button = view_client.findViewWithText('Temporarily show hidden') 35 | if show_hidden_folder_button is not None: 36 | show_hidden_folder_button.touch() 37 | view_client.dump() 38 | return 39 | 40 | def teardown(self, view_client): 41 | filemanager_delete_all_files_under_folder(view_client) 42 | def check_finish(self, view_client, app_event) -> bool: 43 | folder_name = view_client.findViewWithText('.hidden') 44 | if folder_name is not None: 45 | return True 46 | return False 47 | -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/calculator_recalculate.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | 4 | class CalculatorReCalculateTask(Task): 5 | def __init__(self, task_name="calculator_recalculate", 6 | prompt="Calculate the result of '12 × 5'. However, during the input process, the number '4' was " 7 | "mistakenly entered instead of '5'. Correct this by first enter 'C' to delete '4' and re-entering '5' and then perform the " 8 | "calculation", 9 | min_steps=3, 10 | package="com.simplemobiletools.calculator", 11 | max_steps=6, 12 | stop_after_finish=True, 13 | permissions=[]): 14 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish,permissions) 15 | 16 | self.text_filled = False 17 | self.last_event = None 18 | 19 | def setup(self, view_client): 20 | button_1 = view_client.findViewWithText('1') 21 | button_1.touch() 22 | 23 | button_2 = view_client.findViewWithText('2') 24 | button_2.touch() 25 | 26 | button_multiply = view_client.findViewWithText('×') 27 | button_multiply.touch() 28 | 29 | button_4 = view_client.findViewWithText('4') 30 | button_4.touch() 31 | 32 | view_client.dump() 33 | pass 34 | 35 | def check_finish(self, view_client, app_event: AppEvent) -> bool: 36 | result = view_client.findViewById('com.simplemobiletools.calculator:id/result') 37 | if result is not None and result.text().strip() == '60': 38 | self.text_filled = True 39 | 40 | if app_event is not None and app_event.package == self.package: 41 | # last step is = 42 | if app_event.type == AppEventType.Click and len(app_event.text_list) == 1 and \ 43 | app_event.text_list[0] == '=': 44 | if self.last_event is not None and self.last_event.type == AppEventType.Click and len( 45 | self.last_event.text_list) == 1: 46 | return self.text_filled 47 | self.last_event = app_event 48 | return False 49 | 50 | -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/bench_task.py: -------------------------------------------------------------------------------- 1 | from com.dtmilano.android.viewclient import ViewClient 2 | import subprocess 3 | import re 4 | 5 | class ActionLog: 6 | def __init__(self): 7 | self.action = '' 8 | self.step = -1 9 | self.begin_timestamp = 0 10 | self.end_timestamp = 0 11 | self.screenshot_path = '' 12 | self.input_tokens = 0 13 | self.output_tokens = 0 14 | 15 | class Task(object): 16 | def __init__(self, task_name, prompt, min_steps, package, max_steps=20, stop_after_finish=False, permissions=[]): 17 | self.task_name = task_name 18 | self.prompt = prompt 19 | self.min_steps = min_steps 20 | self.package = package 21 | self.max_steps = max_steps 22 | self.stop_after_finish = stop_after_finish 23 | self.permissions = permissions 24 | 25 | self.current_step = 0 26 | self.action_logs = [] 27 | 28 | self.finished = False 29 | 30 | def get_top_activity_name(self, timeout=60) -> str: 31 | """get top activiy name with timeout 32 | When you call view_client getTopActivityName, 33 | you may get a wrong one, this is because the default timeout is too short 34 | 35 | Args: 36 | timeout (int, optional): timeout. Defaults to 60. 37 | 38 | Returns: 39 | str: the top 40 | """ 41 | output = subprocess.run(["adb", "shell", "dumpsys", "-t", str(timeout), "activity", "top"], capture_output=True, text=True) 42 | activity_re = re.compile('\s*ACTIVITY ([A-Za-z0-9_.]+)/([A-Za-z0-9_.\$]+) \w+ pid=(\d+)') 43 | m = activity_re.findall(output.stdout) 44 | if len(m) > 0: 45 | activity_and_pid = m[-1] 46 | return activity_and_pid[0] + '/' + activity_and_pid[1] 47 | else: 48 | return None 49 | 50 | 51 | def setup(self, view_client): 52 | """Setup the task execution environment, either with adb or UI automation 53 | """ 54 | return 55 | 56 | def teardown(self, view_client): 57 | """Restore environment 58 | """ 59 | return 60 | 61 | def check_finish(self, view_client, app_event) -> bool: 62 | raise NotImplementedError 63 | 64 | 65 | 66 | 67 | -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/calendar_delete_tasks.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | from com.dtmilano.android.viewclient import ViewClient 4 | 5 | class CalendarDeleteTasks(Task): 6 | def __init__(self, task_name="calendar_delete_tasks", 7 | prompt="Show events in simple event list, delete the laundry and meeting events.", 8 | min_steps=6, 9 | package="com.simplemobiletools.calendar.pro", 10 | max_steps=14, 11 | stop_after_finish=False, 12 | permissions=["android.permission.POST_NOTIFICATIONS"]): 13 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish, permissions) 14 | 15 | self.text_filled = False 16 | self.last_event = None 17 | 18 | def setup(self, view_client) -> bool: 19 | # grant app permission and prepare 3 events. The task needs to delete two of them 20 | def create_event(name): 21 | add_button = view_client.findViewById('com.simplemobiletools.calendar.pro:id/calendar_fab') 22 | add_button.touch() 23 | view_client.dump() 24 | event_button = view_client.findViewWithText('Event') 25 | event_button.touch() 26 | view_client.dump() 27 | title_view = view_client.findViewById('com.simplemobiletools.calendar.pro:id/event_title') 28 | title_view.type(name) 29 | save_button = view_client.findViewById('com.simplemobiletools.calendar.pro:id/save') 30 | save_button.touch() 31 | create_event("Laundry") 32 | view_client.dump() 33 | ok_button = view_client.findViewWithText('OK') 34 | ok_button.touch() 35 | 36 | view_client.dump() 37 | create_event("Cooking") 38 | 39 | view_client.dump() 40 | create_event("Meeting") 41 | pass 42 | 43 | def check_finish(self, view_client, app_event: AppEvent) -> bool: 44 | cooking = view_client.findViewWithText("Cooking") 45 | laundry = view_client.findViewWithText("Laundry") 46 | meeting = view_client.findViewWithText("Meeting") 47 | 48 | if cooking is not None and laundry is None and meeting is None: 49 | return True 50 | return False 51 | -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/note_open.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from com.dtmilano.android.viewclient import ViewClient 3 | 4 | class OpenNoteTask(Task): 5 | def __init__(self): 6 | super().__init__( 7 | task_name="note_open", 8 | prompt="open the note 'meeting'", 9 | min_steps=1, 10 | package="com.simplemobiletools.notes.pro", 11 | max_steps=2, 12 | stop_after_finish=False, 13 | permissions=[] 14 | ) 15 | 16 | def setup(self, view_client): 17 | """ 18 | Prepares the environment by creating a set of notes that will be manipulated during the task. 19 | This includes interacting with the UI to input the note details and confirm their creation. 20 | """ 21 | def create_note(name,text): 22 | # Access the button to add a new note 23 | new_note_button = view_client.findViewById('com.simplemobiletools.notes.pro:id/new_note') 24 | new_note_button.touch() 25 | view_client.dump() 26 | 27 | title_view = view_client.findViewById('com.simplemobiletools.notes.pro:id/locked_note_title') 28 | title_view.type(name) 29 | 30 | save_button = view_client.findViewById('android:id/button1') 31 | save_button.touch() 32 | 33 | view_client.dump() 34 | 35 | text_view = view_client.findViewById('com.simplemobiletools.notes.pro:id/text_note_view') 36 | text_view.type(text) 37 | view_client.dump() 38 | 39 | # Create example notes 40 | create_note("meeting","9pm May 5th 2024") 41 | create_note("Charles's secrets","I love you") 42 | create_note("to_do_list","Complete the task") 43 | 44 | pass 45 | 46 | def check_finish(self, view_client, app_event): 47 | """ 48 | Verify that specific notes have been deleted by checking their absence. 49 | """ 50 | 51 | text_view = view_client.findViewById('com.simplemobiletools.notes.pro:id/text_note_view') 52 | title_views = view_client.findViewWithText("9pm May 5th 2024") 53 | 54 | 55 | if text_view.text() == "9pm May 5th 2024" and title_views is not None: 56 | return True 57 | return False 58 | -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/contacts_favorite.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | import subprocess 4 | 5 | 6 | class ContactsFavorite(Task): 7 | def __init__(self, task_name="contacts_favorite", 8 | prompt="Set the contact Yuzai to Favorite", 9 | min_steps=2, 10 | package="com.simplemobiletools.contacts.pro", 11 | max_steps=4, 12 | stop_after_finish=False, 13 | permissions=["android.permission.READ_CONTACTS"]): 14 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish, permissions) 15 | 16 | # create contact 17 | def setup(self, view_client): 18 | # touch '+' button 19 | button_add = view_client.findViewById('com.simplemobiletools.contacts.pro:id/fragment_fab') 20 | button_add.touch() 21 | 22 | view_client.dump() 23 | 24 | # input firstname 25 | try: 26 | element = view_client.findViewByIdOrRaise('com.simplemobiletools.contacts.pro:id/contact_first_name') 27 | element.touch() 28 | 29 | view_client.device.type('Yuzai') 30 | except Exception as e: 31 | print("ExceptionName: ", str(e)) 32 | 33 | # input phone number 34 | try: 35 | element = view_client.findViewByIdOrRaise('com.simplemobiletools.contacts.pro:id/contact_number') 36 | element.setText('123456789') 37 | except Exception as e: 38 | print("ExceptionNumber: ", str(e)) 39 | 40 | # click save button 41 | try: 42 | element = view_client.findViewById('com.simplemobiletools.contacts.pro:id/save') 43 | element.touch() 44 | except Exception as e: 45 | print("Exception: ", str(e)) 46 | 47 | pass 48 | 49 | # delete contact 50 | def teardown(self, view_client): 51 | subprocess.run(["adb", "shell", "pm", "clear", "com.android.providers.contacts"]) 52 | 53 | def check_finish(self, view_client, app_event) -> bool: 54 | # Check if the favorite button is clicked 55 | if app_event is not None: 56 | if app_event.type == AppEventType.Click and app_event.id_str == "com.simplemobiletools.contacts.pro:id/contact_toggle_favorite": 57 | return True 58 | return False 59 | -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/contacts_hide_email.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | import subprocess 4 | 5 | class ContactsHideEmail(Task): 6 | def __init__(self, task_name="contacts_hide_email", 7 | prompt="Set not show contact's Email in the contact profile screen", 8 | min_steps=5, 9 | package="com.simplemobiletools.contacts.pro", 10 | max_steps=10, 11 | stop_after_finish=False, 12 | permissions=["android.permission.READ_CONTACTS"]): 13 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish,permissions) 14 | self.unchecked_email = False 15 | 16 | def setup(self, view_client): 17 | button_add = view_client.findViewById('com.simplemobiletools.contacts.pro:id/fragment_fab') 18 | button_add.touch() 19 | 20 | view_client.dump() 21 | 22 | number = view_client.findViewByIdOrRaise('com.simplemobiletools.contacts.pro:id/contact_number') 23 | number.setText('123456789') 24 | 25 | email = view_client.findViewByIdOrRaise('com.simplemobiletools.contacts.pro:id/contact_email') 26 | email.setText('yuzai@gmail.com') 27 | 28 | name = view_client.findViewByIdOrRaise('com.simplemobiletools.contacts.pro:id/contact_first_name') 29 | name.touch() 30 | view_client.device.type('Yuzai') 31 | 32 | view_client.dump() 33 | 34 | save = view_client.findViewById('com.simplemobiletools.contacts.pro:id/save') 35 | save.touch() 36 | 37 | view_client.dump() 38 | 39 | contact_yuzai = view_client.findViewWithText('Yuzai') 40 | contact_yuzai.touch() 41 | 42 | view_client.dump() 43 | 44 | pass 45 | 46 | def teardown(self, view_client): 47 | subprocess.run(["adb", "shell", "pm", "clear", "com.android.providers.contacts"]) 48 | 49 | def check_finish(self, view_client, app_event: AppEvent) -> bool: 50 | email_check = view_client.findViewById('com.simplemobiletools.contacts.pro:id/manage_visible_fields_emails') 51 | if email_check is not None: 52 | self.unchecked_email = not email_check.checked() 53 | 54 | if app_event is not None: 55 | if len(app_event.text_list) > 0: 56 | if 'OK' == app_event.text_list[0]: 57 | return self.unchecked_email 58 | return False 59 | -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/launcher_rename_app.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | from . import task_utils 4 | 5 | 6 | class LauncherRenameApp(Task): 7 | def __init__( 8 | self, 9 | task_name="launcher_rename_app", 10 | prompt="Rename Chrome in Launcher to MyChrome", 11 | min_steps=5, 12 | package="com.simplemobiletools.applauncher", 13 | max_steps=10, 14 | stop_after_finish=False, 15 | permissions=[], 16 | ): 17 | super().__init__( 18 | task_name, 19 | prompt, 20 | min_steps, 21 | package, 22 | max_steps, 23 | stop_after_finish, 24 | permissions, 25 | ) 26 | 27 | self.at_home = False 28 | 29 | # Add Apps 30 | def setup(self, view_client): 31 | # touch '+' button 32 | button_add = view_client.findViewById( 33 | "com.simplemobiletools.applauncher:id/fab" 34 | ) 35 | button_add.touch() 36 | 37 | view_client.dump() 38 | 39 | # Select Chrome 40 | try: 41 | element = view_client.findViewWithText("Chrome") 42 | if not element.checked(): 43 | element.touch() 44 | 45 | except Exception as e: 46 | print("ExceptionFindAppChrome: ", str(e)) 47 | 48 | # Select Camera 49 | try: 50 | element = view_client.findViewWithText("Camera") 51 | if not element.checked(): 52 | element.touch() 53 | except Exception as e: 54 | print("ExceptionFindAppCamera: ", str(e)) 55 | 56 | # click save button 57 | try: 58 | element = view_client.findViewById("android:id/button1") 59 | element.touch() 60 | except Exception as e: 61 | print("Exception: ", str(e)) 62 | 63 | pass 64 | 65 | def check_finish(self, view_client, app_event) -> bool: 66 | # Check if at the main view 67 | launcher_home = view_client.findViewById( 68 | "com.simplemobiletools.applauncher:id/coordinator_layout" 69 | ) 70 | if launcher_home is not None: 71 | self.at_home = True 72 | 73 | chrome_view = view_client.findViewWithText("MyChrome") 74 | if chrome_view is not None and self.at_home: 75 | return True 76 | 77 | self.at_home = False 78 | return False 79 | -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/gallery_filter_by_images_and_videos.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | from com.dtmilano.android.viewclient import ViewClient 4 | from . import task_utils 5 | 6 | class GalleryFilterByImagesAndVideos(Task): 7 | def __init__(self, task_name="gallery_filter_by_images_and_videos", 8 | prompt="filter media in the gallery and only show images and videos", 9 | min_steps=6, 10 | package="com.simplemobiletools.gallery.pro", 11 | max_steps=12, 12 | stop_after_finish=False, 13 | permissions=["android.permission.READ_EXTERNAL_STORAGE", "android.permission.READ_MEDIA_IMAGES", "android.permission.READ_MEDIA_VIDEO", "android.permission.READ_MEDIA_VISUAL_USER_SELECTED", "android.permission.ACCESS_MEDIA_LOCATION"]): 14 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish, permissions) 15 | 16 | self.last_view_box_status = False 17 | 18 | def check_finish(self, view_client: ViewClient, app_event: AppEvent) -> bool: 19 | if app_event is not None: 20 | # Sometimes the "OK" button click message can be missed, so we also check if the window state changes to "Gallery" 21 | if (app_event.type == AppEventType.Click and "OK" in app_event.text_list) or (app_event.type == AppEventType.WindowStateChange and "Gallery" in app_event.text_list) and self.last_view_box_status is True: 22 | return True 23 | 24 | if ( 25 | task_utils.is_box_checked(view_client=view_client, id_str="com.simplemobiletools.gallery.pro:id/filter_media_images") 26 | and task_utils.is_box_checked(view_client=view_client, id_str="com.simplemobiletools.gallery.pro:id/filter_media_videos") 27 | and not task_utils.is_box_checked(view_client=view_client, id_str="com.simplemobiletools.gallery.pro:id/filter_media_gifs") 28 | and not task_utils.is_box_checked(view_client=view_client, id_str="com.simplemobiletools.gallery.pro:id/filter_media_raws") 29 | and not task_utils.is_box_checked(view_client=view_client, id_str="com.simplemobiletools.gallery.pro:id/filter_media_svgs") 30 | and not task_utils.is_box_checked(view_client=view_client, id_str="com.simplemobiletools.gallery.pro:id/filter_media_portraits") 31 | ): 32 | self.last_view_box_status = True 33 | 34 | return False -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/music_player_search_playlist.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | from com.dtmilano.android.viewclient import ViewClient 4 | from . import task_utils 5 | class MusicPlayerSearchPlaylist(Task): 6 | def __init__(self, task_name="music_player_search_playlist", 7 | prompt="search playlist 'Test'", 8 | min_steps=2, 9 | package="com.simplemobiletools.musicplayer", 10 | max_steps=4, 11 | stop_after_finish=False, 12 | permissions = ["android.permission.READ_MEDIA_AUDIO", "android.permission.POST_NOTIFICATIONS"]): 13 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish, permissions) 14 | 15 | def setup(self, view_client): 16 | task_utils.music_player_permissions_for_old_android_version(view_client) 17 | task_utils.music_player_restore_playlist_functioning(view_client) 18 | view_dict = view_client.getViewsById() 19 | for k in view_dict.keys(): 20 | view = view_dict[k] 21 | if view.map['resource-id'] == 'com.simplemobiletools.musicplayer:id/tab_item_label': 22 | if view.text().lower() == 'playlists': 23 | view.touch() 24 | view_client.dump() 25 | break 26 | allView = view_client.getViewsById() 27 | for(viewId, view) in allView.items(): 28 | viewContext = view.map 29 | if viewContext['content-desc'] == 'More options': 30 | view.touch() 31 | break 32 | view_client.dump() 33 | createPlaylistview = view_client.findViewWithText('Create new playlist') 34 | createPlaylistview.touch() 35 | view_client.dump() 36 | playlistTitleView = view_client.findViewById('com.simplemobiletools.musicplayer:id/new_playlist_title') 37 | playlistTitleView.setText('Test') 38 | view_client.dump() 39 | okButton = view_client.findViewWithText('OK') 40 | okButton.touch() 41 | view_client.dump() 42 | pass 43 | def check_finish(self, view_client, app_event) -> bool: 44 | search_bar = view_client.findViewById('com.simplemobiletools.musicplayer:id/top_toolbar_search') 45 | if search_bar is not None: 46 | text = search_bar.text() 47 | if text.lower() == 'test': 48 | return True 49 | return False -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/note_search.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from com.dtmilano.android.viewclient import ViewClient 3 | 4 | class SearchNoteTask(Task): 5 | def __init__(self): 6 | super().__init__( 7 | task_name="note_search", 8 | prompt="search 'secret' in note 'Charles's secrets'", 9 | min_steps=2, 10 | package="com.simplemobiletools.notes.pro", 11 | max_steps=4, 12 | stop_after_finish=False, 13 | permissions=[] 14 | ) 15 | self.search = False 16 | self.content = False 17 | 18 | def setup(self, view_client): 19 | """ 20 | Prepares the environment by creating a set of notes that will be manipulated during the task. 21 | This includes interacting with the UI to input the note details and confirm their creation. 22 | """ 23 | def create_note(name,text): 24 | # Access the button to add a new note 25 | new_note_button = view_client.findViewById('com.simplemobiletools.notes.pro:id/new_note') 26 | new_note_button.touch() 27 | view_client.dump() 28 | 29 | title_view = view_client.findViewById('com.simplemobiletools.notes.pro:id/locked_note_title') 30 | title_view.setText(name) 31 | view_client.dump() 32 | 33 | save_button = view_client.findViewWithText('OK') 34 | save_button.touch() 35 | view_client.dump() 36 | 37 | text_view = view_client.findViewById('com.simplemobiletools.notes.pro:id/text_note_view') 38 | text_view.setText(text) 39 | view_client.dump() 40 | 41 | # Create example notes 42 | create_note("Charles's secrets","The secret is I love you") 43 | pass 44 | 45 | 46 | 47 | def check_finish(self, view_client, app_event) -> bool: 48 | 49 | 50 | search_view = view_client.findViewById('com.simplemobiletools.notes.pro:id/search_query') 51 | if search_view is not None: 52 | if search_view.text().lower() == 'secret': 53 | self.search = True 54 | 55 | note_view = view_client.findViewById('com.simplemobiletools.notes.pro:id/text_note_view') 56 | if note_view is not None: 57 | note_string = note_view.text() 58 | if 'secret' in note_string.lower(): 59 | self.content = True 60 | return self.search and self.content 61 | 62 | -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/contacts_create.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | 4 | class ContactsCreate(Task): 5 | def __init__(self, task_name="contacts_create", 6 | prompt="Create a new contact, his First Name is Yuzai, and his Phone Number is 123456789", 7 | min_steps=6, 8 | package="com.simplemobiletools.contacts.pro", 9 | max_steps=12, 10 | stop_after_finish=False, 11 | permissions=["android.permission.READ_CONTACTS"]): 12 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish,permissions) 13 | 14 | self.found = False 15 | self.phone = False 16 | self.firstname = False 17 | 18 | def teardown(self, view_client): 19 | print("teardown contact...") 20 | pkg = "com.simplemobiletools.contacts.pro" 21 | view_client.device.forceStop(package=pkg) 22 | view_client.device.startActivity(package=pkg) 23 | contact = view_client.findViewById('com.simplemobiletools.contacts.pro:id/item_contact_name') 24 | contact.touch() 25 | view_client.dump() 26 | 27 | delete_button = view_client.findViewById('com.simplemobiletools.contacts.pro:id/delete') 28 | delete_button.touch() 29 | view_client.dump() 30 | ok_button = view_client.findViewById('android:id/button1') 31 | ok_button.touch() 32 | view_client.dump() 33 | pass 34 | 35 | def check_finish(self, view_client, app_event: AppEvent) -> bool: 36 | first_name = view_client.findViewById('com.simplemobiletools.contacts.pro:id/contact_first_name') 37 | if first_name is not None: 38 | text = first_name.text() 39 | if text.lower() == 'yuzai': 40 | self.firstname = True 41 | 42 | phone_number = view_client.findViewById('com.simplemobiletools.contacts.pro:id/contact_number') 43 | if phone_number is not None: 44 | text = phone_number.text() 45 | if text == '123456789': 46 | self.phone = True 47 | 48 | 49 | title_views = view_client.findViewById('com.simplemobiletools.contacts.pro:id/item_contact_name') 50 | 51 | if title_views is not None: 52 | text = title_views.text() 53 | if text.lower() == 'yuzai': 54 | print(f"Detected text: {text}") 55 | self.found = True 56 | return self.phone and self.firstname and self.found 57 | -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/messager_add_block_numbers.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | import subprocess 4 | 5 | class MessagerAddBlockNumberTask(Task): 6 | def __init__(self, task_name="messager_add_block_numbers", 7 | prompt="Add a number '123456789' to block list", 8 | min_steps=5, 9 | package="com.simplemobiletools.smsmessenger", 10 | max_steps=10, 11 | stop_after_finish=False, 12 | permissions=[]): 13 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish,permissions) 14 | self.okButtonClicked = False 15 | self.correctBlockNumberEntered = False 16 | def setup(self, view_client): 17 | subprocess.run(["adb","shell","pm", "disable-user", "--user", "0", "com.google.android.apps.messaging"]) 18 | return 19 | def teardown(self, view_client): 20 | print("teardown sms...") 21 | pkg = "com.simplemobiletools.smsmessenger" 22 | view_client.device.forceStop(package=pkg) 23 | view_client.device.startActivity(package=pkg) 24 | view_client.dump() 25 | settingsButton = view_client.findViewById('com.simplemobiletools.smsmessenger:id/settings') 26 | settingsButton.touch() 27 | view_client.dump() 28 | manageBlockedNumbersButton = view_client.findViewWithText('Manage blocked numbers') 29 | manageBlockedNumbersButton.touch() 30 | view_client.dump() 31 | numberSelection = view_client.findViewWithText('123456789') 32 | if numberSelection is not None: 33 | numberSelection.longTouch() 34 | view_client.dump() 35 | deleteButton = view_client.findViewWithContentDescription('Delete') 36 | deleteButton.touch() 37 | view_client.dump() 38 | pass 39 | def check_finish(self, view_client, app_event: AppEvent) -> bool: 40 | correctBlockNumberEntered = view_client.findViewWithText('123456789') 41 | if correctBlockNumberEntered is not None: 42 | self.correctBlockNumberEntered = True 43 | print('Correct block number entered') 44 | okButton = view_client.findViewWithText('OK') 45 | if okButton is None: 46 | self.okButtonClicked = True 47 | print('OK button clicked') 48 | activity_name = view_client.device.getTopActivityName() 49 | if 'ManageBlockedNumbersActivity' in activity_name: 50 | return self.correctBlockNumberEntered and self.okButtonClicked 51 | return False -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/messager_check_message_properties.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | import subprocess 4 | from . import task_utils 5 | 6 | class MessagerCheckMessagePropertiesTask(Task): 7 | def __init__(self, task_name="messager_check_message_properties", 8 | prompt="open the conversation with contact number '123456789', and check for a random message's properties ", 9 | min_steps=4, 10 | package="com.simplemobiletools.smsmessenger", 11 | max_steps=8, 12 | stop_after_finish=False, 13 | permissions=[]): 14 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish,permissions) 15 | self.receiverCorrect = False 16 | 17 | def setup(self, view_client): 18 | subprocess.run(["adb","shell","pm", "disable-user", "--user", "0", "com.google.android.apps.messaging"]) 19 | addConversationButton = view_client.findViewById("com.simplemobiletools.smsmessenger:id/conversations_fab") 20 | addConversationButton.touch() 21 | view_client.dump() 22 | setContactButton = view_client.findViewById("com.simplemobiletools.smsmessenger:id/new_conversation_address") 23 | setContactButton.setText("123456789") 24 | view_client.dump() 25 | addButton = view_client.findViewById("com.simplemobiletools.smsmessenger:id/new_conversation_confirm") 26 | addButton.touch() 27 | view_client.dump() 28 | editMessage = view_client.findViewById("com.simplemobiletools.smsmessenger:id/thread_type_message") 29 | editMessage.setText("i luv u") 30 | view_client.dump() 31 | sendButton = view_client.findViewById("com.simplemobiletools.smsmessenger:id/thread_send_message") 32 | sendButton.touch() 33 | view_client.dump() 34 | backButton = view_client.findViewWithContentDescription('Back') 35 | backButton.touch() 36 | view_client.dump() 37 | sndBackButton = view_client.findViewWithContentDescription('Back') 38 | sndBackButton.touch() 39 | view_client.dump() 40 | return 41 | 42 | def teardown(self, view_client): 43 | print("teardown sms...") 44 | task_utils.messager_delete_all(view_client) 45 | 46 | def check_finish(self, view_client, app_event) -> bool: 47 | if app_event is not None and app_event.package == 'com.simplemobiletools.smsmessenger': 48 | if app_event.type == AppEventType.WindowStateChange: 49 | if len(app_event.text_list) == 6 and app_event.text_list[0] == 'Message details' and app_event.text_list[2] == '123456789': 50 | return True 51 | return False -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/gallery_set_wallpaper.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | from com.dtmilano.android.viewclient import ViewClient 4 | import subprocess 5 | import importlib.resources as pkg_resources 6 | 7 | 8 | class GallerySetWallpaper(Task): 9 | def __init__(self, task_name="gallery_set_wallpaper", 10 | prompt="Go to Downloads Folder and set the first image as Home screen wallpaper", 11 | min_steps=6, 12 | package="com.simplemobiletools.gallery.pro", 13 | max_steps=12, 14 | stop_after_finish=True, 15 | permissions=["android.permission.READ_EXTERNAL_STORAGE", "android.permission.READ_MEDIA_IMAGES", "android.permission.READ_MEDIA_VIDEO", "android.permission.READ_MEDIA_VISUAL_USER_SELECTED", "android.permission.ACCESS_MEDIA_LOCATION"]): 16 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish, permissions) 17 | self.wallpaper_set = False 18 | 19 | def setup(self, view_client): 20 | #Run adb script to diable system default photos app 21 | command = ["adb", "shell", "pm", "disable-user", "--user", "0", "com.google.android.apps.photos"] 22 | 23 | subprocess.run(["adb", "push", pkg_resources.files("mobile_agent_benchmark") / "assets/wallpaper.png", "/sdcard/Download/wallpaper.png"]) 24 | try: 25 | result = subprocess.run(command, check=True, capture_output=True, text=True) 26 | print("Disabled Google Photos:", result.stdout) 27 | except subprocess.CalledProcessError as e: 28 | print("Command failed with return code:", e.returncode) 29 | print("Error output:", e.stderr) 30 | except Exception as e: 31 | print("An error occurred:", str(e)) 32 | 33 | # refresh 34 | start_x = 540 35 | start_y = 500 36 | end_x = 540 37 | end_y = 1500 38 | view_client.swipe(start_x, start_y, end_x, end_y) 39 | 40 | def teardown(self, view_client): 41 | subprocess.run(["adb", "shell", "rm", "-rf", "/sdcard/Download/wallpaper.png"]) 42 | 43 | 44 | def check_finish(self, view_client, app_event: AppEvent) -> bool: 45 | if app_event is not None: 46 | if app_event.type == AppEventType.WindowStateChange and "Home screen" in app_event.text_list: 47 | self.wallpaper_set = True 48 | # When Home screen is clicked, there's no click event returned, so we need to check if the window state change event has back to the Simple Wallpaper 49 | elif app_event.type == AppEventType.WindowStateChange and "Simple Wallpaper" in app_event.text_list and self.wallpaper_set is True: 50 | return True 51 | return False 52 | -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/calendar_new_task.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | from com.dtmilano.android.viewclient import ViewClient 4 | 5 | from .calendar_laundry import CalendarLaundryTask 6 | 7 | class CalendarNewTask(Task): 8 | def __init__(self, task_name="calendar_new_task", 9 | prompt="Create a new task, named 'laundry', with the description of 'wash all my clothes'. Mark it as all-day.", 10 | min_steps=6, 11 | package="com.simplemobiletools.calendar.pro", 12 | max_steps=14, 13 | stop_after_finish=False, 14 | permissions=["android.permission.POST_NOTIFICATIONS"]): 15 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish, permissions) 16 | 17 | self.text_filled = False 18 | self.last_event = None 19 | 20 | def setup(self, view_client) -> bool: 21 | CalendarLaundryTask().setup(view_client) 22 | 23 | def check_finish(self, view_client, app_event: AppEvent) -> bool: 24 | title_view = view_client.findViewById('com.simplemobiletools.calendar.pro:id/task_title') 25 | description_view = view_client.findViewById('com.simplemobiletools.calendar.pro:id/task_description') 26 | checkbox = view_client.findViewById('com.simplemobiletools.calendar.pro:id/task_all_day') 27 | if title_view is not None and description_view is not None: 28 | title = title_view.text() 29 | description = description_view.text() 30 | checked = checkbox.checked() 31 | if title.lower() == 'laundry' and description.lower() == 'wash all my clothes' and checked: 32 | self.text_filled = True 33 | print('detected text fill') 34 | 35 | if app_event is not None and app_event.package == 'com.simplemobiletools.calendar.pro': 36 | print('app event received') 37 | if app_event.type == AppEventType.WindowStateChange: 38 | if len(app_event.text_list) == 1 and app_event.text_list[0] == 'Calendar': 39 | if self.last_event is not None and self.last_event.type == AppEventType.Click and len(self.last_event.text_list) == 0: 40 | # it's a bit weird, but when the text is filled, sometimes the id_str in the save button is missing 41 | # it should be `com.simplemobiletools.calendar.pro:id/save` 42 | # we use two consecutive events to track the id_button click status: 43 | # 1. a click event with no text 44 | # 2. a new page event with text `Calendar` 45 | return self.text_filled 46 | self.last_event = app_event 47 | return False 48 | -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/server.py: -------------------------------------------------------------------------------- 1 | from http.server import BaseHTTPRequestHandler, HTTPServer 2 | import json 3 | import threading 4 | from enum import Enum 5 | 6 | class AppEventType(Enum): 7 | Unknown = 0 8 | Click = 1 9 | WindowStateChange = 2 10 | 11 | class AppEvent: 12 | def __init__(self): 13 | self.type = AppEventType.Unknown 14 | self.package = '' 15 | self.id_str = '' 16 | self.content_desc = '' 17 | self.text_list = [] 18 | 19 | def __str__(self) -> str: 20 | return f"Event - type: {self.type}, id_str: '{self.id_str}', content_desc: '{self.content_desc}', text_list: {self.text_list}, package: {self.package}" 21 | 22 | def request_handler_class(callback_fn): 23 | class HTTPRequestHandler(BaseHTTPRequestHandler): 24 | def _set_response(self): 25 | self.send_response(200) 26 | self.send_header('Content-type', 'application/json') 27 | self.end_headers() 28 | 29 | def log_message(self, format, *args): 30 | # disable default log prints 31 | pass 32 | 33 | def do_POST(self): 34 | content_length = int(self.headers['Content-Length']) # Gets the size of data 35 | post_data = self.rfile.read(content_length) # Gets the data itself 36 | data = json.loads(post_data.decode('utf-8')) # Decode and load JSON data 37 | 38 | event = AppEvent() 39 | type = data['eventType'] 40 | if type == 0: 41 | event.type = AppEventType.Click 42 | elif type == 1: 43 | event.type = AppEventType.WindowStateChange 44 | 45 | event.package = data['packageStr'] 46 | event.id_str = data['idStr'] 47 | event.text_list = data['text'] 48 | event.content_desc = data['contentDescription'] if data['contentDescription'] != 'null' else '' 49 | 50 | # Print received JSON data 51 | print("Received:", event) 52 | callback_fn(event) 53 | 54 | # Send response back to client 55 | self._set_response() 56 | response = {'status': 'OK'} 57 | self.wfile.write(json.dumps(response).encode('utf-8')) 58 | 59 | return HTTPRequestHandler 60 | 61 | def run(callback_fn, server_class=HTTPServer, port=8888): 62 | server_address = ('', port) 63 | handler_class=request_handler_class(callback_fn) 64 | httpd = server_class(server_address, handler_class) 65 | print(f"Starting httpd server on {port}") 66 | httpd.serve_forever() 67 | 68 | def start_server(callback_fn): 69 | server_thread = threading.Thread(target=run, args=(callback_fn,)) 70 | server_thread.daemon = True # Daemonize thread 71 | server_thread.start() 72 | print("Server started in a background thread.") -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/messager_search_message.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | import subprocess 4 | from . import task_utils 5 | 6 | class MessagerSearchMessageTask(Task): 7 | def __init__(self, task_name="messager_search_message", 8 | prompt="search message 'i luv u' at the top search bar", 9 | min_steps=2, 10 | package="com.simplemobiletools.smsmessenger", 11 | max_steps=4, 12 | stop_after_finish=False, 13 | permissions=[]): 14 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish,permissions) 15 | self.text_correct = False 16 | 17 | def setup(self, view_client): 18 | subprocess.run(["adb","shell","pm", "disable-user", "--user", "0", "com.google.android.apps.messaging"]) 19 | addConversationButton = view_client.findViewById("com.simplemobiletools.smsmessenger:id/conversations_fab") 20 | addConversationButton.touch() 21 | view_client.dump() 22 | setContactButton = view_client.findViewById("com.simplemobiletools.smsmessenger:id/new_conversation_address") 23 | setContactButton.setText("123456789") 24 | view_client.dump() 25 | addButton = view_client.findViewById("com.simplemobiletools.smsmessenger:id/new_conversation_confirm") 26 | addButton.touch() 27 | view_client.dump() 28 | editMessage = view_client.findViewById("com.simplemobiletools.smsmessenger:id/thread_type_message") 29 | editMessage.setText("i luv u") 30 | view_client.dump() 31 | sendButton = view_client.findViewById("com.simplemobiletools.smsmessenger:id/thread_send_message") 32 | sendButton.touch() 33 | view_client.dump() 34 | 35 | backButton = view_client.findViewWithContentDescription('Back') 36 | backButton.touch() 37 | view_client.dump() 38 | 39 | sndBackButton = view_client.findViewWithContentDescription('Back') 40 | sndBackButton.touch() 41 | view_client.dump() 42 | pass 43 | 44 | def teardown(self, view_client): 45 | task_utils.messager_delete_all(view_client) 46 | 47 | def check_finish(self, view_client, app_event): 48 | searchView = view_client.findViewById('com.simplemobiletools.smsmessenger:id/top_toolbar_search') 49 | if searchView is not None: 50 | text = searchView.text() 51 | if text.lower() == 'i luv u': 52 | self.text_correct = True 53 | print('correctly entered') 54 | contactView = view_client.findViewById('com.simplemobiletools.smsmessenger:id/conversation_body_short') 55 | if contactView is not None: 56 | text = contactView.text() 57 | if text.lower() == 'i luv u': 58 | return self.text_correct 59 | return False -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/calendar_laundry.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | from com.dtmilano.android.viewclient import ViewClient 4 | 5 | class CalendarLaundryTask(Task): 6 | def __init__(self, task_name="calendar_laundry", 7 | prompt="Create a new event 'laundry'", 8 | min_steps=4, 9 | package="com.simplemobiletools.calendar.pro", 10 | max_steps=8, 11 | stop_after_finish=True, 12 | permissions=["android.permission.POST_NOTIFICATIONS"]): 13 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish, permissions) 14 | 15 | self.text_filled = False 16 | self.last_event = None 17 | 18 | def setup(self, view_client) -> bool: 19 | # grant app permission by creating a task 20 | add_button = view_client.findViewById('com.simplemobiletools.calendar.pro:id/calendar_fab') 21 | add_button.touch() 22 | view_client.dump() 23 | event_button = view_client.findViewWithText('Event') 24 | event_button.touch() 25 | view_client.dump() 26 | title_view = view_client.findViewById('com.simplemobiletools.calendar.pro:id/event_title') 27 | title_view.type('event1') 28 | save_button = view_client.findViewById('com.simplemobiletools.calendar.pro:id/save') 29 | save_button.touch() 30 | view_client.dump() 31 | ok_button = view_client.findViewWithText('OK') 32 | ok_button.touch() 33 | pass 34 | 35 | def check_finish(self, view_client, app_event: AppEvent) -> bool: 36 | title_view = view_client.findViewById('com.simplemobiletools.calendar.pro:id/event_title') 37 | if title_view is not None: 38 | text = title_view.text() 39 | if text.lower() == 'laundry': 40 | self.text_filled = True 41 | print('detected text fill') 42 | 43 | if app_event is not None and app_event.package == 'com.simplemobiletools.calendar.pro': 44 | print('app event received') 45 | if app_event.type == AppEventType.WindowStateChange: 46 | if len(app_event.text_list) == 1 and app_event.text_list[0] == 'Calendar': 47 | if self.last_event is not None and self.last_event.type == AppEventType.Click and len(self.last_event.text_list) == 0: 48 | # it's a bit weird, but when the text is filled, sometimes the id_str in the save button is missing 49 | # it should be `com.simplemobiletools.calendar.pro:id/save` 50 | # we use two consecutive events to track the id_button click status: 51 | # 1. a click event with no text 52 | # 2. a new page event with text `Calendar` 53 | return self.text_filled 54 | self.last_event = app_event 55 | return False 56 | -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/messager_search_contacts.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | import subprocess 4 | from . import task_utils 5 | 6 | class MessagerSearchContactsTask(Task): 7 | def __init__(self, task_name="messager_search_contacts", 8 | prompt="search for the contact '123456789' at top search bar", 9 | min_steps=2, 10 | package="com.simplemobiletools.smsmessenger", 11 | max_steps=4, 12 | stop_after_finish=False, 13 | permissions=[]): 14 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish,permissions) 15 | self.contactAddressCorrect = False 16 | 17 | def setup(self, view_client): 18 | subprocess.run(["adb","shell","pm", "disable-user", "--user", "0", "com.google.android.apps.messaging"]) 19 | addConversationButton = view_client.findViewById("com.simplemobiletools.smsmessenger:id/conversations_fab") 20 | addConversationButton.touch() 21 | view_client.dump() 22 | setContactButton = view_client.findViewById("com.simplemobiletools.smsmessenger:id/new_conversation_address") 23 | setContactButton.setText("123456789") 24 | view_client.dump() 25 | addButton = view_client.findViewById("com.simplemobiletools.smsmessenger:id/new_conversation_confirm") 26 | addButton.touch() 27 | view_client.dump() 28 | editMessage = view_client.findViewById("com.simplemobiletools.smsmessenger:id/thread_type_message") 29 | editMessage.setText("i luv u") 30 | view_client.dump() 31 | sendButton = view_client.findViewById("com.simplemobiletools.smsmessenger:id/thread_send_message") 32 | sendButton.touch() 33 | view_client.dump() 34 | 35 | backButton = view_client.findViewWithContentDescription('Back') 36 | backButton.touch() 37 | view_client.dump() 38 | 39 | sndBackButton = view_client.findViewWithContentDescription('Back') 40 | sndBackButton.touch() 41 | view_client.dump() 42 | pass 43 | 44 | def teardown(self, view_client): 45 | task_utils.messager_delete_all(view_client) 46 | 47 | def check_finish(self, view_client, app_event: AppEvent) -> bool: 48 | # check if the correct contact is entered into the searchbox 49 | searchView = view_client.findViewById('com.simplemobiletools.smsmessenger:id/top_toolbar_search') 50 | if searchView is not None: 51 | text = searchView.text() 52 | if text == '123456789': 53 | self.contactAddressCorrect = True 54 | print('correctly entered') 55 | contactView = view_client.findViewById('com.simplemobiletools.smsmessenger:id/conversation_address') 56 | if contactView is not None: 57 | text = contactView.text() 58 | if text == '123456789': 59 | return self.contactAddressCorrect 60 | return False -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/__init__.py: -------------------------------------------------------------------------------- 1 | __all__ = [ 2 | "calendar_create_and_search", 3 | "calendar_search", 4 | "calendar_next_month", 5 | "calendar_snooze_time", 6 | "calendar_daily_view", 7 | "calendar_laundry", 8 | "calendar_open_about", 9 | "calendar_new_task", 10 | "calendar_delete_tasks", 11 | "calendar_start_week", 12 | "recorder_delete", 13 | "recorder_delete_all", 14 | "recorder_recycle_bin", 15 | "recorder_rename", 16 | "recorder_rename_all", 17 | "recorder_extension", 18 | "recorder_recycle_settings", 19 | "recorder_empty_trash", 20 | "recorder_bit_rate", 21 | "recorder_theme", 22 | "music_player_open_settings", 23 | "music_player_about_FAQ", 24 | "music_player_create_playlist", 25 | "music_player_search_playlist", 26 | "music_player_create_playlist_and_search", 27 | "music_player_config_equalizer", 28 | "music_player_rescan_media", 29 | "music_player_playlist_sort_desc", 30 | "music_player_album_sort_by_year", 31 | "music_player_create_playlist_and_sort_desc", 32 | "messager_change_font_size", 33 | "messager_start_a_conversation", 34 | "messager_show_archived", 35 | "messager_make_conversation_archived", 36 | "messager_search_contacts", 37 | "messager_add_block_numbers", 38 | "messager_search_message", 39 | "messager_create_conversation_and_search", 40 | "messager_check_message_properties", 41 | "messager_create_conversation_and_check_message_properties", 42 | "calculator_add", 43 | "calculator_minus", 44 | "calculator_multiply", 45 | "calculator_divide", 46 | "calculator_mixed", 47 | "calculator_square", 48 | "calculator_cube", 49 | "calculator_point", 50 | "calculator_recalculate", 51 | "calculator_length", 52 | "gallery_sort_by_size_asc", 53 | "gallery_filter_by_images_and_videos", 54 | "gallery_list_view_type", 55 | "gallery_show_hidden_items", 56 | "gallery_play_videos_automatically", 57 | "gallery_set_favorite", 58 | "gallery_set_wallpaper", 59 | "gallery_use_24_hour_time_format", 60 | "gallery_remember_playback_position", 61 | "gallery_group_by_file_type", 62 | "contacts_about", 63 | "contacts_create", 64 | "contacts_delete", 65 | "contacts_search", 66 | "contacts_favorite", 67 | "contacts_filter", 68 | "contacts_hide_email", 69 | "contacts_modify", 70 | "contacts_remove_dialog", 71 | "contacts_search", 72 | "contacts_sort", 73 | "note_add", 74 | "note_delete", 75 | "note_item_checked", 76 | "note_lock", 77 | "note_new_checklist", 78 | "note_new_checklist_items", 79 | "note_open", 80 | "note_open_about", 81 | "note_rename", 82 | "note_search", 83 | "filemanager_create_new_file", 84 | "filemanager_check_file_properties", 85 | "filemanager_sort_folder_by_size_desc", 86 | "filemanager_rename_file", 87 | "filemanager_delete_file", 88 | "filemanager_search", 89 | "filemanager_hide_folder", 90 | "filemanager_check_storage", 91 | "filemanager_delete_txt", 92 | "filemanager_delete_videos", 93 | "launcher_add_apps", 94 | "launcher_check_contributor", 95 | "launcher_hide_app_name", 96 | "launcher_open_about_FAQ", 97 | "launcher_remove_app", 98 | "launcher_rename_app", 99 | "launcher_search_app", 100 | "launcher_setting_close_app_when_launching", 101 | "launcher_sort_by_custom", 102 | "launcher_sort_by_title_desc", 103 | ] 104 | -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/note_item_checked.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from com.dtmilano.android.viewclient import ViewClient 3 | from ..server import AppEvent, AppEventType 4 | class NoteItemCheck(Task): 5 | def __init__(self): 6 | super().__init__( 7 | task_name="note_item_checked", 8 | prompt="Check the item 'eggs' for shopping_list", 9 | min_steps=1, 10 | package="com.simplemobiletools.notes.pro", 11 | max_steps=2, 12 | stop_after_finish=False, 13 | permissions=[] 14 | ) 15 | def setup(self, view_client): 16 | """ 17 | Prepares the environment by creating a set of notes that will be manipulated during the task. 18 | This includes interacting with the UI to input the note details and confirm their creation. 19 | """ 20 | def create_note(name,item1,item2,item3): 21 | # Access the button to add a new note 22 | new_note_button = view_client.findViewById('com.simplemobiletools.notes.pro:id/new_note') 23 | new_note_button.touch() 24 | view_client.dump() 25 | 26 | type_view = view_client.findViewWithText('Checklist') 27 | type_view.touch() 28 | 29 | title_view = view_client.findViewById('com.simplemobiletools.notes.pro:id/locked_note_title') 30 | title_view.type(name) 31 | 32 | save_button = view_client.findViewById('android:id/button1') 33 | save_button.touch() 34 | 35 | view_client.dump() 36 | 37 | 38 | #input item1 39 | check_view = view_client.findViewById('com.simplemobiletools.notes.pro:id/checklist_fab') 40 | check_view.touch() 41 | view_client.dump() 42 | 43 | title_view = view_client.findViewById('com.simplemobiletools.notes.pro:id/title_edit_text') 44 | title_view.type(item1) 45 | 46 | save_button2 = view_client.findViewById('android:id/button1') 47 | save_button2.touch() 48 | view_client.dump() 49 | 50 | 51 | #input item2 52 | check_view = view_client.findViewById('com.simplemobiletools.notes.pro:id/checklist_fab') 53 | check_view.touch() 54 | view_client.dump() 55 | 56 | title_view = view_client.findViewById('com.simplemobiletools.notes.pro:id/title_edit_text') 57 | title_view.type(item2) 58 | 59 | save_button2 = view_client.findViewById('android:id/button1') 60 | save_button2.touch() 61 | view_client.dump() 62 | 63 | 64 | #input item3 65 | check_view = view_client.findViewById('com.simplemobiletools.notes.pro:id/checklist_fab') 66 | check_view.touch() 67 | view_client.dump() 68 | 69 | 70 | title_view = view_client.findViewById('com.simplemobiletools.notes.pro:id/title_edit_text') 71 | title_view.type(item3) 72 | 73 | save_button2 = view_client.findViewById('android:id/button1') 74 | save_button2.touch() 75 | view_client.dump() 76 | create_note("shopping_list","eggs","beer","steak") 77 | pass 78 | 79 | #Do I need to create two checklist to do further task 80 | def check_finish(self, view_client, app_event) -> bool: 81 | if app_event is not None: 82 | if app_event.type == AppEventType.Click and 'eggs' in app_event.text_list: 83 | return True 84 | return False 85 | -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/log_manager.py: -------------------------------------------------------------------------------- 1 | import os 2 | from datetime import datetime 3 | import numpy as np 4 | import cv2 5 | import json 6 | from math import ceil 7 | 8 | from .bench_task import Task, ActionLog 9 | 10 | class LogManager(): 11 | def __init__(self, log_dir) -> None: 12 | if not os.path.exists(log_dir): 13 | os.makedirs(log_dir) 14 | 15 | now = datetime.now() 16 | time_str = now.strftime("%Y_%m_%d-%H_%M_%S") 17 | self.log_dir = os.path.join(log_dir, time_str) 18 | 19 | os.makedirs(self.log_dir) 20 | 21 | def log_before_action(self, task: Task): 22 | log = ActionLog() 23 | log.begin_timestamp = datetime.timestamp(datetime.now()) 24 | task.action_logs.append(log) 25 | 26 | def log_after_action(self, task: Task, time_stamp, action_desc: str, screenshot, step, model_input_text="", model_input_images=[], model_output_text=""): 27 | task_dir = os.path.join(self.log_dir, task.task_name) 28 | if not os.path.exists(task_dir): 29 | os.makedirs(task_dir) 30 | screenshot_dir = os.path.join(task_dir, 'screenshots') 31 | if not os.path.exists(screenshot_dir): 32 | os.makedirs(screenshot_dir) 33 | 34 | log = task.action_logs[-1] 35 | if log.step != -1: 36 | raise RuntimeError("Did you call `before_one_action`?") 37 | log.action = action_desc 38 | log.step = step 39 | log.end_timestamp = time_stamp 40 | log.input_tokens = self._count_text_tokens(model_input_text) 41 | for each in model_input_images: 42 | log.input_tokens += self._count_image_tokens(each.shape[1], each.shape[0]) 43 | log.output_tokens = self._count_text_tokens(model_output_text) 44 | 45 | img = np.array(screenshot) 46 | img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR) 47 | img_name = f"{step:04d}.png" 48 | img_path = os.path.join(screenshot_dir, img_name) 49 | cv2.imwrite(img_path, img) 50 | 51 | log.screenshot_path = img_path 52 | 53 | 54 | 55 | def save_log(self, task: Task): 56 | # TODO: introduce another int to store multiple runs for the same task 57 | task_dir = os.path.join(self.log_dir, task.task_name) 58 | if not os.path.exists(task_dir): 59 | os.makedirs(task_dir) 60 | json_dict = { 61 | 'task': task.task_name, 62 | 'prompt': task.prompt, 63 | 'finished': task.finished, 64 | 'stop_after_finish': task.stop_after_finish, 65 | 'actions': [] 66 | } 67 | 68 | for each_action in task.action_logs: 69 | action_dict = { 70 | 'action': each_action.action, 71 | 'step': each_action.step, 72 | 'begin_timestamp': each_action.begin_timestamp, 73 | 'end_timestamp': each_action.end_timestamp, 74 | 'input_tokens': each_action.input_tokens, 75 | 'output_tokens': each_action.output_tokens, 76 | 'screenshot': each_action.screenshot_path 77 | } 78 | json_dict['actions'].append(action_dict) 79 | 80 | file_path = os.path.join(task_dir, f"{task.task_name}.json") 81 | with open(file_path, 'w') as file: 82 | # Convert the dictionary to a JSON string with pretty printing 83 | json.dump(json_dict, file, indent=4) 84 | 85 | def _count_text_tokens(self, text: str): 86 | return ceil(len(text) / 4) 87 | 88 | def _count_image_tokens(self, width: int, height: int): 89 | def resize(width, height): 90 | if width > 1024 or height > 1024: 91 | if width > height: 92 | height = int(height * 1024 / width) 93 | width = 1024 94 | else: 95 | width = int(width * 1024 / height) 96 | height = 1024 97 | return width, height 98 | width, height = resize(width, height) 99 | h = ceil(height / 512) 100 | w = ceil(width / 512) 101 | total = 85 + 170 * h * w 102 | return total 103 | 104 | -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/messager_make_conversation_archived.py: -------------------------------------------------------------------------------- 1 | from ..bench_task import Task 2 | from ..server import AppEvent, AppEventType 3 | import subprocess 4 | 5 | class MessagerMakeConversationArchivedTask(Task): 6 | def __init__(self, task_name="messager_make_conversation_archived", 7 | prompt="make the conversation with number '123456789' archived", 8 | min_steps=3, 9 | package="com.simplemobiletools.smsmessenger", 10 | max_steps=6, 11 | stop_after_finish=False, 12 | permissions=[]): 13 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish,permissions) 14 | self.moreOptionButtonClicked = False 15 | self.correctSelection = False 16 | self.alert_showed = False 17 | def setup(self, view_client): 18 | subprocess.run(["adb","shell","pm", "disable-user", "--user", "0", "com.google.android.apps.messaging"]) 19 | addConversationButton = view_client.findViewById("com.simplemobiletools.smsmessenger:id/conversations_fab") 20 | addConversationButton.touch() 21 | view_client.dump() 22 | setContactButton = view_client.findViewById("com.simplemobiletools.smsmessenger:id/new_conversation_address") 23 | setContactButton.setText("123456789") 24 | view_client.dump() 25 | addButton = view_client.findViewById("com.simplemobiletools.smsmessenger:id/new_conversation_confirm") 26 | addButton.touch() 27 | view_client.dump() 28 | editMessage = view_client.findViewById("com.simplemobiletools.smsmessenger:id/thread_type_message") 29 | editMessage.setText("i luv u") 30 | view_client.dump() 31 | sendButton = view_client.findViewById("com.simplemobiletools.smsmessenger:id/thread_send_message") 32 | sendButton.touch() 33 | view_client.dump() 34 | 35 | backButton = view_client.findViewWithContentDescription('Back') 36 | backButton.touch() 37 | view_client.dump() 38 | 39 | sndBackButton = view_client.findViewWithContentDescription('Back') 40 | sndBackButton.touch() 41 | view_client.dump() 42 | return 43 | 44 | def teardown(self, view_client): 45 | moreOptionButton = view_client.findViewWithContentDescription("More options") 46 | if moreOptionButton is None: 47 | return 48 | moreOptionButton.touch() 49 | view_client.dump() 50 | archiveButton = view_client.findViewWithText("Show archived conversations") 51 | archiveButton.touch() 52 | view_client.dump() 53 | archivedMessageView = view_client.findViewById("com.simplemobiletools.smsmessenger:id/conversation_address") 54 | while archivedMessageView is not None: 55 | archivedMessageView.longTouch() 56 | view_client.dump() 57 | deleteButton = view_client.findViewById("com.simplemobiletools.smsmessenger:id/cab_delete") 58 | deleteButton.touch() 59 | view_client.dump() 60 | YesButton = view_client.findViewWithText("Yes") 61 | YesButton.touch() 62 | view_client.dump() 63 | archivedMessageView = view_client.findViewById("com.simplemobiletools.smsmessenger:id/conversation_address") 64 | backButton = view_client.findViewWithContentDescription('Back') 65 | backButton.touch() 66 | view_client.dump() 67 | pass 68 | def check_finish(self, view_client, app_event: AppEvent) -> bool: 69 | correctSelectionView = view_client.findViewById("com.simplemobiletools.smsmessenger:id/conversation_address") 70 | if correctSelectionView is not None: 71 | text = correctSelectionView.text() 72 | if text == '123456789': 73 | print('correct conversation selected') 74 | self.correctSelection = True 75 | 76 | moreOptionView = view_client.findViewWithText("Archive") 77 | if moreOptionView is not None: 78 | self.moreOptionButtonClicked = True 79 | print('detected more options button clicked') 80 | 81 | if view_client.findViewWithText('Are you sure you want to archive 1 conversation?') is not None: 82 | self.alert_showed = True 83 | 84 | if app_event is not None and app_event.package == 'com.simplemobiletools.smsmessenger': 85 | if app_event.type == AppEventType.Click: 86 | if len(app_event.text_list) == 1 and app_event.text_list[0] == 'Yes': 87 | return self.correctSelection and self.alert_showed and self.moreOptionButtonClicked 88 | 89 | return False -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/note_lock.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | 3 | from ..bench_task import Task 4 | from com.dtmilano.android.viewclient import ViewClient 5 | from ..server import AppEvent, AppEventType 6 | 7 | 8 | # from .note_new_checklist import 9 | 10 | class NoteLockTask(Task): 11 | def __init__(self): 12 | super().__init__( 13 | task_name="note_lock", 14 | prompt="use the pin '2580' to open the locked note 'password_list'", 15 | min_steps=6, 16 | package="com.simplemobiletools.notes.pro", 17 | max_steps=12, 18 | stop_after_finish=False, 19 | permissions=[] 20 | ) 21 | 22 | def setup(self, view_client): 23 | def enter_pin(pin): 24 | for digit in pin: 25 | # Find the button for the current digit 26 | pin_button_id = f'com.simplemobiletools.notes.pro:id/pin_{digit}' 27 | pin_button = view_client.findViewById(pin_button_id) 28 | if pin_button: 29 | pin_button.touch() 30 | view_client.dump() 31 | else: 32 | raise ValueError(f"Pin button for digit '{digit}' not found.") 33 | save_button = view_client.findViewById('com.simplemobiletools.notes.pro:id/pin_ok') 34 | save_button.touch() 35 | view_client.dump() 36 | 37 | def open_lock(pin): 38 | moreOptionButton = view_client.findViewWithContentDescription('More options') 39 | moreOptionButton.touch() 40 | 41 | view_client.dump() 42 | 43 | # touch 'Lock note' button 44 | delete_button = view_client.findViewWithText('Lock note') 45 | delete_button.touch() 46 | 47 | view_client.dump() 48 | 49 | # touch 'ok' 50 | ok_button = view_client.findViewById('android:id/button1') 51 | ok_button.touch() 52 | 53 | view_client.dump() 54 | 55 | # touch 'PIN' button 56 | delete_button = view_client.findViewWithText('PIN') 57 | delete_button.touch() 58 | view_client.dump() 59 | 60 | enter_pin(pin) 61 | 62 | # one more time for confirmation 63 | enter_pin(pin) 64 | 65 | def create_checklist(name, item1): 66 | # Access the button to add a new note 67 | new_note_button = view_client.findViewById('com.simplemobiletools.notes.pro:id/new_note') 68 | new_note_button.touch() 69 | view_client.dump() 70 | 71 | type_view = view_client.findViewById('com.simplemobiletools.notes.pro:id/type_checklist') 72 | type_view.touch() 73 | 74 | title_view = view_client.findViewById('com.simplemobiletools.notes.pro:id/locked_note_title') 75 | title_view.type(name) 76 | 77 | save_button = view_client.findViewById('android:id/button1') 78 | save_button.touch() 79 | 80 | view_client.dump() 81 | 82 | # input item1 83 | check_view = view_client.findViewById('com.simplemobiletools.notes.pro:id/checklist_fab') 84 | check_view.touch() 85 | view_client.dump() 86 | 87 | title_view = view_client.findViewById('com.simplemobiletools.notes.pro:id/title_edit_text') 88 | title_view.type(item1) 89 | 90 | save_button2 = view_client.findViewById('android:id/button1') 91 | save_button2.touch() 92 | view_client.dump() 93 | 94 | def add_text(text): 95 | text_view = view_client.findViewById('com.simplemobiletools.notes.pro:id/text_note_view') 96 | text_view.type(text) 97 | view_client.dump() 98 | 99 | create_checklist("password_list","unique2580") 100 | 101 | open_lock("2580") 102 | check_button = view_client.findViewWithText('password_list') 103 | check_button.touch() 104 | 105 | view_client.dump() 106 | 107 | # Return to the main page and come back to note page 108 | 109 | subprocess.run(["adb", "shell", "input", "keyevent", "3"]) 110 | 111 | view_client.dump() 112 | 113 | subprocess.run(["adb", "shell", "monkey", "-p", "com.simplemobiletools.notes.pro", "-c", "android.intent.category.LAUNCHER", "1"]) 114 | 115 | view_client.dump() 116 | 117 | pass 118 | 119 | def check_finish(self, view_client, app_event) -> bool: 120 | note = view_client.findViewWithText("unique2580") 121 | if note is not None: 122 | return True 123 | return False 124 | -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/task_utils.py: -------------------------------------------------------------------------------- 1 | from com.dtmilano.android.viewclient import ViewClient 2 | import subprocess 3 | 4 | def is_box_checked(view_client: ViewClient, id_str: str) -> bool: 5 | view = view_client.findViewById(id_str) 6 | return view is not None and view.checked() 7 | 8 | def music_player_permissions_for_old_android_version(view_client: ViewClient): 9 | # check if the permission dialog is shown 10 | allow_button = view_client.findViewWithText('ALLOW') 11 | if allow_button is not None: 12 | allow_button.touch() 13 | view_client.dump() 14 | pass 15 | 16 | def music_player_restore_playlist_functioning(view_client: ViewClient): 17 | # restore the playlist functioning 18 | view_dict = view_client.getViewsById() 19 | for k in view_dict.keys(): 20 | view = view_dict[k] 21 | if view.map['resource-id'] == 'com.simplemobiletools.musicplayer:id/tab_item_label': 22 | if view.text().lower() == 'albums': 23 | view.touch() 24 | view_client.dump() 25 | break 26 | pass 27 | 28 | def recorder_permissions_for_old_android_version(view_client: ViewClient): 29 | # check if the permission dialog is shown 30 | allow_button = view_client.findViewWithText('ALLOW') 31 | if allow_button is not None: 32 | allow_button.touch() 33 | view_client.dump() 34 | pass 35 | 36 | def filemanager_permissions(view_client: ViewClient): 37 | OK_button = view_client.findViewWithText('OK') 38 | if OK_button is not None: 39 | OK_button.touch() 40 | view_client.dump() 41 | allow_access = view_client.findViewWithText('Allow access to manage all files') 42 | allow_access.touch() 43 | view_client.dump() 44 | navigate_up = view_client.findViewWithContentDescription('Navigate up') 45 | navigate_up.touch() 46 | view_client.dump() 47 | return 48 | else: 49 | return 50 | 51 | def filemanager_delete_all_files_under_folder(view_client: ViewClient): 52 | subprocess.run(['adb', 'shell', 'rm', '-rf', '/sdcard/Download/*']) 53 | 54 | def filemanager_create_file_under_download(view_client: ViewClient, filename='Testfile.txt'): 55 | # create a file called 'Testfile.txt' under the 'Download' folder 56 | view_client.dump() 57 | if view_client.findViewWithText('> Download') is None: 58 | download_folder = view_client.findViewWithText('Download') 59 | if download_folder is None: 60 | raise RuntimeError("Failed to navigate in to Download") 61 | download_folder.touch() 62 | view_client.dump() 63 | create_file_button = view_client.findViewById('com.simplemobiletools.filemanager.pro:id/items_fab') 64 | create_file_button.touch() 65 | view_client.dump() 66 | file_title = view_client.findViewById('com.simplemobiletools.filemanager.pro:id/item_title') 67 | file_title.setText(filename) 68 | File_button = view_client.findViewById('com.simplemobiletools.filemanager.pro:id/dialog_radio_file') 69 | File_button.touch() 70 | Ok_button = view_client.findViewWithText('OK') 71 | Ok_button.touch() 72 | return 73 | 74 | def messager_delete_all(view_client): 75 | print("teardown sms...") 76 | pkg = "com.simplemobiletools.smsmessenger" 77 | view_client.device.forceStop(package=pkg) 78 | view_client.device.startActivity(package=pkg) 79 | 80 | view_client.dump() 81 | # delete the new conversation we started 82 | # use a while loop to delete all possible messages 83 | messageView = view_client.findViewById('com.simplemobiletools.smsmessenger:id/conversation_address') 84 | while messageView is not None: 85 | messageView.longTouch() 86 | view_client.dump() 87 | deleteView = view_client.findViewById('com.simplemobiletools.smsmessenger:id/cab_delete') 88 | deleteView.touch() 89 | view_client.dump() 90 | yesView = view_client.findViewWithText('Yes') 91 | yesView.touch() 92 | view_client.dump() 93 | messageView = view_client.findViewById('com.simplemobiletools.smsmessenger:id/conversation_address') 94 | pass 95 | 96 | def recorder_delete_all(): 97 | subprocess.run(['adb', 'shell', 'rm', '-rf', '/sdcard/Music/Recordings/*']) 98 | 99 | def disable_app(package_name): 100 | command = ["adb", "shell", "pm", "disable-user", "--user", "0", package_name] 101 | try: 102 | result = subprocess.run(command, check=True, capture_output=True, text=True) 103 | print(f"Disabled {package_name}:", result.stdout) 104 | except subprocess.CalledProcessError as e: 105 | print(f"Command failed with return code for {package_name}:", e.returncode) 106 | print("Error output:", e.stderr) 107 | except Exception as e: 108 | print(f"An error occurred while disabling {package_name}:", str(e)) -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/tasks/contacts_sort.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | 3 | from ..bench_task import Task 4 | from ..server import AppEvent, AppEventType 5 | 6 | class ContactsSort(Task): 7 | def __init__(self, task_name="contacts_sort", 8 | prompt="Sort contacts by Data created time, Descending", 9 | min_steps=5, 10 | package="com.simplemobiletools.contacts.pro", 11 | max_steps=10, 12 | stop_after_finish=False, 13 | permissions=["android.permission.READ_CONTACTS"]): 14 | super().__init__(task_name, prompt, min_steps, package, max_steps, stop_after_finish,permissions) 15 | self.yuzai = False 16 | self.zack = False 17 | 18 | # create contact yuzai 19 | def setup(self, view_client): 20 | button_add = view_client.findViewById('com.simplemobiletools.contacts.pro:id/fragment_fab') 21 | button_add.touch() 22 | 23 | view_client.dump() 24 | 25 | try: 26 | element = view_client.findViewByIdOrRaise('com.simplemobiletools.contacts.pro:id/contact_first_name') 27 | element.touch() 28 | 29 | view_client.device.type('Yuzai') 30 | except Exception as e: 31 | print("ExceptionName: ", str(e)) 32 | 33 | try: 34 | element = view_client.findViewByIdOrRaise('com.simplemobiletools.contacts.pro:id/contact_number') 35 | element.setText('123456789') 36 | except Exception as e: 37 | print("ExceptionNumber: ", str(e)) 38 | 39 | try: 40 | element = view_client.findViewById('com.simplemobiletools.contacts.pro:id/save') 41 | element.touch() 42 | except Exception as e: 43 | print("Exception: ", str(e)) 44 | 45 | view_client.dump() 46 | 47 | view_client.sleep(1) 48 | 49 | button_add = view_client.findViewById('com.simplemobiletools.contacts.pro:id/fragment_fab') 50 | button_add.touch() 51 | 52 | view_client.dump() 53 | 54 | try: 55 | element = view_client.findViewByIdOrRaise('com.simplemobiletools.contacts.pro:id/contact_first_name') 56 | element.touch() 57 | 58 | view_client.device.type('Zack') 59 | except Exception as e: 60 | print("ExceptionName: ", str(e)) 61 | 62 | try: 63 | element = view_client.findViewByIdOrRaise('com.simplemobiletools.contacts.pro:id/contact_number') 64 | element.setText('123456789') 65 | except Exception as e: 66 | print("ExceptionNumber: ", str(e)) 67 | 68 | try: 69 | element = view_client.findViewById('com.simplemobiletools.contacts.pro:id/save') 70 | element.touch() 71 | except Exception as e: 72 | print("Exception: ", str(e)) 73 | 74 | view_client.dump() 75 | 76 | pass 77 | 78 | # delete contact Yuzai 79 | def teardown(self, view_client): 80 | subprocess.run(["adb", "shell", "pm", "clear", "com.android.providers.contacts"]) 81 | 82 | def check_finish(self, view_client, app_event: AppEvent) -> bool: 83 | sorted_name = ["zack", "yuzai"] 84 | current = 0 85 | try: 86 | # If there is no contact, return False 87 | recycler_view = view_client.findViewById("com.simplemobiletools.contacts.pro:id/fragment_list") 88 | if recycler_view is not None: 89 | # The List should be ['Zack', 'Yuzai'] 90 | child_zack = recycler_view.children[0] 91 | if (child_zack['class'] == 'android.view.ViewGroup' 92 | and child_zack['resource-id'] == 'com.simplemobiletools.contacts.pro:id/item_contact_frame'): 93 | for subchild in child_zack.children: 94 | if subchild['class'] == 'android.widget.TextView': 95 | contact_name = subchild.getText() 96 | print("current contact name: ", contact_name) 97 | if contact_name.lower() == "zack": 98 | self.zack = True 99 | child_yuzai = recycler_view.children[1] 100 | if (child_yuzai['class'] == 'android.view.ViewGroup' 101 | and child_yuzai['resource-id'] == 'com.simplemobiletools.contacts.pro:id/item_contact_frame'): 102 | for subchild in child_yuzai.children: 103 | if subchild['class'] == 'android.widget.TextView': 104 | contact_name = subchild.getText() 105 | print("current contact name: ", contact_name) 106 | if contact_name.lower() == "yuzai": 107 | self.yuzai = True 108 | 109 | return self.yuzai and self.zack 110 | except Exception as e: 111 | print(f"Exception: {str(e)}") 112 | return False -------------------------------------------------------------------------------- /src/mobile_agent_benchmark/task_orchestrator.py: -------------------------------------------------------------------------------- 1 | from .server import AppEvent, start_server 2 | from .tasks import * 3 | from .log_manager import LogManager 4 | from .bench_task import Task 5 | import time 6 | import queue 7 | import subprocess 8 | import json 9 | import importlib.resources as pkg_resources 10 | from com.dtmilano.android.viewclient import ViewClient 11 | from datetime import datetime 12 | 13 | class TaskOrchestrator(): 14 | def __init__(self, log_dir='./', config_path=None): 15 | 16 | if config_path is None: 17 | config_path = pkg_resources.files("mobile_agent_benchmark") / "configs/default.json" 18 | with open(config_path) as f: 19 | config = json.load(f) 20 | 21 | tasks = [] 22 | name_class_map = {} 23 | for clazz in Task.__subclasses__(): 24 | task = clazz() 25 | name_class_map[task.task_name] = task 26 | for config_task in config["tasks"]: 27 | task = name_class_map[config_task["name"]] 28 | tasks.append(task) 29 | 30 | self.tasks = tasks 31 | 32 | self.current_task_id = 0 33 | self.current_task = None 34 | 35 | self.event_queue = queue.Queue() 36 | 37 | device, serialno = ViewClient.connectToDeviceOrExit() 38 | self.vc = ViewClient(device, serialno, useuiautomatorhelper=False) 39 | 40 | self.log_manager = LogManager(log_dir=log_dir) 41 | 42 | pass 43 | 44 | def _check_task_finish(self, view_client) -> bool: 45 | event = None 46 | finished = False 47 | if self.event_queue.empty(): 48 | finished = self.current_task.check_finish(view_client, None) 49 | 50 | while not self.event_queue.empty(): 51 | event = self.event_queue.get_nowait() # get also pops the element 52 | finished = self.current_task.check_finish(view_client, event) 53 | if finished: 54 | break 55 | 56 | print("-----------------------") 57 | print(f"Task Finished? {finished}") 58 | print("-----------------------") 59 | return finished 60 | 61 | 62 | 63 | def _on_receive_new_event(self, app_event): 64 | # this will be called from anohter thread 65 | # to ensure thread safety, we can only put an event into the queue 66 | # without calling `check_task_finish` function 67 | # in other words, 68 | # app_event will not be delievered at the exact event happending time 69 | self.event_queue.put(app_event) 70 | 71 | # TODO: use `with` keyword 72 | def before_one_action(self): 73 | if self.current_task is None: 74 | return 75 | self.log_manager.log_before_action(self.current_task) 76 | 77 | 78 | def after_one_action(self, action_desc, agent_text_input="", agent_image_input=[], agent_text_output="") -> bool: 79 | """Inform the task orchestrator after one action is performed 80 | We can easily know the initial state, so we don't need `before_one_action` 81 | 82 | Args: 83 | action_desc (str): description of the action, e.g., `click` 84 | agent_text_input(str): text input to the model (to calculate cost) 85 | agent_image_input([np.ndarray]): image input to the model, OpenCV format (to calculate cost) 86 | agent_text_output(str): model output (to calculate cost) 87 | 88 | Returns: 89 | bool: agent should stop (exceed max steps or finished) 90 | """ 91 | if self.current_task is None: 92 | return False 93 | timestamp = datetime.timestamp(datetime.now()) 94 | self.current_task.current_step += 1 95 | 96 | # sleep is important here. Because when we run adb commands (dump) 97 | # accessibility callback doesn't work 98 | # we may lose some action events 99 | time.sleep(1) 100 | self.vc.dump() # update state 101 | 102 | screenshot = self.vc.device.takeSnapshot(reconnect=True) 103 | # TODO: calculate LLM cost 104 | self.log_manager.log_after_action(self.current_task, timestamp, 105 | action_desc, screenshot, 106 | self.current_task.current_step, 107 | model_input_text=agent_text_input, 108 | model_input_images=agent_image_input, 109 | model_output_text=agent_text_output) 110 | 111 | 112 | should_stop = False 113 | 114 | if self.current_task.current_step > self.current_task.max_steps: 115 | should_stop = True 116 | 117 | if not self.current_task.finished: 118 | # once a task is finished, it's always finsihed 119 | self.current_task.finished = self._check_task_finish(self.vc) 120 | 121 | if self.current_task.stop_after_finish and self.current_task.finished: 122 | should_stop = True 123 | 124 | return should_stop 125 | 126 | 127 | def run(self, agent_fn): 128 | start_server(self._on_receive_new_event) 129 | time.sleep(1) 130 | 131 | for i in range(len(self.tasks)): 132 | self.current_task_id = i 133 | self.current_task = self.tasks[i] 134 | 135 | print(f"==== Running: {self.current_task.prompt} ====") 136 | 137 | self.vc.device.forceStop(package=self.current_task.package) 138 | subprocess.run(["adb", "shell", "pm", "clear", self.current_task.package]) # deletes all data to restore app state 139 | # grant permission 140 | for each in self.current_task.permissions: 141 | subprocess.run(["adb", "shell", "pm", "grant", self.current_task.package, each]) 142 | self.vc.device.startActivity(package=self.current_task.package) 143 | 144 | # setup 145 | self.vc.dump() 146 | self.current_task.setup(self.vc) 147 | time.sleep(1) # wait for pending events 148 | with self.event_queue.mutex: # clear events caused by setup 149 | self.event_queue.queue.clear() 150 | 151 | # log initial state 152 | self.vc.dump() # update state 153 | screenshot = self.vc.device.takeSnapshot(reconnect=True) 154 | self.log_manager.log_before_action(self.current_task) 155 | self.log_manager.log_after_action(self.current_task, 0, "init", screenshot, 0) 156 | 157 | # should not run in another thread / process 158 | agent_fn(self.current_task.prompt) 159 | 160 | # when the agent thinks it's done, we still need to wait for events 161 | # for example, the agent click a button and exit 162 | # we need to wait for a bit to make sure the event is received 163 | time.sleep(1) 164 | while not self.event_queue.empty: 165 | print("send pending events...") 166 | self.vc.dump() 167 | if not self.current_task.finished: 168 | self.current_task.finished = self._check_task_finish(self.vc) 169 | 170 | self.log_manager.save_log(self.current_task) 171 | 172 | # teardown 173 | self.vc.dump() 174 | self.current_task.teardown(self.vc) 175 | subprocess.run(["adb", "shell", "pm", "clear", self.current_task.package]) # delete all data one more time to tear down 176 | 177 | 178 | 179 | 180 | 181 | 182 | 183 | --------------------------------------------------------------------------------