111 |
112 |
120 |
114 |
116 |
117 |
119 |
110 |
111 |
121 |
122 |
123 |
168 |
--------------------------------------------------------------------------------
/MLLM_Agent/json_action.py:
--------------------------------------------------------------------------------
1 | # Copyright 2024 The android_world Authors.
2 | #
3 | # Licensed under the Apache License, Version 2.0 (the "License");
4 | # you may not use this file except in compliance with the License.
5 | # You may obtain a copy of the License at
6 | #
7 | # http://www.apache.org/licenses/LICENSE-2.0
8 | #
9 | # Unless required by applicable law or agreed to in writing, software
10 | # distributed under the License is distributed on an "AS IS" BASIS,
11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 | # See the License for the specific language governing permissions and
13 | # limitations under the License.
14 |
15 | """Represents an action for Android interaction, parsed from a JSON format."""
16 |
17 | import dataclasses
18 | import json
19 | from typing import Optional
20 |
21 |
22 | _JSON_SEPARATORS = (',', ':')
23 |
24 | ANSWER = 'answer'
25 | CLICK = 'click'
26 | DOUBLE_TAP = 'double_tap'
27 | INPUT_TEXT = 'input_text'
28 | KEYBOARD_ENTER = 'keyboard_enter'
29 | LONG_PRESS = 'long_press'
30 | NAVIGATE_BACK = 'navigate_back'
31 | NAVIGATE_HOME = 'navigate_home'
32 | OPEN_APP = 'open_app'
33 | SCROLL = 'scroll'
34 | STATUS = 'status'
35 | SWIPE = 'swipe'
36 | UNKNOWN = 'unknown'
37 | WAIT = 'wait'
38 |
39 | _ACTION_TYPES = (
40 | CLICK,
41 | DOUBLE_TAP,
42 | SCROLL,
43 | SWIPE,
44 | INPUT_TEXT,
45 | NAVIGATE_HOME,
46 | NAVIGATE_BACK,
47 | KEYBOARD_ENTER,
48 | OPEN_APP,
49 | STATUS,
50 | WAIT,
51 | LONG_PRESS,
52 | ANSWER,
53 | UNKNOWN,
54 | )
55 |
56 | _SCROLL_DIRECTIONS = ('left', 'right', 'down', 'up')
57 |
58 | # Keys of JSON action.
59 | ACTION_TYPE = 'action_type'
60 | INDEX = 'index'
61 | X = 'x'
62 | Y = 'y'
63 | TEXT = 'text'
64 | DIRECTION = 'direction'
65 | APP_NAME = 'app_name'
66 | GOAL_STATUS = 'goal_status'
67 |
68 |
69 | @dataclasses.dataclass()
70 | class JSONAction:
71 | """Represents a parsed JSON action.
72 |
73 | # Example
74 | result_json = {'action_type': 'click', 'x': %d, 'y': %d}
75 | action = JSONAction(**result_json)
76 |
77 | Attributes:
78 | action_type: The action type.
79 | index: The index to click, if action is a click. Either an index or a
111 |
4 | ![]()
11 |
12 |
4 |
6 |
2 |
3 |
4 |
28 |
29 | ## If you find this work useful for your research, please kindly cite our paper and star our repo.
30 |
31 | ## Updates
32 |
33 | - [05/2025] [Project Page](https://xieincz.github.io/GUI-explorer.github.io/) released.
34 | - [05/2025] [Arxiv paper](https://arxiv.org/abs/2505.16827) released.
35 | - [05/2025] [Code](https://github.com/JiuTian-VL/GUI-explorer) released.
36 |
37 | ## Introduction
38 |
39 | This is the github repository of *GUI-explorer: Autonomous Exploration and Mining of Transition-aware Knowledge for GUI Agent*. In this work, we propose GUI-explorer. It synergizes two key components: (1) Autonomous Exploration of Function-Aware Trajectory; (2) Unsupervised Mining of Transition-Aware Knowledge.
40 |
41 | The overview of the proposed GUI-explorer:
42 |
43 | GUI-explorer: Autonomous Exploration and Mining of Transition-aware Knowledge for GUI Agent
5 |
6 |
18 | *Corresponding author 19 | 20 | Annual Meeting of the Association for Computational Linguistics (**ACL**) 2025 21 | 22 | [[Paper]](https://arxiv.org/abs/2505.16827) [[Code]](https://github.com/JiuTian-VL/GUI-explorer) [[Project Page]](https://xieincz.github.io/GUI-explorer.github.io/) 23 | 24 | :fire: Details will be released. Stay tuned :beers: :+1: 25 | 26 |
27 |
7 | Bin Xie1,
8 | Rui Shao1*,
9 | Gongwei Chen1*,
10 | Kaiwen Zhou2,
11 | Yinchuan Li2,
12 | Jie Liu1,
13 | Min Zhang1,
14 | Liqiang Nie1
15 |
16 |
17 | 1Harbin Institute of Technology, Shenzhen, 2Huawei Noah’s Ark Lab18 | *Corresponding author 19 | 20 | Annual Meeting of the Association for Computational Linguistics (**ACL**) 2025 21 | 22 | [[Paper]](https://arxiv.org/abs/2505.16827) [[Code]](https://github.com/JiuTian-VL/GUI-explorer) [[Project Page]](https://xieincz.github.io/GUI-explorer.github.io/) 23 | 24 | :fire: Details will be released. Stay tuned :beers: :+1: 25 | 26 |
44 | 
45 |
46 |
47 | ## Installation
48 |
49 | ### Download
50 |
51 | ```bash
52 | git clone https://github.com/JiuTian-VL/GUI-explorer.git
53 | cd GUI-explorer
54 | mkdir knowledge_base
55 | cd knowledge_base
56 | wget https://github.com/JiuTian-VL/GUI-explorer/releases/download/knowledge_base/knowledge_data.pkl
57 | ```
58 |
59 | ### Environment
60 |
61 | ```bash
62 | cd GUI-explorer
63 | conda create -n GUI_explorer python=3.12 -y
64 | conda activate GUI_explorer
65 | pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu126
66 | pip install -r requirements.txt
67 | ```
68 |
69 | Duplicate `.env.example` and rename it to `.env`. Then, in the `.env` file, fill in your `OPENAI_API_KEY`.
70 |
71 | ## Usage
72 |
73 | ### Prepare api servers
74 |
75 | ```bash
76 | # Open a new shell window and run
77 | cd GUI-explorer
78 | conda activate GUI_explorer
79 | python -m utils.embedding_pipeline
80 |
81 | # Open a new shell window and run (Need to wait for embedding_pipeline to start up)
82 | cd GUI-explorer
83 | conda activate GUI_explorer
84 | python -m utils.retrieval
85 | ```
86 |
87 | #### Exploration
88 |
89 | ```bash
90 | # After prepare api servers
91 | cd GUI-explorer
92 | conda activate GUI_explorer
93 | python exploration_and_mining.py -device_serial emulator-5554 -max_branching_factor 10 -max_exploration_steps 30 -max_exploration_depth 5 -package_name net.osmand
94 | # After the update of knowledge_base, you need to restart `python -m utils.retrieval` to load the new knowledge_base
95 | ```
96 |
97 | `device_serial` can be obtained by running `adb devices`. (If not, you need to follow the `Setup` section in [this tutorial](https://github.com/ai-agents-2030/SPA-Bench/blob/main/Documentation.md#setup)).
98 |
99 | `package_name` can be obtained from the app's link on the app store. For example, in `https://play.google.com/store/apps/details?id=net.osmand`, `net.osmand` is the `package_name` for this app.
100 |
101 | #### Demo
102 |
103 | ```bash
104 | # After prepare api servers
105 | # Connect an Android device to this computer and make sure you can see it in `adb devices`.
106 | # Open a new shell window and run
107 | cd GUI-explorer
108 | conda activate GUI_explorer
109 | python -m demo.demo_web_backend
110 |
111 | # Open a new shell window and run
112 | cd GUI-explorer
113 | conda activate GUI_explorer
114 | python -m demo.demo_agent_backend
115 |
116 | # Open a new shell window and run
117 | cd GUI-explorer/demo/demo_web_frontend
118 | pnpm install
119 | pnpm run dev
120 | ```
121 |
122 | Open http://localhost:5173 in your browser.
123 |
124 | You should be able to see something like this:
125 |
126 | 
127 |
128 |
129 |
130 | ## Evaluation Results
131 |
132 | Table 1: Main Result of GUI-explorer on SPA-Bench single-app English Level 3 tasks.
133 |
45 | 
134 |
135 | Table 2: Main Result of GUI-explorer on AndroidWorld tasks.
136 | 
137 |
138 | Table 3: Main Result of GUI-explorer on GUI-KRB.
139 | 
140 |
141 |
142 |
143 | ## Showcases
144 |
145 | | Instruction | Video |
146 | | :----------------------------------------------------------: | :----------------------------------------------------------: |
147 | | Open Google Chrome and search for today's weather in Shenzhen. Carefully observe the screen and record the current weather conditions. Then, in Markor, create a note named "today.md" and write the temperature read from the webpage into it. | |
148 | | Get the search results for stay tonight near 'wembley stadium' for 1 adult. Add one result to wishlist. Confirm that this item is in the wishlist. | |
149 |
150 |
151 |
152 | ## More Examples
153 |
154 | 
155 |
156 |
157 |
158 | ## Citation
159 |
160 | If you find this work useful for your research, please kindly cite our paper:
161 | ```
162 | @inproceedings{xie2025gui,
163 | title={GUI-explorer: Autonomous Exploration and Mining of Transition-aware Knowledge for GUI Agent},
164 | author={Bin Xie and Rui Shao and Gongwei Chen and Kaiwen Zhou and Yinchuan Li and Jie Liu and Min Zhang and Liqiang Nie},
165 | booktitle={Annual Meeting of the Association for Computational Linguistics (ACL)},
166 | year={2025}
167 | }
168 | ```
169 |
--------------------------------------------------------------------------------
/utils/prompt_templates.py:
--------------------------------------------------------------------------------
1 | # INPUTS: app_name, package_name, activity_list
2 | TASK_GOAL_GENERATOR = """Given the screenshot of {app_name} and its available activities, generate a comprehensive list of practical user tasks that:
3 |
4 | 1. Start from the current screen shown in the screenshot
5 | 2. Can be completed within 10-30 steps
6 | 3. Utilize the app's full feature set based on the activity list
7 | 4. Are concrete and specific (like searching for a particular item rather than just "search")
8 | 5. Cover different user interaction patterns (viewing, editing, sharing, etc.)
9 | 6. Include both basic and advanced features
10 | 7. Represent realistic user behaviors and goals
11 | 8. Avoid excessive steps on form-filling or scrolling pages
12 |
13 | Important context:
14 | - App name: {app_name}
15 | - Package name: {package_name}
16 | - Available activities (app screens/features):
17 | ```{activity_list}```
18 |
19 | Format requirements:
20 | 1. List only the tasks without explanations or commentary
21 | 2. Each task should be a single, clear directive
22 | 3. Use specific examples (e.g., concrete search terms, actions, settings)
23 | 4. Include the expected outcome where relevant
24 | 5. Tasks should follow this pattern: [Starting action] + [Specific steps] + [End goal]
25 |
26 | Example tasks from other apps (for reference only):
27 | 1. Search for "ocean waves" white noise, then sort results by most played
28 | 2. Open the first recommended video, then post "Great content!" as a comment
29 | 3. Play the trending video, then add it to your "Watch Later" playlist
30 | 4. Navigate to the comments section of a featured video, then like the top comment
31 |
32 | Generate diverse tasks that would help a user explore and utilize all major features visible in the screenshot and implied by the activity list."""
33 |
34 |
35 | # INPUTS: task_description, numeric_tag_of_element, ui_element_attributes, action
36 | KNOWLEDGE_EXTRACTOR = """Objective: Describe the functionality of a specific UI element in a mobile app screenshot.
37 |
38 | Input:
39 | - Two screenshots: Before and after interacting with a UI element
40 | - UI element marked with a numeric tag in the top-left corner
41 | - Element number: {numeric_tag_of_element}
42 | - Broader task context: {task_description}
43 | - Action taken: {action}
44 | - UI Element Attributes:
45 | ```
46 | {ui_element_attributes}
47 | ```
48 |
49 | Requirements for Functionality Description:
50 | 1. Concise: 1-2 sentences
51 | 2. Focus on general function, not specific details
52 | 3. Avoid mentioning the numeric tag
53 | 4. Use generic terms like "UI element" or appropriate pronouns
54 |
55 | Example:
56 | - Incorrect: "Tapping the element #3 displays David's saved recipes in the results panel"
57 | - Correct: "Tapping this element will initiates a search and displays matching results"
58 |
59 | Guidance:
60 | - Describe the core action and immediate result of interacting with the UI element
61 | - Prioritize clarity and generality in the description"""
62 |
63 |
64 | # INPUTS: task_goal, knowledge_a, knowledge_b
65 | RANKER = """Given the user instruction: {task_goal}, determine which of the following two knowledge entries is more useful.
66 | Respond ONLY with a integer value:
67 | 1 means Knowledge A is strictly better.
68 | 2 means Knowledge B is strictly better.
69 |
70 | Knowledge A: {knowledge_a}
71 | Knowledge B: {knowledge_b}
72 |
73 | Please provide your response:
74 | """
75 |
76 |
77 | # INPUTS: task_goal, history, ui_elements, knowledge
78 | REASONING = """## Role Definition
79 | You are an Android operation AI that fulfills user requests through precise screen interactions.
80 | The current screenshot and the same screenshot with bounding boxes and labels added are also given to you.
81 |
82 | ## Action Catalog
83 | Available actions (STRICT JSON FORMAT REQUIRED):
84 | 1. Status Operations:
85 | - Task Complete: {{"action_type": "status", "goal_status": "complete"}}
86 | - Task Infeasible: {{"action_type": "status", "goal_status": "infeasible"}}
87 | 2. Information Actions:
88 | - Answer Question: {{"action_type": "answer", "text": "
3 |
4 |
5 |
6 |
7 |
128 |
129 |
463 |
--------------------------------------------------------------------------------
/demo/demo_web_frontend/src/components/ReasoningPage.vue:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
128 |
129 |
463 |
--------------------------------------------------------------------------------
/utils/memory.py:
--------------------------------------------------------------------------------
1 | """
2 | 首先用 knowledge_generation.py 从执行轨迹生成经验，
3 | 然后用本文件的 load_knowledge_raw_data 把生成的所有app的经验都读取进内存，
4 | 然后用 knowledge_raw_data_to_memory 将这些经验转成向量数据库。（也可以直接用load_memories()完成这两步）
5 | """
6 |
7 | import torch
8 | import os
9 | import shutil
10 | import json
11 | from tqdm import tqdm
12 |
13 | import gc
14 | from utils.embedding_pipeline import SiglipMultimodalEmbeddingPipeline
15 | from typing import List, Dict, Any, Union, Tuple
16 |
17 | from PIL import Image
18 | import numpy as np
19 | import faiss
20 | import numpy as np
21 | from utils.utils import save_object_to_disk, load_object_from_disk
22 |
23 |
24 | class FaissCosineSearcher:
25 | def __init__(
26 | self, vector_database_np: np.ndarray = None, embedding_dim: int = 1152
27 | ):
28 | """
29 | 初始化Faiss余弦相似度搜索器。
30 |
31 | Args:
32 | vector_database_np (np.ndarray): n*m 的二维NumPy数组，作为向量数据库。
33 | embedding_dim (int): 向量的维度，默认为1152。
34 | """
35 | self.m = embedding_dim # 等于vector_database_np.shape[1] # 向量维度
36 |
37 | # 构建Faiss索引 (IndexFlatL2 用于精确搜索)
38 | # 对于余弦相似度，我们在归一化向量上使用L2距离索引，因为将向量归一化之后 cosine_similarity = 1 - (L2_distance^2 / 2)
39 | self.index = faiss.IndexFlatL2(self.m)
40 | if vector_database_np is not None:
41 | self.normalized_db = self._normalize_vectors(
42 | vector_database_np.astype(np.float32)
43 | )
44 | if self.normalized_db.shape[0] > 0: # 只有当数据库非空时才添加
45 | self.index.add(self.normalized_db)
46 |
47 | def _normalize_vectors(self, vectors: np.ndarray) -> np.ndarray:
48 | """
49 | 对向量进行L2归一化。
50 | 如果向量的范数为0，则返回零向量。
51 | """
52 | if vectors.ndim == 1: # 单个向量
53 | norm = np.linalg.norm(vectors)
54 | return vectors / norm if norm > 0 else np.zeros_like(vectors)
55 | elif vectors.ndim == 2: # 向量批次
56 | norms = np.linalg.norm(vectors, axis=1, keepdims=True)
57 | safe_norms = np.where(norms == 0, 1.0, norms) # 范数为0时用1替代，避免除以0
58 | normalized = vectors / safe_norms
59 | normalized[norms.squeeze() == 0] = 0.0 # 将原始范数为0的行设为0向量
60 | return normalized
61 | else:
62 | raise ValueError("vectors must be 1D or 2D NumPy arrays.")
63 |
64 | def add_vectors(self, new_vectors_np: np.ndarray):
65 | """
66 | 向Faiss索引中添加新的向量。
67 |
68 | Args:
69 | new_vectors_np (np.ndarray): n*m 的二维NumPy数组，作为新向量。
70 | """
71 | if not isinstance(new_vectors_np, np.ndarray) or new_vectors_np.ndim != 2:
72 | raise ValueError("new_vectors_np must be a 2D NumPy array.")
73 | if new_vectors_np.shape[0] == 0:
74 | raise ValueError("new_vectors_np cannot be empty.")
75 |
76 | # 归一化新向量
77 | normalized_new_vectors = self._normalize_vectors(
78 | new_vectors_np.astype(np.float32)
79 | )
80 | self.index.add(normalized_new_vectors)
81 |
82 | def search(
83 | self, query_vector_np: np.ndarray, k: int, similarity_threshold: float
84 | ) -> list:
85 | """
86 | 使用Faiss和余弦相似度检索top K个结果 (单个查询)。
87 |
88 | Args:
89 | query_vector_np (np.ndarray): 1*m 的一维或二维NumPy数组，作为查询向量。
90 | k (int): 需要检索的top K结果数量。
91 | similarity_threshold (float): 余弦相似度阈值 (范围通常在 -1.0 到 1.0)。低于此阈值的结果将被去除。
92 |
93 | Returns:
94 | list: 一个列表，包含满足条件的检索结果。每个结果是一个字典，
95 | 格式为 {'index': 数据库中的原始索引, 'similarity': 余弦相似度}。
96 | 列表按相似度降序排列。
97 | """
98 | if not isinstance(query_vector_np, np.ndarray) or query_vector_np.ndim > 2:
99 | raise ValueError("query_vector_np must be a 1D or 2D NumPy array.")
100 |
101 | if query_vector_np.ndim == 1:
102 | query_vector_np_batch = query_vector_np.reshape(1, -1) # 转换为 (1, m)
103 | else: # 已经是二维了
104 | query_vector_np_batch = query_vector_np
105 |
106 | results_batch = self.search_batch(
107 | query_vector_np_batch, k, similarity_threshold
108 | )
109 | return results_batch[0] # 因为是单个查询，所以取第一个结果列表
110 |
111 | def search_batch(
112 | self, query_vectors_np: np.ndarray, k: int, similarity_threshold: float
113 | ) -> list[list[dict]]:
114 | """
115 | 使用Faiss和余弦相似度批量检索top K个结果。
116 |
117 | Args:
118 | query_vectors_np (np.ndarray): b*m 的二维NumPy数组，b是查询数量，m是向量维度。
119 | k (int): 每个查询需要检索的top K结果数量。
120 | similarity_threshold (float): 余弦相似度阈值 (范围通常在 -1.0 到 1.0)。低于此阈值的结果将被去除。
121 |
122 | Returns:
123 | list[list[dict]]: 一个列表的列表。外层列表对应每个查询，内层列表包含该查询满足条件的检索结果。每个结果是一个字典，
124 | 格式为 {'index': 数据库中的原始索引, 'similarity': 余弦相似度}。内层列表按相似度降序排列。
125 | """
126 | if not isinstance(query_vectors_np, np.ndarray) or query_vectors_np.ndim != 2:
127 | raise ValueError("query_vectors_np must be a 2D NumPy array (b * m).")
128 |
129 | num_queries = query_vectors_np.shape[0]
130 | if num_queries == 0:
131 | return []
132 |
133 | if query_vectors_np.shape[1] != self.m:
134 | raise ValueError(
135 | f"Query vector dimension ({query_vectors_np.shape[1]}) does not match database vector dimension ({self.m})."
136 | )
137 | if k <= 0:
138 | raise ValueError("k must be a positive integer.")
139 | if self.index.ntotal == 0: # 如果数据库为空
140 | print("警告：向量数据库为空，无法执行搜索。")
141 | return [[] for _ in range(num_queries)]
142 |
143 | normalized_queries = self._normalize_vectors(
144 | query_vectors_np.astype(np.float32)
145 | )
146 |
147 | is_zero_query = np.all(normalized_queries == 0, axis=1)
148 |
149 | clamped_threshold = max(-1.0, min(1.0, similarity_threshold))
150 | radius_sq = 2.0 - 2.0 * clamped_threshold
151 | if radius_sq < 0:
152 | radius_sq = 0.0
153 |
154 | lims, D_sq_range, I_range = self.index.range_search(
155 | normalized_queries, radius_sq
156 | )
157 |
158 | all_results = []
159 | for i in range(num_queries):
160 | if is_zero_query[i]:
161 | print(
162 | f"警告：批量中的第 {i} 个查询向量归一化后为零向量，该查询返回空结果。"
163 | )
164 | all_results.append([])
165 | continue
166 |
167 | start_offset = lims[i]
168 | end_offset = lims[i + 1]
169 |
170 | query_results_above_threshold = []
171 | for j in range(start_offset, end_offset):
172 | db_index = I_range[j]
173 | dist_sq = D_sq_range[j]
174 |
175 | dist_sq = max(0.0, min(4.0, dist_sq))
176 | cosine_sim = 1.0 - dist_sq / 2.0
177 | cosine_sim = max(-1.0, min(1.0, cosine_sim))
178 |
179 | if cosine_sim >= clamped_threshold:
180 | query_results_above_threshold.append(
181 | {"index": db_index, "similarity": cosine_sim}
182 | )
183 |
184 | # 按相似度降序排序
185 | query_results_above_threshold.sort(
186 | key=lambda x: x["similarity"], reverse=True
187 | )
188 | all_results.append(query_results_above_threshold[:k]) # 取top K
189 |
190 | return all_results
191 |
192 |
193 | class KnowledgeStore:
194 | def __init__(
195 | self,
196 | embedding_pipeline: SiglipMultimodalEmbeddingPipeline,
197 | knowledge_items: list[dict] = None,
198 | ):
199 | self.embedding_pipeline = embedding_pipeline
200 | self.knowledge_items = knowledge_items or []
201 | if len(self.knowledge_items) == 0:
202 | dummy_image = Image.new("RGB", (224, 224), (255, 255, 255))
203 | dummy_embedding = np.array(
204 | self.embedding_pipeline([dummy_image]), dtype=np.float32
205 | )
206 | self.searcher = FaissCosineSearcher(embedding_dim=dummy_embedding.shape[1])
207 | else:
208 | images = []
209 | for item in self.knowledge_items:
210 | img = Image.fromarray(item["image"]).convert("RGB")
211 | images.append(img)
212 | database_vectors = np.array(
213 | self.embedding_pipeline(images), dtype=np.float32
214 | )
215 | self.searcher = FaissCosineSearcher(
216 | database_vectors, embedding_dim=database_vectors.shape[1]
217 | )
218 |
219 | def add_knowledge_items(self, new_knowledge_items: list[dict]):
220 | """
221 | 向知识库中添加新的知识项。
222 |
223 | Args:
224 | new_knowledge_items (list[dict]): 新的知识项列表，每个知识项是一个字典。
225 | """
226 | self.knowledge_items.extend(new_knowledge_items)
227 | images = []
228 | for item in new_knowledge_items:
229 | img = Image.fromarray(item["image"]).convert("RGB")
230 | images.append(img)
231 | new_vectors = np.array(self.embedding_pipeline(images), dtype=np.float32)
232 | self.searcher.add_vectors(new_vectors)
233 |
234 | def search(
235 | self, query_image: Image.Image, k: int = 5, similarity_threshold: float = 0.5
236 | ) -> list[dict]:
237 | query_vector = np.array(self.embedding_pipeline(query_image), dtype=np.float32)
238 | results = self.searcher.search(query_vector, k, similarity_threshold)
239 | return [
240 | {
241 | "knowledge": self.knowledge_items[result["index"]]["info"],
242 | "similarity": result["similarity"],
243 | "index": result["index"],
244 | }
245 | for result in results
246 | ]
247 |
248 | def search_batch(
249 | self,
250 | query_images: list[Image.Image],
251 | k: int = 5,
252 | similarity_threshold: float = 0.5,
253 | ) -> list[list[dict]]:
254 | query_vectors = np.array(
255 | self.embedding_pipeline(query_images), dtype=np.float32
256 | )
257 | results = self.searcher.search_batch(query_vectors, k, similarity_threshold)
258 | return [
259 | [
260 | {
261 | "knowledge": str(self.knowledge_items[result["index"]]["info"]),
262 | "similarity": float(result["similarity"]),
263 | "index": int(result["index"]),
264 | }
265 | for result in result_list
266 | ]
267 | for result_list in results
268 | ]
269 |
270 |
271 | def load_memory(knowledge_items: list[dict]) -> KnowledgeStore:
272 | device = os.getenv(
273 | "CLIENT_EMBEDDING_DEVICE", "cuda" if torch.cuda.is_available() else "cpu"
274 | )
275 | embedding_pipeline = SiglipMultimodalEmbeddingPipeline(
276 | model_id="google/siglip-so400m-patch14-384",
277 | device=device,
278 | server_endpoint=os.getenv("SERVER_EMBEDDING_ENDPOINT"),
279 | )
280 | memory = KnowledgeStore(
281 | knowledge_items=knowledge_items,
282 | embedding_pipeline=embedding_pipeline,
283 | )
284 | return memory
285 |
286 |
287 | def load_knowledge_raw_data(knowledge_base_root_path: str = None):
288 | """
289 | Load knowledge raw data from the specified root path.
290 |
291 | Args:
292 | knowledge_base_root_path (str): The root path of the knowledge base.
293 |
294 | Returns:
295 | Dict[str, Any]: The raw knowledge data.
296 | """
297 | if knowledge_base_root_path is None:
298 | knowledge_base_root_path = os.path.abspath(
299 | os.getenv("KNOWLEDGE_BASE_ABSOLUTE_ROOT_PATH")
300 | )
301 | fp = os.path.join(knowledge_base_root_path, "knowledge_data.pkl")
302 | assert os.path.exists(fp), f"{fp} not exists."
303 | return load_object_from_disk(fp)
304 |
305 |
306 | def knowledge_raw_data_to_memory(
307 | knowledge_raw_data: Dict[str, Any],
308 | ) -> dict[str, KnowledgeStore]:
309 | """Convert knowledge raw data to memory
310 | Args:
311 | knowledge_raw_data: Dict[str, Any]: knowledge raw data. Generated by function load_knowledge_raw_data.
312 | Returns:
313 | dict[str, KnowledgeStore]: memory representation of the knowledge raw data.
314 | """
315 | memories = {}
316 | for package_name, app_knowledge in tqdm(
317 | knowledge_raw_data.items(), desc="Converting knowledge raw data to memory"
318 | ):
319 | knowledge = app_knowledge["knowledge"]
320 | gc.collect()
321 | torch.cuda.empty_cache()
322 | memories[package_name] = load_memory(knowledge)
323 |
324 | print("Fusion knowledge raw data to memory. This may take a while...")
325 | fusion_knowledge = []
326 | for k, v in knowledge_raw_data.items():
327 | fusion_knowledge.extend(v["knowledge"])
328 | gc.collect()
329 | torch.cuda.empty_cache()
330 | memories["fusion"] = load_memory(fusion_knowledge)
331 | gc.collect()
332 | torch.cuda.empty_cache()
333 | print("Done.")
334 | return memories
335 |
336 |
337 | def load_memories(
338 | knowledge_base_root_path: str = None,
339 | ) -> dict[str, KnowledgeStore]:
340 | """
341 | Load memories from the specified root path.
342 |
343 | Args:
344 | knowledge_base_root_path (str): The root path of the knowledge base.
345 |
346 | Returns:
347 | Dict[str, KnowledgeStore]: The loaded memories.
348 | """
349 | knowledge_raw_data = load_knowledge_raw_data(knowledge_base_root_path)
350 | memories = knowledge_raw_data_to_memory(knowledge_raw_data)
351 | return memories
352 |
--------------------------------------------------------------------------------
/exploration_and_mining.py:
--------------------------------------------------------------------------------
1 | import os
2 | import sys
3 | import json
4 | import time
5 | import re
6 | from utils.utils import (
7 | get_apk,
8 | APK,
9 | str_to_md5,
10 | openai_request,
11 | resize_pil_image,
12 | pil_to_webp_base64,
13 | load_object_from_disk,
14 | save_object_to_disk,
15 | )
16 | from utils.device import Device, UIElement
17 | from MLLM_Agent.GUI_explorer import GUI_explorer, execute_adb_action
18 | from PIL import Image
19 | from utils.prompt_templates import TASK_GOAL_GENERATOR
20 | from datetime import datetime
21 | from glob import glob
22 | from utils.memory import load_knowledge_raw_data, load_memory
23 | from utils.knowledge_generation import update_trajectory_to_knowledge
24 | from tqdm import tqdm
25 |
26 |
27 | def parse_task(text: str) -> list[str]:
28 | pattern = r"\d+\.+(.*)"
29 | matches = re.findall(pattern, text)
30 | for i, match in enumerate(matches):
31 | matches[i] = match.strip()
32 | return matches
33 |
34 |
35 | def task_goal_generator(
36 | screenshot: Image.Image,
37 | package_name: str = None,
38 | app_name: str = None,
39 | activity_list: str = None,
40 | usage: dict[str, int] = {"prompt_tokens": 0, "completion_tokens": 0},
41 | ) -> list[str]:
42 | p = TASK_GOAL_GENERATOR.format(
43 | package_name=package_name if package_name else "Not Available",
44 | app_name=app_name if app_name else "Not Available",
45 | activity_list=activity_list if activity_list else "Not Available",
46 | )
47 | # print(f"Prompt: \n{p}")
48 | low_resolution = os.getenv("LOW_RESOLUTION", "False").lower() == "true"
49 | if low_resolution:
50 | screenshot = resize_pil_image(screenshot, 1000)
51 | messages = [
52 | {
53 | "role": "user",
54 | "content": [
55 | {
56 | "type": "image_url",
57 | "image_url": {
58 | "url": f"data:image/webp;base64,{pil_to_webp_base64(screenshot)}",
59 | },
60 | },
61 | {"type": "text", "text": p},
62 | ],
63 | },
64 | ]
65 | rsp_txt = openai_request(
66 | messages=messages, timeout=300, usage=usage, max_tokens=8192
67 | )
68 | return parse_task(rsp_txt)
69 |
70 |
71 | def is_task_explored(root_dir: str, task: str) -> bool:
72 | t = str_to_md5(task)[:16]
73 | res = glob(os.path.join(root_dir, "**", f"*{t}*"), recursive=True)
74 | return len(res) > 0
75 |
76 |
77 | def explore_dfs(
78 | current_task: str,
79 | current_depth: int,
80 | exploration_output_root_dir: str,
81 | max_exploration_tasks: int,
82 | max_exploration_steps: int,
83 | apk_object: APK,
84 | device_controller: Device,
85 | agent: GUI_explorer,
86 | previous_actions: list,
87 | package_name: str,
88 | is_first_task: bool = False,
89 | max_exploration_depth: int = 3,
90 | usage: dict[str, int] = {"prompt_tokens": 0, "completion_tokens": 0},
91 | ):
92 | if (
93 | not is_first_task
94 | ): # 如果不是第一个任务，那么需要根据之前的动作序列来复现到父节点的结束状态
95 | print(f"{'*' * current_depth}Restore To Parent State")
96 | device_controller.stop_all_apps()
97 | device_controller.launch_app(package_name, front=True)
98 | for action in previous_actions:
99 | converted_action, before_ui_elements, logical_screen_size = action
100 | time.sleep(3)
101 | device_controller.wait_to_stabilize()
102 | execute_adb_action(
103 | converted_action,
104 | device_controller,
105 | before_ui_elements,
106 | logical_screen_size,
107 | )
108 | # 运行任务
109 | exploration_output_root_dir = os.path.abspath(exploration_output_root_dir)
110 | if is_task_explored(exploration_output_root_dir, current_task):
111 | print(f"{'*' * current_depth}Task {current_task} already explored. Skip.")
112 | return
113 | records = agent.run(
114 | current_task,
115 | max_rounds=max_exploration_steps,
116 | step_data_output_dir=exploration_output_root_dir,
117 | )
118 |
119 | if current_depth >= max_exploration_depth: # 结束探索
120 | return
121 | # 生成用于恢复到current_task结束后的状态的动作序列
122 | restore_actions = []
123 | for step_data in records:
124 | if isinstance(step_data["converted_action"], str):
125 | continue
126 | action = (
127 | step_data["converted_action"],
128 | [UIElement(**ui_element) for ui_element in step_data["ui_elements"]],
129 | step_data["logical_screen_size"],
130 | )
131 | restore_actions.append(action)
132 | restore_actions = previous_actions + restore_actions
133 | # 生成任务列表
134 | device_controller.wait_to_stabilize()
135 | screenshot = device_controller.get_screenshot()
136 | activity = [act for act in apk_object.get_activities() if "sdk" not in act.lower()]
137 | activity_str = "\n".join(activity)
138 | app_name = apk_object.get_app_name()
139 | task_list = task_goal_generator(
140 | screenshot=screenshot,
141 | package_name=package_name,
142 | app_name=app_name,
143 | activity_list=activity_str,
144 | usage=usage,
145 | )
146 | print(f"{'*' * current_depth}Generated {len(task_list)} tasks: {task_list}")
147 | if len(task_list) == 0:
148 | return
149 | exploration_output_root_dir = os.path.join(
150 | exploration_output_root_dir,
151 | f"depth_{current_depth+1}",
152 | )
153 | for i in range(max_exploration_tasks):
154 | if i < len(task_list):
155 | task = task_list[i]
156 | print(
157 | f"{'*' * current_depth}Exploring task {i+1}/{min(len(task_list),max_exploration_tasks)}: {task}"
158 | )
159 | if is_task_explored(exploration_output_root_dir, task):
160 | print(f"{'*' * current_depth}Task {task} already explored. Skip.")
161 | continue
162 | explore_dfs(
163 | current_task=task,
164 | current_depth=current_depth + 1,
165 | exploration_output_root_dir=exploration_output_root_dir,
166 | max_exploration_tasks=max_exploration_tasks,
167 | max_exploration_steps=max_exploration_steps,
168 | agent=agent,
169 | apk_object=apk_object,
170 | device_controller=device_controller,
171 | previous_actions=restore_actions,
172 | package_name=package_name,
173 | is_first_task=bool(i == 0),
174 | max_exploration_depth=max_exploration_depth,
175 | usage=usage,
176 | )
177 |
178 |
179 | def auto_exploration(
180 | package_name: str,
181 | exploration_output_root_dir: str = "./output",
182 | device_serial: str = None,
183 | max_exploration_tasks: int = 10,
184 | max_exploration_steps: int = 30,
185 | max_exploration_depth: int = 5, # 从首页开始的任务扩展深度（最多扩展 max_exploration_depth-1 代）
186 | usage: dict[str, int] = {"prompt_tokens": 0, "completion_tokens": 0},
187 | ):
188 | exploration_output_root_dir = os.path.abspath(exploration_output_root_dir)
189 | exploration_output_root_dir = os.path.join(
190 | exploration_output_root_dir, package_name
191 | )
192 | os.makedirs(exploration_output_root_dir, exist_ok=True)
193 | apk_path = os.path.join(exploration_output_root_dir, f"{package_name}.apk")
194 | if os.path.exists(apk_path):
195 | os.remove(apk_path)
196 | print(f"Fetching the APK file of {package_name}.")
197 | res = get_apk(package_name, apk_path, device_serial)
198 | if res == "ERROR":
199 | print(f"Failed to get the APK file of {package_name}.")
200 | sys.exit(1)
201 | print("Analyzing the APK file.")
202 | apk_object = APK(apk_path)
203 | app_info = {
204 | "app_name": apk_object.get_app_name(),
205 | "app_version": apk_object.get_androidversion_code(),
206 | "app_version_name": apk_object.get_androidversion_name(),
207 | "app_pkg": apk_object.get_package(),
208 | "app_main_activity": apk_object.get_main_activity(),
209 | }
210 | with open(
211 | os.path.join(exploration_output_root_dir, "app_info.json"),
212 | "w",
213 | encoding="utf-8",
214 | ) as file:
215 | json.dump(app_info, file, indent=2, ensure_ascii=False)
216 | device = Device(device_serial)
217 | agent = GUI_explorer(device_serial=device_serial)
218 | print("Killing all apps.")
219 | device.stop_all_apps()
220 | # device.launch_app(package_name, front=True,activity=app_info["app_main_activity"])
221 | device.launch_app(package_name, front=True)
222 | time.sleep(5) # 等待app启动完成
223 | task_list = []
224 | print("Generating exploration tasks.")
225 | device.wait_to_stabilize()
226 | screenshot = device.get_screenshot()
227 | activity = [act for act in apk_object.get_activities() if "sdk" not in act.lower()]
228 | activity_str = "\n".join(activity)
229 | app_name = apk_object.get_app_name()
230 | task_list = task_goal_generator(
231 | screenshot=screenshot,
232 | package_name=package_name,
233 | app_name=app_name,
234 | activity_list=activity_str,
235 | usage=usage,
236 | )
237 | print(f"Generated {len(task_list)} tasks: {task_list}")
238 | if max_exploration_tasks < len(task_list):
239 | task_list = task_list[:max_exploration_tasks]
240 | print(f"Exploring {len(task_list)} tasks: {task_list}")
241 | for i, task in enumerate(task_list):
242 | print(f"Exploring task {i+1}/{len(task_list)}: {task}")
243 | explore_dfs(
244 | current_task=task,
245 | current_depth=1,
246 | exploration_output_root_dir=exploration_output_root_dir,
247 | max_exploration_tasks=max_exploration_tasks,
248 | max_exploration_steps=max_exploration_steps,
249 | agent=agent,
250 | apk_object=apk_object,
251 | device_controller=device,
252 | previous_actions=[],
253 | package_name=package_name,
254 | is_first_task=bool(i == 0),
255 | max_exploration_depth=max_exploration_depth,
256 | usage=usage,
257 | )
258 | print("Auto exploration finished.")
259 |
260 |
261 | import argparse
262 |
263 | if __name__ == "__main__":
264 | parser = argparse.ArgumentParser()
265 | parser.add_argument(
266 | "-package_name", help="The package name of the APK, e.g. com.android.settings"
267 | )
268 | parser.add_argument(
269 | "-device_serial", help="The serial number of the device, see `adb devices`"
270 | )
271 | parser.add_argument(
272 | "-output_dir",
273 | help="The directory to save the task file",
274 | default="./exploration_output",
275 | )
276 | parser.add_argument(
277 | "-max_branching_factor",
278 | help="The max number of tasks to explore at each node",
279 | default=3,
280 | )
281 | parser.add_argument(
282 | "-max_exploration_steps",
283 | help="The max number of steps to explore for each task",
284 | default=30,
285 | )
286 | parser.add_argument(
287 | "-max_exploration_depth",
288 | help="The max depth of exploration",
289 | default=5,
290 | )
291 | args = parser.parse_args()
292 |
293 | usage: dict[str, int] = {"prompt_tokens": 0, "completion_tokens": 0}
294 |
295 | print(args)
296 | print("Starting auto exploration.")
297 | auto_exploration(
298 | package_name=args.package_name,
299 | exploration_output_root_dir=args.output_dir,
300 | device_serial=args.device_serial,
301 | max_exploration_tasks=int(args.max_branching_factor),
302 | max_exploration_steps=int(args.max_exploration_steps),
303 | max_exploration_depth=int(args.max_exploration_depth),
304 | usage=usage,
305 | )
306 | print(f"Task goal generator token usage: {usage}")
307 |
308 | ## 将轨迹中的知识提取出来
309 | # 如果版本升级，是否清空知识库重新生成
310 | EMPTY_KNOWLEDGE_BASE_WHEN_VERSION_UPGRADE = (
311 | os.getenv("EMPTY_KNOWLEDGE_BASE_WHEN_VERSION_UPGRADE", "False").lower()
312 | == "false"
313 | )
314 | # 如果版本降级，是否清空知识库重新生成
315 | EMPTY_KNOWLEDGE_BASE_WHEN_VERSION_DOWNGRADE = (
316 | os.getenv("EMPTY_KNOWLEDGE_BASE_WHEN_VERSION_DOWNGRADE", "True").lower()
317 | == "true"
318 | )
319 | app_info_json_paths = glob(
320 | os.path.join(args.output_dir, "**", "app_info.json"), recursive=True
321 | )
322 | app_infos = {}
323 | for app_info_json_path in app_info_json_paths:
324 | with open(app_info_json_path, "r", encoding="utf-8") as f:
325 | app_info = json.load(f)
326 | app_infos[app_info["app_pkg"]] = app_info
327 |
328 | # knowledge_raw_data = {}
329 | print("Loading knowledge raw data.")
330 | knowledge_raw_data = load_knowledge_raw_data()
331 | fusion_knowledge = []
332 | locations = []
333 | for k, v in knowledge_raw_data.items():
334 | if k in app_infos:
335 | app_info = app_infos[k]
336 | new_app_version = app_info["app_version"]
337 | old_app_version = v["app_version"]
338 | if new_app_version > old_app_version:
339 | if EMPTY_KNOWLEDGE_BASE_WHEN_VERSION_UPGRADE:
340 | print(
341 | f"App {k} version upgraded from {old_app_version} to {new_app_version}, clear knowledge base."
342 | )
343 | v["knowledge"] = []
344 | if new_app_version < old_app_version:
345 | if EMPTY_KNOWLEDGE_BASE_WHEN_VERSION_DOWNGRADE:
346 | print(
347 | f"App {k} version downgraded from {old_app_version} to {new_app_version}, clear knowledge base."
348 | )
349 | v["knowledge"] = []
350 | fusion_knowledge.extend(v["knowledge"])
351 | locations.extend([(k, i) for i in range(len(v["knowledge"]))])
352 |
353 | for k, app_info in app_infos.items():
354 | if k not in knowledge_raw_data:
355 | print(f"App {k} not in knowledge raw data, add it.")
356 | knowledge_raw_data[k] = app_info
357 | knowledge_raw_data[k]["knowledge"] = []
358 |
359 | print("Loading fusion memory. This may take a while.")
360 | fusion_memory = load_memory(fusion_knowledge)
361 |
362 | for app_info_json_path in tqdm(
363 | app_info_json_paths, ncols=80, desc="Extracting knowledge"
364 | ):
365 | app_info_json_dir = os.path.dirname(app_info_json_path)
366 | pkl_paths = glob(
367 | os.path.join(app_info_json_dir, "**", "*.pkl.zst"), recursive=True
368 | )
369 | for pkl_path in tqdm(
370 | pkl_paths,
371 | ncols=80,
372 | desc=f"Processing pkl files in {os.path.basename(app_info_json_dir)}",
373 | leave=False,
374 | ):
375 | trajectory_data = load_object_from_disk(pkl_path)
376 | update_trajectory_to_knowledge(
377 | trajectory_data=trajectory_data,
378 | locations=locations,
379 | fusion_memory=fusion_memory,
380 | knowledge_data=knowledge_raw_data,
381 | usage=usage,
382 | )
383 | print(f"Total token usage: {usage}")
384 | knowledge_base_root_path = os.path.abspath(os.getenv("KNOWLEDGE_BASE_ABSOLUTE_ROOT_PATH","./knowledge_base"))
385 | fp = os.path.join(knowledge_base_root_path, "knowledge_data.pkl")
386 | save_object_to_disk(knowledge_raw_data, fp, compress_level=20)
387 |
--------------------------------------------------------------------------------
/utils/embedding_pipeline.py:
--------------------------------------------------------------------------------
1 | """
2 | 在项目的根目录运行 python -m utils.embedding_pipeline 即可启动服务，然后在SiglipMultimodalEmbeddingPipeline的构造函数的device填写"server"。
3 | """
4 |
5 | import os
6 |
7 | os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE"
8 | import torch
9 | from PIL import Image
10 | import gradio as gr
11 | from transformers import AutoProcessor, AutoModel
12 | from typing import Any, List, Union, Dict, Tuple
13 | import numpy as np
14 | from gradio_client import Client
15 | import io
16 | import base64
17 | from utils.utils import resize_pil_image
18 | MAX_BATCH_SIZE = 32 # 最大批处理大小，根据显存大小调整，50张最大512像素的图片大约需要2.5GB
19 | MAX_IMAGE_SIZE = 512 # 最大图片大小，512x512，超过这个大小会自动保留比例缩放到最长边为512
20 |
21 |
22 | class SiglipMultimodalEmbeddingPipeline:
23 | IMAGE_PREFIX = "