├── .github └── workflows │ └── python-publish.yml ├── .gitignore ├── 05c8044a-2ac3-45c8-87fa-d1feaa737368.gif ├── MANIFEST.in ├── README.md ├── auto_nico ├── __init__.py ├── android │ ├── __init__.py │ ├── adb_utils.py │ ├── nico_android.py │ ├── nico_android_element.py │ ├── nico_image.py │ ├── package │ │ ├── __init__.py │ │ ├── dump_hierarchy_androidTest_v1.4.apk │ │ └── dump_hierarchy_v1.4.apk │ └── tools │ │ ├── __init__.py │ │ └── format_converter.py ├── common │ ├── __init__.py │ ├── common_utils.py │ ├── error.py │ ├── kmeans_run.py │ ├── logger_config.py │ ├── nico_basic.py │ ├── nico_basic_element.py │ ├── runtime_cache.py │ └── send_request.py ├── console_scripts │ ├── __init__.py │ ├── dump_ui.py │ ├── inspector_web │ │ ├── __init__.py │ │ ├── nico_inspector.py │ │ ├── pyscrcpy │ │ │ ├── __init__.py │ │ │ ├── const.py │ │ │ ├── control.py │ │ │ ├── core.py │ │ │ ├── etsdt.py │ │ │ ├── scrcpy-server.jar │ │ │ └── test.py │ │ ├── static │ │ │ ├── button_script.js │ │ │ ├── script.js │ │ │ └── styles.css │ │ └── templates │ │ │ ├── __init__.py │ │ │ ├── index.html │ │ │ └── xml_template.html │ ├── screenshot.py │ ├── test_ │ │ ├── __init__.py │ │ ├── app.py │ │ ├── nico_utils.py │ │ ├── pyscrcpy │ │ │ ├── __init__.py │ │ │ ├── const.py │ │ │ ├── control.py │ │ │ ├── core.py │ │ │ ├── scrcpy-server.jar │ │ │ └── scrcpy_utils.py │ │ ├── scrcpy_utils.py │ │ ├── static │ │ │ ├── __init__.py │ │ │ ├── api_request.js │ │ │ ├── button_script.js │ │ │ ├── event_listening.js │ │ │ ├── main.js │ │ │ └── styles.css │ │ └── templates │ │ │ ├── __init__.py │ │ │ └── index.html │ └── uninstall_apk.py └── ios │ ├── XCUIElementType.py │ ├── __init__.py │ ├── idb_utils.py │ ├── nico_image.py │ ├── nico_ios.py │ ├── nico_ios_element.py │ ├── received_image.jpg │ └── tools │ ├── __init__.py │ ├── format_converter.py │ └── image_process.py ├── setup.py └── test.py /.github/workflows/python-publish.yml: -------------------------------------------------------------------------------- 1 | # This workflow will upload a Python Package using Twine when a release is created 2 | # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries 3 | 4 | # This workflow uses actions that are not certified by GitHub. 5 | # They are provided by a third-party and are governed by 6 | # separate terms of service, privacy policy, and support 7 | # documentation. 8 | 9 | name: Upload Python Package 10 | 11 | on: 12 | release: 13 | types: [published] 14 | 15 | permissions: 16 | contents: read 17 | 18 | jobs: 19 | deploy: 20 | 21 | runs-on: ubuntu-latest 22 | 23 | steps: 24 | - uses: actions/checkout@v3 25 | - name: Set up Python 26 | uses: actions/setup-python@v3 27 | with: 28 | python-version: '3.11' 29 | - name: Install dependencies 30 | run: | 31 | python -m pip install --upgrade pip 32 | pip install --upgrade setuptools 33 | pip install wheel 34 | - name: Build package 35 | run: python setup.py bdist_wheel 36 | - name: Publish package 37 | uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29 38 | with: 39 | user: __token__ 40 | password: ${{ secrets.PYPI_API_TOKEN }} 41 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | dist/ 3 | .idea/ 4 | *.pyc 5 | build/ 6 | get_root_xml.py 7 | test2.py 8 | test3.py 9 | AutoNico.egg-info/ 10 | selfIdentity.plist 11 | -------------------------------------------------------------------------------- /05c8044a-2ac3-45c8-87fa-d1feaa737368.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/letmeNo1/Nico/79dc1ac061cdeaa7ab74660d42bf85bd93ff0592/05c8044a-2ac3-45c8-87fa-d1feaa737368.gif -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/letmeNo1/Nico/79dc1ac061cdeaa7ab74660d42bf85bd93ff0592/MANIFEST.in -------------------------------------------------------------------------------- /auto_nico/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/letmeNo1/Nico/79dc1ac061cdeaa7ab74660d42bf85bd93ff0592/auto_nico/__init__.py -------------------------------------------------------------------------------- /auto_nico/android/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/letmeNo1/Nico/79dc1ac061cdeaa7ab74660d42bf85bd93ff0592/auto_nico/android/__init__.py -------------------------------------------------------------------------------- /auto_nico/android/adb_utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import re 3 | import time 4 | import subprocess 5 | 6 | from auto_nico.common.send_request import send_http_request 7 | from loguru import logger 8 | 9 | from auto_nico.common.error import ADBServerError, NicoError 10 | from auto_nico.common.runtime_cache import RunningCache 11 | import cv2 12 | import numpy as np 13 | import platform 14 | 15 | 16 | class AdbUtils: 17 | def __init__(self, udid): 18 | self.udid = udid 19 | self.runtime_cache = RunningCache(udid) 20 | self.version = 1.4 21 | 22 | 23 | def get_tcp_forward_port(self): 24 | # Determine the current operating system 25 | system = platform.system() 26 | 27 | # Select appropriate command based on the operating system 28 | if system == "Windows": 29 | # Windows: Use findstr to exclude lines containing "local" and filter by device UDID 30 | cmd = f'''forward --list | findstr /v local | findstr "{self.udid}"''' 31 | else: 32 | # macOS/Linux: Use grep to exclude lines containing "local" and filter by device UDID 33 | cmd = f'''forward --list | grep -v local | grep "{self.udid}"''' 34 | 35 | # Execute the command to list port forwards 36 | rst = self.cmd(cmd) 37 | port = None 38 | 39 | # Process the command output if it's not empty 40 | if rst.strip(): 41 | try: 42 | # Split the output by "tcp:" to extract the port part 43 | parts = rst.split("tcp:") 44 | if len(parts) > 1: 45 | # Extract and clean the port number part 46 | port_part = parts[1].strip() 47 | # Further process to get only the port number 48 | port = port_part.split()[0] if port_part else None 49 | except Exception as e: 50 | print(f"Error parsing port: {e}") 51 | 52 | return port 53 | 54 | def clear_tcp_forward_port(self, port): 55 | logger.info(f"{self.udid}'s forward --remove tcp:{port}") 56 | self.cmd(f"forward --remove tcp:{port}") 57 | 58 | 59 | def set_tcp_forward_port(self, port): 60 | system = platform.system() 61 | # Select appropriate command based on the operating system 62 | if system == "Windows": 63 | # Windows uses findstr command 64 | check_cmd = f'''forward --list | findstr "{port}"''' 65 | else: 66 | # macOS/Linux uses grep command 67 | check_cmd = f'''forward --list | grep "{port}"''' 68 | 69 | # Attempt up to 5 times to set up port forwarding 70 | for _ in range(5): 71 | rst = self.cmd(check_cmd) 72 | 73 | # Use regex to ensure exact port number match 74 | port_pattern = r'tcp:' + re.escape(str(port)) 75 | port_matched = re.search(port_pattern, rst) 76 | # Check if port forwarding exists and is associated with this device 77 | if port_matched and self.udid in rst: 78 | logger.info(f"{self.udid}'s TCP port {port} is already forwarded to TCP:8000") 79 | break 80 | else: 81 | # Configure port forwarding to the device 82 | self.cmd(f'''forward tcp:{port} tcp:8000''') 83 | 84 | def restart_test_server(self, port): 85 | runtime_cache = RunningCache(self.udid) 86 | def __check_server_ready(current_port, timeout): 87 | time_started_sec = time.time() 88 | while time.time() < time_started_sec + timeout: 89 | rst = send_http_request(current_port, "get_root") 90 | if rst and rst !=[]: 91 | logger.info(f"{self.udid}'s test server is ready on {port}") 92 | runtime_cache.set_current_running_port(port) 93 | return True 94 | else: 95 | logger.info(f"{self.udid}'s test server isn't ready on {port}") 96 | time.sleep(0.5) 97 | continue 98 | return False 99 | 100 | for _ in range(5): 101 | logger.debug( 102 | f"""adb -s {self.udid} shell am instrument -r -w -e port {port} -e class nico.dump_hierarchy.HierarchyTest nico.dump_hierarchy.test/androidx.test.runner.AndroidJUnitRunner""") 103 | commands = f"""adb -s {self.udid} shell am instrument -r -w -e port {port} -e class nico.dump_hierarchy.HierarchyTest nico.dump_hierarchy.test/androidx.test.runner.AndroidJUnitRunner""" 104 | subprocess.Popen(commands, shell=True) 105 | if __check_server_ready(port, 10): 106 | logger.info(f"{self.udid}'s uiautomator was initialized successfully") 107 | return True 108 | logger.info(f"wait 3 s") 109 | time.sleep(3) 110 | return False 111 | 112 | def __install(self, udid, version): 113 | for i in [f"dump_hierarchy_v{version}.apk", f"dump_hierarchy_androidTest_v{version}.apk"]: 114 | logger.info(f"{udid}'s start install {i}") 115 | lib_path = (os.path.dirname(__file__) + f"\package\{i}").replace("console_scripts\inspector_web", "") 116 | rst = self.cmd(fr'install -t "{lib_path}"') 117 | if rst.find("Success") >= 0: 118 | logger.info(f"{udid}'s adb install {lib_path} successfully") 119 | else: 120 | logger.info(rst) 121 | 122 | def reinstall_test_server_package(self, version): 123 | for i in ["nico.dump_hierarchy", "nico.dump_hierarchy.test"]: 124 | logger.info(f"{self.udid}'s start uninstall {i}") 125 | rst = self.cmd(f"uninstall {i}") 126 | if rst.find("Success") >= 0: 127 | logger.info(f"{self.udid}'s adb uninstall {i} successfully") 128 | self.__install(self.udid, version) 129 | 130 | def install_test_server_package(self, version): 131 | rst = self.qucik_shell("dumpsys package nico.dump_hierarchy | grep versionName") 132 | if "versionName" not in rst: 133 | self.wait_for_boot_completed() 134 | self.__install(self.udid, version) 135 | elif version > float(rst.split("=")[-1]): 136 | logger.info(float(rst.split("=")[-1])) 137 | logger.info(f"{self.udid}'s New version detected") 138 | for i in ["nico.dump_hierarchy", "nico.dump_hierarchy.test"]: 139 | logger.debug(f"{self.udid}'s start uninstall {i}") 140 | rst = self.cmd(f"uninstall {i}") 141 | if rst.find("Success") >= 0: 142 | logger.debug(f"{self.udid}'s adb uninstall {i} successfully") 143 | else: 144 | logger.debug(rst) 145 | self.__install(self.udid, version) 146 | else: 147 | logger.debug(f"{self.udid}'s version is the latest") 148 | 149 | def is_device_boot_completed(self): 150 | result = self.shell("getprop sys.boot_completed").strip() 151 | return result == "1" 152 | 153 | def wait_for_boot_completed(self): 154 | while not self.is_device_boot_completed(): 155 | logger.debug("Waiting for device to complete booting...") 156 | time.sleep(5) 157 | logger.debug("Device boot completed") 158 | 159 | def check_adb_server(self): 160 | rst = os.popen("adb devices").read() 161 | if self.udid in rst: 162 | pass 163 | else: 164 | raise ADBServerError("no devices connect") 165 | 166 | def is_screen_off(self): 167 | rst = self.shell("dumpsys display | grep 'mScreenState'") 168 | if "mScreenState=ON" in rst: 169 | # logger.info(f"{self.udid}'s screen is on") 170 | return False 171 | else: 172 | logger.info(f"{self.udid}'s screen is off") 173 | return True 174 | 175 | def get_screen_size(self): 176 | command = f'adb -s {self.udid} shell wm size' 177 | output = os.popen(command).read() 178 | size_str = output.split(': ')[-1] 179 | width, height = map(int, size_str.split('x')) 180 | return width, height 181 | 182 | def start_app(self, package_name): 183 | command = f'am start -n {package_name}' 184 | self.qucik_shell(command) 185 | 186 | def stop_app(self, package_name): 187 | command = f'am force-stop {package_name}' 188 | self.qucik_shell(command) 189 | 190 | def qucik_shell(self, cmds): 191 | udid = self.udid 192 | """@Brief: Execute the CMD and return value 193 | @return: bool 194 | """ 195 | try: 196 | result = subprocess.run(f'''adb -s {udid} shell "{cmds}"''', shell=True, capture_output=True, text=True, 197 | check=True).stdout 198 | except subprocess.CalledProcessError as e: 199 | return e.stderr 200 | return result 201 | 202 | def shell(self, cmds, with_root=False, timeout=10): 203 | udid = self.udid 204 | """@Brief: Execute the CMD and return value 205 | @return: bool 206 | """ 207 | commands = "" 208 | if type(cmds) is list: 209 | for cmd in cmds: 210 | commands = commands + cmd + "\n" 211 | else: 212 | commands = cmds 213 | if with_root: 214 | su_commands = "su\n" 215 | commands = su_commands + commands 216 | adb_process = subprocess.Popen("adb -s %s shell" % udid, stdin=subprocess.PIPE, stdout=subprocess.PIPE, 217 | stderr=subprocess.PIPE, text=True, shell=True) 218 | adb_process.stdin.write(commands) 219 | output, error = adb_process.communicate(timeout=timeout) 220 | if output != "": 221 | return output 222 | else: 223 | return error 224 | 225 | def cmd(self, cmd): 226 | udid = self.udid 227 | """@Brief: Execute the CMD and return value 228 | @return: bool 229 | """ 230 | try: 231 | result = subprocess.run(f'''adb -s {udid} {cmd}''', shell=True, capture_output=True, text=True, 232 | check=True, timeout=10).stdout 233 | except subprocess.CalledProcessError as e: 234 | return e.stderr 235 | return result 236 | 237 | def restart_app(self, package_name): 238 | self.stop_app(package_name) 239 | time.sleep(1) 240 | self.start_app(package_name) 241 | 242 | def is_keyboard_shown(self): 243 | """ 244 | Perform `adb shell dumpsys input_method` command and search for information if keyboard is shown 245 | 246 | Returns: 247 | True or False whether the keyboard is shown or not 248 | 249 | """ 250 | dim = self.shell('dumpsys input_method') 251 | if dim: 252 | return "mInputShown=true" in dim 253 | return False 254 | 255 | def is_screenon(self): 256 | """ 257 | Perform `adb shell dumpsys window policy` command and search for information if screen is turned on or off 258 | 259 | Raises: 260 | AirtestError: if screen state can't be detected 261 | 262 | Returns: 263 | True or False whether the screen is turned on or off 264 | 265 | """ 266 | screenOnRE = re.compile('mScreenOnFully=(true|false)') 267 | m = screenOnRE.search(self.qucik_shell('dumpsys window policy')) 268 | if m: 269 | return m.group(1) == 'true' 270 | else: 271 | # MIUI11 272 | screenOnRE = re.compile('screenState=(SCREEN_STATE_ON|SCREEN_STATE_OFF)') 273 | m = screenOnRE.search(self.qucik_shell('dumpsys window policy')) 274 | if m: 275 | return m.group(1) == 'SCREEN_STATE_ON' 276 | raise NicoError("Couldn't determine screen ON state") 277 | 278 | 279 | def is_locked(self): 280 | """ 281 | Perform `adb shell dumpsys window policy` command and search for information if screen is locked or not 282 | 283 | Raises: 284 | AirtestError: if lock screen can't be detected 285 | 286 | Returns: 287 | True or False whether the screen is locked or not 288 | 289 | """ 290 | lockScreenRE = re.compile('(?:mShowingLockscreen|isStatusBarKeyguard|showing)=(true|false)') 291 | m = lockScreenRE.search(self.qucik_shell('dumpsys window policy')) 292 | if not m: 293 | raise NicoError("Couldn't determine screen lock state") 294 | return (m.group(1) == 'true') 295 | 296 | def unlock(self): 297 | """ 298 | Perform `adb shell input keyevent MENU` and `adb shell input keyevent BACK` commands to attempt 299 | to unlock the screen 300 | 301 | Returns: 302 | None 303 | 304 | Warnings: 305 | Might not work on all devices 306 | 307 | """ 308 | self.qucik_shell('input keyevent MENU') 309 | self.qucik_shell('input keyevent BACK') 310 | 311 | def wake_up(self): 312 | self.qucik_shell('input keyevent KEYCODE_WAKEUP') 313 | 314 | def keyevent(self, keyname): 315 | self.qucik_shell(f'input keyevent {keyname}') 316 | 317 | def switch_app(self): 318 | self.keyevent("KEYCODE_APP_SWITCH") 319 | 320 | def back(self): 321 | self.keyevent("BACK") 322 | 323 | def menu(self): 324 | self.keyevent("MENU") 325 | 326 | def home(self): 327 | self.keyevent("HOME") 328 | 329 | def switch_app(self): 330 | self.keyevent("KEYCODE_APP_SWITCH") 331 | 332 | def get_image_object(self, quality=100,use_adb=True): 333 | if use_adb: 334 | result = subprocess.run(['adb', '-s', self.udid, 'exec-out', 'screencap', '-p'], stdout=subprocess.PIPE) 335 | screenshot_data = result.stdout 336 | nparr = np.frombuffer(screenshot_data, np.uint8) 337 | image = cv2.imdecode(nparr, cv2.IMREAD_COLOR) 338 | return image 339 | else: 340 | exists_port = self.runtime_cache.get_current_running_port() 341 | a = send_http_request(exists_port, "screenshot", {"quality": quality}) 342 | nparr = np.frombuffer(a, np.uint8) 343 | image = cv2.imdecode(nparr, cv2.IMREAD_COLOR) 344 | return image 345 | 346 | def get_root_node(self): 347 | exists_port = self.runtime_cache.get_current_running_port() 348 | for _ in range(5): 349 | response = send_http_request(exists_port, "dump", {"compressed": "true"}) 350 | if "" in response: 351 | return response 352 | time.sleep(1) 353 | 354 | def snapshot(self, name, path): 355 | if not os.path.isabs(path): 356 | path = os.path.abspath(path) 357 | self.shell(f'screencap -p /sdcard/{name}.png', with_root=True) 358 | self.cmd(f'pull /sdcard/{name}.png {path}') 359 | self.qucik_shell(f'rm /sdcard/{name}.png') 360 | full_path = f"{path}/{name}.png" 361 | logger.info(full_path) 362 | return full_path 363 | 364 | def swipe(self, direction, scroll_time=1, target_area=None): 365 | x = int(self.get_screen_size()[0] / 2) 366 | y1 = int(self.get_screen_size()[1] / 4) 367 | y2 = int(self.get_screen_size()[1] / 2) 368 | if target_area is not None: 369 | x = int(self.get_screen_size()[0] * target_area.get_position()[0]) 370 | y1 = int((self.get_screen_size()[1] * target_area.get_position()[1]) / 4) 371 | y2 = int((self.get_screen_size()[1] * target_area.get_position()[1]) / 2) 372 | if direction not in ["down", "up"]: 373 | raise TypeError("Please use up or down") 374 | else: 375 | for i in range(int(scroll_time)): 376 | if direction == "down": 377 | self.shell(f"input swipe {x} {y1} {x} {y2}") 378 | elif direction == "up": 379 | self.shell(f"input swipe {x} {y2} {x} {y1}") 380 | -------------------------------------------------------------------------------- /auto_nico/android/nico_android.py: -------------------------------------------------------------------------------- 1 | import random 2 | import time 3 | import subprocess 4 | 5 | from auto_nico.android.nico_android_element import NicoAndroidElement 6 | from loguru import logger 7 | from auto_nico.common.runtime_cache import RunningCache 8 | from auto_nico.common.send_request import send_http_request 9 | from auto_nico.common.nico_basic import NicoBasic 10 | from auto_nico.android.adb_utils import AdbUtils 11 | 12 | 13 | class NicoAndroid(NicoBasic): 14 | def __init__(self, udid, port="random", **query): 15 | super().__init__(udid, **query) 16 | 17 | self.udid = udid 18 | self.adb_utils = AdbUtils(udid) 19 | self.version = 1.4 20 | self.adb_utils.install_test_server_package(self.version) 21 | self.adb_utils.check_adb_server() 22 | self.__set_running_port(port) 23 | self.runtime_cache = RunningCache(udid) 24 | self.runtime_cache.set_initialized(True) 25 | response = send_http_request(RunningCache(udid).get_current_running_port(), "status") 26 | rst = response is not None and "server is running" in response 27 | response = send_http_request(RunningCache(udid).get_current_running_port(), "get_root") 28 | rst2 = response is not None and "[android.view.accessibility.AccessibilityNodeInfo" in response 29 | if rst and rst2: 30 | logger.debug(f"{self.udid}'s test server is ready on {RunningCache(udid).get_current_running_port()}") 31 | else: 32 | logger.debug(f"{self.udid} test server disconnect, restart ") 33 | self.__init_adb_auto(RunningCache(udid).get_current_running_port()) 34 | 35 | self.close_keyboard() 36 | 37 | def __set_running_port(self, port): 38 | exists_port = self.adb_utils.get_tcp_forward_port() 39 | if exists_port is None: 40 | logger.debug(f"{self.udid} no exists port") 41 | if port != "random": 42 | running_port = port 43 | else: 44 | random_number = random.randint(9000, 9999) 45 | running_port = random_number 46 | else: 47 | running_port = int(exists_port) 48 | RunningCache(self.udid).set_current_running_port(running_port)\ 49 | 50 | def __check_server_ready(self,current_port,timeout): 51 | time_started_sec = time.time() 52 | while time.time() < time_started_sec + timeout: 53 | response = send_http_request(current_port, "get_root") 54 | rst = response is not None and "[android.view.accessibility.AccessibilityNodeInfo" in response 55 | logger.info(f"{self.udid}'s response is {response } ") 56 | 57 | if rst: 58 | 59 | logger.info(f"{self.udid}'s test server is ready on {current_port}") 60 | return True 61 | else: 62 | logger.info(f"server is no ready on {current_port}") 63 | time.sleep(0.5) 64 | continue 65 | return False 66 | 67 | def __start_test_server(self): 68 | current_port = RunningCache(self.udid).get_current_running_port() 69 | for _ in range(5): 70 | logger.debug( 71 | f"""adb -s {self.udid} shell am instrument -r -w -e port {current_port} -e class nico.dump_hierarchy.HierarchyTest nico.dump_hierarchy.test/androidx.test.runner.AndroidJUnitRunner""") 72 | commands = f"""adb -s {self.udid} shell am instrument -r -w -e port {current_port} -e class nico.dump_hierarchy.HierarchyTest nico.dump_hierarchy.test/androidx.test.runner.AndroidJUnitRunner""" 73 | subprocess.Popen(commands, shell=True) 74 | time.sleep(3) 75 | if self.__check_server_ready(current_port,10): 76 | break 77 | logger.info(f"wait 3 s") 78 | time.sleep(3) 79 | logger.info(f"{self.udid}'s uiautomator was initialized successfully") 80 | 81 | def __init_adb_auto(self, port): 82 | self.adb_utils.set_tcp_forward_port(port) 83 | self.__start_test_server() 84 | 85 | def close_keyboard(self): 86 | adb_utils = AdbUtils(self.udid) 87 | ime_list = adb_utils.qucik_shell("ime list -s").split("\n")[0:-1] 88 | for ime in ime_list: 89 | adb_utils.qucik_shell(f"ime disable {ime}") 90 | 91 | def __call__(self, **query): 92 | current_port = RunningCache(self.udid).get_current_running_port() 93 | self.adb_utils.check_adb_server() 94 | if self.adb_utils.is_screen_off(): 95 | self.adb_utils.wake_up() 96 | response = send_http_request(current_port, "get_root") 97 | rst = response is not None and "[android.view.accessibility.AccessibilityNodeInfo" in response 98 | if not rst: 99 | logger.info(f"{self.udid} test server disconnect, restart ") 100 | self.adb_utils.install_test_server_package(self.version) 101 | self.__init_adb_auto(current_port) 102 | self.close_keyboard() 103 | else: 104 | logger.info(f"{self.udid} test server connect successful ") 105 | 106 | NAE = NicoAndroidElement(**query) 107 | NAE.set_udid(self.udid) 108 | NAE.set_port(current_port) 109 | return NAE 110 | 111 | # nico = NicoAndroid("RFCXA08RFMM") 112 | -------------------------------------------------------------------------------- /auto_nico/android/nico_android_element.py: -------------------------------------------------------------------------------- 1 | import os 2 | import re 3 | import subprocess 4 | 5 | from loguru import logger 6 | from auto_nico.common.nico_basic_element import NicoBasicElement 7 | from auto_nico.common.runtime_cache import RunningCache 8 | 9 | 10 | class UIStructureError(Exception): 11 | pass 12 | 13 | 14 | class NicoAndroidElement(NicoBasicElement): 15 | def __init__(self, **query): 16 | self.query = query 17 | self.current_node = None 18 | super().__init__(**query) 19 | 20 | def set_seek_bar(self, percentage): 21 | x = self.get_bounds()[0] + self.get_bounds()[2] * percentage 22 | y = self.center_coordinate()[1] 23 | logger.debug(f"set seek bar to {percentage}") 24 | self.click(x, y) 25 | 26 | @property 27 | def index(self): 28 | return self._get_attribute_value("index") 29 | 30 | def get_index(self): 31 | return self.index 32 | 33 | @property 34 | def text(self): 35 | return self._get_attribute_value("text") 36 | 37 | def get_text(self): 38 | return self.text 39 | 40 | @property 41 | def id(self): 42 | return self._get_attribute_value("id") 43 | 44 | def get_id(self): 45 | return self.id 46 | 47 | @property 48 | def class_name(self): 49 | return self._get_attribute_value("class") 50 | 51 | def get_class_name(self): 52 | return self.class_name 53 | 54 | @property 55 | def package(self): 56 | return self._get_attribute_value("package") 57 | 58 | def get_package(self): 59 | return self.package 60 | 61 | @property 62 | def content_desc(self): 63 | return self._get_attribute_value("content_desc") 64 | 65 | def get_content_desc(self): 66 | return self.content_desc 67 | 68 | @property 69 | def checkable(self): 70 | return self._get_attribute_value("checkable") 71 | 72 | def get_checkable(self): 73 | return self.checkable 74 | 75 | @property 76 | def checked(self): 77 | return self._get_attribute_value("checked") 78 | 79 | def get_checked(self): 80 | return self.checked 81 | 82 | @property 83 | def clickable(self): 84 | return self._get_attribute_value("clickable") 85 | 86 | def get_clickable(self): 87 | return self.clickable 88 | 89 | @property 90 | def enabled(self): 91 | return self._get_attribute_value("enabled") 92 | 93 | def get_enabled(self): 94 | return self.enabled 95 | 96 | @property 97 | def focusable(self): 98 | return self._get_attribute_value("focusable") 99 | 100 | def get_focusable(self): 101 | return self.focusable 102 | 103 | @property 104 | def focused(self): 105 | return self._get_attribute_value("focused") 106 | 107 | def get_focused(self): 108 | return self.focused 109 | 110 | @property 111 | def scrollable(self): 112 | return self._get_attribute_value("scrollable") 113 | 114 | def get_scrollable(self): 115 | return self.scrollable 116 | 117 | @property 118 | def long_clickable(self): 119 | return self._get_attribute_value("long-clickable") 120 | 121 | def get_long_clickable(self): 122 | return self.long_clickable 123 | 124 | @property 125 | def password(self): 126 | return self._get_attribute_value("password") 127 | 128 | def get_password(self): 129 | return self.password 130 | 131 | @property 132 | def selected(self): 133 | return self._get_attribute_value("selected") 134 | 135 | def get_selected(self): 136 | return self.selected 137 | 138 | @property 139 | def bounds(self): 140 | pattern = r'\[(\d+),(\d+)\]\[(\d+),(\d+)\]' 141 | matches = re.findall(pattern, self._get_attribute_value("bounds")) 142 | 143 | left = int(matches[0][0]) 144 | top = int(matches[0][1]) 145 | right = int(matches[0][2]) 146 | bottom = int(matches[0][3]) 147 | 148 | # 计算宽度和高度 149 | width = right - left 150 | height = bottom - top 151 | 152 | # 计算左上角坐标（x, y） 153 | x = left 154 | y = top 155 | return x, y, width, height 156 | 157 | @property 158 | def description(self): 159 | import cv2 160 | import numpy as np 161 | logger.debug("Description being generated") 162 | result = subprocess.run(['adb', '-s', self.udid, 'exec-out', 'screencap', '-p'], stdout=subprocess.PIPE) 163 | screenshot_data = result.stdout 164 | nparr = np.frombuffer(screenshot_data, np.uint8) 165 | img = cv2.imdecode(nparr, cv2.IMREAD_COLOR) 166 | text = self._description(img,self.bounds) 167 | return text 168 | 169 | @property 170 | def ocr_id(self): 171 | import cv2 172 | import numpy as np 173 | logger.debug("Description being generated") 174 | result = subprocess.run(['adb', '-s', self.udid, 'exec-out', 'screencap', '-p'], stdout=subprocess.PIPE) 175 | screenshot_data = result.stdout 176 | nparr = np.frombuffer(screenshot_data, np.uint8) 177 | img = cv2.imdecode(nparr, cv2.IMREAD_COLOR) 178 | text = self._ocr_id(img, self.bounds) 179 | return text 180 | 181 | def get_bounds(self): 182 | return self.bounds 183 | 184 | def center_coordinate(self): 185 | x, y, w, h = self.bounds 186 | center_x = x + w // 2 187 | center_y = y + h // 2 188 | return center_x, center_y 189 | 190 | def scroll(self, duration=200, direction='vertical_up'): 191 | if direction not in ('vertical_up', "vertical_down", 'horizontal_left', "horizontal_right"): 192 | raise ValueError( 193 | 'Argument `direction` should be one of "vertical_up" or "vertical_down" or "horizontal_left"' 194 | 'or "horizontal_right". Got {}'.format(repr(direction))) 195 | to_x = 0 196 | to_y = 0 197 | from_x = self.center_coordinate()[0] 198 | from_y = self.center_coordinate()[1] 199 | if direction == "vertical_up": 200 | to_x = from_x 201 | to_y = from_y - from_y / 2 202 | elif direction == "vertical_down": 203 | to_x = from_x 204 | to_y = from_y + from_y / 2 205 | elif direction == "horizontal_left": 206 | to_x = from_x - from_x / 2 207 | to_y = from_y 208 | elif direction == "horizontal_right": 209 | to_x = from_x + from_x / 2 210 | to_y = from_y 211 | command = f'adb -s {self.udid} shell input swipe {from_x} {from_y} {to_x} {to_y} {duration}' 212 | RunningCache(self.udid).clear_current_cache_ui_tree() 213 | os.system(command) 214 | 215 | def swipe(self, to_x, to_y, duration=0): 216 | from_x = self.center_coordinate()[0] 217 | from_y = self.center_coordinate()[1] 218 | duration = duration*1000 + 200 219 | command = f'adb -s {self.udid} shell input swipe {from_x} {from_y} {to_x} {to_y} {duration}' 220 | os.environ[f"{self.udid}_ui_tree"] = "" 221 | os.system(command) 222 | 223 | def drag(self, to_x, to_y, duration=0): 224 | from_x = self.center_coordinate()[0] 225 | from_y = self.center_coordinate()[1] 226 | duration = duration*1000 + 2000 227 | command = f'adb -s {self.udid} shell input swipe {from_x} {from_y} {to_x} {to_y} {duration}' 228 | os.environ[f"{self.udid}_ui_tree"] = "" 229 | os.system(command) 230 | 231 | def click(self, x=None, y=None, x_offset=None, y_offset=None): 232 | if x is None and y is None: 233 | x = self.center_coordinate()[0] 234 | y = self.center_coordinate()[1] 235 | if x_offset is not None: 236 | x = x + x_offset 237 | if y_offset is not None: 238 | y = y + y_offset 239 | command = f'adb -s {self.udid} shell input tap {x} {y}' 240 | os.system(command) 241 | RunningCache(self.udid).clear_current_cache_ui_tree() 242 | logger.debug(f"click {x} {y}") 243 | 244 | def long_click(self, duration, x_offset=None, y_offset=None): 245 | x = self.center_coordinate()[0] 246 | y = self.center_coordinate()[1] 247 | if x_offset is not None: 248 | x = x + x_offset 249 | if y_offset is not None: 250 | y = y + y_offset 251 | command = f'adb -s {self.udid} shell input swipe {x} {y} {x} {y} {int(duration*1000)}' 252 | RunningCache(self.udid).clear_current_cache_ui_tree() 253 | os.system(command) 254 | 255 | def set_text(self, text, append=False, x_offset=None, y_offset=None): 256 | len_of_text = len(self.get_text()) 257 | self.click(x_offset=x_offset, y_offset=y_offset) 258 | os.system(f'adb -s {self.udid} shell input keyevent KEYCODE_MOVE_END') 259 | del_cmd = f'adb -s {self.udid} shell input keyevent' 260 | if not append: 261 | for _ in range(len_of_text + 8): 262 | del_cmd = del_cmd + " KEYCODE_DEL" 263 | os.system(del_cmd) 264 | text = text.replace("&", "\&").replace("\"", "") 265 | RunningCache(self.udid).clear_current_cache_ui_tree() 266 | os.system(f'''adb -s {self.udid} shell input text "{text}"''') 267 | os.system(f'adb -s {self.udid} shell settings put global policy_control immersive.full=*') 268 | 269 | def get(self, index): 270 | node = self._get(index) 271 | NAE = NicoAndroidElement() 272 | NAE.set_current_node(node) 273 | NAE.set_udid(self.udid) 274 | return NAE 275 | 276 | def all(self): 277 | eles = self._all() 278 | if not eles: 279 | return eles 280 | ALL_NAE_LIST = [] 281 | for ele in eles: 282 | NAE = NicoAndroidElement() 283 | NAE.set_current_node(ele) 284 | NAE.set_query(self.query) 285 | NAE.set_udid(self.udid) 286 | ALL_NAE_LIST.append(NAE) 287 | return ALL_NAE_LIST 288 | 289 | def last_sibling(self, index=0): 290 | previous_node = self._last_sibling(index) 291 | NAE = NicoAndroidElement() 292 | NAE.set_udid(self.udid) 293 | NAE.set_query(self.query) 294 | NAE.set_current_node(previous_node) 295 | return NAE 296 | 297 | def next_sibling(self, index=0): 298 | next_node = self._next_sibling(index) 299 | NAE = NicoAndroidElement() 300 | NAE.set_udid(self.udid) 301 | NAE.set_query(self.query) 302 | NAE.set_current_node(next_node) 303 | return NAE 304 | 305 | def parent(self): 306 | parent_node = self._parent() 307 | NAE = NicoAndroidElement() 308 | NAE.set_udid(self.udid) 309 | NAE.set_query(self.query) 310 | NAE.set_current_node(parent_node) 311 | return NAE 312 | 313 | def child(self, index=0): 314 | child_node = self._child(index) 315 | NAE = NicoAndroidElement() 316 | NAE.set_udid(self.udid) 317 | NAE.set_query(self.query) 318 | NAE.set_current_node(child_node) 319 | return NAE 320 | 321 | def children_amount(self) -> int: 322 | child_amount = self._child_amount() 323 | return child_amount -------------------------------------------------------------------------------- /auto_nico/android/nico_image.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | from loguru import logger 4 | from airtest.core.api import * 5 | from airtest.core.settings import Settings as ST 6 | import logging 7 | 8 | airtest_logger = logging.getLogger("airtest") 9 | airtest_logger.setLevel(logging.INFO) 10 | 11 | class NicoImage: 12 | def __init__(self, udid): 13 | logger.info(f"Connecting to device with UDID: {udid}") 14 | connect_device("Android://127.0.0.1:5037/" + udid) 15 | 16 | def __call__(self, v): 17 | self.v = v 18 | return self 19 | 20 | def exists(self, timeout=3): 21 | interval=0.5 22 | logger.info(f"Checking existence of: {self.v} with timeout: {timeout}") 23 | start_time = time.time() 24 | while time.time() - start_time < timeout: 25 | if exists(self.v): 26 | return True 27 | time.sleep(interval) 28 | return False # Return False if the element does not exist within the timeout period 29 | 30 | def click(self, times=1, **kwargs): 31 | logger.info(f"Clicking on: {self.v} for {times} times") 32 | return touch(self.v, times, **kwargs) 33 | 34 | def double_click(self): 35 | logger.info(f"Double clicking on: {self.v}") 36 | return double_click(self.v) 37 | 38 | def find_all(self): 39 | logger.info(f"Finding all instances of: {self.v}") 40 | return find_all(self.v) 41 | 42 | def wait_for_appearance(self, timeout=10, interval=0.5): 43 | logger.info(f"Waiting for appearance of: {self.v} with timeout: {timeout}") 44 | start_time = time.time() 45 | while time.time() - start_time < timeout: 46 | if exists(self.v): 47 | return True 48 | time.sleep(interval) 49 | raise TimeoutError(f"Element {self.v} did not appear within {timeout} seconds") 50 | 51 | def wait_for_disappearance(self, timeout=10, interval=0.5): 52 | logger.info(f"Waiting for disappearance of: {self.v} with timeout: {timeout}") 53 | start_time = time.time() 54 | while time.time() - start_time < timeout: 55 | if not exists(self.v): 56 | return True 57 | time.sleep(interval) 58 | raise TimeoutError(f"Element {self.v} did not disappear within {timeout} seconds") 59 | 60 | def wait_for_any(self, v_list, timeout=10, interval=0.5): 61 | logger.info(f"Waiting for any of: {v_list} with timeout: {timeout}") 62 | start_time = time.time() 63 | while time.time() - start_time < timeout: 64 | for v in v_list: 65 | if exists(v): 66 | return v 67 | time.sleep(interval) 68 | raise TimeoutError(f"None of the elements {v_list} appeared within {timeout} seconds") 69 | 70 | def swipe(self, v2=None, vector=None, **kwargs): 71 | logger.info(f"Swiping from {self.v} to {v2} with vector: {vector}") 72 | return swipe(self.v, v2, vector, **kwargs) 73 | 74 | def set_text(self, enter=True, search=False): 75 | logger.info(f"Setting text: {self.v} with enter: {enter} and search: {search}") 76 | return text(self.v, enter, search) 77 | 78 | def keyevent(self): 79 | logger.info(f"Sending keyevent: {self.v}") 80 | return keyevent(self.v) 81 | 82 | def snapshot(self, filename=None, msg="", quality=None, max_size=None): 83 | logger.info(f"Taking snapshot with filename: {filename}, msg: {msg}, quality: {quality}, max_size: {max_size}") 84 | return snapshot(filename, msg, quality, max_size) 85 | 86 | def sleep(self, secs): 87 | logger.info(f"Sleeping for {secs} seconds") 88 | return sleep(secs) -------------------------------------------------------------------------------- /auto_nico/android/package/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/letmeNo1/Nico/79dc1ac061cdeaa7ab74660d42bf85bd93ff0592/auto_nico/android/package/__init__.py -------------------------------------------------------------------------------- /auto_nico/android/package/dump_hierarchy_androidTest_v1.4.apk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/letmeNo1/Nico/79dc1ac061cdeaa7ab74660d42bf85bd93ff0592/auto_nico/android/package/dump_hierarchy_androidTest_v1.4.apk -------------------------------------------------------------------------------- /auto_nico/android/package/dump_hierarchy_v1.4.apk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/letmeNo1/Nico/79dc1ac061cdeaa7ab74660d42bf85bd93ff0592/auto_nico/android/package/dump_hierarchy_v1.4.apk -------------------------------------------------------------------------------- /auto_nico/android/tools/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/letmeNo1/Nico/79dc1ac061cdeaa7ab74660d42bf85bd93ff0592/auto_nico/android/tools/__init__.py -------------------------------------------------------------------------------- /auto_nico/android/tools/format_converter.py: -------------------------------------------------------------------------------- 1 | from collections import defaultdict 2 | 3 | def add_xpath_att(root, current_path="", parent_class_count=None, level=0): 4 | """ 5 | Traverse all nodes by level and construct XPath using the class_name attribute and its occurrence count, 6 | then add the XPath as an attribute to the node. 7 | 8 | :param root: The current XML element being processed 9 | :param current_path: The XPath path of the current element 10 | :param parent_class_count: A dictionary recording the occurrence count of each class_name attribute at the current level for the parent 11 | :param level: The current level 12 | """ 13 | if parent_class_count is None: 14 | parent_class_count = defaultdict(int) 15 | 16 | # Get the class_name attribute of the current element 17 | class_name = root.attrib.get('class_name', root.tag) 18 | 19 | # Update the count for the current class_name attribute 20 | current_index = parent_class_count[class_name] 21 | parent_class_count[class_name] += 1 22 | 23 | # Construct the XPath path for the current element 24 | current_xpath = f"{current_path}/{str(class_name).split('.')[-1]}[{current_index}]".replace("/hierarchy[0]/","") 25 | 26 | # Add the XPath path as an attribute to the current element 27 | root.set('xpath', current_xpath) 28 | 29 | # Recursively process child elements 30 | child_class_count = defaultdict(int) 31 | for child in root: 32 | add_xpath_att(child, current_xpath, child_class_count, level + 1) 33 | return root -------------------------------------------------------------------------------- /auto_nico/common/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/letmeNo1/Nico/79dc1ac061cdeaa7ab74660d42bf85bd93ff0592/auto_nico/common/__init__.py -------------------------------------------------------------------------------- /auto_nico/common/common_utils.py: -------------------------------------------------------------------------------- 1 | import json 2 | import socket 3 | 4 | 5 | def is_port_in_use(port): 6 | with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: 7 | return s.connect_ex(('localhost', port)) == 0 8 | 9 | 10 | def is_valid_json(s): 11 | try: 12 | return json.loads(s) 13 | except json.JSONDecodeError: 14 | return None 15 | -------------------------------------------------------------------------------- /auto_nico/common/error.py: -------------------------------------------------------------------------------- 1 | class ADBServerError(Exception): 2 | pass 3 | 4 | 5 | class IDBServerError(Exception): 6 | pass 7 | 8 | 9 | class UIStructureError(Exception): 10 | pass 11 | 12 | class NicoError(Exception): 13 | """ 14 | This is NicoError BaseError 15 | When ADB have something wrong 16 | """ 17 | 18 | pass 19 | -------------------------------------------------------------------------------- /auto_nico/common/kmeans_run.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import numpy as np 3 | from sklearn.cluster import MeanShift 4 | 5 | 6 | def kmeans_run(img1, img2, distance=0.7, algorithms_name="SIFT", Scale=1): 7 | img1_gray = cv2.cvtColor(img1, cv2.COLOR_BGR2GRAY) 8 | img2_gray = cv2.cvtColor(img2, cv2.COLOR_BGR2GRAY) 9 | 10 | if algorithms_name == "template_match": 11 | x, y, w, h = template_matching(img1_gray, img2_gray) 12 | if x is None: 13 | return None, None, None, None 14 | else: 15 | return int(x / Scale), int(y / Scale), 10, 10 16 | else: 17 | algorithms_all = { 18 | "SIFT": cv2.SIFT_create(), 19 | "BRISK": cv2.BRISK_create(), 20 | "ORB": cv2.ORB_create() 21 | } 22 | algorithms = algorithms_all[algorithms_name] 23 | 24 | kp1, des1 = algorithms.detectAndCompute(img1_gray, None) 25 | kp2, des2 = algorithms.detectAndCompute(img2_gray, None) 26 | if algorithms_name in ["BRISK", "ORB"]: 27 | BFMatcher = cv2.BFMatcher(cv2.NORM_HAMMING) 28 | matches = BFMatcher.knnMatch(des1, des2, k=2) 29 | else: 30 | FLANN_INDEX_KDTREE = 0 31 | index_params = dict(algorithm=FLANN_INDEX_KDTREE, trees=5) 32 | search_params = dict(checks=50) 33 | flann = cv2.FlannBasedMatcher(index_params, search_params) 34 | matches = flann.knnMatch(des1, des2, k=2) 35 | 36 | good = [m for m, n in matches if m.distance < distance * n.distance] 37 | 38 | if len(good) >= 5: 39 | points = np.float32([kp2[m.trainIdx].pt for m in good]).reshape(-1, 2) 40 | 41 | ms = MeanShift(bandwidth=50) 42 | ms.fit(points) 43 | labels = ms.labels_ 44 | 45 | core_samples_mask = (labels == np.argmax(np.bincount(labels))) 46 | filtered_points = points[core_samples_mask] 47 | 48 | x_mean, y_mean = np.mean(filtered_points, axis=0) 49 | 50 | x = int(x_mean) 51 | y = int(y_mean) 52 | 53 | return int(x / Scale), int(y / Scale), 10, 10 54 | else: 55 | return None, None, None, None 56 | 57 | 58 | def template_matching(img1, img2, method=cv2.TM_CCOEFF_NORMED): 59 | result = cv2.matchTemplate(img2, img1, method) 60 | min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(result) 61 | top_left = max_loc 62 | h, w = img1.shape[:2] 63 | bottom_right = (top_left[0] + w, top_left[1] + h) 64 | cv2.rectangle(img2, top_left, bottom_right, (0, 255, 0), 2) 65 | if max_val < 0.8: 66 | return None, None, None, None 67 | else: 68 | return top_left[0], top_left[1], bottom_right[0], bottom_right[1] 69 | -------------------------------------------------------------------------------- /auto_nico/common/logger_config.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | 4 | class CustomFilter(logging.Filter): 5 | def __init__(self): 6 | super().__init__() 7 | self.is_debug_enabled = False 8 | 9 | def filter(self, record): 10 | # If debug logging is turned off and the current logging level is DEBUG, do not print logging 11 | if not self.is_debug_enabled and record.levelno == logging.DEBUG: 12 | return False 13 | return True 14 | 15 | def enable_debug(self): 16 | # close debug log 17 | self.is_debug_enabled = True 18 | 19 | def disable_debug(self): 20 | # close debug log 21 | self.is_debug_enabled = False 22 | 23 | 24 | logger = logging.getLogger('Nico') 25 | logger.setLevel(logging.DEBUG) 26 | 27 | console_handler = logging.StreamHandler() 28 | console_handler.setLevel(logging.DEBUG) 29 | 30 | formatter = logging.Formatter('%(asctime)s Nico - %(levelname)s - %(message)s') 31 | console_handler.setFormatter(formatter) 32 | 33 | custom_filter = CustomFilter() 34 | 35 | console_handler.addFilter(custom_filter) 36 | 37 | logger.addHandler(console_handler) 38 | 39 | custom_filter.disable_debug() -------------------------------------------------------------------------------- /auto_nico/common/nico_basic_element.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from auto_nico.common.error import UIStructureError 4 | from auto_nico.common.runtime_cache import RunningCache 5 | from lxml.etree import _Element 6 | 7 | from auto_nico.common.nico_basic import NicoBasic 8 | from loguru import logger 9 | 10 | 11 | class NicoBasicElement(NicoBasic): 12 | def __init__(self, **query): 13 | self.udid = None 14 | self.port = None 15 | self.package_name = None 16 | self.current_node = None 17 | self.query = query 18 | super().__init__(self.udid, **query) 19 | 20 | def refresh_ui_tree(self): 21 | RunningCache(self.udid).clear_current_cache_ui_tree() 22 | 23 | def set_udid(self, udid): 24 | self.udid = udid 25 | 26 | def set_port(self, port): 27 | self.port = port 28 | 29 | def set_query(self, query): 30 | self.query = query 31 | 32 | def set_package_name(self, package_name): 33 | RunningCache(self.udid).set_current_running_package_name(package_name) 34 | self.package_name = package_name 35 | 36 | def set_current_node(self, current_node): 37 | self.current_node = current_node 38 | 39 | def _get_attribute_value(self, attribute_name) -> str: 40 | if self.current_node is None: 41 | 42 | self.current_node = self._find_function(self.query) 43 | if self.current_node is None: 44 | raise UIStructureError( 45 | f"Can't found element with query: {self.query}") 46 | elif type(self.current_node) is list: 47 | raise UIStructureError( 48 | "More than one element has been retrieved, use the 'get' method to specify the number you want") 49 | RunningCache(self.udid).set_action_was_taken(False) 50 | if self.current_node.get(attribute_name) is None: 51 | logger.debug(f"Can't found attribute: {attribute_name}") 52 | return self.current_node.get(attribute_name) 53 | 54 | def _get(self, index): 55 | node = self._find_all_function(self.query)[index] 56 | RunningCache(self.udid).set_action_was_taken(False) 57 | if node is None: 58 | raise Exception("No element found") 59 | return node 60 | 61 | def _all(self): 62 | eles = self._find_all_function(self.query) 63 | RunningCache(self.udid).set_action_was_taken(False) 64 | return eles 65 | 66 | def _last_sibling(self, index=0) -> _Element: 67 | if self.current_node is None: 68 | self.current_node = self._find_function(query=self.query, use_xml=True) 69 | if self.current_node is None: 70 | raise UIStructureError(f"Can't found element") 71 | previous_node = self.current_node.getprevious() 72 | if index >= 1: 73 | for i in range(index): 74 | previous_node = previous_node.getprevious() 75 | 76 | return previous_node 77 | 78 | def _next_sibling(self, index=0) -> _Element: 79 | if self.current_node is None: 80 | self.current_node = self._find_function(query=self.query, use_xml=True) 81 | if self.current_node is None: 82 | raise UIStructureError(f"Can't found element") 83 | next_node = self.current_node.getnext() 84 | if index >= 1: 85 | for i in range(index): 86 | next_node = next_node.getnext() 87 | 88 | return next_node 89 | 90 | def _parent(self) -> _Element: 91 | if self.current_node is None: 92 | self.current_node = self._find_function(query=self.query, use_xml=True) 93 | if self.current_node is None: 94 | raise UIStructureError(f"Can't found element") 95 | parent_node = self.current_node.getparent() 96 | return parent_node 97 | 98 | def _child(self, index) -> _Element: 99 | if self.current_node is None: 100 | self.current_node = self._find_function(query=self.query, use_xml=True) 101 | if self.current_node is None: 102 | raise UIStructureError(f"Can't found element") 103 | child_node = self.current_node.getchildren()[index] 104 | return child_node 105 | 106 | def _child_amount(self) -> int: 107 | if self.current_node is None: 108 | self.current_node = self._find_function(query=self.query, use_xml=True) 109 | if self.current_node is None: 110 | raise UIStructureError(f"Can't found element") 111 | 112 | return self.current_node.getchildren().__len__() -------------------------------------------------------------------------------- /auto_nico/common/runtime_cache.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | 4 | import lxml.etree as ET 5 | from auto_nico.common.send_request import send_http_request 6 | 7 | 8 | def set_large_env_var(var_name, var_value, max_length=30000): 9 | parts = [var_value[i:i + max_length] for i in range(0, len(var_value), max_length)] 10 | 11 | for index, part in enumerate(parts, start=1): 12 | env_var_name = f"{var_name}_{index}" 13 | os.environ[env_var_name] = part 14 | 15 | 16 | def get_large_env_var(var_name): 17 | combined_value = "" 18 | index = 1 19 | while True: 20 | env_var_name = f"{var_name}_{index}" 21 | part = os.environ.get(env_var_name) 22 | if part is None: 23 | break 24 | combined_value += part 25 | index += 1 26 | return combined_value 27 | 28 | 29 | def delete_large_env_var(var_name): 30 | index = 1 31 | while True: 32 | env_var_name = f"{var_name}_{index}" 33 | if env_var_name not in os.environ: 34 | break 35 | os.environ.pop(env_var_name) 36 | index += 1 37 | 38 | 39 | class RunningCache: 40 | def __init__(self, udid): 41 | self.udid = udid 42 | 43 | def get_current_cache_ui_tree(self): 44 | ui_tree_part = os.getenv(f"{self.udid}_ui_tree_1") 45 | if ui_tree_part is not None: 46 | tree = ET.fromstring(get_large_env_var(f"{self.udid}_ui_tree").encode('utf-8')) 47 | # root = tree.getroot() 48 | return tree 49 | return None 50 | 51 | def set_current_cache_ui_tree(self, ui_tree_string): 52 | set_large_env_var(f"{self.udid}_ui_tree", ui_tree_string) 53 | 54 | def clear_current_cache_ui_tree(self): 55 | delete_large_env_var(f"{self.udid}_ui_tree") 56 | 57 | def get_current_running_port(self) -> int: 58 | return int(os.getenv(f"{self.udid}_running_port")) 59 | 60 | def set_current_running_port(self, port): 61 | os.environ[f"{self.udid}_running_port"] = str(port) 62 | 63 | def get_ui_change_status(self): 64 | if self.get_current_cache_ui_tree() is None: 65 | return True 66 | if self.is_initialized(): 67 | self.set_initialized(False) 68 | return True 69 | else: 70 | exists_port = self.get_current_running_port() 71 | rst = send_http_request(exists_port, "is_ui_change") 72 | if "false" in rst: 73 | return False 74 | elif "true" in rst: 75 | return True 76 | else: 77 | return False 78 | 79 | def is_initialized(self): 80 | return os.getenv(f"{self.udid}_initialized") == "True" 81 | 82 | def set_initialized(self, initialized: bool): 83 | os.environ[f"{self.udid}_initialized"] = str(initialized) 84 | 85 | def set_action_was_taken(self, action_was_taken: bool): 86 | os.environ[f"{self.udid}_action_was_taken"] = str(action_was_taken) 87 | 88 | def get_action_was_taken(self) -> bool: 89 | return os.getenv(f"{self.udid}_action_was_taken") == "True" 90 | 91 | def set_current_running_package_name(self, package_name: str): 92 | os.environ[f"{self.udid}_running_package"] = package_name 93 | 94 | def get_current_running_package(self): 95 | return os.getenv(f"{self.udid}_running_package") 96 | -------------------------------------------------------------------------------- /auto_nico/common/send_request.py: -------------------------------------------------------------------------------- 1 | import pycurl 2 | from io import BytesIO 3 | from loguru import logger 4 | from urllib.parse import urlencode 5 | 6 | 7 | def send_http_request(port: int, method, params: dict = None, timeout=10): 8 | url = f"http://localhost:{port}/{method}" 9 | logger.debug(f"request:{url}") 10 | 11 | buffer = BytesIO() 12 | c = pycurl.Curl() 13 | c.setopt(c.URL, url) 14 | c.setopt(c.TIMEOUT, timeout) 15 | if params: 16 | param_str = urlencode(params) 17 | logger.debug(f"request:{url}?{param_str}") 18 | 19 | c.setopt(c.URL, f"{url}?{param_str}") 20 | 21 | try: 22 | c.setopt(c.WRITEDATA, buffer) 23 | c.perform() 24 | 25 | response_code = c.getinfo(pycurl.HTTP_CODE) 26 | if response_code == 200: 27 | content_type = c.getinfo(pycurl.CONTENT_TYPE) 28 | buffer.seek(0) 29 | response_content = buffer.read() 30 | if 'image/jpeg' in content_type or 'image/png' in content_type: 31 | logger.debug(f"Request successful, response content: Image content:{response_content[:100]}") 32 | return response_content 33 | else: 34 | response_text = response_content.decode('utf-8') 35 | logger.debug(f"response:{response_text[:100]}") 36 | if "{content}' 47 | # 如果元素有属性，将它们作为文本添加 48 | if element.attrib: 49 | html += " (Attributes: " 50 | html += ", ".join([f'{k}="{v}"' for k, v in element.attrib.items()]) 51 | html += ")" 52 | # 如果元素有文本内容，将其添加到列表项中 53 | if element.text and element.text.strip(): 54 | html += f" - Text: {element.text.strip()}" 55 | 56 | html += "" 57 | # 处理子元素 58 | children = list(element) 59 | if children: 60 | for child in children: 61 | html += xml_to_html_list(child, depth + 1) 62 | return html 63 | 64 | 65 | def dump_ui_tree(): 66 | platform = os.environ.get('nico_ui_platform') 67 | udid = os.environ.get("nico_ui_udid") 68 | idb_utils = IdbUtils(udid) 69 | 70 | port = int(os.environ.get('RemoteServerPort')) 71 | if platform == "android": 72 | xml = send_http_request(port, "dump",{"compressed":"true"}).replace("class", "class_name").replace("resource-id=", 73 | "id=").replace( 74 | "content-desc=", "content_desc=") 75 | root = add_xpath_att(ET.fromstring(xml.encode('utf-8'))) 76 | 77 | else: 78 | package_name = idb_utils.get_current_bundleIdentifier(port) 79 | os.environ['current_package_name'] = package_name 80 | xml = send_http_request(port, f"dump_tree", {"bundle_id": package_name}) 81 | xml = converter(xml) 82 | root = ET.fromstring(xml.encode('utf-8')) 83 | return root 84 | 85 | 86 | @app.route('/refresh_image') 87 | def refresh_image(): 88 | port = int(os.environ.get('RemoteServerPort')) 89 | platform = os.environ.get('nico_ui_platform') 90 | if platform == "android": 91 | new_data = send_http_request(port, "screenshot",{"quality":80}) 92 | else: 93 | new_data = send_http_request(port, "get_jpg_pic", {"compression_quality": 1.0}) 94 | base64_data = base64.b64encode(new_data) 95 | return base64_data 96 | 97 | 98 | @app.route('/refresh_ui_xml') 99 | def refresh_ui_xml(): 100 | root = dump_ui_tree() 101 | 102 | # 构建HTML列表 103 | html_list = xml_to_html_list(root) 104 | 105 | # 渲染模板并传递构建的HTML列表 106 | return html_list 107 | 108 | 109 | @app.route('/image') 110 | def generate_image(): 111 | port = int(os.environ.get('RemoteServerPort')) 112 | platform = os.environ.get('nico_ui_platform') 113 | if platform == "android": 114 | new_data = send_http_request(port, "screenshot",{"quality":80}) 115 | else: 116 | new_data = send_http_request(port, "get_jpg_pic", {"compression_quality": 1.0}) 117 | base64_data = base64.b64encode(new_data) 118 | return base64_data 119 | 120 | 121 | @app.route('/get_element_attribute') 122 | def get_element_attribute(): 123 | id = request.args.get('id') 124 | xpath = request.args.get("xpath") 125 | port = int(os.environ.get('RemoteServerPort')) 126 | if xpath is None or xpath == "null": 127 | return "" 128 | new_data = send_http_request(port, f"find_element_by_query", 129 | {"bundle_id": id, "query_method": "xpath", "query_value": xpath}) 130 | if new_data == "": 131 | return "" 132 | new_data_dict = dict(json.loads(new_data)) 133 | new_data_dict.pop('children', None) 134 | for att in ["title", "label"]: 135 | value = new_data_dict.pop(att) 136 | text = value if value != "" else "" 137 | frame = new_data_dict.pop("frame") 138 | new_data_dict.update({"text": text}) 139 | new_data_dict.update({"class_name": get_element_type_by_value(new_data_dict.pop("elementType"))}) 140 | new_data_dict.update({"bounds": f'[{frame.get("X")},{frame.get("Y")}][{frame.get("Width")},{frame.get("Height")}]'}) 141 | 142 | return new_data_dict 143 | 144 | 145 | @app.route("/android_excute_action") 146 | def android_excute_action(): 147 | action = request.args.get('action') 148 | if action == "click": 149 | x = request.args.get("x") 150 | y = request.args.get("y") 151 | udid = os.environ.get("nico_ui_udid") 152 | adb_utils = AdbUtils(udid) 153 | adb_utils.shell(f'''input tap {x} {y}''') 154 | return "excute sucessful" 155 | elif action == "input": 156 | inputValue = request.args.get("inputValue") 157 | inputValue = inputValue.replace("&", "\&").replace("\"", "") 158 | udid = os.environ.get("nico_ui_udid") 159 | adb_utils = AdbUtils(udid) 160 | adb_utils.shell(f'''input text {inputValue}''') 161 | return "excute sucessful" 162 | 163 | elif action == "home": 164 | udid = os.environ.get("nico_ui_udid") 165 | adb_utils = AdbUtils(udid) 166 | adb_utils.shell(f'''input keyevent KEYCODE_HOME''') 167 | return "excute sucessful" 168 | 169 | elif action == "back": 170 | udid = os.environ.get("nico_ui_udid") 171 | adb_utils = AdbUtils(udid) 172 | adb_utils.shell(f'''input keyevent KEYCODE_BACK''') 173 | return "excute sucessful" 174 | 175 | elif action == "menu": 176 | udid = os.environ.get("nico_ui_udid") 177 | adb_utils = AdbUtils(udid) 178 | adb_utils.shell(f'''input keyevent KEYCODE_MENU''') 179 | return "excute sucessful" 180 | 181 | elif action == "switch_app": 182 | udid = os.environ.get("nico_ui_udid") 183 | adb_utils = AdbUtils(udid) 184 | adb_utils.shell(f'''input keyevent KEYCODE_APP_SWITCH''') 185 | return "excute sucessful" 186 | 187 | elif action == "switch_app": 188 | udid = os.environ.get("nico_ui_udid") 189 | adb_utils = AdbUtils(udid) 190 | adb_utils.shell(f'''input keyevent KEYCODE_APP_SWITCH''') 191 | return "excute sucessful" 192 | 193 | elif action == "volume_up": 194 | udid = os.environ.get("nico_ui_udid") 195 | adb_utils = AdbUtils(udid) 196 | adb_utils.shell(f'''input keyevent KEYCODE_VOLUME_UP''') 197 | return "excute sucessful" 198 | 199 | elif action == "volume_down": 200 | udid = os.environ.get("nico_ui_udid") 201 | adb_utils = AdbUtils(udid) 202 | adb_utils.shell(f'''input keyevent KEYCODE_VOLUME_DOWN''') 203 | return "excute sucessful" 204 | 205 | elif action == "power": 206 | udid = os.environ.get("nico_ui_udid") 207 | adb_utils = AdbUtils(udid) 208 | adb_utils.shell(f'''input keyevent KEYCODE_POWER''') 209 | return "excute sucessful" 210 | 211 | elif action == "delete_text": 212 | udid = os.environ.get("nico_ui_udid") 213 | adb_utils = AdbUtils(udid) 214 | adb_utils.shell(f'''input keyevent KEYCODE_DEL''') 215 | return "excute sucessful" 216 | 217 | 218 | @app.route('/') 219 | def show_xml(): 220 | root = dump_ui_tree() 221 | # 构建HTML列表 222 | html_list = xml_to_html_list(root) 223 | 224 | # 渲染模板并传递构建的HTML列表 225 | return render_template('xml_template.html', xml_content=html_list) 226 | 227 | 228 | def run_app(port): 229 | app.run(debug=False, port=port) 230 | 231 | 232 | def set_tcp_forward_port(udid, port): 233 | platform = os.environ.get('nico_ui_platform') 234 | if platform == "android": 235 | adb_utils = AdbUtils(udid) 236 | adb_utils.cmd(f'''forward tcp:{port} tcp:{port}''') 237 | else: 238 | commands = f"""tidevice --udid {udid} relay {port} {port}""" 239 | subprocess.Popen(commands, shell=True) 240 | print(f'''forward tcp:{port} tcp:{port}''') 241 | 242 | 243 | def main(): 244 | parser = argparse.ArgumentParser() 245 | parser.add_argument('-p1', type=int, help='Remote port to connect to') 246 | parser.add_argument('-p2', type=int, help='Port to run on') 247 | parser.add_argument('-s', type=str, help='device_udid') 248 | parser.add_argument('-plat', type=str, help='platform "i","iOS","a","android"') 249 | 250 | udid = parser.parse_args().s 251 | 252 | platform = parser.parse_args().plat 253 | if platform is None: 254 | if len(udid) > 20: 255 | platform = "iOS" 256 | else: 257 | platform = "android" 258 | elif platform in ["i", "iOS", "a", "android"]: 259 | if platform == "i": 260 | platform = "iOS" 261 | elif platform == "a": 262 | platform = "android" 263 | else: 264 | pass 265 | else: 266 | print('Please enter the correct platform "i","iOS","a","android"') 267 | 268 | os.environ['nico_ui_platform'] = platform 269 | remote_port = parser.parse_args().p1 270 | inspect_port = parser.parse_args().p2 271 | if remote_port is None: 272 | random_number = random.randint(9000, 9999) 273 | remote_port = random_number 274 | 275 | if inspect_port is None: 276 | random_number = random.randint(9000, 9999) 277 | inspect_port = random_number 278 | 279 | if udid is None: 280 | print("Please provide a device_udid") 281 | return 282 | if remote_port is None: 283 | print("Please provide a port to connect remote nico server!!!!") 284 | return 285 | if inspect_port is None: 286 | print("Please provide a port to run inspector UI!!!!") 287 | return 288 | if is_port_in_use(remote_port): 289 | print(f"Port {remote_port} is already in use") 290 | return 291 | if is_port_in_use(inspect_port): 292 | print(f"Port {inspect_port} is already in use") 293 | return 294 | 295 | if platform == "android": 296 | adb_utils = AdbUtils(udid) 297 | adb_utils.clear_tcp_forward_port(remote_port) 298 | adb_utils.cmd(f'''forward tcp:{remote_port} tcp:8000''') 299 | adb_utils.check_adb_server() 300 | adb_utils.install_test_server_package(1.4) 301 | ime_list = adb_utils.qucik_shell("ime list -s").split("\n")[0:-1] 302 | for ime in ime_list: 303 | adb_utils.qucik_shell(f"ime disable {ime}") 304 | commands = f"""adb -s {udid} shell am instrument -r -w -e port {remote_port} -e class nico.dump_hierarchy.HierarchyTest nico.dump_hierarchy.test/androidx.test.runner.AndroidJUnitRunner""" 305 | subprocess.Popen(commands, shell=True) 306 | else: 307 | idb_utils = IdbUtils(udid) 308 | port, pid = idb_utils.get_tcp_forward_port() 309 | if port: 310 | remote_port = port 311 | idb_utils.runtime_cache.set_current_running_port(port) 312 | else: 313 | idb_utils.set_port_forward(remote_port) 314 | idb_utils._init_test_server() 315 | 316 | os.environ['RemoteServerPort'] = str(remote_port) 317 | os.environ['nico_ui_udid'] = udid 318 | time.sleep(5) 319 | 320 | p = multiprocessing.Process(target=run_app, args=(inspect_port,)) 321 | p.start() 322 | -------------------------------------------------------------------------------- /auto_nico/console_scripts/inspector_web/pyscrcpy/__init__.py: -------------------------------------------------------------------------------- 1 | from .const import * 2 | from .core import Client 3 | -------------------------------------------------------------------------------- /auto_nico/console_scripts/inspector_web/pyscrcpy/const.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module includes all consts used in this project 3 | """ 4 | 5 | # Action 6 | ACTION_DOWN = 0 7 | ACTION_UP = 1 8 | ACTION_MOVE = 2 9 | 10 | # KeyCode 11 | KEYCODE_UNKNOWN = 0 12 | KEYCODE_SOFT_LEFT = 1 13 | KEYCODE_SOFT_RIGHT = 2 14 | KEYCODE_HOME = 3 15 | KEYCODE_BACK = 4 16 | KEYCODE_CALL = 5 17 | KEYCODE_ENDCALL = 6 18 | KEYCODE_0 = 7 19 | KEYCODE_1 = 8 20 | KEYCODE_2 = 9 21 | KEYCODE_3 = 10 22 | KEYCODE_4 = 11 23 | KEYCODE_5 = 12 24 | KEYCODE_6 = 13 25 | KEYCODE_7 = 14 26 | KEYCODE_8 = 15 27 | KEYCODE_9 = 16 28 | KEYCODE_STAR = 17 29 | KEYCODE_POUND = 18 30 | KEYCODE_DPAD_UP = 19 31 | KEYCODE_DPAD_DOWN = 20 32 | KEYCODE_DPAD_LEFT = 21 33 | KEYCODE_DPAD_RIGHT = 22 34 | KEYCODE_DPAD_CENTER = 23 35 | KEYCODE_VOLUME_UP = 24 36 | KEYCODE_VOLUME_DOWN = 25 37 | KEYCODE_POWER = 26 38 | KEYCODE_CAMERA = 27 39 | KEYCODE_CLEAR = 28 40 | KEYCODE_A = 29 41 | KEYCODE_B = 30 42 | KEYCODE_C = 31 43 | KEYCODE_D = 32 44 | KEYCODE_E = 33 45 | KEYCODE_F = 34 46 | KEYCODE_G = 35 47 | KEYCODE_H = 36 48 | KEYCODE_I = 37 49 | KEYCODE_J = 38 50 | KEYCODE_K = 39 51 | KEYCODE_L = 40 52 | KEYCODE_M = 41 53 | KEYCODE_N = 42 54 | KEYCODE_O = 43 55 | KEYCODE_P = 44 56 | KEYCODE_Q = 45 57 | KEYCODE_R = 46 58 | KEYCODE_S = 47 59 | KEYCODE_T = 48 60 | KEYCODE_U = 49 61 | KEYCODE_V = 50 62 | KEYCODE_W = 51 63 | KEYCODE_X = 52 64 | KEYCODE_Y = 53 65 | KEYCODE_Z = 54 66 | KEYCODE_COMMA = 55 67 | KEYCODE_PERIOD = 56 68 | KEYCODE_ALT_LEFT = 57 69 | KEYCODE_ALT_RIGHT = 58 70 | KEYCODE_SHIFT_LEFT = 59 71 | KEYCODE_SHIFT_RIGHT = 60 72 | KEYCODE_TAB = 61 73 | KEYCODE_SPACE = 62 74 | KEYCODE_SYM = 63 75 | KEYCODE_EXPLORER = 64 76 | KEYCODE_ENVELOPE = 65 77 | KEYCODE_ENTER = 66 78 | KEYCODE_DEL = 67 79 | KEYCODE_GRAVE = 68 80 | KEYCODE_MINUS = 69 81 | KEYCODE_EQUALS = 70 82 | KEYCODE_LEFT_BRACKET = 71 83 | KEYCODE_RIGHT_BRACKET = 72 84 | KEYCODE_BACKSLASH = 73 85 | KEYCODE_SEMICOLON = 74 86 | KEYCODE_APOSTROPHE = 75 87 | KEYCODE_SLASH = 76 88 | KEYCODE_AT = 77 89 | KEYCODE_NUM = 78 90 | KEYCODE_HEADSETHOOK = 79 91 | KEYCODE_PLUS = 81 92 | KEYCODE_MENU = 82 93 | KEYCODE_NOTIFICATION = 83 94 | KEYCODE_SEARCH = 84 95 | KEYCODE_MEDIA_PLAY_PAUSE = 85 96 | KEYCODE_MEDIA_STOP = 86 97 | KEYCODE_MEDIA_NEXT = 87 98 | KEYCODE_MEDIA_PREVIOUS = 88 99 | KEYCODE_MEDIA_REWIND = 89 100 | KEYCODE_MEDIA_FAST_FORWARD = 90 101 | KEYCODE_MUTE = 91 102 | KEYCODE_PAGE_UP = 92 103 | KEYCODE_PAGE_DOWN = 93 104 | KEYCODE_BUTTON_A = 96 105 | KEYCODE_BUTTON_B = 97 106 | KEYCODE_BUTTON_C = 98 107 | KEYCODE_BUTTON_X = 99 108 | KEYCODE_BUTTON_Y = 100 109 | KEYCODE_BUTTON_Z = 101 110 | KEYCODE_BUTTON_L1 = 102 111 | KEYCODE_BUTTON_R1 = 103 112 | KEYCODE_BUTTON_L2 = 104 113 | KEYCODE_BUTTON_R2 = 105 114 | KEYCODE_BUTTON_THUMBL = 106 115 | KEYCODE_BUTTON_THUMBR = 107 116 | KEYCODE_BUTTON_START = 108 117 | KEYCODE_BUTTON_SELECT = 109 118 | KEYCODE_BUTTON_MODE = 110 119 | KEYCODE_ESCAPE = 111 120 | KEYCODE_FORWARD_DEL = 112 121 | KEYCODE_CTRL_LEFT = 113 122 | KEYCODE_CTRL_RIGHT = 114 123 | KEYCODE_CAPS_LOCK = 115 124 | KEYCODE_SCROLL_LOCK = 116 125 | KEYCODE_META_LEFT = 117 126 | KEYCODE_META_RIGHT = 118 127 | KEYCODE_FUNCTION = 119 128 | KEYCODE_SYSRQ = 120 129 | KEYCODE_BREAK = 121 130 | KEYCODE_MOVE_HOME = 122 131 | KEYCODE_MOVE_END = 123 132 | KEYCODE_INSERT = 124 133 | KEYCODE_FORWARD = 125 134 | KEYCODE_MEDIA_PLAY = 126 135 | KEYCODE_MEDIA_PAUSE = 127 136 | KEYCODE_MEDIA_CLOSE = 128 137 | KEYCODE_MEDIA_EJECT = 129 138 | KEYCODE_MEDIA_RECORD = 130 139 | KEYCODE_F1 = 131 140 | KEYCODE_F2 = 132 141 | KEYCODE_F3 = 133 142 | KEYCODE_F4 = 134 143 | KEYCODE_F5 = 135 144 | KEYCODE_F6 = 136 145 | KEYCODE_F7 = 137 146 | KEYCODE_F8 = 138 147 | KEYCODE_F9 = 139 148 | KEYCODE_F10 = 140 149 | KEYCODE_F11 = 141 150 | KEYCODE_F12 = 142 151 | KEYCODE_NUM_LOCK = 143 152 | KEYCODE_NUMPAD_0 = 144 153 | KEYCODE_NUMPAD_1 = 145 154 | KEYCODE_NUMPAD_2 = 146 155 | KEYCODE_NUMPAD_3 = 147 156 | KEYCODE_NUMPAD_4 = 148 157 | KEYCODE_NUMPAD_5 = 149 158 | KEYCODE_NUMPAD_6 = 150 159 | KEYCODE_NUMPAD_7 = 151 160 | KEYCODE_NUMPAD_8 = 152 161 | KEYCODE_NUMPAD_9 = 153 162 | KEYCODE_NUMPAD_DIVIDE = 154 163 | KEYCODE_NUMPAD_MULTIPLY = 155 164 | KEYCODE_NUMPAD_SUBTRACT = 156 165 | KEYCODE_NUMPAD_ADD = 157 166 | KEYCODE_NUMPAD_DOT = 158 167 | KEYCODE_NUMPAD_COMMA = 159 168 | KEYCODE_NUMPAD_ENTER = 160 169 | KEYCODE_NUMPAD_EQUALS = 161 170 | KEYCODE_NUMPAD_LEFT_PAREN = 162 171 | KEYCODE_NUMPAD_RIGHT_PAREN = 163 172 | KEYCODE_VOLUME_MUTE = 164 173 | KEYCODE_INFO = 165 174 | KEYCODE_CHANNEL_UP = 166 175 | KEYCODE_CHANNEL_DOWN = 167 176 | KEYCODE_ZOOM_IN = 168 177 | KEYCODE_ZOOM_OUT = 169 178 | KEYCODE_TV = 170 179 | KEYCODE_WINDOW = 171 180 | KEYCODE_GUIDE = 172 181 | KEYCODE_DVR = 173 182 | KEYCODE_BOOKMARK = 174 183 | KEYCODE_CAPTIONS = 175 184 | KEYCODE_SETTINGS = 176 185 | KEYCODE_TV_POWER = 177 186 | KEYCODE_TV_INPUT = 178 187 | KEYCODE_STB_POWER = 179 188 | KEYCODE_STB_INPUT = 180 189 | KEYCODE_AVR_POWER = 181 190 | KEYCODE_AVR_INPUT = 182 191 | KEYCODE_PROG_RED = 183 192 | KEYCODE_PROG_GREEN = 184 193 | KEYCODE_PROG_YELLOW = 185 194 | KEYCODE_PROG_BLUE = 186 195 | KEYCODE_APP_SWITCH = 187 196 | KEYCODE_BUTTON_1 = 188 197 | KEYCODE_BUTTON_2 = 189 198 | KEYCODE_BUTTON_3 = 190 199 | KEYCODE_BUTTON_4 = 191 200 | KEYCODE_BUTTON_5 = 192 201 | KEYCODE_BUTTON_6 = 193 202 | KEYCODE_BUTTON_7 = 194 203 | KEYCODE_BUTTON_8 = 195 204 | KEYCODE_BUTTON_9 = 196 205 | KEYCODE_BUTTON_10 = 197 206 | KEYCODE_BUTTON_11 = 198 207 | KEYCODE_BUTTON_12 = 199 208 | KEYCODE_BUTTON_13 = 200 209 | KEYCODE_BUTTON_14 = 201 210 | KEYCODE_BUTTON_15 = 202 211 | KEYCODE_BUTTON_16 = 203 212 | KEYCODE_LANGUAGE_SWITCH = 204 213 | KEYCODE_MANNER_MODE = 205 214 | KEYCODE_3D_MODE = 206 215 | KEYCODE_CONTACTS = 207 216 | KEYCODE_CALENDAR = 208 217 | KEYCODE_MUSIC = 209 218 | KEYCODE_CALCULATOR = 210 219 | KEYCODE_ZENKAKU_HANKAKU = 211 220 | KEYCODE_EISU = 212 221 | KEYCODE_MUHENKAN = 213 222 | KEYCODE_HENKAN = 214 223 | KEYCODE_KATAKANA_HIRAGANA = 215 224 | KEYCODE_YEN = 216 225 | KEYCODE_RO = 217 226 | KEYCODE_KANA = 218 227 | KEYCODE_ASSIST = 219 228 | KEYCODE_BRIGHTNESS_DOWN = 220 229 | KEYCODE_BRIGHTNESS_UP = 221 230 | KEYCODE_MEDIA_AUDIO_TRACK = 222 231 | KEYCODE_SLEEP = 223 232 | KEYCODE_WAKEUP = 224 233 | KEYCODE_PAIRING = 225 234 | KEYCODE_MEDIA_TOP_MENU = 226 235 | KEYCODE_11 = 227 236 | KEYCODE_12 = 228 237 | KEYCODE_LAST_CHANNEL = 229 238 | KEYCODE_TV_DATA_SERVICE = 230 239 | KEYCODE_VOICE_ASSIST = 231 240 | KEYCODE_TV_RADIO_SERVICE = 232 241 | KEYCODE_TV_TELETEXT = 233 242 | KEYCODE_TV_NUMBER_ENTRY = 234 243 | KEYCODE_TV_TERRESTRIAL_ANALOG = 235 244 | KEYCODE_TV_TERRESTRIAL_DIGITAL = 236 245 | KEYCODE_TV_SATELLITE = 237 246 | KEYCODE_TV_SATELLITE_BS = 238 247 | KEYCODE_TV_SATELLITE_CS = 239 248 | KEYCODE_TV_SATELLITE_SERVICE = 240 249 | KEYCODE_TV_NETWORK = 241 250 | KEYCODE_TV_ANTENNA_CABLE = 242 251 | KEYCODE_TV_INPUT_HDMI_1 = 243 252 | KEYCODE_TV_INPUT_HDMI_2 = 244 253 | KEYCODE_TV_INPUT_HDMI_3 = 245 254 | KEYCODE_TV_INPUT_HDMI_4 = 246 255 | KEYCODE_TV_INPUT_COMPOSITE_1 = 247 256 | KEYCODE_TV_INPUT_COMPOSITE_2 = 248 257 | KEYCODE_TV_INPUT_COMPONENT_1 = 249 258 | KEYCODE_TV_INPUT_COMPONENT_2 = 250 259 | KEYCODE_TV_INPUT_VGA_1 = 251 260 | KEYCODE_TV_AUDIO_DESCRIPTION = 252 261 | KEYCODE_TV_AUDIO_DESCRIPTION_MIX_UP = 253 262 | KEYCODE_TV_AUDIO_DESCRIPTION_MIX_DOWN = 254 263 | KEYCODE_TV_ZOOM_MODE = 255 264 | KEYCODE_TV_CONTENTS_MENU = 256 265 | KEYCODE_TV_MEDIA_CONTEXT_MENU = 257 266 | KEYCODE_TV_TIMER_PROGRAMMING = 258 267 | KEYCODE_HELP = 259 268 | KEYCODE_NAVIGATE_PREVIOUS = 260 269 | KEYCODE_NAVIGATE_NEXT = 261 270 | KEYCODE_NAVIGATE_IN = 262 271 | KEYCODE_NAVIGATE_OUT = 263 272 | KEYCODE_STEM_PRIMARY = 264 273 | KEYCODE_STEM_1 = 265 274 | KEYCODE_STEM_2 = 266 275 | KEYCODE_STEM_3 = 267 276 | KEYCODE_DPAD_UP_LEFT = 268 277 | KEYCODE_DPAD_DOWN_LEFT = 269 278 | KEYCODE_DPAD_UP_RIGHT = 270 279 | KEYCODE_DPAD_DOWN_RIGHT = 271 280 | KEYCODE_MEDIA_SKIP_FORWARD = 272 281 | KEYCODE_MEDIA_SKIP_BACKWARD = 273 282 | KEYCODE_MEDIA_STEP_FORWARD = 274 283 | KEYCODE_MEDIA_STEP_BACKWARD = 275 284 | KEYCODE_SOFT_SLEEP = 276 285 | KEYCODE_CUT = 277 286 | KEYCODE_COPY = 278 287 | KEYCODE_PASTE = 279 288 | KEYCODE_SYSTEM_NAVIGATION_UP = 280 289 | KEYCODE_SYSTEM_NAVIGATION_DOWN = 281 290 | KEYCODE_SYSTEM_NAVIGATION_LEFT = 282 291 | KEYCODE_SYSTEM_NAVIGATION_RIGHT = 283 292 | KEYCODE_KEYCODE_ALL_APPS = 284 293 | KEYCODE_KEYCODE_REFRESH = 285 294 | KEYCODE_KEYCODE_THUMBS_UP = 286 295 | KEYCODE_KEYCODE_THUMBS_DOWN = 287 296 | 297 | # Event 298 | EVENT_INIT = "init" 299 | EVENT_FRAME = "frame" 300 | EVENT_DISCONNECT = "disconnect" 301 | EVENT_ONCHANGE = "onchange" 302 | 303 | # Type 304 | TYPE_INJECT_KEYCODE = 0 305 | TYPE_INJECT_TEXT = 1 306 | TYPE_INJECT_TOUCH_EVENT = 2 307 | TYPE_INJECT_SCROLL_EVENT = 3 308 | TYPE_BACK_OR_SCREEN_ON = 4 309 | TYPE_EXPAND_NOTIFICATION_PANEL = 5 310 | TYPE_EXPAND_SETTINGS_PANEL = 6 311 | TYPE_COLLAPSE_PANELS = 7 312 | TYPE_GET_CLIPBOARD = 8 313 | TYPE_SET_CLIPBOARD = 9 314 | TYPE_SET_SCREEN_POWER_MODE = 10 315 | TYPE_ROTATE_DEVICE = 11 316 | 317 | # Lock screen orientation 318 | LOCK_SCREEN_ORIENTATION_UNLOCKED = -1 319 | LOCK_SCREEN_ORIENTATION_INITIAL = -2 320 | LOCK_SCREEN_ORIENTATION_0 = 0 321 | LOCK_SCREEN_ORIENTATION_1 = 1 322 | LOCK_SCREEN_ORIENTATION_2 = 2 323 | LOCK_SCREEN_ORIENTATION_3 = 3 324 | 325 | # Screen power mode 326 | POWER_MODE_OFF = 0 327 | POWER_MODE_NORMAL = 2 328 | -------------------------------------------------------------------------------- /auto_nico/console_scripts/inspector_web/pyscrcpy/control.py: -------------------------------------------------------------------------------- 1 | import functools 2 | import socket 3 | import struct 4 | import time 5 | from . import const 6 | 7 | 8 | def inject(control_type: int): 9 | """ 10 | Inject control code, with this inject, we will be able to do unit test 11 | 12 | Args: 13 | control_type: event to send, TYPE_* 14 | """ 15 | 16 | def wrapper(f): 17 | @functools.wraps(f) 18 | def inner(*args, **kwargs): 19 | package = struct.pack(">B", control_type) + f(*args, **kwargs) 20 | if args[0].parent.control_socket is not None: 21 | with args[0].parent.control_socket_lock: 22 | args[0].parent.control_socket.send(package) 23 | return package 24 | 25 | return inner 26 | 27 | return wrapper 28 | 29 | 30 | class ControlSender: 31 | def __init__(self, parent): 32 | self.parent = parent # client object 33 | self.adbutil_devices = parent.device 34 | 35 | @inject(const.TYPE_INJECT_KEYCODE) 36 | def keycode( 37 | self, keycode: int, action: int = const.ACTION_DOWN, repeat: int = 0 38 | ) -> bytes: 39 | """ 40 | Send keycode to device 41 | 42 | Args: 43 | keycode: const.KEYCODE_* 44 | action: ACTION_DOWN | ACTION_UP 45 | repeat: repeat count 46 | """ 47 | return struct.pack(">Biii", action, keycode, repeat, 0) 48 | 49 | @inject(const.TYPE_INJECT_TEXT) 50 | def text(self, text: str) -> bytes: 51 | """ 52 | Send text to device 53 | 54 | Args: 55 | text: text to send 56 | """ 57 | 58 | buffer = text.encode("utf-8") 59 | return struct.pack(">i", len(buffer)) + buffer 60 | 61 | # @inject(const.TYPE_INJECT_TOUCH_EVENT) 62 | # def touch(self, x: int, y: int, action: int = const.ACTION_DOWN, touch_id: int = -1) -> bytes: 63 | # """ 64 | # Touch screen 65 | # 66 | # Args: 67 | # x: horizontal position 68 | # y: vertical position 69 | # action: ACTION_DOWN | ACTION_UP | ACTION_MOVE 70 | # touch_id: Default using virtual id -1, you can specify it to emulate multi finger touch 71 | # """ 72 | # x, y = max(x, 0), max(y, 0) 73 | # return struct.pack( 74 | # ">BqiiHHHi", 75 | # action, 76 | # touch_id, 77 | # int(x), 78 | # int(y), 79 | # int(self.parent.resolution[0]), 80 | # int(self.parent.resolution[1]), 81 | # 0xFFFF, 82 | # 1, 83 | # ) 84 | def touch(self, x, y): 85 | self.adbutil_devices.shell(f"input tap {x} {y}") 86 | 87 | @inject(const.TYPE_INJECT_SCROLL_EVENT) 88 | def scroll(self, x: int, y: int, h: int, v: int) -> bytes: 89 | """ 90 | Scroll screen 91 | 92 | Args: 93 | x: horizontal position 94 | y: vertical position 95 | h: horizontal movement 96 | v: vertical movement 97 | """ 98 | 99 | x, y = max(x, 0), max(y, 0) 100 | return struct.pack( 101 | ">iiHHii", 102 | int(x), 103 | int(y), 104 | int(self.parent.resolution[0]), 105 | int(self.parent.resolution[1]), 106 | int(h), 107 | int(v), 108 | ) 109 | 110 | @inject(const.TYPE_BACK_OR_SCREEN_ON) 111 | def back_or_turn_screen_on(self, action: int = const.ACTION_DOWN) -> bytes: 112 | """ 113 | If the screen is off, it is turned on only on ACTION_DOWN 114 | 115 | Args: 116 | action: ACTION_DOWN | ACTION_UP 117 | """ 118 | return struct.pack(">B", action) 119 | 120 | @inject(const.TYPE_EXPAND_NOTIFICATION_PANEL) 121 | def expand_notification_panel(self) -> bytes: 122 | """ 123 | Expand notification panel 124 | """ 125 | return b"" 126 | 127 | @inject(const.TYPE_EXPAND_SETTINGS_PANEL) 128 | def expand_settings_panel(self) -> bytes: 129 | """ 130 | Expand settings panel 131 | """ 132 | return b"" 133 | 134 | @inject(const.TYPE_COLLAPSE_PANELS) 135 | def collapse_panels(self) -> bytes: 136 | """ 137 | Collapse all panels 138 | """ 139 | return b"" 140 | 141 | def get_clipboard(self) -> str: 142 | """ 143 | Get clipboard 144 | """ 145 | # Since this function need socket response, we can't auto inject it any more 146 | s: socket.socket = self.parent.control_socket 147 | 148 | with self.parent.control_socket_lock: 149 | # Flush socket 150 | s.setblocking(False) 151 | while True: 152 | try: 153 | s.recv(1024) 154 | except BlockingIOError: 155 | break 156 | s.setblocking(True) 157 | 158 | # Read package 159 | package = struct.pack(">B", const.TYPE_GET_CLIPBOARD) 160 | s.send(package) 161 | (code,) = struct.unpack(">B", s.recv(1)) 162 | assert code == 0 163 | (length,) = struct.unpack(">i", s.recv(4)) 164 | 165 | return s.recv(length).decode("utf-8") 166 | 167 | @inject(const.TYPE_SET_CLIPBOARD) 168 | def set_clipboard(self, text: str, paste: bool = False) -> bytes: 169 | """ 170 | Set clipboard 171 | 172 | Args: 173 | text: the string you want to set 174 | paste: paste now 175 | """ 176 | buffer = text.encode("utf-8") 177 | return struct.pack(">?i", paste, len(buffer)) + buffer 178 | 179 | @inject(const.TYPE_SET_SCREEN_POWER_MODE) 180 | def set_screen_power_mode( 181 | self, mode: int = const.POWER_MODE_NORMAL 182 | ) -> bytes: 183 | """ 184 | Set screen power mode 185 | 186 | Args: 187 | mode: POWER_MODE_OFF | POWER_MODE_NORMAL 188 | """ 189 | return struct.pack(">b", mode) 190 | 191 | @inject(const.TYPE_ROTATE_DEVICE) 192 | def rotate_device(self) -> bytes: 193 | """ 194 | Rotate device 195 | """ 196 | return b"" 197 | 198 | def swipe( 199 | self, 200 | start_x: int, 201 | start_y: int, 202 | end_x: int, 203 | end_y: int, 204 | move_step_length: int = 5, 205 | move_steps_delay: float = 0.005, 206 | ) -> None: 207 | """ 208 | Swipe on screen 209 | 210 | Args: 211 | start_x: start horizontal position 212 | start_y: start vertical position 213 | end_x: start horizontal position 214 | end_y: end vertical position 215 | move_step_length: length per step 216 | move_steps_delay: sleep seconds after each step 217 | :return: 218 | """ 219 | 220 | self.touch(start_x, start_y, const.ACTION_DOWN) 221 | next_x = start_x 222 | next_y = start_y 223 | 224 | if end_x > self.parent.resolution[0]: 225 | end_x = self.parent.resolution[0] 226 | 227 | if end_y > self.parent.resolution[1]: 228 | end_y = self.parent.resolution[1] 229 | 230 | decrease_x = True if start_x > end_x else False 231 | decrease_y = True if start_y > end_y else False 232 | while True: 233 | if decrease_x: 234 | next_x -= move_step_length 235 | if next_x < end_x: 236 | next_x = end_x 237 | else: 238 | next_x += move_step_length 239 | if next_x > end_x: 240 | next_x = end_x 241 | 242 | if decrease_y: 243 | next_y -= move_step_length 244 | if next_y < end_y: 245 | next_y = end_y 246 | else: 247 | next_y += move_step_length 248 | if next_y > end_y: 249 | next_y = end_y 250 | 251 | self.touch(next_x, next_y, const.ACTION_MOVE) 252 | 253 | if next_x == end_x and next_y == end_y: 254 | self.touch(next_x, next_y, const.ACTION_UP) 255 | break 256 | time.sleep(move_steps_delay) 257 | -------------------------------------------------------------------------------- /auto_nico/console_scripts/inspector_web/pyscrcpy/core.py: -------------------------------------------------------------------------------- 1 | import os 2 | import abc 3 | from pathlib import Path 4 | import socket 5 | import struct 6 | import threading 7 | import time 8 | from time import sleep 9 | from typing import Any, Callable, Optional, Tuple, Union 10 | 11 | import numpy as np 12 | import numpy.typing as npt 13 | from adbutils import AdbConnection, AdbDevice, AdbError, Network, adb 14 | from av.codec import CodecContext # type: ignore 15 | from av.error import InvalidDataError # type: ignore 16 | import cv2 as cv 17 | import cv2 18 | from loguru import logger 19 | 20 | from .const import EVENT_DISCONNECT, EVENT_FRAME, EVENT_INIT, LOCK_SCREEN_ORIENTATION_UNLOCKED, EVENT_ONCHANGE 21 | from .control import ControlSender 22 | 23 | Frame = npt.NDArray[np.int8] 24 | 25 | VERSION = "1.20" 26 | HERE = Path(__file__).resolve().parent 27 | JAR = HERE / f"scrcpy-server.jar" 28 | 29 | 30 | class Client: 31 | def __init__( 32 | self, 33 | device: Optional[Union[AdbDevice, str]] = None, 34 | max_size: int = 0, 35 | bitrate: int = 8000000, 36 | max_fps: int = 0, 37 | block_frame: bool = True, 38 | stay_awake: bool = True, 39 | lock_screen_orientation: int = LOCK_SCREEN_ORIENTATION_UNLOCKED, 40 | skip_same_frame=False 41 | ): 42 | """ 43 | [ok]Create a scrcpy client. The client won't be started until you call .start() 44 | 45 | Args: 46 | device: Android device to coennect to. Colud be also specify by 47 | serial string. If device is None the client try to connect 48 | to the first available device in adb deamon. 49 | max_size: Specify the maximum dimension of the video stream. This 50 | dimensioin refer both to width and hight.0: no limit[已校验, max size of width or height] 51 | bitrate: bitrate 52 | max_fps: Maximum FPS (Frame Per Second) of the video stream. If it 53 | is set to 0 it means that there is not limit to FPS. 54 | This feature is supported by android 10 or newer. 55 | [flip]: 没有这个参数, 会自动处理 56 | block_frame: If set to true, the on_frame callbacks will be only 57 | apply on not empty frames. Otherwise try to apply on_frame 58 | callbacks on every frame, but this could raise exceptions in 59 | callbacks if they are not able to handle None value for frame. 60 | True:跳过空白帧 61 | stay_awake: keep Android device awake while the client-server 62 | connection is alive. 63 | lock_screen_orientation: lock screen in a particular orientation. 64 | The available screen orientation are specify in const.py 65 | in variables LOCK_SCREEN_ORIENTATION* 66 | """ 67 | # Params挪到后面去 68 | self.max_size = max_size 69 | self.bitrate = bitrate 70 | self.max_fps = max_fps 71 | self.block_frame = block_frame 72 | self.stay_awake = stay_awake 73 | self.lock_screen_orientation = lock_screen_orientation 74 | self.skip_same_frame = skip_same_frame 75 | self.min_frame_interval = 1 / max_fps 76 | 77 | if device is None: 78 | try: 79 | device = adb.device_list()[0] 80 | except IndexError: 81 | raise Exception("Cannot connect to phone") 82 | elif isinstance(device, str): 83 | device = adb.device(serial=device) 84 | 85 | self.device = device 86 | self.listeners = dict(frame=[], init=[], disconnect=[], onchange=[]) 87 | 88 | # User accessible 89 | self.last_frame: Optional[np.ndarray] = None 90 | self.resolution: Optional[Tuple[int, int]] = None 91 | self.device_name: Optional[str] = None 92 | self.control = ControlSender(self) 93 | 94 | # Need to destroy 95 | self.alive = False 96 | self.__server_stream: Optional[AdbConnection] = None 97 | self.__video_socket: Optional[socket.socket] = None 98 | self.control_socket: Optional[socket.socket] = None 99 | self.control_socket_lock = threading.Lock() 100 | 101 | def __init_server_connection(self) -> None: 102 | """ 103 | Connect to android server, there will be two sockets: video and control socket. 104 | This method will also set resolution property. 105 | """ 106 | for _ in range(30): # 超时 写死 107 | try: 108 | self.__video_socket = self.device.create_connection( 109 | Network.LOCAL_ABSTRACT, "scrcpy" 110 | ) 111 | break 112 | except AdbError: 113 | sleep(0.1) 114 | pass 115 | else: 116 | raise ConnectionError("Failed to connect scrcpy-server after 3 seconds") 117 | 118 | dummy_byte = self.__video_socket.recv(1) 119 | if not len(dummy_byte): 120 | raise ConnectionError("Did not receive Dummy Byte!") 121 | 122 | self.control_socket = self.device.create_connection( 123 | Network.LOCAL_ABSTRACT, "scrcpy" 124 | ) 125 | self.device_name = self.__video_socket.recv(64).decode("utf-8").rstrip("\x00") 126 | if not len(self.device_name): 127 | raise ConnectionError("Did not receive Device Name!") 128 | 129 | res = self.__video_socket.recv(4) 130 | self.resolution = struct.unpack(">HH", res) 131 | self.__video_socket.setblocking(False) 132 | 133 | def __deploy_server(self) -> None: 134 | """ 135 | Deploy server to android device. 136 | Push the scrcpy-server.jar into the Android device using 137 | the adb.push(...). Then a basic connection between client and server 138 | is established. 139 | """ 140 | cmd = [ 141 | "CLASSPATH=/data/local/tmp/scrcpy-server.jar", 142 | "app_process", 143 | "/", 144 | "com.genymobile.scrcpy.Server", 145 | VERSION, # Scrcpy server version 146 | "info", # Log level: info, verbose... 147 | f"{self.max_size}", # Max screen width (long side) 148 | f"{self.bitrate}", # Bitrate of video 149 | f"{self.max_fps}", # Max frame per second 150 | f"{self.lock_screen_orientation}", # Lock screen orientation 151 | "true", # Tunnel forward 152 | "-", # Crop screen 153 | "false", # Send frame rate to client 154 | "true", # Control enabled 155 | "0", # Display id 156 | "false", # Show touches 157 | "true" if self.stay_awake else "false", # Stay awake 158 | "-", # Codec (video encoding) options 159 | "-", # Encoder name 160 | "false", # Power off screen after server closed 161 | ] 162 | self.device.push(JAR, "/data/local/tmp/") 163 | self.__server_stream: AdbConnection = self.device.shell(cmd, stream=True) 164 | 165 | def start(self, threaded: bool = False) -> None: 166 | """ 167 | Start the client-server connection. 168 | In order to avoid unpredictable behaviors, this method must be called 169 | after the on_init and on_frame callback are specify. 170 | 171 | Args: 172 | threaded : If set to True the stream loop willl run in a separated 173 | thread. This mean that the code after client.strart() will be 174 | run. Otherwise the client.start() method starts a endless loop 175 | and the code after this method will never run. todo new_thread 176 | """ 177 | assert self.alive is False 178 | 179 | self.__deploy_server() 180 | self.__init_server_connection() 181 | self.alive = True 182 | for func in self.listeners[EVENT_INIT]: 183 | func(self) 184 | 185 | if threaded: # 不阻塞当前thread 186 | threading.Thread(target=self.__stream_loop).start() 187 | else: 188 | self.__stream_loop() 189 | 190 | def stop(self) -> None: 191 | """ 192 | [ok]Close the various socket connection. 193 | Stop listening (both threaded and blocked) 194 | """ 195 | self.alive = False 196 | try: 197 | self.__server_stream.close() 198 | except Exception: 199 | pass 200 | try: 201 | self.control_socket.close() 202 | except Exception: 203 | pass 204 | try: 205 | self.__video_socket.close() 206 | except Exception: 207 | pass 208 | 209 | def __del__(self): 210 | self.stop() 211 | 212 | def __calculate_diff(self, img1, img2): 213 | if img1 is None: 214 | return 1 215 | gray1 = cv.cvtColor(img1, cv.COLOR_BGR2GRAY) 216 | gray2 = cv.cvtColor(img2, cv.COLOR_BGR2GRAY) 217 | 218 | # 计算两张灰度图像的差异 219 | diff = cv2.absdiff(gray1, gray2) 220 | 221 | # 设置阈值，忽略差异值较小的像素 222 | threshold = 30 223 | _, thresholded_diff = cv2.threshold(diff, threshold, 255, cv2.THRESH_BINARY) 224 | 225 | # 计算差异像素的总数 226 | total_diff_pixels = np.sum(thresholded_diff / 255) # 除以255得到二值图像中白色像素的数量 227 | 228 | # 计算图像的总像素数 229 | total_pixels = gray1.size 230 | 231 | # 计算变化率 232 | change_rate = total_diff_pixels / total_pixels 233 | return change_rate 234 | 235 | def __stream_loop(self) -> None: 236 | """ 237 | Core loop for video parsing. 238 | While the connection is open (self.alive == True) recive raw h264 video 239 | stream and decode it into frames. These frame are those passed to 240 | on_frame callbacks. 241 | """ 242 | codec = CodecContext.create("h264", "r") 243 | while self.alive: 244 | try: 245 | raw = self.__video_socket.recv(0x10000) 246 | if raw == b"": 247 | raise ConnectionError("Video stream is disconnected") 248 | for packet in codec.parse(raw): 249 | for frame in codec.decode(packet): # codec.decode(packet)包含多帧 250 | frame = frame.to_ndarray(format="bgr24") 251 | 252 | if len(self.listeners[EVENT_ONCHANGE]) == 0 and not self.skip_same_frame: 253 | self.last_frame = frame 254 | elif self.__calculate_diff(self.last_frame, frame) > 0.1: 255 | logger.debug("different frame detected") 256 | self.last_frame = frame 257 | for func in self.listeners[EVENT_ONCHANGE]: 258 | func(self, frame) 259 | else: # no_change and should skip this frame 260 | continue 261 | 262 | self.resolution = (frame.shape[1], frame.shape[0]) 263 | for func in self.listeners[EVENT_FRAME]: # 发送给用户自定义的函数 264 | func(self, frame) 265 | except (BlockingIOError, InvalidDataError): # empty frame 266 | time.sleep(0.01) 267 | if not self.block_frame: # init时允许空白帧 268 | for func in self.listeners[EVENT_FRAME]: 269 | func(self, None) 270 | except (ConnectionError, OSError) as e: # Socket Closed 271 | if self.alive: 272 | # todo on_disconnect event 273 | self.stop() 274 | raise e 275 | 276 | def on_init(self, func: Callable[[Any], None]) -> None: 277 | """ 278 | Add funtion to on_init listeners. 279 | Your function is run after client.start() is called. 280 | 281 | Args: 282 | func: callback to be called after the server starts. 参数:这个class的obj 283 | """ 284 | self.listeners[EVENT_INIT].append(func) 285 | 286 | def on_frame(self, func: Callable[[Any, Frame], None]): 287 | """ 288 | Add functoin to on-frame listeners. 289 | Your function will be run on every valid frame recived. 290 | 291 | Args: 292 | func: callback to be called on every frame. 293 | 294 | Returns: 295 | The list of on-frame callbacks. 296 | """ 297 | self.listeners[EVENT_FRAME].append(func) 298 | 299 | def on_change(self, func: Callable[[Any, Frame], None]): 300 | self.listeners[EVENT_ONCHANGE].append(func) 301 | -------------------------------------------------------------------------------- /auto_nico/console_scripts/inspector_web/pyscrcpy/etsdt.py: -------------------------------------------------------------------------------- 1 | import cv2 as cv 2 | 3 | from auto_nico.console_scripts.test.pyscrcpy import Client 4 | 5 | 6 | def on_frame(client, frame): 7 | """ 8 | 帧处理回调函数，在每次接收到有效帧时被调用 9 | :param client: 客户端对象 10 | :param frame: 接收到的帧 11 | """ 12 | # 模拟在屏幕坐标 (300, 500) 处点击 13 | # client.control.touch(300, 500) 14 | # 显示接收到的帧 15 | cv.imshow('Video', frame) 16 | # 等待 1 毫秒，处理按键事件 17 | cv.waitKey(1) 18 | 19 | def main(): 20 | """ 21 | 主函数，创建客户端并启动屏幕镜像 22 | """ 23 | # 创建一个 Client 对象，设置最大帧率为 1，最大尺寸为 900 24 | client = Client(max_fps=24, max_size=900) 25 | # 添加帧处理回调函数 26 | client.on_frame(on_frame) 27 | # 启动客户端，开始屏幕镜像 28 | client.start() 29 | 30 | if __name__ == "__main__": 31 | main() -------------------------------------------------------------------------------- /auto_nico/console_scripts/inspector_web/pyscrcpy/scrcpy-server.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/letmeNo1/Nico/79dc1ac061cdeaa7ab74660d42bf85bd93ff0592/auto_nico/console_scripts/inspector_web/pyscrcpy/scrcpy-server.jar -------------------------------------------------------------------------------- /auto_nico/console_scripts/inspector_web/pyscrcpy/test.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | from auto_nico.console_scripts.test.pyscrcpy import Client 3 | from fastapi import FastAPI 4 | from fastapi.responses import Response 5 | import threading 6 | 7 | app = FastAPI() 8 | client = None 9 | frame_buffer = None 10 | 11 | # 初始化设备连接（自动选择首个设备） 12 | def on_frame(client, frame, cv=None): 13 | global frame_buffer 14 | frame_buffer = frame 15 | 16 | def start_client(): 17 | global client 18 | client = Client(max_fps=20) 19 | client.start(threaded=True) # create a new thread for scrcpy 20 | while True: 21 | if client.last_frame is not None: 22 | on_frame(client, client.last_frame) 23 | 24 | @app.get("/get_image") 25 | async def get_image(): 26 | global frame_buffer 27 | if frame_buffer is not None: 28 | ret, buffer = cv2.imencode('.jpg', frame_buffer) 29 | frame = buffer.tobytes() 30 | return Response(content=frame, media_type="image/jpeg") 31 | return {"message": "No frame available yet"} 32 | 33 | def start_scrcpy(): 34 | import uvicorn 35 | # 启动客户端线程 36 | client_thread = threading.Thread(target=start_client) 37 | client_thread.daemon = True 38 | client_thread.start() 39 | 40 | # 启动 FastAPI 服务 41 | uvicorn.run(app, host="0.0.0.0", port=5000) -------------------------------------------------------------------------------- /auto_nico/console_scripts/inspector_web/static/button_script.js: -------------------------------------------------------------------------------- 1 | $(document).ready(function() { 2 | // 监听开关状态变化 3 | $('#realtime-switch').change(function() { 4 | if ($(this).is(':checked')) { 5 | // 开启实时显示 6 | startRealtimeUpdates(); 7 | } else { 8 | // 关闭实时显示 9 | stopRealtimeUpdates(); 10 | } 11 | }); 12 | 13 | // 刷新按钮点击事件 14 | $('#refresh-button').click(function() { 15 | refreshData(); 16 | }); 17 | 18 | // 模拟实时更新的函数 19 | function startRealtimeUpdates() { 20 | console.log("实时显示已开启"); 21 | removeTextControlHoverListeners() 22 | removeImageListeners() 23 | refreshInterval = setInterval(refreshImage, 500); // 每秒调用一次refreshData 24 | } 25 | 26 | function stopRealtimeUpdates() { 27 | console.log("实时显示已关闭"); 28 | 29 | clearInterval(refreshInterval); // 停止调用refreshData 30 | } 31 | 32 | // 提取的请求函数 33 | function sendRequest(url) { 34 | console.log(url); 35 | $.ajax({ 36 | url: url, 37 | type: 'GET', 38 | async: true, // 确保请求是异步的 39 | success: function(data) { 40 | setTimeout(function() { 41 | refreshData(); 42 | }, 500); // 500毫秒 = 0.5秒 43 | }, 44 | error: function(xhr, status, error) { 45 | console.error('Request failed:', status, error); 46 | // 即使请求失败，也可以选择调用refreshData 47 | refreshData(); 48 | } 49 | }); 50 | } 51 | 52 | var first_node = document.getElementById('Title'); 53 | var platform = first_node.getAttribute("nico_ui_platform"); 54 | 55 | if (platform !== "iOS") { 56 | // 处理提交按钮的点击事件 57 | $('#submit-button').click(function() { 58 | var inputValue = $('#text-input').val(); 59 | if (inputValue) { 60 | var url = `/android_excute_action?action=input&inputValue=${inputValue}`; 61 | sendRequest(url); 62 | } else { 63 | alert('Please enter some text.'); 64 | } 65 | }); 66 | 67 | // 处理 home 按钮的点击事件 68 | $('#home-button').click(function() { 69 | var url = `/android_excute_action?action=home`; 70 | sendRequest(url); 71 | }); 72 | 73 | // 处理 back 按钮的点击事件 74 | $('#back-button').click(function() { 75 | var url = `/android_excute_action?action=back`; 76 | sendRequest(url); 77 | }); 78 | 79 | // 处理 menu 按钮的点击事件 80 | $('#menu-button').click(function() { 81 | var url = `/android_excute_action?action=menu`; 82 | sendRequest(url); 83 | }); 84 | 85 | $('#switch-button').click(function() { 86 | var url = `/android_excute_action?action=switch_app`; 87 | sendRequest(url); 88 | }); 89 | 90 | $('#volume-up').click(function() { 91 | var url = `/android_excute_action?action=volume_up`; 92 | sendRequest(url); 93 | }); 94 | 95 | $('#volume-down').click(function() { 96 | var url = `/android_excute_action?action=volume_down`; 97 | sendRequest(url); 98 | }); 99 | 100 | $('#power').click(function() { 101 | var url = `/android_excute_action?action=power`; 102 | sendRequest(url); 103 | }); 104 | 105 | $('#delete_text').click(function() { 106 | var url = `/android_excute_action?action=delete_text`; 107 | sendRequest(url); 108 | }); 109 | } 110 | }); -------------------------------------------------------------------------------- /auto_nico/console_scripts/inspector_web/static/styles.css: -------------------------------------------------------------------------------- 1 | body { 2 | font-family: Arial, sans-serif; 3 | background-color: #2c3e50; 4 | color: #ecf0f1; 5 | margin: 0; 6 | padding: 0; 7 | height: 100vh; 8 | width: 100vw; 9 | box-sizing: border-box; 10 | } 11 | 12 | .hovered { 13 | color: red; 14 | } 15 | 16 | .content::after { 17 | content: ""; 18 | display: table; 19 | clear: both; 20 | } 21 | 22 | .content { 23 | float: left; 24 | width: 50%; /* 修改为50% */ 25 | height: 100%; /* 设置高度为视口的100% */ 26 | box-sizing: border-box; /* 包含内边距和边框在内的宽度 */ 27 | } 28 | 29 | .content-inner { 30 | width: 100%; /* 修改为100% */ 31 | height: 50%; /* 修改为50% */ 32 | overflow: auto; /* 添加这一行以在内容超出时显示滚动条 */ 33 | } 34 | 35 | .image img { 36 | width: 100%; 37 | height: 100%; 38 | object-fit: contain; 39 | } 40 | 41 | .image { 42 | position: relative; /* 添加这一行 */ 43 | float: right; 44 | width: 50%; /* 修改为50% */ 45 | height: 100vh; /* 设置高度为视口的100% */ 46 | box-sizing: border-box; 47 | } 48 | 49 | #myCanvas { 50 | position: absolute; 51 | top: 0; 52 | left: 0; 53 | width: 100%; 54 | height: 100%; 55 | } 56 | 57 | #info-list { 58 | font-size: 20px; /* 设置字体大小为20像素 */ 59 | } 60 | 61 | .switch-container { 62 | display: flex; 63 | align-items: center; 64 | margin: 20px; 65 | } 66 | 67 | .switch { 68 | position: relative; 69 | display: inline-block; 70 | width: 60px; 71 | height: 34px; 72 | margin-right: 10px; /* 添加间距 */ 73 | } 74 | 75 | .switch input { 76 | opacity: 0; 77 | width: 0; 78 | height: 0; 79 | } 80 | 81 | .slider { 82 | position: absolute; 83 | cursor: pointer; 84 | top: 0; 85 | left: 0; 86 | right: 0; 87 | bottom: 0; 88 | background-color: #ccc; 89 | transition: .4s; 90 | border-radius: 34px; 91 | } 92 | 93 | .slider:before { 94 | position: absolute; 95 | content: ""; 96 | height: 26px; 97 | width: 26px; 98 | left: 4px; 99 | bottom: 4px; 100 | background-color: white; 101 | transition: .4s; 102 | border-radius: 50%; 103 | } 104 | 105 | input:checked + .slider { 106 | background-color: #2196F3; 107 | } 108 | 109 | input:checked + .slider:before { 110 | transform: translateX(26px); 111 | } 112 | 113 | .c_button { 114 | padding: 10px 20px; 115 | background-color: #2196F3; 116 | color: white; 117 | border: none; 118 | border-radius: 5px; 119 | cursor: pointer; 120 | margin-right: 20px; /* 添加间距 */ 121 | } 122 | 123 | .c_button:hover { 124 | background-color: #1976D2; 125 | } 126 | 127 | .controls { 128 | display: flex; 129 | flex-wrap: wrap; /* 添加这一行 */ 130 | align-items: center; 131 | justify-content: flex-end; 132 | padding: 20px; 133 | box-sizing: border-box; 134 | width: 50%; /* 设置宽度为50% */ 135 | position: absolute; 136 | top: 0; 137 | right: 0; 138 | z-index: 1000; /* 确保在最前端 */ 139 | } 140 | 141 | /* 为提交按钮添加相同的样式 */ 142 | .button { 143 | padding: 10px 20px; 144 | background-color: #2196F3; 145 | color: white; 146 | border: none; 147 | border-radius: 5px; 148 | cursor: pointer; 149 | margin-right: 20px; /* 添加间距 */ 150 | } 151 | 152 | .button:hover { 153 | background-color: #1976D2; 154 | } 155 | 156 | /* 为输入框添加样式 */ 157 | .styled-input { 158 | padding: 10px; 159 | margin: 4px 2px; 160 | font-size: 16px; 161 | border-radius: 4px; 162 | border: 1px solid #ccc; 163 | box-sizing: border-box; 164 | } -------------------------------------------------------------------------------- /auto_nico/console_scripts/inspector_web/templates/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/letmeNo1/Nico/79dc1ac061cdeaa7ab74660d42bf85bd93ff0592/auto_nico/console_scripts/inspector_web/templates/__init__.py -------------------------------------------------------------------------------- /auto_nico/console_scripts/inspector_web/templates/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Device Screen Streaming 6 | 7 | 8 |

Device Screen Streaming

9 | Device Screen 10 | 11 | -------------------------------------------------------------------------------- /auto_nico/console_scripts/inspector_web/templates/xml_template.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | XML Content 6 | 7 | 8 | 9 |
10 |

XML Content as HTML

11 |
12 | {{ xml_content|safe }} 13 |
14 | 15 |
16 |
17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 |
31 | 35 | 36 |
37 | 38 | 39 |
40 |
41 | Image 42 | 43 |
44 | 45 | 46 | 47 | 48 | 49 | 50 | -------------------------------------------------------------------------------- /auto_nico/console_scripts/screenshot.py: -------------------------------------------------------------------------------- 1 | import os 2 | from pathlib import Path 3 | from datetime import datetime 4 | 5 | import cv2 6 | 7 | from auto_nico.android.adb_utils import AdbUtils 8 | from auto_nico.android.nico_android import NicoAndroid 9 | 10 | 11 | def main(): 12 | import argparse 13 | 14 | parser = argparse.ArgumentParser() 15 | parser.add_argument('-s', type=str, help='device_udid') 16 | 17 | parser.add_argument('-m', action='store_true', 18 | help='Activate special mode.') 19 | args = parser.parse_args() 20 | adb_utils = AdbUtils(args.u) 21 | desktop_path = Path.home() / 'Desktop' 22 | timestamp = datetime.now().strftime("Screenshot_%Y-%m-%d_%H%M%S") 23 | pic_path = f"{desktop_path}\{timestamp}.png" 24 | 25 | if args.m: 26 | nico = NicoAndroid(args.s) 27 | eles = nico(text_matches=r'^(?=(?:.*?\d){2})').all() 28 | adb_utils.snapshot(timestamp, desktop_path) 29 | image = cv2.imread(f"{pic_path}") 30 | for ele in eles: 31 | x, y, w, h = ele.get_bounds 32 | if h < 10: 33 | h = 50 34 | cv2.rectangle(image, (x, y), (x + w, y + h), (0, 0, 0), -1) 35 | cv2.imwrite(pic_path, image) 36 | os.startfile(desktop_path) 37 | 38 | 39 | else: 40 | adb_utils.snapshot(timestamp, desktop_path) 41 | os.startfile(desktop_path) 42 | -------------------------------------------------------------------------------- /auto_nico/console_scripts/test_/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/letmeNo1/Nico/79dc1ac061cdeaa7ab74660d42bf85bd93ff0592/auto_nico/console_scripts/test_/__init__.py -------------------------------------------------------------------------------- /auto_nico/console_scripts/test_/app.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import base64 3 | import os 4 | import socket 5 | import subprocess 6 | 7 | from auto_nico.console_scripts.inspector_web.nico_inspector import xml_to_html_list,dump_ui_tree 8 | from auto_nico.android.adb_utils import AdbUtils 9 | from auto_nico.common.send_request import send_http_request 10 | from auto_nico.ios.idb_utils import IdbUtils 11 | from flask import Flask, Response, jsonify, render_template, request 12 | import io 13 | import time 14 | from scrcpy_utils import ( 15 | frame_buffers, buffer_lock, caches, clients, starting, request_cache, 16 | on_frame, start_client_by_threading, is_device_connected 17 | ) 18 | 19 | app = Flask(__name__) 20 | 21 | def find_available_port(): 22 | with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: 23 | s.bind(('', 0)) 24 | return s.getsockname()[1] 25 | 26 | 27 | @app.route("/dynamic_image") 28 | def get_image(): 29 | device_id = os.getenv("nico_ui_udid") # 从请求参数中获取 device_id 30 | # 非第一次请求设备画面，判断距离第一次请求是否超过 2 秒，没超过则等待，超过则大概率 pyscrcpy server 已经启动好了 31 | if device_id in request_cache: 32 | sequence, request_time = request_cache[device_id] 33 | if sequence == 1 and (time.time() - request_time) <= 2: 34 | return jsonify({"message": "正在初始化，请稍等几秒中！"}) 35 | if sequence == 1 and (time.time() - request_time) > 2: 36 | new_sequence = sequence + 1 37 | request_cache[device_id] = (new_sequence, time.time()) 38 | # 第一次请求设备画面，设置请求顺序为 1 39 | if device_id not in request_cache: 40 | request_cache[device_id] = (1, time.time()) 41 | 42 | # 1. 当设备未连接到电脑的情况 43 | if not is_device_connected(device_id): 44 | return jsonify({"message": "设备未连接"}) 45 | # 2. 当 pyscrcpy server 没启动的情况 46 | if device_id not in starting or device_id not in clients: 47 | starting[device_id] = (True, time.time()) 48 | start_client_by_threading(device_id) 49 | return jsonify({"message": "scrcpy server 尚未启动，启动", "starting": starting}) 50 | # 3. pyscrcpy server 已启动，设备连接正常的情况 51 | if clients[device_id].alive: 52 | starting[device_id] = (False, time.time()) 53 | 54 | with buffer_lock: 55 | if device_id in caches and 'image' in caches[device_id]: 56 | image_data = caches[device_id]['image'] 57 | return Response(io.BytesIO(image_data), mimetype="image/jpeg") 58 | # 4. 利用缓存，避免拔插导致的一直 keep 在 starting 状态 59 | is_starting, starting_time = starting[device_id] 60 | if is_starting and (time.time() - starting_time) <= 2: 61 | return jsonify({"message": "scrcpy server 启动中", "starting": starting}) 62 | if is_starting and (time.time() - starting_time) > 2: 63 | starting[device_id] = (True, time.time()) 64 | start_client_by_threading(device_id) 65 | return jsonify({"message": "scrcpy server 启动超时，再次启动", "starting": starting}) 66 | # 5. 设备中途掉线、恢复的情况 67 | if not is_starting and not clients[device_id].alive: 68 | starting[device_id] = (True, time.time()) 69 | start_client_by_threading(device_id) 70 | return jsonify({"message": "scrcpy server 掉线，启动中", "starting": starting}) 71 | # 6. 画面超过缓存时间（120 秒），强制重启 72 | starting[device_id] = (True, time.time()) 73 | start_client_by_threading(device_id) 74 | return jsonify({"message": "画面超时，强制重启 scrcpy server", "starting": starting}) 75 | 76 | 77 | @app.route('/static_image') 78 | def generate_image(): 79 | port = int(os.environ.get('RemoteServerPort')) 80 | platform = os.environ.get('nico_ui_platform') 81 | if platform == "android": 82 | new_data = send_http_request(port, "screenshot", {"quality": 80}) 83 | else: 84 | new_data = send_http_request(port, "get_jpg_pic", {"compression_quality": 1.0}) 85 | if new_data is None: 86 | return jsonify({"error": "Failed to get image data"}), 500 87 | base64_data = base64.b64encode(new_data).decode('utf-8') 88 | return jsonify({"image": base64_data}) 89 | 90 | @app.route('/', methods=['GET', 'POST']) 91 | def index(): 92 | return render_template('index.html') 93 | 94 | 95 | @app.route('/refresh_ui_xml') 96 | def refresh_ui_xml(): 97 | root = dump_ui_tree() 98 | 99 | # 构建HTML列表 100 | html_list = xml_to_html_list(root) 101 | 102 | # 渲染模板并传递构建的HTML列表 103 | return html_list 104 | 105 | 106 | @app.route("/action", methods=["POST"]) 107 | def action(): 108 | device_id = os.getenv("nico_ui_udid") 109 | if device_id not in clients: 110 | return jsonify({"message": f"设备 {device_id} 的客户端未找到"}), 400 111 | client = clients[device_id] 112 | data = request.get_json() 113 | x_percent = data.get('xPercent') 114 | y_percent = data.get('yPercent') 115 | x_percent = float(x_percent) / 100 116 | y_percent = float(y_percent) / 100 117 | x = int(x_percent * int(client.resolution[0])) 118 | y = int(y_percent * int(client.resolution[1])) 119 | if data.get('actionType') == 'touch_down': 120 | client.control.touch_down(x, y) 121 | elif data.get('actionType') == 'touch_up': 122 | client.control.touch_up(x, y) 123 | elif data.get("actionType") == "touch_move": 124 | client.control.touch_move(x, y) 125 | 126 | return jsonify({"message": f"{data.get('actionType')} sent successfully"}) 127 | 128 | 129 | if __name__ == '__main__': 130 | parser = argparse.ArgumentParser(description='设备控制服务') 131 | parser.add_argument('-s', '--udid', required=True, help='设备 UDID') 132 | parser.add_argument('-plat', type=str, help='platform "i","iOS","a","android"') 133 | 134 | args = parser.parse_args() 135 | udid = args.udid 136 | platform = parser.parse_args().plat 137 | if platform is None: 138 | if len(args.udid) > 20: 139 | platform = "iOS" 140 | else: 141 | platform = "android" 142 | elif platform in ["i", "iOS", "a", "android"]: 143 | if platform == "i": 144 | platform = "iOS" 145 | elif platform == "a": 146 | platform = "android" 147 | else: 148 | pass 149 | else: 150 | print('Please enter the correct platform "i","iOS","a","android"') 151 | os.environ['nico_ui_udid'] = udid 152 | remote_port = find_available_port() 153 | os.environ['RemoteServerPort'] = str(remote_port) 154 | os.environ['nico_ui_platform'] = platform 155 | 156 | if platform == "android": 157 | adb_utils = AdbUtils(udid) 158 | adb_utils.clear_tcp_forward_port(remote_port) 159 | adb_utils.cmd(f'''forward tcp:{remote_port} tcp:8000''') 160 | adb_utils.check_adb_server() 161 | adb_utils.install_test_server_package(1.4) 162 | ime_list = adb_utils.qucik_shell("ime list -s").split("\n")[0:-1] 163 | for ime in ime_list: 164 | adb_utils.qucik_shell(f"ime disable {ime}") 165 | commands = f"""adb -s {udid} shell am instrument -r -w -e port {remote_port} -e class nico.dump_hierarchy.HierarchyTest nico.dump_hierarchy.test/androidx.test.runner.AndroidJUnitRunner""" 166 | subprocess.Popen(commands, shell=True) 167 | else: 168 | idb_utils = IdbUtils(udid) 169 | port, pid = idb_utils.get_tcp_forward_port() 170 | if port: 171 | remote_port = port 172 | idb_utils.runtime_cache.set_current_running_port(port) 173 | else: 174 | idb_utils.set_port_forward(remote_port) 175 | idb_utils._init_test_server() 176 | # 启动服务 177 | port = find_available_port() 178 | 179 | # 启动服务 180 | app.run(host="0.0.0.0", port=port, threaded=True) -------------------------------------------------------------------------------- /auto_nico/console_scripts/test_/nico_utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from auto_nico.android.tools.format_converter import add_xpath_att 4 | from auto_nico.common.send_request import send_http_request 5 | from auto_nico.ios.idb_utils import IdbUtils 6 | from auto_nico.ios.tools.format_converter import converter 7 | import lxml.etree as ET 8 | 9 | 10 | def dump_ui_tree(): 11 | platform = os.environ.get('nico_ui_platform') 12 | udid = os.environ.get("nico_ui_udid") 13 | idb_utils = IdbUtils(udid) 14 | 15 | port = int(os.environ.get('RemoteServerPort')) 16 | if platform == "android": 17 | xml = send_http_request(port, "dump",{"compressed":"true"}).replace("class", "class_name").replace("resource-id=", 18 | "id=").replace( 19 | "content-desc=", "content_desc=") 20 | root = add_xpath_att(ET.fromstring(xml.encode('utf-8'))) 21 | 22 | else: 23 | package_name = idb_utils.get_current_bundleIdentifier(port) 24 | os.environ['current_package_name'] = package_name 25 | xml = send_http_request(port, f"dump_tree", {"bundle_id": package_name}) 26 | xml = converter(xml) 27 | root = ET.fromstring(xml.encode('utf-8')) 28 | return root -------------------------------------------------------------------------------- /auto_nico/console_scripts/test_/pyscrcpy/__init__.py: -------------------------------------------------------------------------------- 1 | from .const import * 2 | from .core import Client 3 | -------------------------------------------------------------------------------- /auto_nico/console_scripts/test_/pyscrcpy/const.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module includes all consts used in this project 3 | """ 4 | 5 | # Action 6 | ACTION_DOWN = 0 7 | ACTION_UP = 1 8 | ACTION_MOVE = 2 9 | 10 | # KeyCode 11 | KEYCODE_UNKNOWN = 0 12 | KEYCODE_SOFT_LEFT = 1 13 | KEYCODE_SOFT_RIGHT = 2 14 | KEYCODE_HOME = 3 15 | KEYCODE_BACK = 4 16 | KEYCODE_CALL = 5 17 | KEYCODE_ENDCALL = 6 18 | KEYCODE_0 = 7 19 | KEYCODE_1 = 8 20 | KEYCODE_2 = 9 21 | KEYCODE_3 = 10 22 | KEYCODE_4 = 11 23 | KEYCODE_5 = 12 24 | KEYCODE_6 = 13 25 | KEYCODE_7 = 14 26 | KEYCODE_8 = 15 27 | KEYCODE_9 = 16 28 | KEYCODE_STAR = 17 29 | KEYCODE_POUND = 18 30 | KEYCODE_DPAD_UP = 19 31 | KEYCODE_DPAD_DOWN = 20 32 | KEYCODE_DPAD_LEFT = 21 33 | KEYCODE_DPAD_RIGHT = 22 34 | KEYCODE_DPAD_CENTER = 23 35 | KEYCODE_VOLUME_UP = 24 36 | KEYCODE_VOLUME_DOWN = 25 37 | KEYCODE_POWER = 26 38 | KEYCODE_CAMERA = 27 39 | KEYCODE_CLEAR = 28 40 | KEYCODE_A = 29 41 | KEYCODE_B = 30 42 | KEYCODE_C = 31 43 | KEYCODE_D = 32 44 | KEYCODE_E = 33 45 | KEYCODE_F = 34 46 | KEYCODE_G = 35 47 | KEYCODE_H = 36 48 | KEYCODE_I = 37 49 | KEYCODE_J = 38 50 | KEYCODE_K = 39 51 | KEYCODE_L = 40 52 | KEYCODE_M = 41 53 | KEYCODE_N = 42 54 | KEYCODE_O = 43 55 | KEYCODE_P = 44 56 | KEYCODE_Q = 45 57 | KEYCODE_R = 46 58 | KEYCODE_S = 47 59 | KEYCODE_T = 48 60 | KEYCODE_U = 49 61 | KEYCODE_V = 50 62 | KEYCODE_W = 51 63 | KEYCODE_X = 52 64 | KEYCODE_Y = 53 65 | KEYCODE_Z = 54 66 | KEYCODE_COMMA = 55 67 | KEYCODE_PERIOD = 56 68 | KEYCODE_ALT_LEFT = 57 69 | KEYCODE_ALT_RIGHT = 58 70 | KEYCODE_SHIFT_LEFT = 59 71 | KEYCODE_SHIFT_RIGHT = 60 72 | KEYCODE_TAB = 61 73 | KEYCODE_SPACE = 62 74 | KEYCODE_SYM = 63 75 | KEYCODE_EXPLORER = 64 76 | KEYCODE_ENVELOPE = 65 77 | KEYCODE_ENTER = 66 78 | KEYCODE_DEL = 67 79 | KEYCODE_GRAVE = 68 80 | KEYCODE_MINUS = 69 81 | KEYCODE_EQUALS = 70 82 | KEYCODE_LEFT_BRACKET = 71 83 | KEYCODE_RIGHT_BRACKET = 72 84 | KEYCODE_BACKSLASH = 73 85 | KEYCODE_SEMICOLON = 74 86 | KEYCODE_APOSTROPHE = 75 87 | KEYCODE_SLASH = 76 88 | KEYCODE_AT = 77 89 | KEYCODE_NUM = 78 90 | KEYCODE_HEADSETHOOK = 79 91 | KEYCODE_PLUS = 81 92 | KEYCODE_MENU = 82 93 | KEYCODE_NOTIFICATION = 83 94 | KEYCODE_SEARCH = 84 95 | KEYCODE_MEDIA_PLAY_PAUSE = 85 96 | KEYCODE_MEDIA_STOP = 86 97 | KEYCODE_MEDIA_NEXT = 87 98 | KEYCODE_MEDIA_PREVIOUS = 88 99 | KEYCODE_MEDIA_REWIND = 89 100 | KEYCODE_MEDIA_FAST_FORWARD = 90 101 | KEYCODE_MUTE = 91 102 | KEYCODE_PAGE_UP = 92 103 | KEYCODE_PAGE_DOWN = 93 104 | KEYCODE_BUTTON_A = 96 105 | KEYCODE_BUTTON_B = 97 106 | KEYCODE_BUTTON_C = 98 107 | KEYCODE_BUTTON_X = 99 108 | KEYCODE_BUTTON_Y = 100 109 | KEYCODE_BUTTON_Z = 101 110 | KEYCODE_BUTTON_L1 = 102 111 | KEYCODE_BUTTON_R1 = 103 112 | KEYCODE_BUTTON_L2 = 104 113 | KEYCODE_BUTTON_R2 = 105 114 | KEYCODE_BUTTON_THUMBL = 106 115 | KEYCODE_BUTTON_THUMBR = 107 116 | KEYCODE_BUTTON_START = 108 117 | KEYCODE_BUTTON_SELECT = 109 118 | KEYCODE_BUTTON_MODE = 110 119 | KEYCODE_ESCAPE = 111 120 | KEYCODE_FORWARD_DEL = 112 121 | KEYCODE_CTRL_LEFT = 113 122 | KEYCODE_CTRL_RIGHT = 114 123 | KEYCODE_CAPS_LOCK = 115 124 | KEYCODE_SCROLL_LOCK = 116 125 | KEYCODE_META_LEFT = 117 126 | KEYCODE_META_RIGHT = 118 127 | KEYCODE_FUNCTION = 119 128 | KEYCODE_SYSRQ = 120 129 | KEYCODE_BREAK = 121 130 | KEYCODE_MOVE_HOME = 122 131 | KEYCODE_MOVE_END = 123 132 | KEYCODE_INSERT = 124 133 | KEYCODE_FORWARD = 125 134 | KEYCODE_MEDIA_PLAY = 126 135 | KEYCODE_MEDIA_PAUSE = 127 136 | KEYCODE_MEDIA_CLOSE = 128 137 | KEYCODE_MEDIA_EJECT = 129 138 | KEYCODE_MEDIA_RECORD = 130 139 | KEYCODE_F1 = 131 140 | KEYCODE_F2 = 132 141 | KEYCODE_F3 = 133 142 | KEYCODE_F4 = 134 143 | KEYCODE_F5 = 135 144 | KEYCODE_F6 = 136 145 | KEYCODE_F7 = 137 146 | KEYCODE_F8 = 138 147 | KEYCODE_F9 = 139 148 | KEYCODE_F10 = 140 149 | KEYCODE_F11 = 141 150 | KEYCODE_F12 = 142 151 | KEYCODE_NUM_LOCK = 143 152 | KEYCODE_NUMPAD_0 = 144 153 | KEYCODE_NUMPAD_1 = 145 154 | KEYCODE_NUMPAD_2 = 146 155 | KEYCODE_NUMPAD_3 = 147 156 | KEYCODE_NUMPAD_4 = 148 157 | KEYCODE_NUMPAD_5 = 149 158 | KEYCODE_NUMPAD_6 = 150 159 | KEYCODE_NUMPAD_7 = 151 160 | KEYCODE_NUMPAD_8 = 152 161 | KEYCODE_NUMPAD_9 = 153 162 | KEYCODE_NUMPAD_DIVIDE = 154 163 | KEYCODE_NUMPAD_MULTIPLY = 155 164 | KEYCODE_NUMPAD_SUBTRACT = 156 165 | KEYCODE_NUMPAD_ADD = 157 166 | KEYCODE_NUMPAD_DOT = 158 167 | KEYCODE_NUMPAD_COMMA = 159 168 | KEYCODE_NUMPAD_ENTER = 160 169 | KEYCODE_NUMPAD_EQUALS = 161 170 | KEYCODE_NUMPAD_LEFT_PAREN = 162 171 | KEYCODE_NUMPAD_RIGHT_PAREN = 163 172 | KEYCODE_VOLUME_MUTE = 164 173 | KEYCODE_INFO = 165 174 | KEYCODE_CHANNEL_UP = 166 175 | KEYCODE_CHANNEL_DOWN = 167 176 | KEYCODE_ZOOM_IN = 168 177 | KEYCODE_ZOOM_OUT = 169 178 | KEYCODE_TV = 170 179 | KEYCODE_WINDOW = 171 180 | KEYCODE_GUIDE = 172 181 | KEYCODE_DVR = 173 182 | KEYCODE_BOOKMARK = 174 183 | KEYCODE_CAPTIONS = 175 184 | KEYCODE_SETTINGS = 176 185 | KEYCODE_TV_POWER = 177 186 | KEYCODE_TV_INPUT = 178 187 | KEYCODE_STB_POWER = 179 188 | KEYCODE_STB_INPUT = 180 189 | KEYCODE_AVR_POWER = 181 190 | KEYCODE_AVR_INPUT = 182 191 | KEYCODE_PROG_RED = 183 192 | KEYCODE_PROG_GREEN = 184 193 | KEYCODE_PROG_YELLOW = 185 194 | KEYCODE_PROG_BLUE = 186 195 | KEYCODE_APP_SWITCH = 187 196 | KEYCODE_BUTTON_1 = 188 197 | KEYCODE_BUTTON_2 = 189 198 | KEYCODE_BUTTON_3 = 190 199 | KEYCODE_BUTTON_4 = 191 200 | KEYCODE_BUTTON_5 = 192 201 | KEYCODE_BUTTON_6 = 193 202 | KEYCODE_BUTTON_7 = 194 203 | KEYCODE_BUTTON_8 = 195 204 | KEYCODE_BUTTON_9 = 196 205 | KEYCODE_BUTTON_10 = 197 206 | KEYCODE_BUTTON_11 = 198 207 | KEYCODE_BUTTON_12 = 199 208 | KEYCODE_BUTTON_13 = 200 209 | KEYCODE_BUTTON_14 = 201 210 | KEYCODE_BUTTON_15 = 202 211 | KEYCODE_BUTTON_16 = 203 212 | KEYCODE_LANGUAGE_SWITCH = 204 213 | KEYCODE_MANNER_MODE = 205 214 | KEYCODE_3D_MODE = 206 215 | KEYCODE_CONTACTS = 207 216 | KEYCODE_CALENDAR = 208 217 | KEYCODE_MUSIC = 209 218 | KEYCODE_CALCULATOR = 210 219 | KEYCODE_ZENKAKU_HANKAKU = 211 220 | KEYCODE_EISU = 212 221 | KEYCODE_MUHENKAN = 213 222 | KEYCODE_HENKAN = 214 223 | KEYCODE_KATAKANA_HIRAGANA = 215 224 | KEYCODE_YEN = 216 225 | KEYCODE_RO = 217 226 | KEYCODE_KANA = 218 227 | KEYCODE_ASSIST = 219 228 | KEYCODE_BRIGHTNESS_DOWN = 220 229 | KEYCODE_BRIGHTNESS_UP = 221 230 | KEYCODE_MEDIA_AUDIO_TRACK = 222 231 | KEYCODE_SLEEP = 223 232 | KEYCODE_WAKEUP = 224 233 | KEYCODE_PAIRING = 225 234 | KEYCODE_MEDIA_TOP_MENU = 226 235 | KEYCODE_11 = 227 236 | KEYCODE_12 = 228 237 | KEYCODE_LAST_CHANNEL = 229 238 | KEYCODE_TV_DATA_SERVICE = 230 239 | KEYCODE_VOICE_ASSIST = 231 240 | KEYCODE_TV_RADIO_SERVICE = 232 241 | KEYCODE_TV_TELETEXT = 233 242 | KEYCODE_TV_NUMBER_ENTRY = 234 243 | KEYCODE_TV_TERRESTRIAL_ANALOG = 235 244 | KEYCODE_TV_TERRESTRIAL_DIGITAL = 236 245 | KEYCODE_TV_SATELLITE = 237 246 | KEYCODE_TV_SATELLITE_BS = 238 247 | KEYCODE_TV_SATELLITE_CS = 239 248 | KEYCODE_TV_SATELLITE_SERVICE = 240 249 | KEYCODE_TV_NETWORK = 241 250 | KEYCODE_TV_ANTENNA_CABLE = 242 251 | KEYCODE_TV_INPUT_HDMI_1 = 243 252 | KEYCODE_TV_INPUT_HDMI_2 = 244 253 | KEYCODE_TV_INPUT_HDMI_3 = 245 254 | KEYCODE_TV_INPUT_HDMI_4 = 246 255 | KEYCODE_TV_INPUT_COMPOSITE_1 = 247 256 | KEYCODE_TV_INPUT_COMPOSITE_2 = 248 257 | KEYCODE_TV_INPUT_COMPONENT_1 = 249 258 | KEYCODE_TV_INPUT_COMPONENT_2 = 250 259 | KEYCODE_TV_INPUT_VGA_1 = 251 260 | KEYCODE_TV_AUDIO_DESCRIPTION = 252 261 | KEYCODE_TV_AUDIO_DESCRIPTION_MIX_UP = 253 262 | KEYCODE_TV_AUDIO_DESCRIPTION_MIX_DOWN = 254 263 | KEYCODE_TV_ZOOM_MODE = 255 264 | KEYCODE_TV_CONTENTS_MENU = 256 265 | KEYCODE_TV_MEDIA_CONTEXT_MENU = 257 266 | KEYCODE_TV_TIMER_PROGRAMMING = 258 267 | KEYCODE_HELP = 259 268 | KEYCODE_NAVIGATE_PREVIOUS = 260 269 | KEYCODE_NAVIGATE_NEXT = 261 270 | KEYCODE_NAVIGATE_IN = 262 271 | KEYCODE_NAVIGATE_OUT = 263 272 | KEYCODE_STEM_PRIMARY = 264 273 | KEYCODE_STEM_1 = 265 274 | KEYCODE_STEM_2 = 266 275 | KEYCODE_STEM_3 = 267 276 | KEYCODE_DPAD_UP_LEFT = 268 277 | KEYCODE_DPAD_DOWN_LEFT = 269 278 | KEYCODE_DPAD_UP_RIGHT = 270 279 | KEYCODE_DPAD_DOWN_RIGHT = 271 280 | KEYCODE_MEDIA_SKIP_FORWARD = 272 281 | KEYCODE_MEDIA_SKIP_BACKWARD = 273 282 | KEYCODE_MEDIA_STEP_FORWARD = 274 283 | KEYCODE_MEDIA_STEP_BACKWARD = 275 284 | KEYCODE_SOFT_SLEEP = 276 285 | KEYCODE_CUT = 277 286 | KEYCODE_COPY = 278 287 | KEYCODE_PASTE = 279 288 | KEYCODE_SYSTEM_NAVIGATION_UP = 280 289 | KEYCODE_SYSTEM_NAVIGATION_DOWN = 281 290 | KEYCODE_SYSTEM_NAVIGATION_LEFT = 282 291 | KEYCODE_SYSTEM_NAVIGATION_RIGHT = 283 292 | KEYCODE_KEYCODE_ALL_APPS = 284 293 | KEYCODE_KEYCODE_REFRESH = 285 294 | KEYCODE_KEYCODE_THUMBS_UP = 286 295 | KEYCODE_KEYCODE_THUMBS_DOWN = 287 296 | 297 | # Event 298 | EVENT_INIT = "init" 299 | EVENT_FRAME = "frame" 300 | EVENT_DISCONNECT = "disconnect" 301 | EVENT_ONCHANGE = "onchange" 302 | 303 | # Type 304 | TYPE_INJECT_KEYCODE = 0 305 | TYPE_INJECT_TEXT = 1 306 | TYPE_INJECT_TOUCH_EVENT = 2 307 | TYPE_INJECT_SCROLL_EVENT = 3 308 | TYPE_BACK_OR_SCREEN_ON = 4 309 | TYPE_EXPAND_NOTIFICATION_PANEL = 5 310 | TYPE_EXPAND_SETTINGS_PANEL = 6 311 | TYPE_COLLAPSE_PANELS = 7 312 | TYPE_GET_CLIPBOARD = 8 313 | TYPE_SET_CLIPBOARD = 9 314 | TYPE_SET_SCREEN_POWER_MODE = 10 315 | TYPE_ROTATE_DEVICE = 11 316 | 317 | # Lock screen orientation 318 | LOCK_SCREEN_ORIENTATION_UNLOCKED = -1 319 | LOCK_SCREEN_ORIENTATION_INITIAL = -2 320 | LOCK_SCREEN_ORIENTATION_0 = 0 321 | LOCK_SCREEN_ORIENTATION_1 = 1 322 | LOCK_SCREEN_ORIENTATION_2 = 2 323 | LOCK_SCREEN_ORIENTATION_3 = 3 324 | 325 | # Screen power mode 326 | POWER_MODE_OFF = 0 327 | POWER_MODE_NORMAL = 2 328 | -------------------------------------------------------------------------------- /auto_nico/console_scripts/test_/pyscrcpy/control.py: -------------------------------------------------------------------------------- 1 | import functools 2 | import socket 3 | import struct 4 | import time 5 | from . import const 6 | 7 | 8 | def inject(control_type: int): 9 | """ 10 | Inject control code, with this inject, we will be able to do unit test_ 11 | 12 | Args: 13 | control_type: event to send, TYPE_* 14 | """ 15 | 16 | def wrapper(f): 17 | @functools.wraps(f) 18 | def inner(*args, **kwargs): 19 | package = struct.pack(">B", control_type) + f(*args, **kwargs) 20 | if args[0].parent.control_socket is not None: 21 | with args[0].parent.control_socket_lock: 22 | args[0].parent.control_socket.send(package) 23 | return package 24 | 25 | return inner 26 | 27 | return wrapper 28 | 29 | 30 | class ControlSender: 31 | def __init__(self, parent): 32 | self.parent = parent # client object 33 | self.adbutil_devices = parent.device 34 | 35 | @inject(const.TYPE_INJECT_KEYCODE) 36 | def keycode( 37 | self, keycode: int, action: int = const.ACTION_DOWN, repeat: int = 0 38 | ) -> bytes: 39 | """ 40 | Send keycode to device 41 | 42 | Args: 43 | keycode: const.KEYCODE_* 44 | action: ACTION_DOWN | ACTION_UP 45 | repeat: repeat count 46 | """ 47 | return struct.pack(">Biii", action, keycode, repeat, 0) 48 | 49 | @inject(const.TYPE_INJECT_TEXT) 50 | def text(self, text: str) -> bytes: 51 | """ 52 | Send text to device 53 | 54 | Args: 55 | text: text to send 56 | """ 57 | 58 | buffer = text.encode("utf-8") 59 | return struct.pack(">i", len(buffer)) + buffer 60 | 61 | @inject(const.TYPE_INJECT_TOUCH_EVENT) 62 | def touch(self, x: int, y: int, action: int = const.ACTION_DOWN, touch_id: int = -1) -> bytes: 63 | """ 64 | Touch screen 65 | 66 | Args: 67 | x: horizontal position 68 | y: vertical position 69 | action: ACTION_DOWN | ACTION_UP | ACTION_MOVE 70 | touch_id: Default using virtual id -1, you can specify it to emulate multi finger touch 71 | """ 72 | x, y = max(x, 0), max(y, 0) 73 | print(self.parent.resolution) 74 | return struct.pack( 75 | ">BqiiHHHi", 76 | action, 77 | touch_id, 78 | int(x), 79 | int(y), 80 | int(self.parent.resolution[0]), 81 | int(self.parent.resolution[1]), 82 | 0xFFFF, 83 | 1, 84 | ) 85 | 86 | def touch_down(self, x, y): 87 | self.touch(x, y, const.ACTION_DOWN) 88 | 89 | def touch_up(self, x, y): 90 | self.touch(x, y, const.ACTION_UP) 91 | 92 | def touch_move(self, x, y): 93 | self.touch(x, y, const.ACTION_MOVE) 94 | 95 | @inject(const.TYPE_INJECT_SCROLL_EVENT) 96 | def scroll(self, x: int, y: int, h: int, v: int) -> bytes: 97 | """ 98 | Scroll screen 99 | 100 | Args: 101 | x: horizontal position 102 | y: vertical position 103 | h: horizontal movement 104 | v: vertical movement 105 | """ 106 | 107 | x, y = max(x, 0), max(y, 0) 108 | return struct.pack( 109 | ">iiHHii", 110 | int(x), 111 | int(y), 112 | int(self.parent.resolution[0]), 113 | int(self.parent.resolution[1]), 114 | int(h), 115 | int(v), 116 | ) 117 | 118 | @inject(const.TYPE_BACK_OR_SCREEN_ON) 119 | def back_or_turn_screen_on(self, action: int = const.ACTION_DOWN) -> bytes: 120 | """ 121 | If the screen is off, it is turned on only on ACTION_DOWN 122 | 123 | Args: 124 | action: ACTION_DOWN | ACTION_UP 125 | """ 126 | return struct.pack(">B", action) 127 | 128 | @inject(const.TYPE_EXPAND_NOTIFICATION_PANEL) 129 | def expand_notification_panel(self) -> bytes: 130 | """ 131 | Expand notification panel 132 | """ 133 | return b"" 134 | 135 | @inject(const.TYPE_EXPAND_SETTINGS_PANEL) 136 | def expand_settings_panel(self) -> bytes: 137 | """ 138 | Expand settings panel 139 | """ 140 | return b"" 141 | 142 | @inject(const.TYPE_COLLAPSE_PANELS) 143 | def collapse_panels(self) -> bytes: 144 | """ 145 | Collapse all panels 146 | """ 147 | return b"" 148 | 149 | def get_clipboard(self) -> str: 150 | """ 151 | Get clipboard 152 | """ 153 | # Since this function need socket response, we can't auto inject it any more 154 | s: socket.socket = self.parent.control_socket 155 | 156 | with self.parent.control_socket_lock: 157 | # Flush socket 158 | s.setblocking(False) 159 | while True: 160 | try: 161 | s.recv(1024) 162 | except BlockingIOError: 163 | break 164 | s.setblocking(True) 165 | 166 | # Read package 167 | package = struct.pack(">B", const.TYPE_GET_CLIPBOARD) 168 | s.send(package) 169 | (code,) = struct.unpack(">B", s.recv(1)) 170 | assert code == 0 171 | (length,) = struct.unpack(">i", s.recv(4)) 172 | 173 | return s.recv(length).decode("utf-8") 174 | 175 | @inject(const.TYPE_SET_CLIPBOARD) 176 | def set_clipboard(self, text: str, paste: bool = False) -> bytes: 177 | """ 178 | Set clipboard 179 | 180 | Args: 181 | text: the string you want to set 182 | paste: paste now 183 | """ 184 | buffer = text.encode("utf-8") 185 | return struct.pack(">?i", paste, len(buffer)) + buffer 186 | 187 | @inject(const.TYPE_SET_SCREEN_POWER_MODE) 188 | def set_screen_power_mode( 189 | self, mode: int = const.POWER_MODE_NORMAL 190 | ) -> bytes: 191 | """ 192 | Set screen power mode 193 | 194 | Args: 195 | mode: POWER_MODE_OFF | POWER_MODE_NORMAL 196 | """ 197 | return struct.pack(">b", mode) 198 | 199 | @inject(const.TYPE_ROTATE_DEVICE) 200 | def rotate_device(self) -> bytes: 201 | """ 202 | Rotate device 203 | """ 204 | return b"" 205 | 206 | def swipe( 207 | self, 208 | start_x: int, 209 | start_y: int, 210 | end_x: int, 211 | end_y: int, 212 | move_step_length: int = 5, 213 | move_steps_delay: float = 0.005, 214 | ) -> None: 215 | """ 216 | Swipe on screen 217 | 218 | Args: 219 | start_x: start horizontal position 220 | start_y: start vertical position 221 | end_x: start horizontal position 222 | end_y: end vertical position 223 | move_step_length: length per step 224 | move_steps_delay: sleep seconds after each step 225 | :return: 226 | """ 227 | 228 | self.touch(start_x, start_y, const.ACTION_DOWN) 229 | next_x = start_x 230 | next_y = start_y 231 | 232 | if end_x > self.parent.resolution[0]: 233 | end_x = self.parent.resolution[0] 234 | 235 | if end_y > self.parent.resolution[1]: 236 | end_y = self.parent.resolution[1] 237 | 238 | decrease_x = True if start_x > end_x else False 239 | decrease_y = True if start_y > end_y else False 240 | while True: 241 | if decrease_x: 242 | next_x -= move_step_length 243 | if next_x < end_x: 244 | next_x = end_x 245 | else: 246 | next_x += move_step_length 247 | if next_x > end_x: 248 | next_x = end_x 249 | 250 | if decrease_y: 251 | next_y -= move_step_length 252 | if next_y < end_y: 253 | next_y = end_y 254 | else: 255 | next_y += move_step_length 256 | if next_y > end_y: 257 | next_y = end_y 258 | 259 | self.touch(next_x, next_y, const.ACTION_MOVE) 260 | 261 | if next_x == end_x and next_y == end_y: 262 | self.touch(next_x, next_y, const.ACTION_UP) 263 | break 264 | time.sleep(move_steps_delay) 265 | -------------------------------------------------------------------------------- /auto_nico/console_scripts/test_/pyscrcpy/core.py: -------------------------------------------------------------------------------- 1 | import os 2 | import abc 3 | from pathlib import Path 4 | import socket 5 | import struct 6 | import threading 7 | import time 8 | from time import sleep 9 | from typing import Any, Callable, Optional, Tuple, Union 10 | 11 | import numpy as np 12 | import numpy.typing as npt 13 | from adbutils import AdbConnection, AdbDevice, AdbError, Network, adb 14 | from av.codec import CodecContext # type: ignore 15 | from av.error import InvalidDataError # type: ignore 16 | import cv2 as cv 17 | import cv2 18 | from loguru import logger 19 | 20 | from .const import EVENT_DISCONNECT, EVENT_FRAME, EVENT_INIT, LOCK_SCREEN_ORIENTATION_UNLOCKED, EVENT_ONCHANGE 21 | from .control import ControlSender 22 | 23 | Frame = npt.NDArray[np.int8] 24 | 25 | VERSION = "1.20" 26 | HERE = Path(__file__).resolve().parent 27 | JAR = HERE / f"scrcpy-server.jar" 28 | 29 | 30 | class Client: 31 | def __init__( 32 | self, 33 | device: Optional[Union[AdbDevice, str]] = None, 34 | max_size: int = 0, 35 | bitrate: int = 8000000, 36 | max_fps: int = 0, 37 | block_frame: bool = True, 38 | stay_awake: bool = True, 39 | lock_screen_orientation: int = LOCK_SCREEN_ORIENTATION_UNLOCKED, 40 | skip_same_frame=False 41 | ): 42 | """ 43 | [ok]Create a scrcpy client. The client won't be started until you call .start() 44 | 45 | Args: 46 | device: Android device to coennect to. Colud be also specify by 47 | serial string. If device is None the client try to connect 48 | to the first available device in adb deamon. 49 | max_size: Specify the maximum dimension of the video stream. This 50 | dimensioin refer both to width and hight.0: no limit[已校验, max size of width or height] 51 | bitrate: bitrate 52 | max_fps: Maximum FPS (Frame Per Second) of the video stream. If it 53 | is set to 0 it means that there is not limit to FPS. 54 | This feature is supported by android 10 or newer. 55 | [flip]: 没有这个参数, 会自动处理 56 | block_frame: If set to true, the on_frame callbacks will be only 57 | apply on not empty frames. Otherwise try to apply on_frame 58 | callbacks on every frame, but this could raise exceptions in 59 | callbacks if they are not able to handle None value for frame. 60 | True:跳过空白帧 61 | stay_awake: keep Android device awake while the client-server 62 | connection is alive. 63 | lock_screen_orientation: lock screen in a particular orientation. 64 | The available screen orientation are specify in const.py 65 | in variables LOCK_SCREEN_ORIENTATION* 66 | """ 67 | # Params挪到后面去 68 | self.max_size = max_size 69 | self.bitrate = bitrate 70 | self.max_fps = max_fps 71 | self.block_frame = block_frame 72 | self.stay_awake = stay_awake 73 | self.lock_screen_orientation = lock_screen_orientation 74 | self.skip_same_frame = skip_same_frame 75 | self.min_frame_interval = 1 / max_fps 76 | 77 | if device is None: 78 | try: 79 | device = adb.device_list()[0] 80 | except IndexError: 81 | raise Exception("Cannot connect to phone") 82 | elif isinstance(device, str): 83 | device = adb.device(serial=device) 84 | 85 | self.device = device 86 | self.listeners = dict(frame=[], init=[], disconnect=[], onchange=[]) 87 | 88 | # User accessible 89 | self.last_frame: Optional[np.ndarray] = None 90 | self.resolution: Optional[Tuple[int, int]] = None 91 | self.device_name: Optional[str] = None 92 | self.control = ControlSender(self) 93 | 94 | # Need to destroy 95 | self.alive = False 96 | self.__server_stream: Optional[AdbConnection] = None 97 | self.__video_socket: Optional[socket.socket] = None 98 | self.control_socket: Optional[socket.socket] = None 99 | self.control_socket_lock = threading.Lock() 100 | 101 | def __init_server_connection(self) -> None: 102 | """ 103 | Connect to android server, there will be two sockets: video and control socket. 104 | This method will also set resolution property. 105 | """ 106 | for _ in range(30): # 超时 写死 107 | try: 108 | self.__video_socket = self.device.create_connection( 109 | Network.LOCAL_ABSTRACT, "scrcpy" 110 | ) 111 | break 112 | except AdbError: 113 | sleep(0.1) 114 | pass 115 | else: 116 | raise ConnectionError("Failed to connect scrcpy-server after 3 seconds") 117 | 118 | dummy_byte = self.__video_socket.recv(1) 119 | if not len(dummy_byte): 120 | raise ConnectionError("Did not receive Dummy Byte!") 121 | 122 | self.control_socket = self.device.create_connection( 123 | Network.LOCAL_ABSTRACT, "scrcpy" 124 | ) 125 | self.device_name = self.__video_socket.recv(64).decode("utf-8").rstrip("\x00") 126 | if not len(self.device_name): 127 | raise ConnectionError("Did not receive Device Name!") 128 | 129 | res = self.__video_socket.recv(4) 130 | self.resolution = struct.unpack(">HH", res) 131 | self.__video_socket.setblocking(False) 132 | 133 | def __deploy_server(self) -> None: 134 | """ 135 | Deploy server to android device. 136 | Push the scrcpy-server.jar into the Android device using 137 | the adb.push(...). Then a basic connection between client and server 138 | is established. 139 | """ 140 | cmd = [ 141 | "CLASSPATH=/data/local/tmp/scrcpy-server.jar", 142 | "app_process", 143 | "/", 144 | "com.genymobile.scrcpy.Server", 145 | VERSION, # Scrcpy server version 146 | "info", # Log level: info, verbose... 147 | f"{self.max_size}", # Max screen width (long side) 148 | f"{self.bitrate}", # Bitrate of video 149 | f"{self.max_fps}", # Max frame per second 150 | f"{self.lock_screen_orientation}", # Lock screen orientation 151 | "true", # Tunnel forward 152 | "-", # Crop screen 153 | "false", # Send frame rate to client 154 | "true", # Control enabled 155 | "0", # Display id 156 | "false", # Show touches 157 | "true" if self.stay_awake else "false", # Stay awake 158 | "-", # Codec (video encoding) options 159 | "-", # Encoder name 160 | "false", # Power off screen after server closed 161 | ] 162 | self.device.push(JAR, "/data/local/tmp/") 163 | self.__server_stream: AdbConnection = self.device.shell(cmd, stream=True) 164 | 165 | def start(self, threaded: bool = False) -> None: 166 | """ 167 | Start the client-server connection. 168 | In order to avoid unpredictable behaviors, this method must be called 169 | after the on_init and on_frame callback are specify. 170 | 171 | Args: 172 | threaded : If set to True the stream loop willl run in a separated 173 | thread. This mean that the code after client.strart() will be 174 | run. Otherwise the client.start() method starts a endless loop 175 | and the code after this method will never run. todo new_thread 176 | """ 177 | assert self.alive is False 178 | 179 | self.__deploy_server() 180 | self.__init_server_connection() 181 | self.alive = True 182 | for func in self.listeners[EVENT_INIT]: 183 | func(self) 184 | 185 | if threaded: # 不阻塞当前thread 186 | threading.Thread(target=self.__stream_loop).start() 187 | else: 188 | self.__stream_loop() 189 | 190 | def stop(self) -> None: 191 | """ 192 | [ok]Close the various socket connection. 193 | Stop listening (both threaded and blocked) 194 | """ 195 | self.alive = False 196 | try: 197 | self.__server_stream.close() 198 | except Exception: 199 | pass 200 | try: 201 | self.control_socket.close() 202 | except Exception: 203 | pass 204 | try: 205 | self.__video_socket.close() 206 | except Exception: 207 | pass 208 | 209 | def __del__(self): 210 | self.stop() 211 | 212 | def __calculate_diff(self, img1, img2): 213 | if img1 is None: 214 | return 1 215 | gray1 = cv.cvtColor(img1, cv.COLOR_BGR2GRAY) 216 | gray2 = cv.cvtColor(img2, cv.COLOR_BGR2GRAY) 217 | 218 | # 计算两张灰度图像的差异 219 | diff = cv2.absdiff(gray1, gray2) 220 | 221 | # 设置阈值，忽略差异值较小的像素 222 | threshold = 30 223 | _, thresholded_diff = cv2.threshold(diff, threshold, 255, cv2.THRESH_BINARY) 224 | 225 | # 计算差异像素的总数 226 | total_diff_pixels = np.sum(thresholded_diff / 255) # 除以255得到二值图像中白色像素的数量 227 | 228 | # 计算图像的总像素数 229 | total_pixels = gray1.size 230 | 231 | # 计算变化率 232 | change_rate = total_diff_pixels / total_pixels 233 | return change_rate 234 | 235 | def __stream_loop(self) -> None: 236 | """ 237 | Core loop for video parsing. 238 | While the connection is open (self.alive == True) recive raw h264 video 239 | stream and decode it into frames. These frame are those passed to 240 | on_frame callbacks. 241 | """ 242 | codec = CodecContext.create("h264", "r") 243 | while self.alive: 244 | try: 245 | raw = self.__video_socket.recv(0x10000) 246 | if raw == b"": 247 | raise ConnectionError("Video stream is disconnected") 248 | for packet in codec.parse(raw): 249 | for frame in codec.decode(packet): # codec.decode(packet)包含多帧 250 | frame = frame.to_ndarray(format="bgr24") 251 | 252 | if len(self.listeners[EVENT_ONCHANGE]) == 0 and not self.skip_same_frame: 253 | self.last_frame = frame 254 | elif self.__calculate_diff(self.last_frame, frame) > 0.1: 255 | logger.debug("different frame detected") 256 | self.last_frame = frame 257 | for func in self.listeners[EVENT_ONCHANGE]: 258 | func(self, frame) 259 | else: # no_change and should skip this frame 260 | continue 261 | 262 | self.resolution = (frame.shape[1], frame.shape[0]) 263 | for func in self.listeners[EVENT_FRAME]: # 发送给用户自定义的函数 264 | func(self, frame) 265 | except (BlockingIOError, InvalidDataError): # empty frame 266 | time.sleep(0.01) 267 | if not self.block_frame: # init时允许空白帧 268 | for func in self.listeners[EVENT_FRAME]: 269 | func(self, None) 270 | except (ConnectionError, OSError) as e: # Socket Closed 271 | if self.alive: 272 | # todo on_disconnect event 273 | self.stop() 274 | raise e 275 | 276 | def on_init(self, func: Callable[[Any], None]) -> None: 277 | """ 278 | Add funtion to on_init listeners. 279 | Your function is run after client.start() is called. 280 | 281 | Args: 282 | func: callback to be called after the server starts. 参数:这个class的obj 283 | """ 284 | self.listeners[EVENT_INIT].append(func) 285 | 286 | def on_frame(self, func: Callable[[Any, Frame], None]): 287 | """ 288 | Add functoin to on-frame listeners. 289 | Your function will be run on every valid frame recived. 290 | 291 | Args: 292 | func: callback to be called on every frame. 293 | 294 | Returns: 295 | The list of on-frame callbacks. 296 | """ 297 | self.listeners[EVENT_FRAME].append(func) 298 | 299 | def on_change(self, func: Callable[[Any, Frame], None]): 300 | self.listeners[EVENT_ONCHANGE].append(func) 301 | -------------------------------------------------------------------------------- /auto_nico/console_scripts/test_/pyscrcpy/scrcpy-server.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/letmeNo1/Nico/79dc1ac061cdeaa7ab74660d42bf85bd93ff0592/auto_nico/console_scripts/test_/pyscrcpy/scrcpy-server.jar -------------------------------------------------------------------------------- /auto_nico/console_scripts/test_/pyscrcpy/scrcpy_utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | import subprocess 4 | import cv2 as cv 5 | from pyscrcpy import Client 6 | import io 7 | import threading 8 | from cachetools import TTLCache 9 | from io import BytesIO 10 | from PIL import Image 11 | 12 | # 存储每个设备的帧缓存 13 | frame_buffers = {} 14 | # 用于线程安全的锁 15 | buffer_lock = threading.Lock() 16 | 17 | # 缓存字典 18 | caches = {} 19 | clients = {} 20 | starting = {} 21 | request_cache = {} 22 | 23 | 24 | def on_frame(aa, frame, device_id): 25 | global frame_buffers 26 | 27 | # 将帧保存到内存中 28 | _, buffer = cv.imencode('.jpg', frame) 29 | with buffer_lock: 30 | # 降低图片质量为 50% 31 | output_stream = BytesIO() 32 | img = Image.open(BytesIO(buffer)) 33 | img.convert("RGB").save(output_stream, format='JPEG', quality=50) 34 | output_stream.seek(0) 35 | frame_buffer = output_stream 36 | # 更新设备的帧缓存 37 | frame_buffers[device_id] = frame_buffer.getvalue() 38 | # 更新设备的缓存 39 | if device_id not in caches: 40 | # 设置缓存画面最多 120 秒 41 | caches[device_id] = TTLCache(maxsize=1, ttl=120) 42 | # 缓存图像数据 43 | caches[device_id]['image'] = frame_buffers[device_id] 44 | 45 | 46 | client_started_event = threading.Event() 47 | 48 | 49 | def start_client(device_id): 50 | if device_id in clients: 51 | clients[device_id].stop() 52 | del clients[device_id] 53 | if device_id in caches: 54 | del caches[device_id] 55 | 56 | try: 57 | client = Client(device=device_id, max_fps=24, max_size=900, stay_awake=True) 58 | 59 | client.on_frame(lambda c, f: on_frame(c, f, device_id)) # 传递设备 ID 60 | clients[device_id] = client 61 | client.start() 62 | # 通知主线程客户端已启动 63 | client_started_event.set() 64 | except Exception as e: 65 | print(e) 66 | client_started_event.clear() 67 | 68 | 69 | def start_client_by_threading(device_id): 70 | client_thread = threading.Thread(target=start_client, args=(device_id,)) 71 | client_thread.daemon = True 72 | client_thread.start() 73 | 74 | while device_id not in clients: 75 | time.sleep(0.5) 76 | 77 | print("pyscrcpy server 已启动~") 78 | 79 | 80 | def is_device_connected(device_id): 81 | try: 82 | result = subprocess.run(['adb', 'devices'], capture_output=True, text=True, check=True) 83 | 84 | devices_output = result.stdout.strip() 85 | return device_id in devices_output 86 | except subprocess.CalledProcessError as e: 87 | print(f"Error executing adb command: {e}") 88 | return False 89 | 90 | 91 | def get_image(device_id): 92 | # 非第一次请求设备画面，判断距离第一次请求是否超过 2 秒，没超过则等待，超过则大概率 pyscrcpy server 已经启动好了 93 | if device_id in request_cache: 94 | sequence, request_time = request_cache[device_id] 95 | if sequence == 1 and (time.time() - request_time) <= 2: 96 | return {"message": "正在初始化，请稍等几秒中！"} 97 | if sequence == 1 and (time.time() - request_time) > 2: 98 | new_sequence = sequence + 1 99 | request_cache[device_id] = (new_sequence, time.time()) 100 | # 第一次请求设备画面，设置请求顺序为 1 101 | if device_id not in request_cache: 102 | request_cache[device_id] = (1, time.time()) 103 | 104 | # 1. 当设备未连接到电脑的情况 105 | if not is_device_connected(device_id): 106 | return {"message": "设备未连接"} 107 | # 2. 当 pyscrcpy server 没启动的情况 108 | if device_id not in starting or device_id not in clients: 109 | starting[device_id] = (True, time.time()) 110 | start_client_by_threading(device_id) 111 | return {"message": "scrcpy server 尚未启动，启动", "starting": starting} 112 | # 3. pyscrcpy server 已启动，设备连接正常的情况 113 | if clients[device_id].alive: 114 | starting[device_id] = (False, time.time()) 115 | 116 | with buffer_lock: 117 | if device_id in caches and 'image' in caches[device_id]: 118 | image_data = caches[device_id]['image'] 119 | return image_data 120 | # 4. 利用缓存，避免拔插导致的一直 keep 在 starting 状态 121 | is_starting, starting_time = starting[device_id] 122 | if is_starting and (time.time() - starting_time) <= 2: 123 | return {"message": "scrcpy server 启动中", "starting": starting} 124 | if is_starting and (time.time() - starting_time) > 2: 125 | starting[device_id] = (True, time.time()) 126 | start_client_by_threading(device_id) 127 | return {"message": "scrcpy server 启动超时，再次启动", "starting": starting} 128 | # 5. 设备中途掉线、恢复的情况 129 | if not is_starting and not clients[device_id].alive: 130 | starting[device_id] = (True, time.time()) 131 | start_client_by_threading(device_id) 132 | return {"message": "scrcpy server 掉线，启动中", "starting": starting} 133 | # 6. 画面超过缓存时间（120 秒），强制重启 134 | starting[device_id] = (True, time.time()) 135 | start_client_by_threading(device_id) 136 | return {"message": "画面超时，强制重启 scrcpy server", "starting": starting} 137 | 138 | 139 | def action(data, device_id): 140 | client = clients.get(device_id) 141 | if client is None: 142 | return {"message": "Client not found"} 143 | x_percent = data.get('xPercent') 144 | y_percent = data.get('yPercent') 145 | x_percent = float(x_percent) / 100 146 | y_percent = float(y_percent) / 100 147 | x = int(x_percent * int(client.resolution[0])) 148 | y = int(y_percent * int(client.resolution[1])) 149 | if data.get('actionType') == 'touch_down': 150 | client.control.touch_down(x, y) 151 | elif data.get('actionType') == 'touch_up': 152 | client.control.touch_up(x, y) 153 | elif data.get("actionType") == "touch_move": 154 | client.control.touch_move(x, y) 155 | 156 | return {"message": "Click action sent successfully"} 157 | -------------------------------------------------------------------------------- /auto_nico/console_scripts/test_/scrcpy_utils.py: -------------------------------------------------------------------------------- 1 | import time 2 | import subprocess 3 | import cv2 as cv 4 | from pyscrcpy import Client 5 | import threading 6 | from cachetools import TTLCache 7 | from io import BytesIO 8 | from PIL import Image 9 | 10 | # 存储每个设备的帧缓存 11 | frame_buffers = {} 12 | # 用于线程安全的锁 13 | buffer_lock = threading.Lock() 14 | 15 | # 缓存字典 16 | caches = {} 17 | clients = {} 18 | starting = {} 19 | request_cache = {} 20 | client: Client 21 | 22 | 23 | def on_frame(aa, frame, device_id): 24 | global frame_buffers 25 | 26 | # 将帧保存到内存中 27 | _, buffer = cv.imencode('.jpg', frame) 28 | with buffer_lock: 29 | # 降低图片质量为 50% 30 | output_stream = BytesIO() 31 | img = Image.open(BytesIO(buffer)) 32 | img.convert("RGB").save(output_stream, format='JPEG', quality=50) 33 | output_stream.seek(0) 34 | frame_buffer = output_stream 35 | # 更新设备的帧缓存 36 | frame_buffers[device_id] = frame_buffer.getvalue() 37 | # 更新设备的缓存 38 | if device_id not in caches: 39 | # 设置缓存画面最多 120 秒 40 | caches[device_id] = TTLCache(maxsize=24, ttl=120) 41 | # 缓存图像数据 42 | caches[device_id]['image'] = frame_buffers[device_id] 43 | 44 | 45 | client_started_event = threading.Event() 46 | 47 | 48 | def start_client(device_id): 49 | if device_id in clients: 50 | clients[device_id].stop() 51 | del clients[device_id] 52 | if device_id in caches: 53 | del caches[device_id] 54 | 55 | try: 56 | global client 57 | client = Client(device=device_id, max_fps=24, max_size=900, stay_awake=True) 58 | 59 | client.on_frame(lambda c, f: on_frame(c, f, device_id)) # 传递设备 ID 60 | clients[device_id] = client 61 | client.start() 62 | # 通知主线程客户端已启动 63 | client_started_event.set() 64 | except Exception as e: 65 | print(e) 66 | client_started_event.clear() 67 | 68 | 69 | def start_client_by_threading(device_id): 70 | client_thread = threading.Thread(target=start_client, args=(device_id,)) 71 | client_thread.daemon = True 72 | client_thread.start() 73 | 74 | while device_id not in clients: 75 | time.sleep(0.5) 76 | 77 | print("pyscrcpy server 已启动~") 78 | 79 | 80 | def is_device_connected(device_id): 81 | try: 82 | result = subprocess.run(['adb', 'devices'], capture_output=True, text=True, check=True) 83 | 84 | devices_output = result.stdout.strip() 85 | return device_id in devices_output 86 | except subprocess.CalledProcessError as e: 87 | print(f"Error executing adb command: {e}") 88 | return False 89 | -------------------------------------------------------------------------------- /auto_nico/console_scripts/test_/static/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/letmeNo1/Nico/79dc1ac061cdeaa7ab74660d42bf85bd93ff0592/auto_nico/console_scripts/test_/static/__init__.py -------------------------------------------------------------------------------- /auto_nico/console_scripts/test_/static/api_request.js: -------------------------------------------------------------------------------- 1 | let intervalId; 2 | let longPressTimeout; 3 | const longPressDuration = 500; // 500 milliseconds for long press 4 | let startX, startY; 5 | 6 | async function fetchImage() { 7 | const imageElement = document.getElementById('dynamicImage'); 8 | const overlay = document.getElementById('action-layer'); 9 | // 保存原来图片的 src、尺寸和位置 10 | const originalSrc = imageElement.src; 11 | const originalWidth = imageElement.offsetWidth; 12 | const originalHeight = imageElement.offsetHeight; 13 | const originalLeft = imageElement.offsetLeft; 14 | const originalTop = imageElement.offsetTop; 15 | 16 | try { 17 | const response = await fetch(`/dynamic_image`); 18 | if (!response.ok) throw new Error('Network response was not ok'); 19 | 20 | const blob = await response.blob(); 21 | const imageUrl = URL.createObjectURL(blob); 22 | 23 | // 处理图片加载 24 | await new Promise((resolve) => { 25 | const tempImage = new Image(); 26 | tempImage.src = imageUrl; 27 | tempImage.onload = () => { 28 | // 新图片加载完成后更新 src 和尺寸 29 | imageElement.src = imageUrl; 30 | imageElement.style.width = `${originalWidth}px`; 31 | imageElement.style.height = `${originalHeight}px`; 32 | // 恢复原图片位置 33 | imageElement.style.position = 'absolute'; 34 | imageElement.style.left = `${originalLeft}px`; 35 | imageElement.style.top = `${originalTop}px`; 36 | resolve(); 37 | }; 38 | tempImage.onerror = () => { 39 | throw new Error('Image loading failed'); 40 | }; 41 | }); 42 | 43 | console.log(imageElement.offsetWidth); 44 | // 关键：设置遮罩尺寸与图片一致 45 | overlay.style.width = `${originalWidth}px`; 46 | overlay.style.height = `${originalHeight}px`; 47 | overlay.style.left = `${originalLeft}px`; 48 | overlay.style.top = `${originalTop}px`; 49 | overlay.style.display = 'block'; 50 | 51 | } catch (error) { 52 | console.log('Error fetching image:', error); 53 | // 加载失败恢复原来的图片 54 | imageElement.src = originalSrc; 55 | overlay.style.display = 'flex'; 56 | } 57 | } 58 | 59 | 60 | 61 | function fetchStaticImage() { 62 | const imageElement = document.getElementById('dynamicImage'); 63 | const overlay = document.getElementById('action-layer'); 64 | const errorMessage = document.getElementById('error-message'); 65 | $.get('/static_image', function(data) { 66 | var img = document.querySelector('img'); 67 | 68 | // 处理可能的 JSON 格式 69 | if (typeof data === 'object') { 70 | if (data.image) { 71 | img.src = 'data:image/png;base64,' + data.image; 72 | } else { 73 | console.error('无效数据格式，缺少 image 字段'); 74 | return; 75 | } 76 | } else { 77 | // 直接使用字符串 78 | img.src = 'data:image/png;base64,' + data; 79 | } 80 | }).fail(function(jqXHR, textStatus, errorThrown) { 81 | console.error('请求失败:', errorThrown); 82 | }); 83 | } 84 | 85 | 86 | function startFetching() { 87 | if (!intervalId) { 88 | intervalId = setInterval(fetchImage, 30); // Fetch image every 30 milliseconds 89 | } 90 | } 91 | 92 | function stopFetching() { 93 | if (intervalId) { 94 | clearInterval(intervalId); 95 | intervalId = null; 96 | } 97 | } 98 | 99 | fetchStaticImage() 100 | 101 | $.get('/refresh_ui_xml', function(data) { 102 | var xmlContainer = document.querySelector('.content-inner'); 103 | xmlContainer.innerHTML = data; 104 | // initImageControl() 105 | // addTextControlHoverListeners(); // 添加新的事件监听器 106 | // addImageListeners(); // 添加新的事件监听器 107 | }); 108 | 109 | -------------------------------------------------------------------------------- /auto_nico/console_scripts/test_/static/button_script.js: -------------------------------------------------------------------------------- 1 | var RealtimeUpdates = false 2 | 3 | $(document).ready(function() { 4 | // 监听开关状态变化 5 | $('#realtime-switch').change(function() { 6 | if ($(this).is(':checked')) { 7 | // 开启实时显示 8 | startRealtimeUpdates(); 9 | } else { 10 | // 关闭实时显示 11 | stopRealtimeUpdates(); 12 | } 13 | }); 14 | 15 | // 刷新按钮点击事件 16 | $('#refresh-button').click(function() { 17 | refreshData(); 18 | }); 19 | 20 | // 模拟实时更新的函数 21 | function startRealtimeUpdates() { 22 | console.log("实时显示已开启"); 23 | startFetching(); // 每秒调用一次refreshData 24 | RealtimeUpdates= true 25 | } 26 | 27 | function stopRealtimeUpdates() { 28 | console.log("实时显示已关闭"); 29 | stopFetching(); // 停止调用refreshData 30 | RealtimeUpdates= false 31 | 32 | } 33 | 34 | // 提取的请求函数 35 | function sendRequest(url) { 36 | console.log(url); 37 | $.ajax({ 38 | url: url, 39 | type: 'GET', 40 | async: true, // 确保请求是异步的 41 | success: function(data) { 42 | setTimeout(function() { 43 | refreshData(); 44 | }, 500); // 500毫秒 = 0.5秒 45 | }, 46 | error: function(xhr, status, error) { 47 | console.error('Request failed:', status, error); 48 | // 即使请求失败，也可以选择调用refreshData 49 | refreshData(); 50 | } 51 | }); 52 | } 53 | 54 | var first_node = document.getElementById('Title'); 55 | var platform = first_node.getAttribute("nico_ui_platform"); 56 | 57 | if (platform !== "iOS") { 58 | // 处理提交按钮的点击事件 59 | $('#submit-button').click(function() { 60 | var inputValue = $('#text-input').val(); 61 | if (inputValue) { 62 | var url = `/android_excute_action?action=input&inputValue=${inputValue}`; 63 | sendRequest(url); 64 | } else { 65 | alert('Please enter some text.'); 66 | } 67 | }); 68 | 69 | // 处理 home 按钮的点击事件 70 | $('#home-button').click(function() { 71 | var url = `/android_excute_action?action=home`; 72 | sendRequest(url); 73 | }); 74 | 75 | // 处理 back 按钮的点击事件 76 | $('#back-button').click(function() { 77 | var url = `/android_excute_action?action=back`; 78 | sendRequest(url); 79 | }); 80 | 81 | // 处理 menu 按钮的点击事件 82 | $('#menu-button').click(function() { 83 | var url = `/android_excute_action?action=menu`; 84 | sendRequest(url); 85 | }); 86 | 87 | $('#switch-button').click(function() { 88 | var url = `/android_excute_action?action=switch_app`; 89 | sendRequest(url); 90 | }); 91 | 92 | $('#volume-up').click(function() { 93 | var url = `/android_excute_action?action=volume_up`; 94 | sendRequest(url); 95 | }); 96 | 97 | $('#volume-down').click(function() { 98 | var url = `/android_excute_action?action=volume_down`; 99 | sendRequest(url); 100 | }); 101 | 102 | $('#power').click(function() { 103 | var url = `/android_excute_action?action=power`; 104 | sendRequest(url); 105 | }); 106 | 107 | $('#delete_text').click(function() { 108 | var url = `/android_excute_action?action=delete_text`; 109 | sendRequest(url); 110 | }); 111 | } 112 | }); -------------------------------------------------------------------------------- /auto_nico/console_scripts/test_/static/event_listening.js: -------------------------------------------------------------------------------- 1 | const actionLayer = document.getElementById('action-layer'); 2 | 3 | function getPercentageCoordinates(x, y) { 4 | const rect = actionLayer.getBoundingClientRect(); 5 | const xPercent = ((x - rect.left) / rect.width) * 100; 6 | const yPercent = ((y - rect.top) / rect.height) * 100; 7 | return { xPercent, yPercent }; 8 | } 9 | 10 | function sendActionToServer(actionType, xPercent, yPercent) { 11 | fetch('/action', { 12 | method: 'POST', 13 | headers: { 14 | 'Content-Type': 'application/json' 15 | }, 16 | body: JSON.stringify({ actionType, xPercent, yPercent }) 17 | }) 18 | .then(response => response.json()) 19 | .then(data => console.log('Server response:', data)) 20 | .catch(error => console.error('Error:', error)); 21 | } 22 | 23 | let isDragging = false; 24 | let lastX = 0; 25 | let lastY = 0; 26 | 27 | actionLayer.addEventListener('mousedown', (event) => { 28 | if (RealtimeUpdates){ 29 | isDragging = true; 30 | lastX = event.clientX; 31 | lastY = event.clientY; 32 | 33 | const { xPercent, yPercent } = getPercentageCoordinates(lastX, lastY); 34 | console.log('Press detected at:', xPercent.toFixed(2) + '%', yPercent.toFixed(2) + '%'); 35 | sendActionToServer("touch_down", xPercent.toFixed(2), yPercent.toFixed(2)); 36 | } 37 | 38 | }); 39 | 40 | document.addEventListener('mousemove', (event) => { 41 | if (isDragging) { 42 | const { xPercent, yPercent } = getPercentageCoordinates(event.clientX, event.clientY); 43 | console.log('Move detected at:', xPercent.toFixed(2) + '%', yPercent.toFixed(2) + '%'); 44 | sendActionToServer('touch_move', xPercent.toFixed(2), yPercent.toFixed(2)); 45 | lastX = event.clientX; 46 | lastY = event.clientY; 47 | } 48 | }); 49 | 50 | document.addEventListener('mouseup', (event) => { 51 | if (isDragging) { 52 | isDragging = false; 53 | const { xPercent, yPercent } = getPercentageCoordinates(lastX, lastY); 54 | console.log('Release detected at:', xPercent.toFixed(2) + '%', yPercent.toFixed(2) + '%'); 55 | sendActionToServer("touch_up", xPercent.toFixed(2), yPercent.toFixed(2)); 56 | } 57 | }); -------------------------------------------------------------------------------- /auto_nico/console_scripts/test_/static/main.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/letmeNo1/Nico/79dc1ac061cdeaa7ab74660d42bf85bd93ff0592/auto_nico/console_scripts/test_/static/main.js -------------------------------------------------------------------------------- /auto_nico/console_scripts/test_/static/styles.css: -------------------------------------------------------------------------------- 1 | body { 2 | margin: 0; 3 | padding: 0; 4 | display: flex; 5 | height: 100vh; 6 | } 7 | 8 | .section { 9 | box-sizing: border-box; 10 | padding: 20px; 11 | color: white; 12 | text-align: center; 13 | display: flex; 14 | flex-direction: column; 15 | justify-content: flex-start; 16 | align-items: center; 17 | overflow-y: auto; 18 | } 19 | 20 | .content::after { 21 | content: ""; 22 | display: table; 23 | clear: both; 24 | } 25 | 26 | .content { 27 | float: left; 28 | width: 100%; /* 修改为50% */ 29 | height: 100%; /* 设置高度为视口的100% */ 30 | box-sizing: border-box; /* 包含内边距和边框在内的宽度 */ 31 | overflow-y: auto; 32 | 33 | } 34 | h1 { 35 | background-color: rgb(255, 218, 51);; 36 | color: black; 37 | } 38 | .section1 { 39 | width: 40%; 40 | background-color: #2c3e50; 41 | display: flex; 42 | flex-direction: column 43 | } 44 | 45 | .section2 { 46 | width: 50%; 47 | background-color: #2c3e50; 48 | overflow-y: auto; 49 | 50 | } 51 | 52 | .section3 { 53 | width: 10%; 54 | background-color: #2c3e50; 55 | } 56 | 57 | .section2 img { 58 | max-width: 100%; 59 | max-height: 100%; 60 | object-fit: contain; 61 | } 62 | 63 | .c_button { 64 | margin: 5px; 65 | padding: 5px 10px; 66 | width: 90%; 67 | } 68 | 69 | .styled-input { 70 | margin: 5px; 71 | padding: 5px; 72 | width: 90%; 73 | } 74 | 75 | .switch { 76 | position: relative; 77 | display: inline-block; 78 | width: 60px; 79 | height: 34px; 80 | } 81 | 82 | .switch input { 83 | opacity: 0; 84 | width: 0; 85 | height: 0; 86 | } 87 | 88 | .slider { 89 | position: absolute; 90 | cursor: pointer; 91 | top: 0; 92 | left: 0; 93 | right: 0; 94 | bottom: 0; 95 | background-color: #ccc; 96 | -webkit-transition: .4s; 97 | transition: .4s; 98 | } 99 | 100 | .slider:before { 101 | position: absolute; 102 | content: ""; 103 | height: 26px; 104 | width: 26px; 105 | left: 4px; 106 | bottom: 4px; 107 | background-color: white; 108 | -webkit-transition: .4s; 109 | transition: .4s; 110 | } 111 | 112 | input:checked + .slider { 113 | background-color: #2196F3; 114 | } 115 | 116 | input:focus + .slider { 117 | box-shadow: 0 0 1px #2196F3; 118 | } 119 | 120 | input:checked + .slider:before { 121 | -webkit-transform: translateX(26px); 122 | -ms-transform: translateX(26px); 123 | transform: translateX(26px); 124 | } 125 | 126 | .slider.round { 127 | border-radius: 34px; 128 | } 129 | 130 | .slider.round:before { 131 | border-radius: 50%; 132 | } 133 | 134 | .switch-container { 135 | margin-top: 10px; 136 | display: flex; 137 | align-items: center; 138 | justify-content: center; 139 | width: 90%; 140 | } 141 | 142 | .content-inner div { 143 | margin: 0; 144 | padding: 0; 145 | } 146 | 147 | /* 加载动画样式 */ 148 | .content-inner { 149 | width: 100%; /* 使其宽度与父节点一致，不超出 */ 150 | height: auto; /* 根据内容自动调整高度，也可设置固定值 */ 151 | text-align: left; /* 左对齐 */ 152 | word-wrap: break-word; /* 自动换行 */ 153 | overflow-wrap: break-word; /* 自动换行，更现代的属性 */ 154 | overflow: auto; 155 | 156 | } 157 | 158 | .container { 159 | width: 100%; 160 | height: 100vh; /* 容器占满视口高度 */ 161 | } 162 | 163 | #dynamicImage { 164 | width: auto; 165 | height: 100%; 166 | z-index: 0; 167 | 168 | } 169 | #action-layer { 170 | position: absolute; 171 | width: auto; 172 | height: 100%; 173 | z-index: 1; 174 | 175 | } 176 | 177 | .toggle-button { 178 | float: right; 179 | margin-left: 10px; 180 | background-color: #f4f4f4; 181 | border: 1px solid #ccc; 182 | padding: 5px 10px; 183 | cursor: pointer; 184 | } 185 | 186 | .content-wrapper { 187 | overflow: hidden; 188 | max-height: 3em; /* 初始显示高度 */ 189 | } 190 | 191 | .node { 192 | position: relative; 193 | } 194 | 195 | .node::after { 196 | content: ''; 197 | position: absolute; 198 | top: 0; 199 | left: 0; 200 | width: 100%; 201 | height: 100%; 202 | background-color: rgba(0, 0, 0, 0.5); 203 | opacity: 0; 204 | transition: opacity 0.3s ease; 205 | } 206 | 207 | .node:hover::after { 208 | opacity: 1; 209 | } -------------------------------------------------------------------------------- /auto_nico/console_scripts/test_/templates/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/letmeNo1/Nico/79dc1ac061cdeaa7ab74660d42bf85bd93ff0592/auto_nico/console_scripts/test_/templates/__init__.py -------------------------------------------------------------------------------- /auto_nico/console_scripts/test_/templates/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | Three Section Layout 8 | 9 | 10 | 11 | 12 |
13 |
14 |

XML Content as HTML

15 |
16 | {{ xml_content|safe }} 17 |
18 |
19 | 20 | 21 | 22 | 23 |
24 |
25 | 26 |
27 | 28 |
29 | 30 | placeholder 31 |
32 | 33 |
34 |
35 |
36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 |
49 | 53 | 54 |
55 |
56 |
57 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | -------------------------------------------------------------------------------- /auto_nico/console_scripts/uninstall_apk.py: -------------------------------------------------------------------------------- 1 | from auto_nico.android.adb_utils import AdbUtils 2 | 3 | 4 | def main(): 5 | import argparse 6 | 7 | parser = argparse.ArgumentParser() 8 | parser.add_argument('-s', type=str, help='device_udid') 9 | 10 | args = parser.parse_args() 11 | adb_utils = AdbUtils(args.s) 12 | adb_utils.qucik_shell("pm uninstall nico.dump-hierarchy") 13 | adb_utils.qucik_shell("pm uninstall nico.dump-hierarchy.test") -------------------------------------------------------------------------------- /auto_nico/ios/XCUIElementType.py: -------------------------------------------------------------------------------- 1 | def get_element_type_by_value(value): 2 | element__mapping = { 3 | 0: "Any", 4 | 1: "Other", 5 | 2: "Application", 6 | 3: "Group", 7 | 4: "Window", 8 | 5: "Sheet", 9 | 6: "Drawer", 10 | 7: "Alert", 11 | 8: "Dialog", 12 | 9: "Button", 13 | 10: "RadioButton", 14 | 11: "RadioGroup", 15 | 12: "CheckBox", 16 | 13: "DisclosureTriangle", 17 | 14: "PopUpButton", 18 | 15: "ComboBox", 19 | 16: "MenuButton", 20 | 17: "ToolbarButton", 21 | 18: "Popover", 22 | 19: "Keyboard", 23 | 20: "Key", 24 | 21: "NavigationBar", 25 | 22: "TabBar", 26 | 23: "TabGroup", 27 | 24: "Toolbar", 28 | 25: "StatusBar", 29 | 26: "Table", 30 | 27: "TableRow", 31 | 28: "TableColumn", 32 | 29: "Outline", 33 | 30: "OutlineRow", 34 | 31: "Browser", 35 | 32: "CollectionView", 36 | 33: "Slider", 37 | 34: "PageIndicator", 38 | 35: "ProgressIndicator", 39 | 36: "ActivityIndicator", 40 | 37: "SegmentedControl", 41 | 38: "Picker", 42 | 39: "PickerWheel", 43 | 40: "Switch", 44 | 41: "Toggle", 45 | 42: "Link", 46 | 43: "Image", 47 | 44: "Icon", 48 | 45: "SearchField", 49 | 46: "ScrollView", 50 | 47: "ScrollBar", 51 | 48: "StaticText", 52 | 49: "TextField", 53 | 50: "SecureTextField", 54 | 51: "DatePicker", 55 | 52: "TextView", 56 | 53: "Menu", 57 | 54: "MenuItem", 58 | 55: "MenuBar", 59 | 56: "MenuBarItem", 60 | 57: "Map", 61 | 58: "WebView", 62 | 59: "IncrementArrow", 63 | 60: "DecrementArrow", 64 | 61: "Timeline", 65 | 62: "RatingIndicator", 66 | 63: "ValueIndicator", 67 | 64: "SplitGroup", 68 | 65: "Splitter", 69 | 66: "RelevanceIndicator", 70 | 67: "ColorWell", 71 | 68: "HelpTag", 72 | 69: "Matte", 73 | 70: "DockItem", 74 | 71: "Ruler", 75 | 72: "RulerMarker", 76 | 73: "Grid", 77 | 74: "LevelIndicator", 78 | 75: "Cell", 79 | 76: "LayoutArea", 80 | 77: "LayoutItem", 81 | 78: "Handle", 82 | 79: "Stepper", 83 | 80: "Tab", 84 | 81: "TouchBar", 85 | 82: "StatusItem", 86 | } 87 | return element__mapping.get(value, None) 88 | 89 | def get_value_by_element_type(element_type): 90 | element_mapping = { 91 | "Any": 0, 92 | "Other": 1, 93 | "Application": 2, 94 | "Group": 3, 95 | "Window": 4, 96 | "Sheet": 5, 97 | "Drawer": 6, 98 | "Alert": 7, 99 | "Dialog": 8, 100 | "Button": 9, 101 | "RadioButton": 10, 102 | "RadioGroup": 11, 103 | "CheckBox": 12, 104 | "DisclosureTriangle": 13, 105 | "PopUpButton": 14, 106 | "ComboBox": 15, 107 | "MenuButton": 16, 108 | "ToolbarButton": 17, 109 | "Popover": 18, 110 | "Keyboard": 19, 111 | "Key": 20, 112 | "NavigationBar": 21, 113 | "TabBar": 22, 114 | "TabGroup": 23, 115 | "Toolbar": 24, 116 | "StatusBar": 25, 117 | "Table": 26, 118 | "TableRow": 27, 119 | "TableColumn": 28, 120 | "Outline": 29, 121 | "OutlineRow": 30, 122 | "Browser": 31, 123 | "CollectionView": 32, 124 | "Slider": 33, 125 | "PageIndicator": 34, 126 | "ProgressIndicator": 35, 127 | "ActivityIndicator": 36, 128 | "SegmentedControl": 37, 129 | "Picker": 38, 130 | "PickerWheel": 39, 131 | "Switch": 40, 132 | "Toggle": 41, 133 | "Link": 42, 134 | "Image": 43, 135 | "Icon": 44, 136 | "SearchField": 45, 137 | "ScrollView": 46, 138 | "ScrollBar": 47, 139 | "StaticText": 48, 140 | "TextField": 49, 141 | "SecureTextField": 50, 142 | "DatePicker": 51, 143 | "TextView": 52, 144 | "Menu": 53, 145 | "MenuItem": 54, 146 | "MenuBar": 55, 147 | "MenuBarItem": 56, 148 | "Map": 57, 149 | "WebView": 58, 150 | "IncrementArrow": 59, 151 | "DecrementArrow": 60, 152 | "Timeline": 61, 153 | "RatingIndicator": 62, 154 | "ValueIndicator": 63, 155 | "SplitGroup": 64, 156 | "Splitter": 65, 157 | "RelevanceIndicator": 66, 158 | "ColorWell": 67, 159 | "HelpTag": 68, 160 | "Matte": 69, 161 | "DockItem": 70, 162 | "Ruler": 71, 163 | "RulerMarker": 72, 164 | "Grid": 73, 165 | "LevelIndicator": 74, 166 | "Cell": 75, 167 | "LayoutArea": 76, 168 | "LayoutItem": 77, 169 | "Handle": 78, 170 | "Stepper": 79, 171 | "Tab": 80, 172 | "TouchBar": 81, 173 | "StatusItem": 82, 174 | } 175 | return element_mapping.get(element_type, None) -------------------------------------------------------------------------------- /auto_nico/ios/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/letmeNo1/Nico/79dc1ac061cdeaa7ab74660d42bf85bd93ff0592/auto_nico/ios/__init__.py -------------------------------------------------------------------------------- /auto_nico/ios/idb_utils.py: -------------------------------------------------------------------------------- 1 | import json 2 | import os 3 | import platform 4 | import random 5 | import re 6 | import time 7 | import subprocess 8 | 9 | import cv2 10 | import numpy as np 11 | import psutil 12 | 13 | from loguru import logger 14 | from auto_nico.common.runtime_cache import RunningCache 15 | from auto_nico.common.send_request import send_http_request 16 | from auto_nico.common.error import NicoError 17 | 18 | 19 | class IdbUtils: 20 | def __init__(self, udid): 21 | self.udid = udid 22 | self.runtime_cache = RunningCache(udid) 23 | 24 | def get_tcp_forward_port(self): 25 | if platform.system() == "Windows": 26 | result = subprocess.run(f'netstat -ano | findstr "LISTENING"', capture_output=True, text=True, shell=True) 27 | output = result.stdout 28 | lines = output.split('\n') 29 | pids = [line.split()[4] for line in lines if line.strip()] 30 | 31 | elif platform.system() == "Darwin" or platform.system() == "Linux": 32 | result = subprocess.run(f'lsof -i | grep LISTEN', capture_output=True, text=True, shell=True) 33 | output = result.stdout 34 | lines = output.split('\n') 35 | pids = [line.split()[1] for line in lines if line.strip()] 36 | else: 37 | raise Exception("Unsupported platform") 38 | for index, pid in enumerate(pids): 39 | try: 40 | if "tidevice" in psutil.Process(int(pid)).cmdline()[1]: 41 | if platform.system() == "Windows": 42 | return lines[index].split()[1].split("->")[0].split(":")[-1], pid 43 | else: 44 | result = subprocess.run(f"lsof -Pan -p {pid} -i", capture_output=True, text=True, shell=True) 45 | output = result.stdout 46 | ports = re.findall(r':(\d+)', output) 47 | return ports[0], pid 48 | except: 49 | continue 50 | return None, None 51 | 52 | def is_greater_than_ios_17(self): 53 | from packaging.version import Version 54 | return Version(self.get_system_info().get("ProductVersion")) >= Version("17.0.0") 55 | 56 | def device_list(self): 57 | command = f'tidevice list' 58 | return os.popen(command).read() 59 | 60 | def set_port_forward(self, port): 61 | 62 | commands = f"""tidevice --udid {self.udid} relay {port} {port}""" 63 | subprocess.Popen(commands, shell=True) 64 | self.runtime_cache.set_current_running_port(port) 65 | 66 | def get_app_list(self): 67 | os.environ['PYTHONIOENCODING'] = 'utf-8' 68 | result = subprocess.run(f"tidevice --udid {self.udid} applist", capture_output=True, text=True, 69 | encoding='utf-8') 70 | result_list = result.stdout.splitlines() 71 | return result_list 72 | 73 | def get_test_server_package(self): 74 | app_list = self.get_app_list() 75 | xctrunner_package_name = [s for s in app_list if "dump_hierarchyUITests-Runner" in s][0].split(" ")[0] 76 | return {"test_server_package": xctrunner_package_name} 77 | 78 | def get_wda_server_package(self): 79 | app_list = self.get_app_list() 80 | test_server_package = [s for s in app_list if s.startswith('com.facebook')] 81 | return test_server_package[0].split(" ")[0] 82 | 83 | def start_app(self, package_name): 84 | command = f'launch {package_name}' 85 | self.cmd(command) 86 | self.runtime_cache.set_current_running_package_name(package_name) 87 | 88 | def _init_test_server(self): 89 | self._set_tcp_forward_port() 90 | if self.get_system_info().get("ProductVersion") >= "17.0.0": 91 | self._start_tunnel() 92 | self.__start_test_server() 93 | 94 | def __start_test_server(self): 95 | current_port = RunningCache(self.udid).get_current_running_port() 96 | test_server_package_dict = self.get_test_server_package() 97 | logger.debug( 98 | f"ios runwda --bundleid {test_server_package_dict.get('test_server_package')} --testrunnerbundleid {test_server_package_dict.get('test_server_package')} --xctestconfig=dump_hierarchyUITests.xctest --udid={self.udid} --env=USE_PORT={current_port}") 99 | 100 | commands = f"ios runwda --bundleid {test_server_package_dict.get('test_server_package')} --testrunnerbundleid {test_server_package_dict.get('test_server_package')} --xctestconfig=dump_hierarchyUITests.xctest --udid={self.udid} --env=USE_PORT={current_port}" 101 | subprocess.Popen(commands, shell=True) 102 | for _ in range(10): 103 | response = send_http_request(current_port, "check_status") 104 | if response is not None: 105 | logger.debug(f"{self.udid}'s test server is ready") 106 | break 107 | time.sleep(1) 108 | logger.debug(f"{self.udid}'s uiautomator was initialized successfully") 109 | 110 | def _start_tunnel(self): 111 | # self.kill_process_by_port() 112 | logger.debug(f"ios tunnel ls --udid={self.udid}") 113 | rst = os.popen(f"ios tunnel ls --udid={self.udid}").read() 114 | if str(self.udid).strip() in rst: 115 | logger.debug(f"tunnel for {self.udid} is started") 116 | else: 117 | logger.debug(f"ios tunnel start --udid={self.udid}") 118 | 119 | command = f"ios tunnel start --udid={self.udid}" 120 | subprocess.Popen(command, shell=True) 121 | for _ in range(10): 122 | rst = os.popen("ios tunnel ls").read() 123 | if self.udid in rst: 124 | logger.debug(f"tunnel for {self.udid} is started") 125 | return 126 | time.sleep(1) 127 | raise NicoError(f"tunnel for {self.udid} is not started") 128 | 129 | def _set_tcp_forward_port(self): 130 | current_port = RunningCache(self.udid).get_current_running_port() 131 | logger.debug( 132 | f"""tidevice --udid {self.udid} relay {current_port} {current_port}""") 133 | commands = f"""tidevice --udid {self.udid} relay {current_port} {current_port}""" 134 | try: 135 | subprocess.Popen(commands, shell=True) 136 | except OSError: 137 | logger.error("start fail") 138 | subprocess.Popen(commands, shell=True) 139 | 140 | def _set_running_port(self, port): 141 | exists_port, pid = self.get_tcp_forward_port() 142 | if exists_port is None: 143 | logger.debug(f"{self.udid} no exists port") 144 | if port != "random": 145 | running_port = port 146 | else: 147 | random_number = random.randint(9000, 9999) 148 | running_port = random_number 149 | else: 150 | running_port = int(exists_port) 151 | RunningCache(self.udid).set_current_running_port(running_port) 152 | 153 | def activate_app(self, package_name): 154 | exists_port = self.runtime_cache.get_current_running_port() 155 | send_http_request(exists_port, "activate_app", {"bundle_id": package_name}) 156 | 157 | def terminate_app(self, package_name): 158 | exists_port = self.runtime_cache.get_current_running_port() 159 | send_http_request(exists_port, "terminate_app", {"bundle_id": package_name}) 160 | 161 | def get_output_device_name(self): 162 | exists_port = self.runtime_cache.get_current_running_port() 163 | respo = send_http_request(exists_port, "device_info", {"value": "get_output_device_name"}) 164 | return respo 165 | 166 | def stop_app(self, package_name): 167 | command = f'kill {package_name}' 168 | self.cmd(command) 169 | 170 | def get_system_info(self): 171 | data_string = os.popen(f"tidevice --udid {self.udid} info").read() 172 | data_dict = {} 173 | for line in data_string.strip().split('\n'): 174 | if ':' in line: 175 | key, value = line.split(':', 1) 176 | data_dict[key.strip()] = value.strip() 177 | return data_dict 178 | 179 | def cmd(self, cmd): 180 | udid = self.udid 181 | """@Brief: Execute the CMD and return value 182 | @return: bool 183 | """ 184 | try: 185 | if self.is_greater_than_ios_17(): 186 | result = subprocess.run(f'''ios {cmd} --udid={udid}''', shell=True, capture_output=True, text=True, 187 | check=True, timeout=10).stdout 188 | else: 189 | result = subprocess.run(f'''tidevice --udid {udid} {cmd}''', shell=True, capture_output=True, text=True,check=True, timeout=10).stdout 190 | except subprocess.CalledProcessError as e: 191 | return e.stderr 192 | return result 193 | 194 | def restart_app(self, package_name): 195 | self.stop_app(package_name) 196 | time.sleep(1) 197 | self.start_app(package_name) 198 | 199 | def unlock(self): 200 | pass 201 | 202 | def home(self): 203 | exists_port = self.runtime_cache.get_current_running_port() 204 | return send_http_request(exists_port, "device_action", {"action": "home"}) 205 | 206 | def get_volume(self): 207 | exists_port = self.runtime_cache.get_current_running_port() 208 | return send_http_request(exists_port, "device_info", {"value": "get_output_volume"}) 209 | 210 | def turn_volume_up(self): 211 | exists_port = self.runtime_cache.get_current_running_port() 212 | send_http_request(exists_port, "device_action", {"action": "volume_up"}) 213 | 214 | def turn_volume_down(self): 215 | exists_port = self.runtime_cache.get_current_running_port() 216 | send_http_request(exists_port, "device_action", {"action": "volume_down"}) 217 | 218 | def snapshot(self, name, path): 219 | self.cmd(f'screenshot {path}/{name}.jpg') 220 | 221 | def get_pic(self, quality=1.0): 222 | exists_port = self.runtime_cache.get_current_running_port() 223 | return send_http_request(exists_port, f"get_jpg_pic", {"compression_quality": quality}) 224 | 225 | def get_image_object(self, quality=100): 226 | exists_port = self.runtime_cache.get_current_running_port() 227 | a = send_http_request(exists_port, f"get_jpg_pic", {"compression_quality": quality}) 228 | nparr = np.frombuffer(a, np.uint8) 229 | image = cv2.imdecode(nparr, cv2.IMREAD_COLOR) 230 | return image 231 | 232 | def click(self, x, y,bundleIdentifier=None): 233 | current_bundleIdentifier = bundleIdentifier if bundleIdentifier is not None else self.runtime_cache.get_current_running_package() 234 | if current_bundleIdentifier is None: 235 | current_bundleIdentifier = self.get_current_bundleIdentifier( 236 | self.runtime_cache.get_current_running_port()) 237 | 238 | send_http_request(self.runtime_cache.get_current_running_port(), 239 | f"coordinate_action", 240 | {"bundle_id": current_bundleIdentifier, "action": "click", "xPixel": x, "yPixel": y, 241 | "action_parms": "none"}) 242 | self.runtime_cache.clear_current_cache_ui_tree() 243 | 244 | def get_current_bundleIdentifier(self, port): 245 | bundle_list = self.get_app_list() 246 | method = "get_current_bundleIdentifier" 247 | params = { 248 | "bundle_ids": "" 249 | } 250 | command = [] # Use a list to collect bundle IDs 251 | for item in bundle_list: 252 | if item: 253 | item = item.split(" ")[0] 254 | command.append(item) # Append item to the list 255 | params["bundle_ids"] = ",".join(command) # Join list items with commas 256 | 257 | return send_http_request(port, method, params) 258 | 259 | def get_xpaths(self,id,xpath): 260 | exists_port = self.runtime_cache.get_current_running_port() 261 | 262 | return send_http_request(exists_port, f"find_element_by_query", 263 | {"bundle_id": id, "query_method": "predicate", "query_value": xpath}) 264 | 265 | # a= IdbUtils("00008140-001C7CD80202801C") 266 | # a.restart_app("com.apple.Preferences") -------------------------------------------------------------------------------- /auto_nico/ios/nico_image.py: -------------------------------------------------------------------------------- 1 | import tempfile 2 | 3 | 4 | 5 | class NicoImage: 6 | def __init__(self, udid): 7 | self.udid = udid 8 | self.source_image_path = tempfile.gettempdir() + "/test.png" 9 | 10 | -------------------------------------------------------------------------------- /auto_nico/ios/nico_ios.py: -------------------------------------------------------------------------------- 1 | import os 2 | import random 3 | import tempfile 4 | import subprocess 5 | 6 | from auto_nico.common.error import IDBServerError 7 | from auto_nico.common.runtime_cache import RunningCache 8 | from auto_nico.ios.idb_utils import IdbUtils 9 | from auto_nico.ios.nico_ios_element import NicoIOSElement 10 | from auto_nico.common.send_request import send_http_request 11 | from loguru import logger 12 | from auto_nico.common.nico_basic import NicoBasic 13 | 14 | 15 | class NicoIOS(NicoBasic): 16 | def __init__(self, udid, package_name=None, port="random", **query): 17 | super().__init__(udid, **query) 18 | self.udid = udid 19 | logger.debug(f"{self.udid}'s test server is being initialized, please wait") 20 | self.idb_utils = IdbUtils(udid) 21 | self.__check_idb_server(udid) 22 | self.idb_utils._set_running_port(port) 23 | self.runtime_cache = RunningCache(udid) 24 | rst = send_http_request(RunningCache(udid).get_current_running_port(), "check_status") is not None 25 | if rst: 26 | logger.debug(f"{self.udid}'s test server is ready") 27 | else: 28 | logger.debug(f"{self.udid} test server disconnect, restart ") 29 | self.idb_utils._init_test_server() 30 | if package_name is None: 31 | self.package_name = self.__get_current_bundleIdentifier(RunningCache(udid).get_current_running_port()) 32 | self.runtime_cache.set_action_was_taken(True) 33 | 34 | def __check_idb_server(self, udid): 35 | result = subprocess.run("tidevice list", shell=True, stdout=subprocess.PIPE).stdout 36 | decoded_result = result.decode('utf-8', errors='ignore') 37 | if udid in decoded_result: 38 | pass 39 | else: 40 | raise IDBServerError("no devices connect") 41 | 42 | def __get_current_bundleIdentifier(self, port): 43 | bundle_list = self.idb_utils.get_app_list() 44 | method = "get_current_bundleIdentifier" 45 | params = { 46 | "bundle_ids": "" 47 | } 48 | command = [] # Use a list to collect bundle IDs 49 | for item in bundle_list: 50 | if item: 51 | item = item.split(" ")[0] 52 | command.append(item) # Append item to the list 53 | params["bundle_ids"] = ",".join(command) # Join list items with commas 54 | 55 | return send_http_request(port, method, params) 56 | 57 | def __remove_ui_xml(self, udid): 58 | temp_folder = tempfile.gettempdir() 59 | path = temp_folder + f"/{udid}_ui.xml" 60 | os.remove(path) 61 | 62 | def __call__(self, **query): 63 | current_port = RunningCache(self.udid).get_current_running_port() 64 | self.__check_idb_server(self.udid) 65 | rst = send_http_request(current_port, "check_status") is not None 66 | if not rst: 67 | logger.debug(f"{self.udid} test server disconnect, restart ") 68 | self.idb_utils._init_test_server() 69 | if self.runtime_cache.get_current_running_package(): 70 | self.package_name = self.runtime_cache.get_current_running_package() 71 | else: 72 | if self.package_name is None: 73 | self.package_name = self.__get_current_bundleIdentifier(current_port) 74 | NIE = NicoIOSElement(**query) 75 | NIE.set_udid(self.udid) 76 | NIE.set_port(current_port) 77 | NIE.set_package_name(self.package_name) 78 | return NIE -------------------------------------------------------------------------------- /auto_nico/ios/nico_ios_element.py: -------------------------------------------------------------------------------- 1 | import os 2 | import re 3 | 4 | from auto_nico.common.nico_basic_element import NicoBasicElement 5 | from auto_nico.common.runtime_cache import RunningCache 6 | from auto_nico.common.send_request import send_http_request 7 | from auto_nico.ios.XCUIElementType import get_element_type_by_value 8 | from loguru import logger 9 | import cv2 10 | from auto_nico.ios.idb_utils import IdbUtils 11 | 12 | 13 | class UIStructureError(Exception): 14 | pass 15 | 16 | 17 | class NicoIOSElement(NicoBasicElement): 18 | def __init__(self, **query): 19 | self.query = query 20 | super().__init__(**query) 21 | 22 | 23 | @property 24 | def index(self): 25 | return self._get_attribute_value("index") 26 | 27 | def get_index(self): 28 | return self.index 29 | 30 | @property 31 | def text(self): 32 | if self._get_attribute_value("label") is not None: 33 | return self._get_attribute_value("label") 34 | elif self._get_attribute_value("title") is not None: 35 | return self._get_attribute_value("title") 36 | elif self._get_attribute_value("text") is not None: 37 | return self._get_attribute_value("text") 38 | return None 39 | 40 | def get_text(self): 41 | return self.text 42 | 43 | @property 44 | def identifier(self): 45 | return self._get_attribute_value("identifier") 46 | 47 | def get_identifier(self): 48 | return self.identifier 49 | 50 | @property 51 | def value(self): 52 | return self._get_attribute_value("value") 53 | 54 | def get_value(self): 55 | return self.value 56 | 57 | @property 58 | def xpath(self): 59 | return self._get_attribute_value("xpath") 60 | 61 | def get_xpath(self): 62 | return self.xpath 63 | 64 | @property 65 | def class_name(self): 66 | class_name = get_element_type_by_value(self._get_attribute_value("elementType")) 67 | if class_name is not None: 68 | return class_name 69 | else: 70 | return self._get_attribute_value("class_name") 71 | 72 | def get_class_name(self): 73 | return self.class_name 74 | 75 | @property 76 | def bounds(self): 77 | pattern = r'\[(\d+),(\d+)\]\[(\d+),(\d+)\]' 78 | bounds = self._get_attribute_value("bounds") 79 | if bounds is None: 80 | frame = self._get_attribute_value("frame") 81 | bounds = f'[{int(frame.get("X"))},{int(frame.get("Y"))}][{int(frame.get("Width"))},{int(frame.get("Height"))}]' 82 | matches = re.findall(pattern, bounds) 83 | x = int(matches[0][0]) 84 | y = int(matches[0][1]) 85 | w = int(matches[0][2]) 86 | h = int(matches[0][3]) 87 | l = int(matches[0][2]) + x 88 | t = int(matches[0][3]) + y 89 | 90 | return x, y, w, h, l, t 91 | 92 | def __image_rescale(self,image): 93 | rst = self._find_function(query={"xpath":"Window[1]"},use_xml=True).get("bounds") 94 | matches = re.findall(r'\[(\d+),(\d+)\]', rst) 95 | target_w = int(matches[-1][0]) 96 | original_h, original_w = image.shape[:2] 97 | aspect_ratio = original_w / original_h 98 | target_h = int(target_w / aspect_ratio) 99 | if matches: 100 | print( matches[-1]) 101 | w, h = map(int, matches[-1]) 102 | scaled_image = cv2.resize(image, (target_w, target_h)) 103 | return scaled_image 104 | return None 105 | 106 | 107 | @property 108 | def description(self): 109 | self.idb_utils = IdbUtils(self.udid) 110 | logger.debug("Description being generated") 111 | img = self.idb_utils.get_image_object() 112 | img = self.__image_rescale(img) 113 | text = self._description(img, self.bounds[:4]) 114 | return text 115 | 116 | @property 117 | def ocr_id(self): 118 | self.idb_utils = IdbUtils(self.udid) 119 | logger.debug("Description being generated") 120 | img = self.idb_utils.get_image_object() 121 | img = self.__image_rescale(img) 122 | text = self._ocr_id(img, self.bounds[:4]) 123 | return text 124 | 125 | def center_coordinate(self): 126 | x, y, w, h, l, t = self.bounds 127 | center_x = x + w // 2 128 | center_y = y + h // 2 129 | return center_x, center_y 130 | 131 | def click(self, x=None, y=None, x_offset=None, y_offset=None): 132 | RunningCache(self.udid).get_current_running_port() 133 | 134 | if x is None and y is None: 135 | x = self.center_coordinate()[0] 136 | y = self.center_coordinate()[1] 137 | if x_offset is not None: 138 | x = x + x_offset 139 | if y_offset is not None: 140 | y = y + y_offset 141 | send_http_request(RunningCache(self.udid).get_current_running_port(), 142 | f"coordinate_action", 143 | {"bundle_id": self.package_name, "action": "click", "xPixel": x, "yPixel": y, 144 | "action_parms": "none"}) 145 | RunningCache(self.udid).clear_current_cache_ui_tree() 146 | logger.debug(f"click {x} {y}") 147 | 148 | def long_click(self, duration, x_offset=None, y_offset=None): 149 | x = self.center_coordinate()[0] 150 | y = self.center_coordinate()[1] 151 | if x_offset is not None: 152 | x = x + x_offset 153 | if y_offset is not None: 154 | y = y + y_offset 155 | send_http_request(RunningCache(self.udid).get_current_running_port(), 156 | f"coordinate_action", 157 | {"bundle_id": self.package_name, "action": "press", "xPixel": x, "yPixel": y, 158 | "action_parms": float(duration)}) 159 | RunningCache(self.udid).clear_current_cache_ui_tree() 160 | logger.debug(f"click {x} {y}") 161 | 162 | def set_text(self, text): 163 | send_http_request(RunningCache(self.udid).get_current_running_port(), 164 | f"coordinate_action", 165 | {"bundle_id": self.package_name, "action": "enter_text", 166 | "action_parms": text}) 167 | RunningCache(self.udid).clear_current_cache_ui_tree() 168 | 169 | def get(self, index): 170 | node = self._get(index) 171 | NAE = NicoIOSElement() 172 | NAE.set_current_node(node) 173 | NAE.set_udid(self.udid) 174 | NAE.set_package_name(self.package_name) 175 | NAE.set_port(RunningCache(self.udid).get_current_running_port()) 176 | return NAE 177 | 178 | def all(self): 179 | eles = self._find_all_function(self.query) 180 | RunningCache(self.udid).set_action_was_taken(False) 181 | if not eles: 182 | return eles 183 | ALL_NAE_LIST = [] 184 | for ele in eles: 185 | NAE = NicoIOSElement() 186 | NAE.set_query(self.query) 187 | NAE.set_port(RunningCache(self.udid).get_current_running_port()) 188 | NAE.set_udid(self.udid) 189 | NAE.set_package_name(self.package_name) 190 | NAE.set_current_node(ele) 191 | 192 | ALL_NAE_LIST.append(NAE) 193 | return ALL_NAE_LIST 194 | 195 | def last_sibling(self, index=0): 196 | previous_node = self._last_sibling(index) 197 | NAE = NicoIOSElement() 198 | NAE.set_query(self.query) 199 | NAE.set_port(RunningCache(self.udid).get_current_running_port()) 200 | NAE.set_udid(self.udid) 201 | NAE.set_package_name(self.package_name) 202 | NAE.set_current_node(previous_node) 203 | return NAE 204 | 205 | def next_sibling(self, index=0): 206 | next_node = self._next_sibling(index) 207 | NAE = NicoIOSElement() 208 | NAE.set_query(self.query) 209 | NAE.set_port(RunningCache(self.udid).get_current_running_port()) 210 | NAE.set_udid(self.udid) 211 | NAE.set_package_name(self.package_name) 212 | NAE.set_current_node(next_node) 213 | return NAE 214 | 215 | def parent(self): 216 | parent_node = self._parent() 217 | NAE = NicoIOSElement() 218 | NAE.set_query(self.query) 219 | NAE.set_port(RunningCache(self.udid).get_current_running_port()) 220 | NAE.set_udid(self.udid) 221 | NAE.set_package_name(self.package_name) 222 | NAE.set_current_node(parent_node) 223 | return NAE 224 | 225 | def child(self, index=0): 226 | child_node = self._child(index) 227 | NAE = NicoIOSElement() 228 | NAE.set_query(self.query) 229 | NAE.set_port(RunningCache(self.udid).get_current_running_port()) 230 | NAE.set_udid(self.udid) 231 | NAE.set_package_name(self.package_name) 232 | NAE.set_current_node(child_node) 233 | return NAE 234 | -------------------------------------------------------------------------------- /auto_nico/ios/received_image.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/letmeNo1/Nico/79dc1ac061cdeaa7ab74660d42bf85bd93ff0592/auto_nico/ios/received_image.jpg -------------------------------------------------------------------------------- /auto_nico/ios/tools/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/letmeNo1/Nico/79dc1ac061cdeaa7ab74660d42bf85bd93ff0592/auto_nico/ios/tools/__init__.py -------------------------------------------------------------------------------- /auto_nico/ios/tools/format_converter.py: -------------------------------------------------------------------------------- 1 | import lxml.etree as ET 2 | from lxml import etree 3 | 4 | 5 | # 用于计算缩进的函数 6 | def count_leading_spaces(s): 7 | return len(s) - len(s.lstrip(' ')) 8 | 9 | 10 | def get_element_of_attr(search_str, attr_list): 11 | for element in attr_list: 12 | if search_str in element: 13 | return element.split(": ")[1].strip().replace("'", "") 14 | return "" 15 | 16 | 17 | def exclude_invalid_rows(hierarchy_string, front, end): 18 | lines = hierarchy_string.split('\n') 19 | 20 | trimmed_lines = lines[front:end] 21 | 22 | trimmed_string = '\n'.join(trimmed_lines) 23 | 24 | return trimmed_string 25 | 26 | 27 | def generate_xpath(element): 28 | parts = [] 29 | ancestors = list(element.iterancestors()) # Convert the iterator to a list 30 | ancestors.reverse() 31 | ancestors.pop(0) # Reverse the list 32 | ancestors.append(element) 33 | 34 | for ancestor in ancestors: 35 | class_name = str(ancestor.get('class_name')) 36 | index = sum(1 for sibling in ancestor.itersiblings(preceding=True) if sibling.get('class_name') == class_name) 37 | parts.append(f"{class_name}[{index}]") 38 | 39 | return '/'.join(parts) 40 | 41 | 42 | def converter(hierarchy_string, front=3, end=-8) -> str: 43 | hierarchy_string = exclude_invalid_rows(hierarchy_string, front, end).replace("Window (Main)", "Window") 44 | elements_str = hierarchy_string.split('\n') 45 | # 创建 XML 根元素 46 | root = ET.Element(f"hierarchy") 47 | 48 | # 用于存储上一级的元素引用 49 | parent_stack = [(root, 0)] 50 | 51 | # 遍历数组中的每个字符串 52 | for node_index, element_str in enumerate(elements_str): 53 | # 计算前导空格数，以确定层级 54 | element_str = element_str[4:] 55 | indent = count_leading_spaces(element_str) 56 | parent, index = parent_stack[-1] 57 | element_attrs = element_str.strip().replace("{", "").replace("}", "").split(",") 58 | class_name = element_attrs[0] 59 | x = round(float(element_attrs[2].strip())) 60 | y = round(float(element_attrs[3].strip())) 61 | w = round(float(element_attrs[4].strip())) 62 | h = round(float(element_attrs[5].strip())) 63 | bounds = f"[{x},{y}][{w},{h}]" 64 | left = round(float(element_attrs[4].strip())) + x 65 | top = round(float(element_attrs[5].strip())) + y 66 | label = get_element_of_attr("label", element_attrs) 67 | identifier = get_element_of_attr("identifier", element_attrs) 68 | value = get_element_of_attr("value", element_attrs) 69 | title = get_element_of_attr("title", element_attrs) 70 | level = int(indent / 2) # 假设每个层级缩进两个空格 71 | 72 | # 如果当前层级为0，重置父栈 73 | if level == 0 and node_index != 0: 74 | parent_stack = [(root, 0)] 75 | 76 | # 获取当前的父元素 77 | parent, index = parent_stack[-1] 78 | 79 | # 创建 XML 元素 80 | element = ET.Element("node") 81 | # 如果当前层级小于或等于栈的大小，需要回退到正确的父元素 82 | while level <= len(parent_stack) - 1: 83 | parent_stack.pop() 84 | if not parent_stack: # break the loop if parent_stack is empty 85 | break 86 | parent, index = parent_stack[-1] 87 | 88 | # 将当前元素添加到父元素下 89 | parent.append(element) 90 | text = "" 91 | _index = len(parent) - 1 92 | for att in [label, title]: 93 | if att != "": 94 | text = att 95 | break 96 | element.set("index", str(len(parent) - 1)) 97 | element.set("class_name", class_name) 98 | element.set("bounds", bounds) 99 | element.set("letf", str(left)) 100 | element.set("top", str(top)) 101 | element.set("text", text) 102 | element.set("value", value) 103 | element.set("identifier", identifier) 104 | element.set("xpath", generate_xpath(element)) 105 | 106 | # print(ET.ElementTree(element).getpath(element)) 107 | # print(element_str) 108 | 109 | # 更新父栈和当前元素 110 | parent_stack.append((element, index + 1)) 111 | 112 | # 打印 XML 字符串 113 | tree = ET.ElementTree(root) 114 | # 获取根元素 115 | root_element = tree.getroot() 116 | tree_string = etree.tostring(root_element, pretty_print=True, xml_declaration=True, encoding="utf-8") 117 | return tree_string.decode("utf-8") 118 | 119 | 120 | srt = '''Attributes: Application, 0x105a2f7b0, pid: 994, label: 'Phone call' 121 | Element subtree: 122 | Application, 0x105a2f7b0, pid: 994, label: 'Phone call' 123 | Window, 0x105a2fd90, {{0.0, 0.0}, {414.0, 736.0}} 124 | Other, 0x105a2feb0, {{0.0, 0.0}, {414.0, 736.0}} 125 | Other, 0x105a30270, {{0.0, 0.0}, {414.0, 736.0}} 126 | Other, 0x105a2ffd0, {{0.0, 0.0}, {414.0, 736.0}} 127 | Other, 0x105a300f0, {{0.0, 0.0}, {414.0, 736.0}} 128 | Other, 0x105a30810, {{0.0, 0.0}, {414.0, 736.0}} 129 | Window (Main), 0x105a30390, {{0.0, 0.0}, {414.0, 736.0}} 130 | Other, 0x105a304b0, {{0.0, 0.0}, {414.0, 736.0}} 131 | Other, 0x105a305d0, {{0.0, 0.0}, {414.0, 736.0}} 132 | Other, 0x105a306f0, {{0.0, 0.0}, {414.0, 736.0}} 133 | Other, 0x105a30930, {{0.0, 0.0}, {414.0, 736.0}} 134 | Other, 0x105a30a50, {{0.0, 0.0}, {414.0, 736.0}} 135 | Other, 0x105a30b70, {{0.0, 0.0}, {414.0, 736.0}} 136 | Other, 0x105a30c90, {{0.0, 0.0}, {414.0, 736.0}} 137 | Other, 0x105a30db0, {{0.0, 0.0}, {414.0, 736.0}} 138 | Other, 0x105a30f90, {{20.0, 64.0}, {374.0, 69.0}} 139 | Other, 0x105a310b0, {{20.0, 64.0}, {374.0, 69.0}}, identifier: 'PHSingleCallParticipantLabelView' 140 | StaticText, 0x105a311d0, {{88.0, 64.0}, {238.3, 40.7}}, identifier: 'PHMarqueeView', label: '‪158 8020 6986‬' 141 | Other, 0x105a312f0, {{138.0, 106.7}, {138.0, 26.3}} 142 | StaticText, 0x105a31410, {{138.0, 106.7}, {138.0, 26.3}}, identifier: 'PHSingleCallParticipantLabelView_StatusLabel', label: 'Xiamen, Fujian' 143 | Other, 0x105a31530, {{0.0, 463.7}, {414.0, 272.3}} 144 | Button, 0x105a31650, {{52.0, 463.7}, {82.0, 82.0}}, label: 'Remind Me' 145 | StaticText, 0x105a31770, {{52.0, 495.7}, {82.0, 19.1}}, label: 'Remind Me' 146 | Button, 0x105a31890, {{52.0, 575.7}, {82.0, 82.0}}, identifier: 'Decline', label: 'Decline' 147 | StaticText, 0x105a319b0, {{66.0, 665.7}, {55.0, 20.0}}, label: 'Decline' 148 | Button, 0x105a31ad0, {{280.0, 463.7}, {82.0, 82.0}}, label: 'Message' 149 | StaticText, 0x105a31bf0, {{280.0, 496.0}, {82.0, 19.1}}, label: 'Message' 150 | Button, 0x105a31d10, {{280.0, 575.7}, {82.0, 82.0}}, identifier: 'Accept', label: 'Answer call' 151 | StaticText, 0x105a31e30, {{295.0, 665.7}, {52.0, 20.0}}, label: 'Accept' 152 | Path to element: 153 | →Application, 0x105a2f7b0, pid: 994, label: 'Phone call' 154 | Query chain: 155 | →Find: Application 'com.apple.InCallService' 156 | Output: { 157 | Application, 0x105637a60, pid: 994, label: 'Phone call' 158 | } 159 | ''' 160 | 161 | # print(converter(srt)) -------------------------------------------------------------------------------- /auto_nico/ios/tools/image_process.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | import numpy as np 3 | 4 | 5 | def bytes_to_image(byte_data): 6 | # 将字节数据转换为 NumPy 数组 7 | nparr = np.frombuffer(byte_data, np.uint8) 8 | 9 | # 使用 OpenCV 解码图像 10 | img_np = cv2.imdecode(nparr, cv2.IMREAD_COLOR) 11 | 12 | return img_np 13 | def images_to_video(images, output_file, fps=10): 14 | # 获取图像的大小 15 | height, width, _ = images[0].shape 16 | 17 | # 创建 VideoWriter 对象 18 | video = cv2.VideoWriter(output_file, cv2.VideoWriter_fourcc(*'mp4v'), fps, (width, height)) 19 | 20 | # 将每个图像写入视频 21 | for image in images: 22 | video.write(image) 23 | 24 | # 释放 VideoWriter 25 | video.release() -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | 3 | with open("README.md", "r", encoding='UTF-8') as fh: 4 | long_description = fh.read() 5 | 6 | 7 | def parse_requirements(filename): 8 | lineiter = (line.strip() for line in open(filename)) 9 | return [line for line in lineiter if line and not line.startswith("#")] 10 | 11 | 12 | setuptools.setup( 13 | name="AutoNico", 14 | version="1.3.9", 15 | author="Hank Hang", 16 | author_email="hanhuang@jabra.com", 17 | description="Provide Basic Interface to conrol Mobile UI.", 18 | long_description=long_description, 19 | long_description_content_type="text/markdown", 20 | url="https://github.com/letmeNo1/nico", 21 | packages=setuptools.find_packages(), 22 | package_data={ 23 | 'auto_nico': ['android/package/*','console_scripts/inspector_web/templates/xml_template.html', 24 | 'console_scripts/inspector_web/static/*'] 25 | }, 26 | install_requires=[ 27 | "flask==3.0.3", 28 | 'lxml==5.1.0', 29 | 'numpy>=1.24.4', 30 | 'opencv-python==4.9.0.80', 31 | 'loguru==0.7.2', 32 | 'py-ios==0.1.3' 33 | ], 34 | classifiers=[ 35 | "Programming Language :: Python :: 3", 36 | "License :: OSI Approved :: MIT License", 37 | "Operating System :: OS Independent", 38 | ], 39 | entry_points={ 40 | 'console_scripts': [ 41 | 'nico_dump = auto_nico.console_scripts.dump_ui:main', 42 | 'nico_screenshot = auto_nico.console_scripts.screenshot:main', 43 | 'nico_ui = auto_nico.console_scripts.inspector_web.nico_inspector:main', 44 | 'nico_uninstall_apk = auto_nico.console_scripts.uninstall_apk:main', 45 | 46 | ], 47 | }, 48 | python_requires='>=3.6', 49 | include_package_data=True) 50 | -------------------------------------------------------------------------------- /test.py: -------------------------------------------------------------------------------- 1 | import requests 2 | # 3 | # 服务器地址和端口 4 | url = "http://localhost:8200/get_current_bundleIdentifier" 5 | # 若需要传递参数，可在字典中指定 6 | params = { 7 | "bundle_ids": "com.example.app1,com.example.app2" 8 | } 9 | 10 | try: 11 | response = requests.get(url, params=params) 12 | # 检查响应状态码 13 | print(response.text) 14 | 15 | except requests.RequestException as e: 16 | print(f"请求发生错误：{e}") 17 | 18 | # url = "http://localhost:8200/check_status" 19 | # # 若需要传递参数，可在字典中指定 20 | # 21 | # 22 | # try: 23 | # response = requests.get(url) 24 | # # 检查响应状态码 25 | # print(response) 26 | # 27 | # except requests.RequestException as e: 28 | # print(f"请求发生错误：{e}") 29 | 30 | from auto_nico.android.adb_utils import AdbUtils 31 | --------------------------------------------------------------------------------