├── best.pt
├── README.md
└── main.py


/best.pt:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/hlwgy/juejin_rpa/HEAD/best.pt


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | 掘金配套文件代码
2 | 
3 | 《AI加持下的RPA：模拟人工动鼠标的方式打开滑块验证码》
4 | 
5 | https://juejin.cn/post/7270028440035524663
6 | 


--------------------------------------------------------------------------------
/main.py:
--------------------------------------------------------------------------------
 1 | 
 2 | # %% ======================== 屏幕截取
 3 | # 导入库
 4 | import pyautogui
 5 | from ultralytics import YOLO
 6 | import numpy as np
 7 | 
 8 | # 加载模型
 9 | model = YOLO('best.pt')
10 | # 定位大体区域
11 | x1,y1,x2,y2 = 140, 110, 570, 510
12 | # 分类名称
13 | names = {0: 'start', 1: 'target', 2: 'fill', 3: 'operate', 4: 'refresh'}
14 | 
15 | # 截取整个屏幕并检测结果
16 | def get_current():
17 |     screenshot = pyautogui.screenshot()
18 |     cropped = screenshot.crop((x1,y1,x2,y2))
19 |     results = model.predict(source=cropped)
20 |     # 获取检测到的矩形框
21 |     b_datas = results[0].boxes.data
22 |     print("检测原始数据:", b_datas)
23 |     points = {}
24 |     # 将数据按照{"start":[x1,y1,x2,y2]}的格式重组，方便调用
25 |     for box in b_datas:
26 |         if box[4] > 0.65: # 概率大于80%才记录
27 |             name = names[int(box[5])]
28 |             points[name] = np.array(box[:4], np.int32)
29 |     print("加工后:", points)
30 |     return points
31 | 
32 | # 根据结果获取移动信息
33 | def get_move_info(points):
34 | 
35 |     if "operate" not in points or "target" not in points:
36 |         raise ValueError("找不到元素")
37 |     operate_box = points["operate"]
38 |     # 找到操作块的中心点
39 |     centerx_op = (operate_box[0] + operate_box[2])/2 + x1
40 |     centery_op = (operate_box[1] + operate_box[3])/2 + y1
41 |     # 找到终点块的中心
42 |     target_box = points["target"]
43 |     centerx_target = (target_box[0] + target_box[2])//2
44 |     # 如果没有起点块，操作充当
45 |     if "start" not in points:
46 |         start_box = operate_box
47 |     else: # 有起点块
48 |         start_box = points["start"]
49 |         # 找到起点块的中心
50 |     centerx_start = (start_box[0] + start_box[2])//2
51 |     # 计算距离
52 |     drag_distance = centerx_target - centerx_start
53 | 
54 |     return centerx_op, centery_op, drag_distance
55 | 
56 | # 拖动鼠标
57 | def darg_mouse(centerx_op, centery_op, drag_distance):
58 |     print(f"从 ({centerx_op}, {centery_op}) 拖动到 {drag_distance}")
59 |     # 移动到操作块中心点
60 |     pyautogui.moveTo(centerx_op, centery_op, duration=1)
61 |     # 按下鼠标左键
62 |     pyautogui.mouseDown()
63 |     # 拖动鼠标
64 |     pyautogui.moveRel(drag_distance, 0, duration=1)
65 |     # 松开鼠标左键
66 |     pyautogui.mouseUp()
67 | # %%
68 | # 获取并识别当前验证码信息
69 | points = get_current()
70 | # 分析结果，找到拖动的参数
71 | centerx_op, centery_op, drag_distance = get_move_info(points)
72 | # 移动鼠标
73 | darg_mouse(centerx_op, centery_op, drag_distance)
74 | # %%
75 | 


--------------------------------------------------------------------------------