├── simplerpa ├── __init__.py ├── core │ ├── __init__.py │ ├── action │ │ ├── __init__.py │ │ ├── ActionDo.py │ │ ├── ActionSystem.py │ │ ├── ActionData.py │ │ ├── ActionClipboard.py │ │ ├── ActionError.py │ │ ├── ActionKeyboard.py │ │ ├── ActionMouse.py │ │ ├── ActionScreen.py │ │ └── ActionWindow.py │ ├── data │ │ ├── __init__.py │ │ ├── StateBlockBase.py │ │ ├── Project.py │ │ ├── Transition.py │ │ ├── Misc.py │ │ ├── Find.py │ │ ├── Action.py │ │ └── ScreenRect.py │ ├── share │ │ ├── __init__.py │ │ ├── yaml.py │ │ ├── list_util.py │ │ └── str_util.py │ ├── detection │ │ ├── __init__.py │ │ ├── ColorDetection.py │ │ ├── Detection.py │ │ ├── OcrDetection.py │ │ ├── WindowDetection.py │ │ └── ImageDetection.py │ ├── extractor │ │ ├── __init__.py │ │ ├── FormExtractor.py │ │ └── Extractor.py │ ├── monitor │ │ ├── __init__.py │ │ ├── Monitor.py │ │ └── ImageMonitor.py │ ├── const.py │ ├── Option.py │ ├── App.py │ ├── Variable.py │ ├── ProjectLoader.py │ └── Executor.py ├── objtyping │ ├── __init__.py │ └── objtyping.py ├── conf │ ├── auto_wechat │ │ ├── angle.png │ │ ├── jindi.png │ │ ├── one_msg.png │ │ ├── chat_head.png │ │ ├── completed.png │ │ ├── form_end.png │ │ ├── form_start.png │ │ ├── group_name.png │ │ ├── h+_title.png │ │ ├── icon_rmb.png │ │ ├── icon_time.png │ │ ├── snapshot.png │ │ ├── chat_toolbar.png │ │ ├── icon_end_big.png │ │ ├── wechat_logo.png │ │ ├── chat_splitter.png │ │ ├── completed_78%.png │ │ ├── completed_blue.png │ │ ├── completed_gray.png │ │ ├── icon_end_small.png │ │ ├── icon_rmb_small.png │ │ ├── icon_start_big.png │ │ ├── icon_time_bin.png │ │ ├── icon_time_small.png │ │ ├── wangyue_title.png │ │ ├── chat_head_circle.png │ │ ├── contactor_name_h.png │ │ ├── icon_start_small.png │ │ ├── line_in_snapshot.png │ │ └── snapshot_thumbnail.png │ ├── auto_trello │ │ ├── trello.png │ │ └── detect_target.png │ ├── auto_playvalkyr │ │ ├── fight.png │ │ ├── select.png │ │ ├── strong.png │ │ ├── weak.png │ │ ├── confirm.png │ │ ├── veryweak.png │ │ └── plugin_metamask.png │ ├── auto_pingcode │ │ ├── find_brief.png │ │ ├── find_task.png │ │ ├── detect_brief.png │ │ ├── find_workitem.png │ │ ├── detect_pingcode.png │ │ └── detect_in_sprint.png │ ├── auto_wemeeting │ │ ├── h+_title.png │ │ ├── icon_pre.png │ │ ├── btn_confirm.png │ │ └── logo_wemeeting.png │ ├── auto_dingding │ │ ├── dingding_logo.png │ │ ├── notepad_logo.png │ │ ├── notepad_title.png │ │ ├── work_report.png │ │ ├── context_select_all.png │ │ ├── dingding_msg_icon.png │ │ ├── tzding_logo_on_dingding.png │ │ └── work_report_text_snippet.png │ ├── auto_snapshot.yaml │ ├── auto_refresh.yaml │ ├── auto_scroll.yaml │ ├── auto_browser.yaml │ ├── auto_trello.yaml │ ├── auto_pingcode.yaml │ ├── auto_wemeeting.yaml │ ├── auto_playvalkyr.yaml │ ├── project.yaml │ ├── test_rect.yaml │ ├── auto_dingding.yaml │ └── auto_wechat.yaml └── main.py ├── test ├── test_for_typing │ ├── __init__.py │ └── test_for_typing.py ├── test_for_dynamic_exe │ ├── __init__.py │ ├── test_exec.py │ ├── test_eval.py │ └── test_compile.py ├── find_template │ ├── result.png │ ├── seg_sharp.png │ ├── seg_course_menu.png │ ├── seg_course_menu_long.png │ ├── seg_course_menu_small.png │ ├── seg_course_whole_page.png │ ├── top_k.py │ ├── test_find_template.py │ ├── get_sorted_top_k.py │ └── find_all_template.py ├── test_find_image │ ├── result.png │ ├── icon_rmb.png │ ├── book_cover.jpg │ ├── chat_snapshot.jpg │ ├── img_src_soble.png │ ├── icon_rmb_small.png │ ├── img_temp_soble.png │ ├── book_cover_rotated.jpg │ ├── test_sift.py │ ├── orb_flann_matcher.py │ ├── test_diff_method.py │ └── test_Sobel.py ├── test_import_class │ ├── test_import.py │ ├── ClassC.py │ ├── ClassB.py │ ├── test.py │ └── ClassA.py ├── test_for_circular_reference │ ├── Module1.py │ ├── Module2.py │ └── ref_in_one_file.py ├── test_selenium │ ├── deploy.cmd │ ├── option.py │ ├── test_selenium.py │ ├── test_shein.py │ ├── download.py │ ├── pexels_download.py │ └── facebook_download.py ├── test_connectedComponents │ ├── img_erode.png │ └── test_connectedComponents.py ├── test_for_clipboard │ └── test_clipboard.py ├── test_eval │ └── test_eval.py ├── test_for_fucn_instance_variable │ └── fun_instance_variable.py └── test_for_win32 │ └── test_win32.py ├── apidocs ├── docs │ ├── api_reference.md │ └── index.md └── mkdocs.yml ├── docs ├── yaml_demo │ ├── project.yaml │ ├── project.json │ └── project.xml ├── simplerpa_sample_object_diagram.md ├── simplerpa_reference.md └── simplerpa_class_diagram.md ├── deploy └── install_package.cmd ├── main.py ├── LICENSE ├── requirements.txt ├── .gitignore └── README.md /simplerpa/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /simplerpa/core/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /simplerpa/core/action/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /simplerpa/core/data/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /simplerpa/core/share/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /simplerpa/objtyping/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/test_for_typing/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /simplerpa/core/detection/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /simplerpa/core/extractor/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /simplerpa/core/monitor/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/test_for_dynamic_exe/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /apidocs/docs/api_reference.md: -------------------------------------------------------------------------------- 1 | # SimpleRPA 参考手册 2 | ::: simplerpa -------------------------------------------------------------------------------- /simplerpa/core/share/yaml.py: -------------------------------------------------------------------------------- 1 | from ruamel.yaml import YAML 2 | 3 | yaml = YAML() 4 | -------------------------------------------------------------------------------- /test/find_template/result.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/test/find_template/result.png -------------------------------------------------------------------------------- /test/test_find_image/result.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/test/test_find_image/result.png -------------------------------------------------------------------------------- /test/find_template/seg_sharp.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/test/find_template/seg_sharp.png -------------------------------------------------------------------------------- /test/test_find_image/icon_rmb.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/test/test_find_image/icon_rmb.png -------------------------------------------------------------------------------- /test/test_import_class/test_import.py: -------------------------------------------------------------------------------- 1 | from ClassA import C 2 | 3 | 4 | def test2(): 5 | c = C() 6 | c.walk() 7 | -------------------------------------------------------------------------------- /simplerpa/conf/auto_wechat/angle.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/simplerpa/conf/auto_wechat/angle.png -------------------------------------------------------------------------------- /simplerpa/conf/auto_wechat/jindi.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/simplerpa/conf/auto_wechat/jindi.png -------------------------------------------------------------------------------- /test/test_find_image/book_cover.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/test/test_find_image/book_cover.jpg -------------------------------------------------------------------------------- /test/test_for_circular_reference/Module1.py: -------------------------------------------------------------------------------- 1 | from . import Module2 2 | 3 | 4 | class A: 5 | x: str 6 | y: Module2.B 7 | -------------------------------------------------------------------------------- /simplerpa/conf/auto_trello/trello.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/simplerpa/conf/auto_trello/trello.png -------------------------------------------------------------------------------- /simplerpa/conf/auto_wechat/one_msg.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/simplerpa/conf/auto_wechat/one_msg.png -------------------------------------------------------------------------------- /test/find_template/seg_course_menu.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/test/find_template/seg_course_menu.png -------------------------------------------------------------------------------- /test/test_find_image/chat_snapshot.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/test/test_find_image/chat_snapshot.jpg -------------------------------------------------------------------------------- /test/test_find_image/img_src_soble.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/test/test_find_image/img_src_soble.png -------------------------------------------------------------------------------- /test/test_import_class/ClassC.py: -------------------------------------------------------------------------------- 1 | from ClassA import A 2 | 3 | 4 | class C(A): 5 | def walk(self): 6 | print(A.test) 7 | -------------------------------------------------------------------------------- /test/test_selenium/deploy.cmd: -------------------------------------------------------------------------------- 1 | # tested on python 3.7 venv 2 | pip install selenium 3 | pip install pandas 4 | pip install wget 5 | -------------------------------------------------------------------------------- /simplerpa/conf/auto_playvalkyr/fight.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/simplerpa/conf/auto_playvalkyr/fight.png -------------------------------------------------------------------------------- /simplerpa/conf/auto_playvalkyr/select.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/simplerpa/conf/auto_playvalkyr/select.png -------------------------------------------------------------------------------- /simplerpa/conf/auto_playvalkyr/strong.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/simplerpa/conf/auto_playvalkyr/strong.png -------------------------------------------------------------------------------- /simplerpa/conf/auto_playvalkyr/weak.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/simplerpa/conf/auto_playvalkyr/weak.png -------------------------------------------------------------------------------- /simplerpa/conf/auto_wechat/chat_head.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/simplerpa/conf/auto_wechat/chat_head.png -------------------------------------------------------------------------------- /simplerpa/conf/auto_wechat/completed.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/simplerpa/conf/auto_wechat/completed.png -------------------------------------------------------------------------------- /simplerpa/conf/auto_wechat/form_end.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/simplerpa/conf/auto_wechat/form_end.png -------------------------------------------------------------------------------- /simplerpa/conf/auto_wechat/form_start.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/simplerpa/conf/auto_wechat/form_start.png -------------------------------------------------------------------------------- /simplerpa/conf/auto_wechat/group_name.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/simplerpa/conf/auto_wechat/group_name.png -------------------------------------------------------------------------------- /simplerpa/conf/auto_wechat/h+_title.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/simplerpa/conf/auto_wechat/h+_title.png -------------------------------------------------------------------------------- /simplerpa/conf/auto_wechat/icon_rmb.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/simplerpa/conf/auto_wechat/icon_rmb.png -------------------------------------------------------------------------------- /simplerpa/conf/auto_wechat/icon_time.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/simplerpa/conf/auto_wechat/icon_time.png -------------------------------------------------------------------------------- /simplerpa/conf/auto_wechat/snapshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/simplerpa/conf/auto_wechat/snapshot.png -------------------------------------------------------------------------------- /test/test_find_image/icon_rmb_small.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/test/test_find_image/icon_rmb_small.png -------------------------------------------------------------------------------- /test/test_find_image/img_temp_soble.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/test/test_find_image/img_temp_soble.png -------------------------------------------------------------------------------- /simplerpa/conf/auto_pingcode/find_brief.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/simplerpa/conf/auto_pingcode/find_brief.png -------------------------------------------------------------------------------- /simplerpa/conf/auto_pingcode/find_task.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/simplerpa/conf/auto_pingcode/find_task.png -------------------------------------------------------------------------------- /simplerpa/conf/auto_playvalkyr/confirm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/simplerpa/conf/auto_playvalkyr/confirm.png -------------------------------------------------------------------------------- /simplerpa/conf/auto_playvalkyr/veryweak.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/simplerpa/conf/auto_playvalkyr/veryweak.png -------------------------------------------------------------------------------- /simplerpa/conf/auto_wechat/chat_toolbar.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/simplerpa/conf/auto_wechat/chat_toolbar.png -------------------------------------------------------------------------------- /simplerpa/conf/auto_wechat/icon_end_big.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/simplerpa/conf/auto_wechat/icon_end_big.png -------------------------------------------------------------------------------- /simplerpa/conf/auto_wechat/wechat_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/simplerpa/conf/auto_wechat/wechat_logo.png -------------------------------------------------------------------------------- /simplerpa/conf/auto_wemeeting/h+_title.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/simplerpa/conf/auto_wemeeting/h+_title.png -------------------------------------------------------------------------------- /simplerpa/conf/auto_wemeeting/icon_pre.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/simplerpa/conf/auto_wemeeting/icon_pre.png -------------------------------------------------------------------------------- /test/find_template/seg_course_menu_long.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/test/find_template/seg_course_menu_long.png -------------------------------------------------------------------------------- /test/test_connectedComponents/img_erode.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/test/test_connectedComponents/img_erode.png -------------------------------------------------------------------------------- /test/test_find_image/book_cover_rotated.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/test/test_find_image/book_cover_rotated.jpg -------------------------------------------------------------------------------- /simplerpa/conf/auto_dingding/dingding_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/simplerpa/conf/auto_dingding/dingding_logo.png -------------------------------------------------------------------------------- /simplerpa/conf/auto_dingding/notepad_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/simplerpa/conf/auto_dingding/notepad_logo.png -------------------------------------------------------------------------------- /simplerpa/conf/auto_dingding/notepad_title.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/simplerpa/conf/auto_dingding/notepad_title.png -------------------------------------------------------------------------------- /simplerpa/conf/auto_dingding/work_report.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/simplerpa/conf/auto_dingding/work_report.png -------------------------------------------------------------------------------- /simplerpa/conf/auto_pingcode/detect_brief.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/simplerpa/conf/auto_pingcode/detect_brief.png -------------------------------------------------------------------------------- /simplerpa/conf/auto_pingcode/find_workitem.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/simplerpa/conf/auto_pingcode/find_workitem.png -------------------------------------------------------------------------------- /simplerpa/conf/auto_trello/detect_target.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/simplerpa/conf/auto_trello/detect_target.png -------------------------------------------------------------------------------- /simplerpa/conf/auto_wechat/chat_splitter.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/simplerpa/conf/auto_wechat/chat_splitter.png -------------------------------------------------------------------------------- /simplerpa/conf/auto_wechat/completed_78%.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/simplerpa/conf/auto_wechat/completed_78%.png -------------------------------------------------------------------------------- /simplerpa/conf/auto_wechat/completed_blue.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/simplerpa/conf/auto_wechat/completed_blue.png -------------------------------------------------------------------------------- /simplerpa/conf/auto_wechat/completed_gray.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/simplerpa/conf/auto_wechat/completed_gray.png -------------------------------------------------------------------------------- /simplerpa/conf/auto_wechat/icon_end_small.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/simplerpa/conf/auto_wechat/icon_end_small.png -------------------------------------------------------------------------------- /simplerpa/conf/auto_wechat/icon_rmb_small.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/simplerpa/conf/auto_wechat/icon_rmb_small.png -------------------------------------------------------------------------------- /simplerpa/conf/auto_wechat/icon_start_big.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/simplerpa/conf/auto_wechat/icon_start_big.png -------------------------------------------------------------------------------- /simplerpa/conf/auto_wechat/icon_time_bin.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/simplerpa/conf/auto_wechat/icon_time_bin.png -------------------------------------------------------------------------------- /simplerpa/conf/auto_wechat/icon_time_small.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/simplerpa/conf/auto_wechat/icon_time_small.png -------------------------------------------------------------------------------- /simplerpa/conf/auto_wechat/wangyue_title.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/simplerpa/conf/auto_wechat/wangyue_title.png -------------------------------------------------------------------------------- /simplerpa/conf/auto_wemeeting/btn_confirm.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/simplerpa/conf/auto_wemeeting/btn_confirm.png -------------------------------------------------------------------------------- /test/find_template/seg_course_menu_small.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/test/find_template/seg_course_menu_small.png -------------------------------------------------------------------------------- /test/find_template/seg_course_whole_page.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/test/find_template/seg_course_whole_page.png -------------------------------------------------------------------------------- /simplerpa/conf/auto_pingcode/detect_pingcode.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/simplerpa/conf/auto_pingcode/detect_pingcode.png -------------------------------------------------------------------------------- /simplerpa/conf/auto_wechat/chat_head_circle.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/simplerpa/conf/auto_wechat/chat_head_circle.png -------------------------------------------------------------------------------- /simplerpa/conf/auto_wechat/contactor_name_h.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/simplerpa/conf/auto_wechat/contactor_name_h.png -------------------------------------------------------------------------------- /simplerpa/conf/auto_wechat/icon_start_small.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/simplerpa/conf/auto_wechat/icon_start_small.png -------------------------------------------------------------------------------- /simplerpa/conf/auto_wechat/line_in_snapshot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/simplerpa/conf/auto_wechat/line_in_snapshot.png -------------------------------------------------------------------------------- /simplerpa/conf/auto_wemeeting/logo_wemeeting.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/simplerpa/conf/auto_wemeeting/logo_wemeeting.png -------------------------------------------------------------------------------- /simplerpa/core/const.py: -------------------------------------------------------------------------------- 1 | STATE: str = 'state' 2 | FIND_RESULT: str = 'find_result' 3 | MONITOR_RESULT: str = 'monitor_result' 4 | MOUSE: str = 'mouse' 5 | -------------------------------------------------------------------------------- /simplerpa/core/monitor/Monitor.py: -------------------------------------------------------------------------------- 1 | from simplerpa.core.data.StateBlockBase import StateBlockBase 2 | 3 | 4 | class Monitor(StateBlockBase): 5 | pass -------------------------------------------------------------------------------- /test/test_for_circular_reference/Module2.py: -------------------------------------------------------------------------------- 1 | from typing import List 2 | 3 | from . import Module1 4 | 5 | 6 | class B: 7 | w: List[Module1.A] 8 | -------------------------------------------------------------------------------- /simplerpa/conf/auto_dingding/context_select_all.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/simplerpa/conf/auto_dingding/context_select_all.png -------------------------------------------------------------------------------- /simplerpa/conf/auto_dingding/dingding_msg_icon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/simplerpa/conf/auto_dingding/dingding_msg_icon.png -------------------------------------------------------------------------------- /simplerpa/conf/auto_pingcode/detect_in_sprint.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/simplerpa/conf/auto_pingcode/detect_in_sprint.png -------------------------------------------------------------------------------- /simplerpa/conf/auto_playvalkyr/plugin_metamask.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/simplerpa/conf/auto_playvalkyr/plugin_metamask.png -------------------------------------------------------------------------------- /simplerpa/conf/auto_wechat/snapshot_thumbnail.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/simplerpa/conf/auto_wechat/snapshot_thumbnail.png -------------------------------------------------------------------------------- /test/test_import_class/ClassB.py: -------------------------------------------------------------------------------- 1 | from ClassA import A 2 | 3 | 4 | class B(A): 5 | def walk(self): 6 | A.test = 8 7 | print(A.test) 8 | -------------------------------------------------------------------------------- /simplerpa/conf/auto_dingding/tzding_logo_on_dingding.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/simplerpa/conf/auto_dingding/tzding_logo_on_dingding.png -------------------------------------------------------------------------------- /simplerpa/conf/auto_dingding/work_report_text_snippet.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/songofhawk/simplerpa/HEAD/simplerpa/conf/auto_dingding/work_report_text_snippet.png -------------------------------------------------------------------------------- /test/test_import_class/test.py: -------------------------------------------------------------------------------- 1 | from ClassA import B,A 2 | from test_import import test2 3 | 4 | if __name__ == '__main__': 5 | b = B() 6 | A.test = 8 7 | b.walk() 8 | test2() 9 | -------------------------------------------------------------------------------- /simplerpa/core/data/StateBlockBase.py: -------------------------------------------------------------------------------- 1 | 2 | class StateBlockBase(object): 3 | name: str = None 4 | project = None 5 | # executor: Executor.Executor = None 6 | 7 | debug: bool = False 8 | -------------------------------------------------------------------------------- /simplerpa/core/Option.py: -------------------------------------------------------------------------------- 1 | class Option: 2 | project: str = './conf/auto_dingding.yaml' 3 | 4 | def __init__(self, project): 5 | self.project = project if project is not None else self.project 6 | 7 | 8 | -------------------------------------------------------------------------------- /test/test_for_circular_reference/ref_in_one_file.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | from typing import List 3 | 4 | 5 | class A: 6 | x: str 7 | y: B 8 | 9 | 10 | class B: 11 | w: List[A] 12 | -------------------------------------------------------------------------------- /test/test_import_class/ClassA.py: -------------------------------------------------------------------------------- 1 | class A: 2 | test = 5 3 | 4 | 5 | class B(A): 6 | def walk(self): 7 | print(A.test) 8 | 9 | 10 | class C(A): 11 | def walk(self): 12 | print(A.test) 13 | -------------------------------------------------------------------------------- /simplerpa/core/share/list_util.py: -------------------------------------------------------------------------------- 1 | def append_to(the_list, item): 2 | if isinstance(item, list): 3 | the_list.extend(item) 4 | elif item is None: 5 | return 6 | else: 7 | the_list.append(item) 8 | -------------------------------------------------------------------------------- /test/test_for_dynamic_exe/test_exec.py: -------------------------------------------------------------------------------- 1 | def pr(x): 2 | print('My result: {}'.format(x)) 3 | 4 | 5 | if __name__ == "__main__": 6 | s = ''' 7 | a = 15 8 | b = 3 9 | if a > b: 10 | pr(a+b) 11 | ''' 12 | exec(s) 13 | -------------------------------------------------------------------------------- /docs/yaml_demo/project.yaml: -------------------------------------------------------------------------------- 1 | id: 100 2 | name: "测试项目" 3 | version: "3.1" 4 | steps: 5 | - id: 18 6 | action: "Prepare" 7 | expects: 8 | - id: 238, 9 | result: GOOD 10 | - id: 239, 11 | result: PERFECT 12 | -------------------------------------------------------------------------------- /simplerpa/core/action/ActionDo.py: -------------------------------------------------------------------------------- 1 | class ActionDo: 2 | def __init__(self, context): 3 | self._context = context 4 | 5 | def do(self, params): 6 | raise NotImplementedError("'do' method should be implemented by sub class!") 7 | -------------------------------------------------------------------------------- /test/test_for_dynamic_exe/test_eval.py: -------------------------------------------------------------------------------- 1 | def select_max(x, y): 2 | return x if x > y else y 3 | 4 | 5 | if __name__ == "__main__": 6 | a = 3 7 | b = 5 8 | c = eval('select_max(a , b)') 9 | print("c is {}".format(c)) 10 | -------------------------------------------------------------------------------- /simplerpa/core/action/ActionSystem.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | 4 | class ActionSystem: 5 | @classmethod 6 | def wait(cls, sec): 7 | """ 8 | 暂停 9 | Returns: 10 | float: 时间（单位秒） 11 | """ 12 | time.sleep(sec) 13 | 14 | -------------------------------------------------------------------------------- /test/test_for_clipboard/test_clipboard.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | import pyautogui 4 | import pyperclip 5 | 6 | if __name__ == "__main__": 7 | print('wait') 8 | time.sleep(2) 9 | print('copy') 10 | pyperclip.copy("可能性") 11 | print('paste') 12 | pyautogui.hotkey('ctrl', 'v') 13 | -------------------------------------------------------------------------------- /deploy/install_package.cmd: -------------------------------------------------------------------------------- 1 | pip install pypiwin32 2 | pip install pyautogui 3 | pip install pyperclip 4 | pip install opencv-python 5 | pip install aircv 6 | pip install ruamel.yaml 7 | pip install pandas 8 | pip install openpyxl 9 | pip install cnocr 10 | # 如果安装cnocr之后，启动程序报错，提示“ WinError 126] 找不到指定的模块”，可以安装vc_redist_2015和vc_redist_2012来解决 -------------------------------------------------------------------------------- /test/test_for_dynamic_exe/test_compile.py: -------------------------------------------------------------------------------- 1 | exp = compile('select_max(a , b)', '', 'eval') 2 | 3 | 4 | def select_max(x, y): 5 | return x if x > y else y 6 | 7 | 8 | if __name__ == "__main__": 9 | for i in range(10): 10 | a = i 11 | b = i + 10 12 | c = eval(exp) 13 | print("c is {}".format(c)) 14 | -------------------------------------------------------------------------------- /test/test_eval/test_eval.py: -------------------------------------------------------------------------------- 1 | _call_env = {'print': print} 2 | exp = compile('a = 3', '', 'exec') 3 | exec(exp, {"__builtins__": {}}, _call_env) 4 | 5 | exp3 = compile('print(a)', '', 'exec') 6 | exec(exp3, {"__builtins__": {}}, _call_env) 7 | 8 | exp2 = compile('a', '', 'eval') 9 | x = eval(exp2, {"__builtins__": {}}, _call_env) 10 | print(x) 11 | -------------------------------------------------------------------------------- /simplerpa/conf/auto_snapshot.yaml: -------------------------------------------------------------------------------- 1 | name: "测试截图到剪贴板" 2 | ver: 0.1 3 | #screen_width: 3440 4 | #screen_height: 1440 5 | range: !rect l:0, r:1920, t:0, b:1080 6 | time_scale: 1 7 | states: 8 | - name: "截图到剪贴板" 9 | id: 1 10 | action: 11 | - snapshot_pil(ScreenRect(0,200,0,100), to_clipboard=True) 12 | transition: 13 | # 点击 14 | to: end 15 | -------------------------------------------------------------------------------- /apidocs/mkdocs.yml: -------------------------------------------------------------------------------- 1 | site_name: simplerpa 2 | 3 | nav: 4 | - 首页: index.md 5 | - 参考手册: api_reference.md 6 | - 历史: history.md 7 | 8 | # theme: readthedocs 9 | 10 | plugins: 11 | - search 12 | - mkdocstrings: 13 | handlers: 14 | python: 15 | setup_commands: 16 | - import sys 17 | - sys.path.insert(0, "..") -------------------------------------------------------------------------------- /simplerpa/core/action/ActionData.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | 3 | 4 | class ActionData: 5 | def __init__(self, context): 6 | self._context = context 7 | 8 | @staticmethod 9 | def create_dataframe(column_names): 10 | if isinstance(column_names,list): 11 | return pd.DataFrame(columns=column_names) 12 | else: 13 | return pd.DataFrame() 14 | -------------------------------------------------------------------------------- /docs/yaml_demo/project.json: -------------------------------------------------------------------------------- 1 | { 2 | "id": 100, 3 | "name": "测试项目", 4 | "version": "3.1", 5 | "steps": [ 6 | { 7 | "id": 18, 8 | "action": "Prepare", 9 | "expects": [ 10 | { 11 | "id": 238, 12 | "result": "GOOD" 13 | }, 14 | { 15 | "id": 239, 16 | "result": "PERFECT" 17 | } 18 | ] 19 | } 20 | ] 21 | } 22 | -------------------------------------------------------------------------------- /test/test_for_typing/test_for_typing.py: -------------------------------------------------------------------------------- 1 | from typing import Tuple, get_type_hints 2 | 3 | 4 | class a: 5 | x: Tuple[int, int] = (5, 19) 6 | 7 | 8 | clazz_dict = get_type_hints(a) 9 | clazz = clazz_dict['x'] 10 | if clazz._name == 'Tuple': 11 | print('a is a tuple, checked by name') 12 | 13 | if hasattr(clazz, '__origin__') and clazz.__origin__ == tuple: 14 | print('a is a tuple, checked by origin') 15 | -------------------------------------------------------------------------------- /simplerpa/conf/auto_refresh.yaml: -------------------------------------------------------------------------------- 1 | name: "浏览器自动刷新" 2 | ver: 0.1 3 | #screen_width: 3440 4 | #screen_height: 1440 5 | range: !rect l:0, r:1920, t:0, b:1080 6 | time_scale: 1 7 | states: 8 | - name: "切换当前窗口" 9 | id: 1 10 | transition: 11 | # 点击 12 | action: hotkey('alt', 'tab') 13 | wait: 1 14 | - name: "刷新" 15 | id: 2 16 | transition: 17 | # 右击第一个卡片 18 | action: hotkey('f5') 19 | wait: 60 20 | to: 2 21 | -------------------------------------------------------------------------------- /simplerpa/conf/auto_scroll.yaml: -------------------------------------------------------------------------------- 1 | name: "测试scroll的使用" 2 | ver: 0.1 3 | #screen_width: 3440 4 | #screen_height: 1440 5 | range: !rect l:0, r:1000, t:0, b:800 6 | time_scale: 1 7 | states: 8 | - name: "当前窗口" 9 | id: 0 10 | find: 11 | image: 12 | snapshot: !rect l:70, r:250, t:76, b:760 13 | template: auto_dingding/work_report.png 14 | scroll: 15 | one_page: -500 16 | page_count: 3 17 | fail_action: raise_error('没找到工作汇报消息') 18 | -------------------------------------------------------------------------------- /test/test_for_fucn_instance_variable/fun_instance_variable.py: -------------------------------------------------------------------------------- 1 | def test(): 2 | print('I am a function') 3 | 4 | 5 | class A: 6 | fun_dict = { 7 | 'test': test 8 | } 9 | 10 | def __init__(self): 11 | self.attr1 = 'xxx' 12 | self.func = test 13 | # self.func = A.fun_dict['test'] 14 | pass 15 | 16 | def call(self): 17 | self.func() 18 | 19 | 20 | if __name__ == '__main__': 21 | a = A() 22 | a.call() 23 | -------------------------------------------------------------------------------- /apidocs/docs/index.md: -------------------------------------------------------------------------------- 1 | # SimpleRPA 简介 2 | 3 | SimpleRPA是一款python语言编写的开源RPA工具（桌面自动控制工具）。 4 | 5 | 我们做自动化任务的时候，通常都会经历这样几个步骤： 6 | 1. 检查当前桌面上是否显示了需要的页面（比如查看特定位置的图像，或者比对OCR识别出的文字） 7 | 2. 如果确实是，就收集一些文字或图像的信息（这一步未必会有，要看具体任务类型，有些自动化只要把页面流程走通就可以） 8 | 3. 查找页面上特定的控件（比如某个按钮），对它进行操作（如点击） 9 | 4. 跳转到下一个页面，回到步骤1，反复循环，直到最终页面出现 10 | 11 | SimpleRPA 把这个过程，抽象为一个状态机模型：每个页面是一个状态（state），通过“action”触发，可以跳转到下一个状态； 12 | 在每一个State内部，可以做check（检查是否需要的页面），可以find（查找特定控件，或者收集信息）； 13 | 针对find的结果，还可以形成子状态，来实现复杂的操作。 14 | 15 | ## 文档结构 16 | 17 | docs/ 18 | index.md # 首页 19 | api_reference.md # API参考手册 20 | 21 | -------------------------------------------------------------------------------- /test/test_connectedComponents/test_connectedComponents.py: -------------------------------------------------------------------------------- 1 | from cv2 import cv2 2 | 3 | if __name__ == '__main__': 4 | img = cv2.imread('./img_erode.png', cv2.IMREAD_UNCHANGED) 5 | black_back = cv2.bitwise_not(img) 6 | num_labels, labels, stats, centroids = cv2.connectedComponentsWithStats(black_back) 7 | print(stats) 8 | # num_labels, labels, stats, centroids = cv2.connectedComponentsWithStatsWithAlgorithm(img, 4, ltype=cv2.CV_16U, 9 | # ccltype=cv2.CCL_GRANA) 10 | # print(stats) 11 | -------------------------------------------------------------------------------- /simplerpa/core/action/ActionClipboard.py: -------------------------------------------------------------------------------- 1 | import pyperclip 2 | 3 | 4 | class ActionClipboard: 5 | """ 6 | 剪贴板相关操作 7 | """ 8 | @classmethod 9 | def copy(cls, str_to_copy): 10 | """ 11 | 复制指定字符串到剪贴板 12 | Args: 13 | str_to_copy: 要复制的字符串 14 | 15 | Returns: 16 | None 17 | """ 18 | pyperclip.copy(str_to_copy) 19 | 20 | @classmethod 21 | def paste(cls): 22 | """ 23 | 提取当前剪贴板中的文本，作为字符串返回 24 | Returns: 25 | str: 当前剪贴板中的文本字符串 26 | """ 27 | pyperclip.paste() 28 | -------------------------------------------------------------------------------- /test/test_selenium/option.py: -------------------------------------------------------------------------------- 1 | class Option: 2 | SPLITTER: str = '/' 3 | output_dir: str = '.' 4 | result_file: str = 'result.csv' 5 | limit: int = 100 6 | 7 | def __init__(self, output_dir: str, result_file: str, limit: int): 8 | # 这里不用参数缺省值，是为了简化调用方，不需要非空判断 9 | self.output_dir = output_dir + self.SPLITTER if output_dir is not None else self.output_dir + self.SPLITTER 10 | self.result_file = result_file if result_file is not None else self.result_file 11 | self.output_result = self.output_dir + self.result_file 12 | 13 | self.limit = limit if limit is not None else self.limit 14 | -------------------------------------------------------------------------------- /simplerpa/core/App.py: -------------------------------------------------------------------------------- 1 | from simplerpa.core.Option import Option 2 | from simplerpa.core.ProjectLoader import ProjectLoader 3 | from simplerpa.core.Executor import Executor 4 | from simplerpa.core.data.Project import Project 5 | 6 | 7 | class App: 8 | option: Option = None 9 | project: Project = None 10 | 11 | def __init__(self, option: Option): 12 | self.option = option 13 | self.load_project(option.project) 14 | 15 | def load_project(self, project_file: str): 16 | self.project = ProjectLoader.load(project_file) 17 | 18 | def execute(self): 19 | executor = Executor(self.project) 20 | executor.run() 21 | -------------------------------------------------------------------------------- /simplerpa/conf/auto_browser.yaml: -------------------------------------------------------------------------------- 1 | name: "浏览器打开网页" 2 | ver: 0.1 3 | #screen_width: 3440 4 | #screen_height: 1440 5 | range: !rect l:0, r:1920, t:0, b:1080 6 | time_scale: 1 7 | states: 8 | - name: "切换当前窗口" 9 | id: 1 10 | transition: 11 | # 点击 12 | action: hotkey('alt', 'tab') 13 | wait: 1 14 | - name: "点击地址栏" 15 | id: 2 16 | transition: 17 | # 右击第一个卡片 18 | action: click(300, 60) 19 | wait: 3 20 | - name: "输入地址" 21 | id: 3 22 | transition: 23 | # 右击第一个卡片 24 | action: type('http://www.baidu.com') 25 | wait: 2 26 | - name: "点击回车键" 27 | id: 4 28 | transition: 29 | # 右击第一个卡片 30 | action: press('enter') 31 | wait: 1 32 | 33 | -------------------------------------------------------------------------------- /test/test_selenium/test_selenium.py: -------------------------------------------------------------------------------- 1 | from selenium import webdriver 2 | from selenium.webdriver.common.by import By 3 | from selenium.webdriver.common.keys import Keys 4 | from selenium.webdriver.support.ui import WebDriverWait 5 | from selenium.webdriver.support.expected_conditions import presence_of_element_located 6 | 7 | #This example requires Selenium WebDriver 3.13 or newer 8 | with webdriver.Chrome() as driver: 9 | wait = WebDriverWait(driver, 10) 10 | driver.get("https://google.com/ncr") 11 | driver.find_element(By.NAME, "q").send_keys("cheese" + Keys.RETURN) 12 | first_result = wait.until(presence_of_element_located((By.CSS_SELECTOR, "a>h3"))) 13 | print(first_result.get_attribute("textContent")) 14 | -------------------------------------------------------------------------------- /test/test_selenium/test_shein.py: -------------------------------------------------------------------------------- 1 | from selenium import webdriver 2 | from selenium.webdriver.common.by import By 3 | from selenium.webdriver.common.keys import Keys 4 | from selenium.webdriver.support.ui import WebDriverWait 5 | from selenium.webdriver.support.expected_conditions import presence_of_element_located 6 | 7 | #This example requires Selenium WebDriver 3.13 or newer 8 | with webdriver.Chrome() as driver: 9 | wait = WebDriverWait(driver, 10) 10 | driver.get("https://google.com/ncr") 11 | driver.find_element(By.NAME, "q").send_keys("cheese" + Keys.RETURN) 12 | first_result = wait.until(presence_of_element_located((By.CSS_SELECTOR, "a>h3"))) 13 | print(first_result.get_attribute("textContent")) 14 | -------------------------------------------------------------------------------- /docs/yaml_demo/project.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 100: 100 4 | 测试项目 5 | 3.1 6 | 7 | 8 | 18, 9 | Prepare, 10 | 11 | 12 | 238 13 | GOOD 14 | 15 | 16 | 239 17 | PERFECT 18 | 19 | 20 | 21 | 22 | -------------------------------------------------------------------------------- /docs/simplerpa_sample_object_diagram.md: -------------------------------------------------------------------------------- 1 | 这是示例配置文件的对象实例图 2 | 3 | ```puml 4 | @startuml 5 | object project 6 | 7 | object 当前窗口 8 | 当前窗口 : id = 1 9 | object 浏览器窗口 10 | 浏览器窗口 : id = 2 11 | object check 12 | object image 13 | image : snapshot 14 | image : template 15 | object transition1 16 | transition1 : wait = 1 17 | transition1 : to = 2 18 | object transition2 19 | transition2 : wait = 60 20 | transition2 : to = 2 21 | object action1 22 | object action2 23 | 24 | project o-- 当前窗口 25 | project o-- 浏览器窗口 26 | 27 | transition1 .. 浏览器窗口 28 | transition2 .. 浏览器窗口 29 | 30 | 浏览器窗口 *-- check 31 | check o-- image 32 | 33 | 当前窗口 *-- transition1 34 | 浏览器窗口 *-- transition2 35 | 36 | transition1 *.. action1 : alt+tab 37 | transition2 *.. action2 : F5 38 | 39 | @enduml 40 | ``` 41 | -------------------------------------------------------------------------------- /test/find_template/top_k.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | 4 | def top_k(a, k): 5 | ravel = a.ravel() 6 | idx = np.argpartition(ravel, a.size - k)[-k:] 7 | idx2 = np.argsort(-ravel[idx]) 8 | return np.unravel_index(idx[idx2], a.shape) 9 | 10 | 11 | if __name__ == "__main__": 12 | # a = np.array([[38, 14, 81, 50], 13 | # [17, 65, 60, 24], 14 | # [64, 73, 25, 95]]) 15 | # print(a) 16 | # row, col = top_k(a, 2) 17 | # print(row, col) 18 | 19 | import time 20 | from sklearn.metrics.pairwise import cosine_similarity 21 | 22 | x = np.random.rand(10, 128) 23 | y = np.random.rand(1000000, 128) 24 | z = cosine_similarity(x, y) 25 | start_time = time.time() 26 | top = top_k(z, 3) 27 | print(top) 28 | print(time.time() - start_time) 29 | -------------------------------------------------------------------------------- /test/find_template/test_find_template.py: -------------------------------------------------------------------------------- 1 | import simplerpa.aircv as ac 2 | from cv2 import cv2 3 | import time 4 | 5 | from find_all_template import find_all_template_v2, find_all_template_v1 6 | 7 | image_origin = cv2.imread('seg_course_whole_page.png') 8 | image_template = cv2.imread('seg_sharp.png') 9 | 10 | start_time = time.time() 11 | # match_results = ac.find_all_template(image_origin, image_template, 0.5) 12 | # match_results = find_all_template_v2(image_origin, image_template, 0.5, 50) 13 | match_results = find_all_template_v1(image_origin, image_template, 0.5, 50, debug=True) 14 | print("total time: {}".format(time.time() - start_time)) 15 | 16 | img_result = image_origin.copy() 17 | for match_result in match_results: 18 | rect = match_result['rectangle'] 19 | cv2.rectangle(img_result, (rect[0][0], rect[0][1]), (rect[3][0], rect[3][1]), (0, 0, 220), 2) 20 | cv2.imwrite('result.png', img_result) 21 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 可以让这个py文件直接在Unix/Linux/Mac上运行 2 | # -*- coding: utf-8 -*- 使用标准UTF-8编码； 3 | 4 | # ' main entry point ' #表示模块的文档注释 5 | import argparse 6 | # import pretty_errors 7 | 8 | from simplerpa.core.App import App 9 | from simplerpa.core.Option import Option 10 | 11 | __author__ = 'Song Hui' # 作者名 12 | 13 | 14 | def get_options_from_command_line(): 15 | # Initialize parser 16 | parser = argparse.ArgumentParser() 17 | 18 | # Adding optional argument 19 | parser.add_argument("-p", "--project", help="Use specified project file") 20 | 21 | # Read arguments from command line 22 | args = parser.parse_args() 23 | 24 | if args: 25 | print("parsing arguments: {}".format(args)) 26 | return Option(args.project) 27 | 28 | 29 | if __name__ == '__main__': 30 | option = get_options_from_command_line() 31 | app = App(option) 32 | app.execute() 33 | -------------------------------------------------------------------------------- /simplerpa/conf/auto_trello.yaml: -------------------------------------------------------------------------------- 1 | name: "自动归档Trello" 2 | ver: 0.5 3 | #screen_width: 3440 4 | #screen_height: 1440 5 | range: !rect l:0, r:1920, t:0, b:1080 6 | time_scale: 1 7 | states: 8 | - name: "点击获取窗口焦点" 9 | id: 1 10 | transition: 11 | # 点击 12 | action: click(300, 20) 13 | wait: 1.5 14 | to: next 15 | - name: "已完成列表" 16 | id: 2 17 | transition: 18 | # 右击第一个卡片 19 | action: rightclick(1540, 290) 20 | wait: 1 21 | to: next 22 | - name: "右键菜单" 23 | id: 3 24 | find: 25 | image: 26 | snapshot: !rect l:1415, r:1805, t:239, b:609 27 | template: auto_trello/detect_target.png 28 | confidence: 0.8 29 | fail_action: raise_error('找不到归档按钮') 30 | transition: 31 | # 左击归档按钮 32 | action: click(1415 + state.find_result.center_x, 239 + state.find_result.center_y) 33 | wait: 1 34 | to: 2 35 | max_time: 2 36 | -------------------------------------------------------------------------------- /simplerpa/core/data/Project.py: -------------------------------------------------------------------------------- 1 | from typing import List, Dict 2 | 3 | from .Misc import State 4 | from .ScreenRect import ScreenRect 5 | 6 | 7 | class Project: 8 | """ 9 | 配置根节点，对应自动化项目的基本信息 10 | 11 | Attributes: 12 | name (str): 项目名称 13 | ver (str): 字符串版本号 14 | screen_width (int): 屏幕宽度（如果设置了这个属性，项目运行时会首先调整分辨率） 15 | screen_height (int): 屏幕高度（如果设置了这个属性，项目运行时会首先调整分辨率） 16 | states (List[State]): 状态列表，项目启动后，将从第一个状态开始执行 17 | """ 18 | name: str = None 19 | ver: str = 0 20 | screen_width: int 21 | screen_height: int 22 | range: ScreenRect = None 23 | time_scale: float = 1.0 24 | states: List[State] = [] 25 | 26 | def __init__(self): 27 | self.all_states: Dict[int, State] = {} 28 | self.path_root = None 29 | self.executor = None 30 | 31 | # @property 32 | # def executor(self): 33 | # return StateBlockBase.executor 34 | # 35 | # @executor.setter 36 | # def executor(self, executor): 37 | # StateBlockBase.executor = executor 38 | 39 | -------------------------------------------------------------------------------- /simplerpa/main.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 可以让这个py文件直接在Unix/Linux/Mac上运行 2 | # -*- coding: utf-8 -*- 使用标准UTF-8编码； 3 | 4 | # ' main entry point ' #表示模块的文档注释 5 | 6 | """ 7 | 程序启动入口。启动后，程序将按照-project(-p)参数指定的配置文件，执行自动化项目 8 | 9 | Examples: 10 | ```console 11 | python simplerpa -p ./conf/auto_dingding.yaml 12 | ``` 13 | """ 14 | 15 | from simplerpa.core.App import App 16 | from simplerpa.core.Option import Option 17 | 18 | __author__ = 'Song Hui' # 作者名 19 | 20 | 21 | def get_options_from_command_line(): 22 | import argparse 23 | # Initialize parser 24 | parser = argparse.ArgumentParser() 25 | 26 | # Adding optional argument 27 | parser.add_argument("-p", "--project", help="Use specified project file") 28 | 29 | # Read arguments from command line 30 | args = parser.parse_args() 31 | 32 | if args: 33 | print("parsing arguments: {}".format(args)) 34 | return Option(args.project) 35 | 36 | 37 | if __name__ == '__main__': 38 | option = get_options_from_command_line() 39 | app = App(option) 40 | -------------------------------------------------------------------------------- /simplerpa/core/action/ActionError.py: -------------------------------------------------------------------------------- 1 | class ActionError: 2 | """ 3 | 异常处理相关方法 4 | """ 5 | 6 | @staticmethod 7 | def trigger(params: tuple): 8 | """ 9 | 主动触发一个运行时异常 10 | Args: 11 | params: 要触发的异常内容 12 | 13 | Returns: 14 | None 15 | """ 16 | (msg) = params 17 | # print("ERROR: {}".format(msg)) 18 | raise RuntimeError(msg) 19 | 20 | @staticmethod 21 | def locate_state(params: tuple): 22 | """ 23 | 这个方法的意思，重新查找当前界面处于哪个状态，但是真正的查找逻辑是在Executor中，这里只是输出一下 24 | Args: 25 | params: 定位当前界面所属状态前，要输出的内容 26 | 27 | Returns: 28 | None 29 | """ 30 | (msg) = params 31 | print("ERROR: {}".format(msg)) 32 | 33 | @staticmethod 34 | def level_return(current_state: int): 35 | """ 36 | 这个方法的意思，是退出当前的状态层级，返回上一层，真正的执行逻辑在Executor中，这里只是输出一下 37 | Args: 38 | params: 定位当前界面所属状态前，要输出的内容 39 | 40 | Returns: 41 | None 42 | """ 43 | print("return to upper level from state: {}".format(current_state)) 44 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Song Hui 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /simplerpa/core/share/str_util.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | 4 | # class Expression(object): 5 | # def __init__(self, param_str): 6 | # match_result = re.match('\b$(.*?)\b', param_str) 7 | # if match_result is None: 8 | # self.has_variable = False 9 | # self.value = param_str 10 | # else: 11 | # self.has_variable = True 12 | 13 | 14 | def parse_function_call(call_str: str, parse_variable: bool = False): 15 | left_pos: int = call_str.find('(') 16 | if left_pos == -1: 17 | return call_str.strip(), [] 18 | else: 19 | name: str = call_str[0:left_pos].strip() 20 | right_pos: int = call_str.rfind(')') 21 | param_str: str = call_str[left_pos + 1:right_pos].strip() 22 | params = map(lambda s: s.strip("'") if s.startswith("'") else s, 23 | map(str.strip, param_str.split(','))) 24 | return name, tuple(params) 25 | 26 | # if not parse_variable: 27 | # return name, list(params) 28 | # 29 | # param_list = [] 30 | # for param in params: 31 | # param_list.append(Expression(param)) 32 | # return name, param_list 33 | -------------------------------------------------------------------------------- /simplerpa/core/Variable.py: -------------------------------------------------------------------------------- 1 | class Variable(object): 2 | def __init__(self): 3 | self._store_vars = {} 4 | self._store_index_vars = {} 5 | 6 | def get_vars(self, var_name): 7 | if var_name not in self._store_vars: 8 | return None 9 | return self._store_vars[var_name] 10 | 11 | def set_vars(self, var_name, value): 12 | self._store_vars[var_name] = value 13 | 14 | def get_index_vars(self, index, var_name=None): 15 | if index not in self._store_index_vars: 16 | self._store_index_vars[index] = {} 17 | var_dict = self._store_index_vars[index] 18 | if var_name is None: 19 | return var_dict 20 | else: 21 | if not isinstance(var_dict, dict): 22 | raise RuntimeError('The variable from index "{}" is not a dict, can not be accessed by a var_name "{}"!'.format(index, var_name)) 23 | 24 | if var_name in var_dict: 25 | var_value = var_dict[var_name] 26 | return var_value 27 | else: 28 | raise RuntimeError('The variable from index"{}" with var_name "{}" not found!'.format(index, var_name)) 29 | 30 | def set_index_vars(self, index, var_name, value): 31 | var_dict = self.get_index_vars(index) 32 | var_dict[var_name] = value 33 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | absl-py==0.14.0 2 | aiohttp==3.7.4.post0 3 | aircv==1.4.6 4 | async-timeout==3.0.1 5 | attrs==21.2.0 6 | cachetools==4.2.2 7 | certifi==2021.5.30 8 | chardet==4.0.0 9 | charset-normalizer==2.0.6 10 | click==8.0.1 11 | cnocr>=2.1.0 12 | colorama==0.4.4 13 | fsspec==2021.9.0 14 | future==0.18.2 15 | google-auth==1.35.0 16 | google-auth-oauthlib==0.4.6 17 | grpcio==1.41.0 18 | idna==3.2 19 | importlib-metadata==4.8.1 20 | Markdown==3.3.4 21 | MouseInfo==0.1.3 22 | multidict==5.1.0 23 | numpy>=1.21.4 24 | oauthlib==3.1.1 25 | opencv-python>=4.5.4.60 26 | packaging==21.0 27 | Pillow>=8.4.0 28 | protobuf==3.18.0 29 | pyasn1==0.4.8 30 | pyasn1-modules==0.2.8 31 | PyAutoGUI==0.9.53 32 | pyDeprecate==0.3.1 33 | PyGetWindow==0.0.9 34 | PyMsgBox==1.0.9 35 | pyparsing==2.4.7 36 | pyperclip==1.8.2 37 | pypiwin32==223 38 | PyRect==0.1.4 39 | PyScreeze==0.1.28 40 | pytorch-lightning==1.4.8 41 | pytweening==1.0.4 42 | pywin32>=302 43 | PyYAML==5.4.1 44 | requests==2.26.0 45 | requests-oauthlib==1.3.0 46 | rsa==4.7.2 47 | ruamel.yaml==0.17.16 48 | ruamel.yaml.clib==0.2.6 49 | six==1.16.0 50 | tensorboard==2.6.0 51 | tensorboard-data-server==0.6.1 52 | tensorboard-plugin-wit==1.8.0 53 | torch==1.9.1 54 | torchmetrics==0.5.1 55 | torchvision==0.10.1 56 | tqdm==4.62.3 57 | typing-extensions==3.10.0.2 58 | urllib3==1.26.7 59 | Werkzeug==2.0.1 60 | yarl==1.6.3 61 | zipp==3.5.1 62 | 63 | pandas~=1.3.4 64 | selenium>=4.1.0 65 | wget>=3.2 -------------------------------------------------------------------------------- /simplerpa/core/data/Transition.py: -------------------------------------------------------------------------------- 1 | from simplerpa.core.data.Action import Execution 2 | from simplerpa.core.data.StateBlockBase import StateBlockBase 3 | 4 | 5 | class To(StateBlockBase): 6 | def __init__(self, _to='next'): 7 | self.is_next = False 8 | self.id = None 9 | self._to = _to 10 | self.parse() 11 | 12 | def parse(self): 13 | _to = self._to 14 | if _to == 'next': 15 | self.is_next = True 16 | return 17 | else: 18 | self.is_next = False 19 | 20 | if _to == 'end': 21 | self.id = None 22 | 23 | if isinstance(_to, int): 24 | self.id = _to 25 | elif _to.isdigit(): 26 | self.id = int(self._to) 27 | 28 | 29 | class Transition(StateBlockBase): 30 | """ 31 | 状态迁移配置节点，指定什么动作会触发迁移 32 | """ 33 | action: Execution = None 34 | wait_before: int = None 35 | wait: int = None 36 | to: To 37 | # sub_states: List[State] = None 38 | max_time: int = None 39 | 40 | def __init__(self, to_str='next', action_str=None): 41 | self._trans_time = 0 42 | self.to = To(to_str) 43 | if action_str is not None: 44 | self.action = Execution(action_str) 45 | 46 | def count(self): 47 | self._trans_time += 1 48 | 49 | def reach_max_time(self): 50 | return False if self.max_time is None else self._trans_time >= self.max_time 51 | -------------------------------------------------------------------------------- /simplerpa/core/action/ActionKeyboard.py: -------------------------------------------------------------------------------- 1 | import pyautogui 2 | 3 | 4 | class ActionKeyboard: 5 | """键盘操作 6 | 7 | """ 8 | @staticmethod 9 | def hotkey(*keys): 10 | """ 11 | 发送热键 12 | Examples: 13 | ```python 14 | action: hotkey("ctrl","c") 15 | ``` 16 | ```python 17 | action: hotkey("alt","tab") 18 | ``` 19 | Args: 20 | *keys: 热键组合，多键组合用多个参数表示 21 | 22 | Returns: 23 | None 24 | 25 | """ 26 | pyautogui.hotkey(*keys) 27 | 28 | @staticmethod 29 | def type(str_to_type, interval=0): 30 | """ 31 | 把指定字符串以键盘敲击的形式输入 32 | Args: 33 | str_to_type (str): 要输入的字符串 34 | interval (float): 每个输入字符之间的时间间隔，单位是秒 35 | 36 | Returns: 37 | None 38 | """ 39 | if interval == 0: 40 | pyautogui.write(str_to_type) 41 | else: 42 | pyautogui.write(str_to_type, interval) 43 | 44 | @staticmethod 45 | def press(key_or_keys, interval=0): 46 | """ 47 | 点击特定的键 48 | Args: 49 | key_or_keys (str,List[str]): 表示键的字符串或者字符串数组 50 | interval (float): 第一个参数是数组，那么这里表示多个敲击之间的时间间隔，单位是秒 51 | 52 | Returns: 53 | 54 | """ 55 | if interval == 0: 56 | pyautogui.press(key_or_keys) 57 | else: 58 | pyautogui.press(key_or_keys, interval) 59 | -------------------------------------------------------------------------------- /simplerpa/core/detection/ColorDetection.py: -------------------------------------------------------------------------------- 1 | from typing import Tuple 2 | 3 | from simplerpa.core.action.ActionScreen import ActionScreen 4 | from simplerpa.core.detection.Detection import Detection 5 | 6 | 7 | class ColorDetection(Detection): 8 | """ 9 | 颜色检测，查看当前页面上，指定位置的像素，是否满足指定的颜色值。 10 | 11 | Example: 12 | ``` 13 | check: 14 | color: 15 | pos: (12,92) 16 | color: (209, 211, 213) 17 | fail_action: locate_state('当前页面没有检测到指定颜色') 18 | ``` 19 | 20 | Attributes: 21 | pos (Tuple[int,int]): 像素坐标位置 22 | color (Tuple[int,int]): rgb表示的颜色值，3组0~255之间的数值 23 | tolerance (int): 容忍度，rgb三色差绝对值之和，如果小于容忍度，就认为颜色相同 24 | """ 25 | pos: Tuple[int, int] 26 | color: Tuple[int, int, int] 27 | tolerance: int = 0 28 | 29 | def do(self, find_all=False): 30 | x, y = self.pos 31 | pix = ActionScreen.pick_color(x, y) 32 | tolerance = self.tolerance 33 | r, g, b = pix[:3] 34 | exR, exG, exB = self.color 35 | diff = abs(r - exR) + abs(g - exG) + abs(b - exB) 36 | 37 | if self.debug: 38 | print("检测颜色{}，坐标点({},{}) 颜色{}，与预期颜色({},{},{})差异值‘{}’".format( 39 | '成功' if diff <= tolerance else '失败', 40 | x, y, 41 | pix, 42 | r, g, b, 43 | diff 44 | )) 45 | if diff <= tolerance: 46 | return diff 47 | else: 48 | return None 49 | -------------------------------------------------------------------------------- /test/test_for_win32/test_win32.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | import win32con 4 | import win32gui 5 | 6 | 7 | def get_current_window(): 8 | return win32gui.GetForegroundWindow() 9 | 10 | 11 | def set_current_window(hwnd): 12 | if win32gui.IsIconic(hwnd): 13 | win32gui.ShowWindow(hwnd, win32con.SW_RESTORE) 14 | win32gui.SetForegroundWindow(hwnd) 15 | 16 | 17 | def get_window_title(hwnd): 18 | return win32gui.GetWindowText(hwnd) 19 | 20 | 21 | def get_current_window_title(): 22 | return get_window_title(get_current_window()) 23 | 24 | 25 | def find_window_by_title(title): 26 | try: 27 | return win32gui.FindWindow(None, title) 28 | except Exception as ex: 29 | print('error calling win32gui.FindWindow ' + str(ex)) 30 | return -1 31 | 32 | 33 | def set_window_pos(hwnd, x, y, width, height): 34 | win32gui.SetWindowPos(hwnd, win32con.HWND_TOPMOST, x, y, width, height, win32con.SWP_SHOWWINDOW) 35 | 36 | 37 | if __name__ == "__main__": 38 | # 获取当前窗口句柄(是一个整数) 39 | print(get_current_window()) 40 | # 获取当前窗口标题 41 | print(get_current_window_title()) 42 | 43 | # 给定一个标题, 查找这个窗口, 如果找到就放到最前 44 | hwnd = find_window_by_title('文档') 45 | set_current_window(hwnd) 46 | time.sleep(2) 47 | set_window_pos(hwnd, 0, 0, 1024, 768) 48 | # 打印刚刚切换到最前的窗口标题 49 | print(get_current_window_title()) 50 | 51 | # https://www.programcreek.com/python/example/89828/win32gui.SetForegroundWindow 52 | # def find_window_movetop(cls): 53 | # hwnd = win32gui.FindWindow(None, cls.processname) 54 | # win32gui.ShowWindow(hwnd, 5) 55 | # win32gui.SetForegroundWindow(hwnd) 56 | # rect = win32gui.GetWindowRect(hwnd) 57 | # sleep(0.2) 58 | # return rect 59 | -------------------------------------------------------------------------------- /test/test_find_image/test_sift.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import simplerpa.aircv as ac 3 | import cv2 4 | 5 | if __name__ == "__main__": 6 | parser = argparse.ArgumentParser(add_help=False) 7 | parser.add_argument("--src", default='book_cover_rotated.jpg', help="path for the object image") 8 | parser.add_argument("--dest", default='book_cover.jpg', help="path for image containing the object") 9 | args = parser.parse_args() 10 | 11 | img1 = cv2.imread(args.src) 12 | img2 = cv2.imread(args.dest) 13 | 14 | 15 | # MIN_MATCHES = 30 16 | # 17 | # orb = cv2.ORB_create(nfeatures=500) 18 | # kp1, des1 = orb.detectAndCompute(img1, None) 19 | # kp2, des2 = orb.detectAndCompute(img2, None) 20 | # 21 | # index_params = dict(algorithm=6, 22 | # table_number=6, 23 | # key_size=12, 24 | # multi_probe_level=2) 25 | # search_params = {} 26 | # flann = cv2.FlannBasedMatcher(index_params, search_params) 27 | # matches = flann.knnMatch(des1, des2, k=2) 28 | 29 | 30 | result_list = ac.find_all_sift(img1, img2, 4, 1) 31 | # 最终证明用特征点的方法查找logo类图标的效果不好，可能由于图像简单，特征点太少，无法匹配 32 | for index, result in enumerate(result_list): 33 | rect = result['rectangle'] 34 | print('result-{}: result-{}, rectangle-{}, confidence-{}'.format(index, 35 | result['result'], 36 | rect, 37 | result['confidence'])) 38 | cv2.rectangle(img1, result['lt'], result['br'], (0, 0, 220), 2) 39 | 40 | cv2.imwrite('result.png', img1) 41 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /simplerpa/core/detection/Detection.py: -------------------------------------------------------------------------------- 1 | from ..data.StateBlockBase import StateBlockBase 2 | from simplerpa.core.data.Action import Action, Execution 3 | 4 | 5 | class DetectResult: 6 | """ 7 | Attributes: 8 | detected (bool): 标识是否检测到了指定内容 9 | """ 10 | # detected: bool = False 11 | 12 | 13 | class Detection(StateBlockBase): 14 | """ 15 | 检测基础类，有ColorDetection, ImageDetection, OcrDetection, WindowDetection等子类，用于检测页面上的特定内容。 16 | Attributes: 17 | template (str): 模板 18 | find_all (bool): 是否查找所有结果,如果为False, 那么只返回第一个，缺省为False 19 | for_not_exist (bool): 检测结果取反，也就是检测不到的情况下，正常返回，检测到了触发fail_action，缺省为False；如果这个属性设置为True，那么find_all属性就失效了 20 | debug (bool): 执行检测do方法的时候，将相关信息记录到日志中（文本），或者单独保存文件（图片） 21 | """ 22 | detect_all: bool = False 23 | for_not_exist: bool = False 24 | debug: bool = False 25 | fail_action: Execution = None 26 | 27 | def __init__(self): 28 | self.project = None 29 | self._template_full_path = None 30 | 31 | def do(self): 32 | result = self.do_detection() 33 | if self.for_not_exist: 34 | if result is not None: 35 | if self.fail_action is not None: 36 | Action.call(self.fail_action) 37 | return None 38 | else: 39 | # pass 40 | # # result本来应该有具体内容，但这里是个特殊的检测，只有检测不到才通过，所以也不可能有真正的result存在，但设置为True，是为了避免返回以后，Find相关逻辑把空结果滤掉 41 | result = True 42 | else: 43 | if result is None and self.fail_action is not None: 44 | Action.call(self.fail_action) 45 | 46 | return result 47 | 48 | def do_detection(self): 49 | raise NotImplementedError('"do_detection" method must be implemented by a sub class of Detection') 50 | -------------------------------------------------------------------------------- /test/test_find_image/orb_flann_matcher.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | 3 | import cv2 4 | import numpy as np 5 | 6 | 7 | def get_corrected_img(img1, img2): 8 | MIN_MATCHES = 30 9 | 10 | orb = cv2.ORB_create(nfeatures=500) 11 | kp1, des1 = orb.detectAndCompute(img1, None) 12 | kp2, des2 = orb.detectAndCompute(img2, None) 13 | 14 | index_params = dict(algorithm=6, 15 | table_number=6, 16 | key_size=12, 17 | multi_probe_level=2) 18 | search_params = {} 19 | flann = cv2.FlannBasedMatcher(index_params, search_params) 20 | matches = flann.knnMatch(des1, des2, k=2) 21 | 22 | # As per Lowe's ratio test to filter good matches 23 | good_matches = [] 24 | for m, n in matches: 25 | if m.distance < 0.75 * n.distance: 26 | good_matches.append(m) 27 | 28 | if len(good_matches) > MIN_MATCHES: 29 | src_points = np.float32([kp1[m.queryIdx].pt for m in good_matches]).reshape(-1, 1, 2) 30 | dst_points = np.float32([kp2[m.trainIdx].pt for m in good_matches]).reshape(-1, 1, 2) 31 | m, mask = cv2.findHomography(src_points, dst_points, cv2.RANSAC, 5.0) 32 | corrected_img = cv2.warpPerspective(img1, m, (img2.shape[1], img2.shape[0])) 33 | 34 | return corrected_img 35 | return img2 36 | 37 | 38 | if __name__ == "__main__": 39 | parser = argparse.ArgumentParser(add_help=False) 40 | parser.add_argument("--src", default='book_cover.jpg', help="path for the object image") 41 | parser.add_argument("--dest", default='book_cover_rotated.jpg', help="path for image containing the object") 42 | args = parser.parse_args() 43 | 44 | im1 = cv2.imread(args.src) 45 | im2 = cv2.imread(args.dest) 46 | 47 | img = get_corrected_img(im2, im1) 48 | cv2.imshow('Corrected image', img) 49 | cv2.waitKey() 50 | -------------------------------------------------------------------------------- /test/find_template/get_sorted_top_k.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | def get_sorted_top_k(array, top_k=1, axis=-1, reverse=False): 4 | """ 5 | 多维数组排序 6 | Args: 7 | array: 多维数组 8 | top_k: 取数 9 | axis: 轴维度 10 | reverse: 是否倒序 11 | 12 | Returns: 13 | top_sorted_scores: 值 14 | top_sorted_indexes: 位置 15 | """ 16 | if reverse: 17 | # argpartition分区排序，在给定轴上找到最小的值对应的idx，partition同理找对应的值 18 | # kth表示在前的较小值的个数，带来的问题是排序后的结果两个分区间是仍然是无序的 19 | # kth绝对值越小，分区排序效果越明显 20 | axis_length = array.shape[axis] 21 | partition_index = np.take(np.argpartition(array, kth=-top_k, axis=axis), 22 | range(axis_length - top_k, axis_length), axis) 23 | else: 24 | partition_index = np.take(np.argpartition(array, kth=top_k, axis=axis), range(0, top_k), axis) 25 | top_scores = np.take_along_axis(array, partition_index, axis) 26 | # 分区后重新排序 27 | sorted_index = np.argsort(top_scores, axis=axis) 28 | if reverse: 29 | sorted_index = np.flip(sorted_index, axis=axis) 30 | top_sorted_scores = np.take_along_axis(top_scores, sorted_index, axis) 31 | top_sorted_indexes = np.take_along_axis(partition_index, sorted_index, axis) 32 | return top_sorted_scores, top_sorted_indexes 33 | 34 | 35 | if __name__ == "__main__": 36 | import time 37 | from sklearn.metrics.pairwise import cosine_similarity 38 | 39 | x = np.random.rand(10, 128) 40 | y = np.random.rand(1000000, 128) 41 | z = cosine_similarity(x, y) 42 | start_time = time.time() 43 | sorted_index_1 = get_sorted_top_k(z, top_k=3, axis=1, reverse=True)[1] 44 | print(time.time() - start_time) 45 | start_time = time.time() 46 | sorted_index_2 = np.flip(np.argsort(z, axis=1)[:, -3:], axis=1) 47 | print(time.time() - start_time) 48 | print((sorted_index_1 == sorted_index_2).all()) 49 | 50 | -------------------------------------------------------------------------------- /docs/simplerpa_reference.md: -------------------------------------------------------------------------------- 1 | # simplerpa 配置参考文档 2 | 3 | ## Project 4 | ### name: 5 | #### 名称 6 | #### 类型：str 7 | 8 | ### ver: 9 | 版本 10 | 11 | ### screen_width: 12 | 屏幕宽度-横向分辨率 13 | 14 | ### screen_height: 15 | 屏幕高度-纵向分辨率 16 | 17 | ### states: 18 | 状态集合 19 | 20 | 21 | 22 | ## State 23 | ### name: 24 | 名称 25 | 26 | ### id: 27 | 微一标识 28 | 29 | ### check: 30 | Find-确认页面状态 31 | 32 | ### find: 33 | Find-查找页面内容 34 | 35 | ### action: 36 | 进入页面后要执行的动作 37 | 38 | ### transition: 39 | 迁移 40 | 41 | ### foreach: 42 | 遍历 43 | 44 | 45 | 46 | ## Find 47 | ### image: 48 | 图像检测 49 | 50 | ### ocr: 51 | 文本检测 52 | 53 | ### color: 54 | 色彩检测 55 | 56 | ### window: 57 | 窗口句柄检测 58 | 59 | ### scroll: 60 | 检测的同时滚动窗口 61 | 62 | ### fail_action: 63 | 检测失败以后采取的动作 64 | 65 | ### find_all: 66 | bool = 查找所有结果 67 | 68 | 69 | 70 | ## Detection 71 | ### template: 72 | 目标模板 73 | 74 | 75 | 76 | ## ImageDetection 77 | ### snapshot: 78 | 截图区域 79 | 80 | ### confidence: 81 | 置信度 82 | 83 | ### keep_clip: 84 | 指定额外的截图 85 | 86 | 87 | 88 | ## OcrDetection 89 | ### snapshot: 90 | 截图区域 91 | 92 | ### confidence: 93 | 置信度 94 | 95 | ### text: 96 | 目标文本 97 | 98 | 99 | 100 | ## WindowDetection 101 | ### hwnd: 102 | 窗口句柄 103 | 104 | 105 | 106 | ## Transition 107 | ### action: 108 | 迁移的触发动作 109 | 110 | ### wait_before: 111 | 执行action之前的等待时间 112 | 113 | ### wait: 114 | 执行action之后的等待时间 115 | 116 | ### to: 117 | 迁移目标状态 118 | 119 | ### sub_states: 120 | 子状态集合 121 | 122 | ### max_time: 123 | 最多迁移次数(用于终止循环) 124 | 125 | 126 | 127 | ## Action 128 | ### func_dict: 129 | 定义了所有可用的自动化控制函数 130 | 131 | 132 | 133 | ## Evaluation 134 | # 有返回值的Action 135 | 136 | 137 | ## Execution 138 | # 无返回值的Action 139 | 140 | 141 | 142 | ## ForEach 143 | ### in_items: 144 | 需要遍历的集合表达式 145 | 146 | ### item: 147 | 遍历集合时,单个元素的变量名 148 | 149 | ### action: 150 | 遍历每个元素时执行的动作 151 | 152 | ### sub_states: 153 | 遍历每个元素经历的子状态 154 | 155 | 156 | 157 | 158 | -------------------------------------------------------------------------------- /test/test_find_image/test_diff_method.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import simplerpa.aircv as ac 3 | import cv2 4 | import numpy as np 5 | from simplerpa.core.data.Action import Action 6 | from simplerpa.core.action.ActionImage import ActionImage 7 | 8 | 9 | def sobel(image): 10 | gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) 11 | # img_original = gray / 255 # 归一化 12 | # 分别求X,Y方向的梯度 13 | grad_X = cv2.Sobel(gray, -1, 1, 0) 14 | grad_Y = cv2.Sobel(gray, -1, 0, 1) 15 | # 求梯度图像 16 | grad = cv2.addWeighted(grad_X, 0.5, grad_Y, 0.5, 0) 17 | return grad 18 | 19 | 20 | if __name__ == "__main__": 21 | parser = argparse.ArgumentParser(add_help=False) 22 | parser.add_argument("--src", default='book_cover_rotated.jpg', help="path for the object image") 23 | parser.add_argument("--temp", default='book_cover.jpg', help="path for image containing the object") 24 | args = parser.parse_args() 25 | 26 | img_src = cv2.imread(args.src) 27 | img_temp = cv2.imread(args.temp) 28 | 29 | # 载入灰度原图，并且归一化 30 | 31 | result_list = ActionImage.find_all_template(img_src, img_temp, 0.8, auto_scale=(1.5, 1.9)) 32 | # 最终证明用特征点的方法查找logo类图标的效果不好，可能由于图像简单，特征点太少，无法匹配 33 | for index, result in enumerate(result_list): 34 | rect = result.rect 35 | print('result-{}: confidence-{}, scale-{}, priority-{}， {}'.format(index, 36 | result.confidence if result is not None else None, 37 | result.scale, 38 | result.priority if hasattr( 39 | result, 40 | 'priority') else None, 41 | rect if result is not None else None)) 42 | cv2.rectangle(img_src, (rect.left, rect.top), (rect.right, rect.bottom), (0, 0, 220), 2) 43 | 44 | cv2.imwrite('result.png', img_src) 45 | 46 | 47 | 48 | -------------------------------------------------------------------------------- /simplerpa/core/detection/OcrDetection.py: -------------------------------------------------------------------------------- 1 | from simplerpa.core.action.ActionScreen import ActionScreen 2 | from simplerpa.core.action.ActionImage import ActionImage 3 | from simplerpa.core.data.ScreenRect import ScreenRect 4 | from simplerpa.core.detection.Detection import Detection 5 | 6 | 7 | class OcrDetection(Detection): 8 | """ 9 | 文本检测，在当前页面的指定范围内，首先用OCR识别文字，查找是否与给定文本匹配。如果识别文本和给定文本的匹配度，大于给定值，则认为找到了，并且将其返回 10 | 11 | Example: 12 | ```yaml 13 | # State的一个属性 14 | ocr: 15 | snapshot: !rect l:75, r:137, t:129, b:157 16 | text: "看日志" 17 | ``` 18 | 19 | Attributes: 20 | snapshot (ScreenRect): 屏幕截图位置，限定查找范围，可以指定得大一些，程序会在指定范围内查找图像 21 | text (str): 要查找的文本字符串 22 | confidence (float): 置信度，查找文本的时候，并不需要严格一致，程序会模糊匹配，并返回一个置信度（0 ~ 1之间的一个数值），置信度大于给定的数值，才会认为找到了 23 | 24 | ```yaml 25 | result: 如果找到了，就返回OCR识别的文本本身，如果没有找到，就返回None 26 | ``` 27 | """ 28 | 29 | snapshot: ScreenRect 30 | text: str 31 | confidence: float = 0.8 32 | 33 | def do_detection(self, find_all=False): 34 | image_current = ActionScreen.snapshot(self.snapshot.evaluate()) 35 | confidence, text = self.text_similar(self.text, image_current) 36 | if confidence >= self.confidence: 37 | return text 38 | else: 39 | return None 40 | 41 | def text_similar(self, source_text, target_pillow_image): 42 | """ 43 | 检查指定图像中是否包含特定的文字 44 | :param source_text: 要查找的文字 45 | :param target_pillow_image: 目标图像，函数将从这个图像提取文字， 46 | :return:相似度，完全相同是1，完全不同是0，其他是 source_text 与识别出来的文字的比例 47 | """ 48 | 49 | if len(source_text) == 0: 50 | '''如果source_text是空字符，就认为永远能识别不出来''' 51 | return 0, None 52 | 53 | cv_image = ActionImage.pil_to_cv(target_pillow_image) 54 | ActionImage.log_image('target', cv_image, debug=self.debug) 55 | text_from_image = ActionImage.ocr(cv_image, debug=self.debug) 56 | 57 | if source_text in text_from_image: 58 | return len(source_text) / len(text_from_image), text_from_image 59 | else: 60 | return 0, None 61 | -------------------------------------------------------------------------------- /simplerpa/conf/auto_pingcode.yaml: -------------------------------------------------------------------------------- 1 | name: "自动归档Trello" 2 | ver: 0.5 3 | #screen_width: 3440 4 | #screen_height: 1440 5 | range: !rect l:0, r:1920, t:0, b:1080 6 | time_scale: 1 7 | states: 8 | - name: "切换焦点" 9 | id: 1 10 | transition: 11 | # 点击 12 | action: hotkey('alt', 'tab') 13 | wait: 1.5 14 | - name: "浏览器PingCode页面" 15 | id: 2 16 | check: 17 | - image: 18 | snapshot: !rect l:0, r:60, t:113, b:182 19 | template: auto_pingcode/detect_pingcode.png 20 | # debug: True 21 | fail_action: raise_error('当前页面不是PingCode') 22 | - image: 23 | snapshot: !rect l:66, r:317, t:105, b:204 24 | template: auto_pingcode/detect_in_sprint.png 25 | # debug: True 26 | fail_action: raise_error('当前不在特定迭代中') 27 | - image: 28 | snapshot: !rect l:279, r:410, t:150, b:203 29 | template: auto_pingcode/detect_brief.png 30 | # debug: True 31 | fail_action: raise_error('当前标签页不是概览') 32 | transition: 33 | to: next 34 | - name: "概览" 35 | id: 3 36 | action: wait(300) 37 | find: 38 | image: 39 | snapshot: !rect l:70, r:699, t:156, b:195 40 | template: auto_pingcode/find_workitem.png 41 | # debug: True 42 | fail_action: raise_error('没找到"工作项"标签页标题') 43 | transition: 44 | action: click(find_result.rect_on_screen.center_x, find_result.rect_on_screen.center_y) 45 | - name: "工作项" 46 | id: 4 47 | action: wait(120) 48 | find: 49 | image: 50 | snapshot: !rect l:70, r:699, t:156, b:195 51 | template: auto_pingcode/find_task.png 52 | fail_action: raise_error('没找到"任务板"标签页标题') 53 | transition: 54 | action: click(find_result.rect_on_screen.center_x, find_result.rect_on_screen.center_y) 55 | - name: "任务板" 56 | id: 5 57 | action: wait(120) 58 | find: 59 | image: 60 | snapshot: !rect l:70, r:699, t:156, b:195 61 | template: auto_pingcode/find_brief.png 62 | fail_action: raise_error('没找到"概览"标签页标题') 63 | transition: 64 | action: click(find_result.rect_on_screen.center_x, find_result.rect_on_screen.center_y) 65 | to: 3 66 | -------------------------------------------------------------------------------- /simplerpa/core/data/Misc.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from typing import List 4 | 5 | from .StateBlockBase import StateBlockBase 6 | from simplerpa.core.data.Action import Action, Execution, Evaluation 7 | from simplerpa.core.data.Find import Find 8 | from .Transition import Transition 9 | from ..extractor import FormExtractor 10 | from ..monitor.ImageMonitor import ImageMonitor 11 | 12 | 13 | class State(StateBlockBase): 14 | """ 15 | 状态节点，用于配置界面流中一个特定的页面 16 | 17 | Attributes: 18 | name (str): 状态（页面）名称 19 | id (int): 状态唯一标识 20 | check (Find): 检查是否真属于当前页面 21 | find (Find): 在当前页面中查找特定元素或者片段 22 | action (Action): 进入当前页面状态后，会采取什么操作（比如鼠标点击），但不会引起状态迁移，这里可以是一个操作，也可以是多个；如果需要多个操作，就配置为列表，操作会按列表顺序执行 23 | transition (simplerpa.core.data.Transition.Transition): 状态迁移配置 24 | foreach (ForEach): 在当前状态下，针对特定数据做循环 25 | """ 26 | name: str = None 27 | id: int = -1 28 | check: Find 29 | monitor: ImageMonitor 30 | find: Find 31 | action: Execution 32 | transition: Transition 33 | foreach: ForEach 34 | form: FormExtractor.FormExtractor = None 35 | 36 | def __init__(self): 37 | # 缺省的Transition是to end 38 | self.transition = Transition() 39 | 40 | 41 | class ForEach(StateBlockBase): 42 | in_items: Evaluation 43 | item: str 44 | action: Execution 45 | sub_states: List[State] 46 | 47 | def __init__(self): 48 | self.call_env = {} 49 | 50 | def do(self): 51 | items = self.in_items.call_once() 52 | item_name = 'item' if self.item is None else self.item 53 | if isinstance(items, list): 54 | for item in items: 55 | self.call_env[item_name] = item 56 | self._do_one() 57 | elif items is not None: 58 | self.call_env[item_name] = items 59 | self._do_one() 60 | else: 61 | return 62 | 63 | def _do_one(self): 64 | Action.call(self.action, self.call_env) 65 | 66 | sub_states = self.sub_states 67 | if sub_states is not None and len(sub_states) > 0: 68 | self.project.executor.drill_into_substates(sub_states) 69 | -------------------------------------------------------------------------------- /test/test_selenium/download.py: -------------------------------------------------------------------------------- 1 | import contextlib 2 | import tempfile 3 | import urllib 4 | import urllib.request as request 5 | from urllib.error import ContentTooShortError 6 | from urllib.request import Request 7 | 8 | _url_tempfiles = [] 9 | 10 | 11 | def urlretrieve(req, filename=None, reporthook=None, data=None): 12 | if isinstance(req, str): 13 | headers = { 14 | 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.76 Safari/537.36'} 15 | url = req 16 | req = Request(url=url, headers=headers) 17 | 18 | with contextlib.closing(request.urlopen(req, data)) as fp: 19 | headers = fp.info() 20 | 21 | # Just return the local path and the "headers" for file:// 22 | # URLs. No sense in performing a copy unless requested. 23 | 24 | # Handle temporary file setup. 25 | if filename: 26 | tfp = open(filename, 'wb') 27 | else: 28 | tfp = tempfile.NamedTemporaryFile(delete=False) 29 | filename = tfp.name 30 | _url_tempfiles.append(filename) 31 | 32 | with tfp: 33 | result = filename, headers 34 | bs = 1024 * 8 35 | size = -1 36 | read = 0 37 | blocknum = 0 38 | if "content-length" in headers: 39 | size = int(headers["Content-Length"]) 40 | 41 | if reporthook: 42 | reporthook(blocknum, bs, size) 43 | 44 | while True: 45 | block = fp.read(bs) 46 | if not block: 47 | break 48 | read += len(block) 49 | tfp.write(block) 50 | blocknum += 1 51 | if reporthook: 52 | reporthook(blocknum, bs, size) 53 | 54 | if size >= 0 and read < size: 55 | raise ContentTooShortError( 56 | "retrieval incomplete: got only %i out of %i bytes" 57 | % (read, size), result) 58 | 59 | return result 60 | 61 | 62 | if __name__ == '__main__': 63 | urlretrieve( 64 | "https://images.pexels.com/videos/9917770/camp-sky-sunrise-sunset-9917770.jpeg?auto=compress&cs=tinysrgb&dpr=1&w=500", 65 | "./9917770.jpeg") 66 | -------------------------------------------------------------------------------- /test/test_find_image/test_Sobel.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import simplerpa.aircv as ac 3 | import cv2 4 | import numpy as np 5 | from simplerpa.core.data.Action import Action 6 | from simplerpa.core.action.ActionImage import ActionImage 7 | 8 | 9 | def sobel(image): 10 | gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) 11 | # img_original = gray / 255 # 归一化 12 | # 分别求X,Y方向的梯度 13 | grad_X = cv2.Sobel(gray, -1, 1, 0) 14 | grad_Y = cv2.Sobel(gray, -1, 0, 1) 15 | # 求梯度图像 16 | grad = cv2.addWeighted(grad_X, 0.5, grad_Y, 0.5, 0) 17 | return grad 18 | 19 | 20 | if __name__ == "__main__": 21 | parser = argparse.ArgumentParser(add_help=False) 22 | parser.add_argument("--src", default='book_cover_rotated.jpg', help="path for the object image") 23 | parser.add_argument("--temp", default='book_cover.jpg', help="path for image containing the object") 24 | args = parser.parse_args() 25 | 26 | img_src = cv2.imread(args.src) 27 | img_temp = cv2.imread(args.temp) 28 | 29 | img_src_sobel = sobel(img_src) 30 | cv2.imwrite('img_src_soble.png', img_src_sobel) 31 | img_temp_sobel = sobel(img_temp) 32 | cv2.imwrite('img_temp_soble.png', img_temp_sobel) 33 | 34 | # 载入灰度原图，并且归一化 35 | 36 | result_list = ActionImage.find_all_template(img_src_sobel, img_temp_sobel, 0.8, auto_scale=(1.5, 1.9)) 37 | # 最终证明用特征点的方法查找logo类图标的效果不好，可能由于图像简单，特征点太少，无法匹配 38 | for index, result in enumerate(result_list): 39 | rect = result['rectangle'] 40 | # print('result-{}: confidence-{}, scale-{}, priority-{}， {}'.format(index, 41 | # result.confidence if result is not None else None, 42 | # result.scale, 43 | # result.priority if hasattr( 44 | # result, 45 | # 'priority') else None, 46 | # rect if result is not None else None)) 47 | cv2.rectangle(img_src, rect[0], rect[3], (0, 0, 220), 2) 48 | 49 | cv2.imwrite('result.png', img_src) 50 | 51 | 52 | 53 | -------------------------------------------------------------------------------- /simplerpa/core/action/ActionMouse.py: -------------------------------------------------------------------------------- 1 | import pyautogui 2 | 3 | from simplerpa.core.data import ScreenRect 4 | 5 | 6 | def click(x, y=None): 7 | """ 8 | 鼠标点击，可以指定点击的位置 9 | Args: 10 | x (int): x坐标 11 | y (int): y坐标 12 | 13 | Returns: 14 | None 15 | """ 16 | if isinstance(x, ScreenRect.Vector): 17 | p = x 18 | pyautogui.click(p.x, p.y) 19 | else: 20 | pyautogui.click(int(x), int(y)) 21 | 22 | 23 | def dbclick(x, y=None): 24 | """ 25 | 鼠标双击，可以指定点击的位置 26 | Args: 27 | x (int): x坐标 28 | y (int): y坐标 29 | 30 | Returns: 31 | None 32 | """ 33 | if isinstance(x, ScreenRect.Vector): 34 | p = x 35 | pyautogui.doubleClick(p.x, p.y) 36 | else: 37 | pyautogui.doubleClick(int(x), int(y)) 38 | 39 | 40 | def move(x, y=None): 41 | """ 42 | 移动鼠标指针到指定的位置 43 | Args: 44 | x (int): x坐标 45 | y (int): y坐标 46 | 47 | Returns: 48 | None 49 | """ 50 | if isinstance(x, ScreenRect.Vector): 51 | p = x 52 | pyautogui.moveTo(p.x, p.y) 53 | else: 54 | pyautogui.moveTo(int(x), int(y), duration=0.25) 55 | 56 | 57 | def rightclick(x, y=None): 58 | """ 59 | 鼠标右键点击，可以指定点击的位置 60 | Args: 61 | x (int): x坐标 62 | y (int): y坐标 63 | 64 | Returns: 65 | None 66 | """ 67 | if isinstance(x, ScreenRect.Vector): 68 | p = x 69 | pyautogui.rightClick(p.x, p.y) 70 | else: 71 | pyautogui.rightClick(int(x), int(y)) 72 | 73 | 74 | def drag(x, y=None): 75 | """ 76 | 鼠标从当前位置，拖拽到指定的位置 77 | Args: 78 | x (int): 目标位置x坐标 79 | y (int): 目标位置y坐标 80 | 81 | Returns: 82 | None 83 | """ 84 | if isinstance(x, ScreenRect.Vector): 85 | p = x 86 | pyautogui.dragTo(p.x, p.y, 0.5) 87 | else: 88 | pyautogui.dragTo(int(x), int(y), 0.5) 89 | 90 | 91 | def scroll(clicks): 92 | """ 93 | 鼠标滚轮滚动指定的距离 94 | Args: 95 | clicks (int): 滚动的格数，正数表示滚动条向上，负数表示滚动条向下 96 | 97 | Returns: 98 | None 99 | """ 100 | pyautogui.scroll(clicks) 101 | 102 | 103 | def position(): 104 | """ 105 | 获取鼠标当前位置 106 | 107 | Returns: 108 | MousePosition对象，有x,y两个属性 109 | """ 110 | 111 | pos = pyautogui.position() 112 | return ScreenRect.Vector(pos[0], pos[1]) 113 | -------------------------------------------------------------------------------- /simplerpa/core/ProjectLoader.py: -------------------------------------------------------------------------------- 1 | import os 2 | from types import CodeType 3 | 4 | from simplerpa.core.data import Project 5 | from simplerpa.core.data.Misc import State 6 | from simplerpa.core.data.Transition import Transition, To 7 | from simplerpa.core.data.Project import Project 8 | from simplerpa.core.share.yaml import yaml 9 | from simplerpa.objtyping import objtyping 10 | 11 | 12 | class ProjectLoader: 13 | @classmethod 14 | def load(cls, project_file): 15 | path_root, file = os.path.split(project_file) 16 | with open(project_file, encoding='utf-8') as f: 17 | yaml_obj = yaml.load(f) 18 | project = objtyping.from_dict_list(yaml_obj, Project) 19 | project.path_root = path_root 20 | # noinspection PyTypeChecker 21 | cls.parse(project) 22 | return project 23 | 24 | @classmethod 25 | def parse(cls, project: Project): 26 | cls._traverse_set(project, project, 'project') 27 | 28 | @classmethod 29 | def _traverse_set(cls, obj, root_node, ref_root_name, reserved_classes=None): 30 | """ 31 | 递归处理，把项目配置对象中，每个子节点，都加上根节点的引用 32 | 同时，生成一个包含所有state的字典，方便做跳转 33 | :param obj: 主对象 34 | :param root_node: 根节点 35 | :param ref_root_name: 根节点属性名 36 | :param reserved_classes: 保留的类（不解析，不赋值） 37 | :return: 38 | """ 39 | if reserved_classes is None: 40 | reserved_classes = [] 41 | if obj is None: 42 | return None 43 | if type(obj) in reserved_classes: 44 | return obj 45 | if objtyping.is_basic_type(obj): 46 | return obj 47 | 48 | all_states = root_node.all_states 49 | 50 | attributes = cls._get_all_children(obj) 51 | for attr in attributes: 52 | if attr is None: 53 | continue 54 | if objtyping.is_basic_type(attr): 55 | continue 56 | if type(attr) in reserved_classes: 57 | continue 58 | if isinstance(attr, State) and attr.id is not None: 59 | all_states[attr.id] = State 60 | if isinstance(attr, Transition) and attr.to is None: 61 | attr.to = To() 62 | cls._traverse_set(attr, root_node, ref_root_name, reserved_classes) 63 | 64 | if not obj == root_node and not isinstance(obj, list) and not isinstance(obj, dict) and not isinstance(obj, tuple) and not callable(obj) and not isinstance(obj, CodeType): 65 | setattr(obj, ref_root_name, root_node) 66 | 67 | @classmethod 68 | def _get_all_children(cls, obj): 69 | if isinstance(obj, list): 70 | return obj 71 | elif hasattr(obj, '__dict__'): 72 | return obj.__dict__.values() 73 | else: 74 | return [] 75 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | 111 | # Spyder project settings 112 | .spyderproject 113 | .spyproject 114 | 115 | # Rope project settings 116 | .ropeproject 117 | 118 | # mkdocs documentation 119 | /site 120 | 121 | # mypy 122 | .mypy_cache/ 123 | .dmypy.json 124 | dmypy.json 125 | 126 | # Pyre type checker 127 | .pyre/ 128 | .idea/ 129 | log/* 130 | /simplerpa/conf/log/ 131 | .vscode/ 132 | /simplerpa/log/ 133 | 134 | /simplerpa/result.xlsx 135 | /simplerpa/result.csv 136 | /test/test_selenium/*.mp4 137 | /test/test_selenium/*.jpeg 138 | /test/test_selenium/*.csv 139 | /test/test_selenium/output/ 140 | /test/test_selenium/*.zip 141 | /test/test_selenium/*.jpg 142 | /test/test_selenium/facebook/ 143 | -------------------------------------------------------------------------------- /simplerpa/core/extractor/FormExtractor.py: -------------------------------------------------------------------------------- 1 | from typing import Tuple, List 2 | 3 | from ..detection.ImageDetection import ImageDetection 4 | 5 | from .Extractor import Extractor 6 | from ..action.ActionData import ActionData 7 | from ..action.ActionImage import ActionImage 8 | from ..data.Action import Evaluation, Action 9 | from ..data.StateBlockBase import StateBlockBase 10 | 11 | 12 | class FormField(StateBlockBase): 13 | name: str = None 14 | feature: ImageDetection = None 15 | position: Evaluation = None 16 | foreground: Tuple[int, int, int] = (0, 0, 0) 17 | background: Tuple[int, int, int] = (255, 255, 255) 18 | tolerance: float = 0.1 19 | 20 | def __init__(self): 21 | self.content = None 22 | 23 | def get_content(self, image): 24 | result = self.feature.do_detection(image) 25 | if result is None: 26 | raise RuntimeError('field "{}" not found!'.format(self.name)) 27 | rect = Action.call_once(self.position, {'feature_rect': result.rect_on_image}) 28 | content_img = ActionImage.sub_image(image, rect) 29 | ActionImage.log_image('field_{}_content'.format(self.name), content_img, debug=self.debug) 30 | main_part, main_part_bin = ActionImage.find_main_part(content_img, self.foreground, self.tolerance, 31 | debug=self.debug) 32 | ActionImage.log_image('field_{}_main_part'.format(self.name), main_part, debug=self.debug) 33 | 34 | rows = ActionImage.split_rows(main_part_bin, 255) 35 | rows_img = [] 36 | for row in rows: 37 | rows_img.append(main_part[row[0]:row[1], :]) 38 | 39 | # 这里已经是二值图像了，所以背景颜色一定是255（只能取0和255两种颜色） 40 | content = "" 41 | for row_img in rows_img: 42 | ActionImage.log_image('field_{}_row'.format(self.name), row_img, debug=self.debug) 43 | row_txt = ActionImage.ocr(row_img) 44 | content += row_txt 45 | return content 46 | 47 | 48 | class FormExtractor(Extractor): 49 | foreground: Tuple[int, int, int] = (0, 0, 0) 50 | tolerance: float = 0.1 51 | fields: List[FormField] = None 52 | 53 | def __init__(self): 54 | super().__init__() 55 | self.field_names = None 56 | 57 | def prepare(self): 58 | df = super(FormExtractor, self).prepare() 59 | 60 | if self.field_names is None: 61 | self.field_names = list(map(lambda x: x.name, self.fields)) 62 | df1 = ActionData.create_dataframe(self.field_names) 63 | df = df.join(df1, how="outer") 64 | return df 65 | 66 | def do_once(self, image): 67 | data_dict = {} 68 | for field in self.fields: 69 | content = field.get_content(image) 70 | data_dict[field.name] = content 71 | print("{}: {}".format(field.name, content)) 72 | return data_dict 73 | -------------------------------------------------------------------------------- /simplerpa/conf/auto_wemeeting.yaml: -------------------------------------------------------------------------------- 1 | name: "测试截图到剪贴板" 2 | ver: 0.1 3 | #screen_width: 3440 4 | #screen_height: 1440 5 | range: !rect l:0, r:1920, t:0, b:1080 6 | time_scale: 1 7 | states: 8 | - name: "当前窗口" 9 | id: 0 10 | transition: 11 | # 显示桌面的热键 12 | action: hotkey('win', 'd') 13 | wait: 1 14 | - name: "桌面" 15 | id: 10 16 | find: 17 | image: 18 | snapshot: !rect l:0, r:800, t:0, b:600 19 | template: auto_wemeeting/logo_wemeeting.png 20 | # debug: True 21 | fail_action: raise_error('找不到腾讯会议图标') 22 | transition: 23 | # 双击腾讯会议图标 24 | action: dbclick(find_result.rect_on_screen.center_x, find_result.rect_on_screen.center_y) 25 | wait: 3 26 | - name: "腾讯会议主窗口" 27 | id: 20 28 | find: 29 | window: 30 | win_class: "TXGuiFoundation" 31 | title: "腾讯会议" 32 | fail_action: raise_error('没有找到腾讯会议窗口！') 33 | # activate: True 34 | wait: 1 35 | transition: 36 | # 点击 37 | action: 38 | - wemeet = get_window_rect(find_result.hwnd) 39 | - print(wemeet) 40 | wait: 1 41 | - name: "腾讯会议主窗口-step2" 42 | id: 30 43 | find: 44 | image: 45 | snapshot: wemeet 46 | template: auto_wemeeting/icon_pre.png 47 | fail_action: raise_error('没有找到预定会议按钮！') 48 | # activate: True 49 | wait: 1 50 | transition: 51 | # 点击 52 | action: 53 | - arrow = find_result.rect_on_screen.bottomright.offset(8,-5) 54 | - move(arrow) 55 | - wait(1) 56 | - click(arrow.offset(0,40)) 57 | wait: 1 58 | - name: "预定会议窗口" 59 | id: 40 60 | find: 61 | image: 62 | snapshot: wemeet 63 | template: auto_wemeeting/btn_confirm.png 64 | fail_action: raise_error('没有找到预定确认按钮！') 65 | # activate: True 66 | wait: 1 67 | transition: 68 | # 点击 69 | action: 70 | - click(find_result.rect_on_screen.center) 71 | wait: 3 72 | - name: "会议二维码" 73 | id: 50 74 | action: 75 | - snapshot_pil(wemeet, to_clipboard=True) 76 | transition: 77 | wait: 0.5 78 | - name: "微信聊天窗口" 79 | id: 60 80 | find: 81 | window: 82 | win_class: "WeChatMainWndForPC" 83 | title: "微信" 84 | fail_action: raise_error('没有找到微信！') 85 | activate: True 86 | wait: 1 87 | transition: 88 | # 点击 89 | action: 90 | - wechat = get_window_rect(find_result.hwnd) 91 | - print(wechat) 92 | wait: 1 93 | - name: "h+" 94 | id: 70 95 | find: 96 | image: 97 | template: auto_wemeeting/h+_title.png 98 | snapshot: wechat 99 | fail_action: raise_error('没找到h+聊天') 100 | transition: 101 | action: 102 | - click(find_result.rect_on_screen.center) 103 | - wait(1) 104 | - hotkey('ctrl','v') 105 | - wait(0.5) 106 | - hotkey('enter') 107 | to: end 108 | 109 | 110 | 111 | -------------------------------------------------------------------------------- /simplerpa/conf/auto_playvalkyr.yaml: -------------------------------------------------------------------------------- 1 | name: "PlayValkyr" 2 | ver: 0.1 3 | #screen_width: 3440 4 | #screen_height: 1440 5 | #range: !rect l:0, r:1920, t:0, b:1080 6 | time_scale: 1 7 | states: 8 | - id: 1 9 | name: "桌面" 10 | find: 11 | # 查找浏览器PlayValkyr窗口 12 | window: 13 | title: "Play Valkyrio - Google Chrome" 14 | # win_class: 15 | fail_action: raise_error('没有找到Play Valkyrio窗口！') 16 | activate: True 17 | wait: 1 18 | set_pos: !rect l:0, r:1440, t:0, b:1200 19 | transition: 20 | action: 21 | - win_rect = get_window_rect(find_result.hwnd) 22 | - id: 2 23 | name: "首页" 24 | find: 25 | # 查找select按钮 26 | image: 27 | snapshot: !rect l:596, r:827, t:880, b:1007 28 | template: auto_playvalkyr/select.png 29 | fail_action: raise_error('没找到select按钮') 30 | debug: True 31 | transition: 32 | action: 33 | - click(find_result.rect_on_screen.center) 34 | wait: 2 35 | - id: 3 36 | name: "战斗" 37 | find: 38 | # 查找最弱的敌人 39 | detections: 40 | - snapshot: !rect l:88, r:1402, t:687, b:878 41 | template: auto_playvalkyr/veryweak.png 42 | - snapshot: !rect l:88, r:1402, t:687, b:878 43 | template: auto_playvalkyr/weak.png 44 | - snapshot: !rect l:88, r:1402, t:687, b:878 45 | template: auto_playvalkyr/strong.png 46 | fail: 47 | action: raise_error('没找到最弱,弱或者强壮的敌人') 48 | transition: 49 | action: 50 | - search_rect = find_result.rect_on_screen.snap_bottom(400, 120) 51 | wait: 3 52 | - id: 3 53 | name: "战斗-next" 54 | find: 55 | # 查找最弱的敌人 56 | image: 57 | snapshot: search_rect 58 | template: auto_playvalkyr/fight.png 59 | fail_action: raise_error('没找到战斗按钮') 60 | transition: 61 | action: 62 | - click(find_result.rect_on_screen.center) 63 | wait: 1 64 | - id: 4 65 | name: "MetaMask插件" 66 | find: 67 | # 查找付费按钮 68 | image: 69 | snapshot: !rect l:962, r:1318, t:76, b:676 70 | template: auto_playvalkyr/confirm.png 71 | # fail_action: raise_error('没找到确认付费按钮') 72 | fail: 73 | to: 10 74 | wait: 3 75 | transition: 76 | action: 77 | - click(find_result.rect_on_screen.center) 78 | wait: 30 79 | to: 5 80 | - id: 10 81 | name: "MetaMask插件" 82 | find: 83 | # 查找付费按钮 84 | image: 85 | snapshot: !rect l:1292, r:1330, t:45, b:81 86 | template: auto_playvalkyr/plugin_metamask.png 87 | fail_action: raise_error('没找到插件按钮') 88 | debug: True 89 | transition: 90 | action: 91 | - click(find_result.rect_on_screen.center) 92 | wait: 3 93 | to: 4 94 | - id: 5 95 | name: "返回首页" 96 | find: 97 | # 查找selector按钮 98 | image: 99 | snapshot: !rect l:596, r:827, t:888, b:1007 100 | template: auto_playvalkyr/select.png 101 | fail_action: raise_error('没找到select按钮') 102 | -------------------------------------------------------------------------------- /simplerpa/core/action/ActionScreen.py: -------------------------------------------------------------------------------- 1 | import win32api 2 | import pyautogui as autogui 3 | 4 | from simplerpa.core.action.ActionImage import ActionImage 5 | import win32clipboard as clip 6 | import win32con 7 | from io import BytesIO 8 | 9 | 10 | class ActionScreen: 11 | @staticmethod 12 | def change_resolution(params: tuple): 13 | """ 14 | 改变屏幕分辨率 15 | Args: 16 | params (Tuple[int,int]): 指定分辨率的宽和高 17 | 18 | Returns: 19 | None 20 | """ 21 | (width, height) = params 22 | dm = win32api.EnumDisplaySettings(None, 0) 23 | dm.PelsWidth = width 24 | dm.PelsHeight = height 25 | dm.BitsPerPel = 32 26 | dm.DisplayFixedOutput = 1 # 0:缺省; 1:居中; 2:拉伸 27 | win32api.ChangeDisplaySettings(dm, 0) 28 | 29 | @classmethod 30 | def snapshot(cls, rect=None, to_clipboard=False): 31 | """ 32 | 根据跟定的ScreenRect区域截图 33 | Args: 34 | rect: 遵从一般系统坐标系的矩形区域(左上角为0,0点), autogui和Pillow都适用 35 | to_clipboard: 是否把截图copy到剪贴板 36 | Returns: 37 | 返回PIL格式的指定区域截图 38 | """ 39 | screen_shot = autogui.screenshot() 40 | ret_image = screen_shot 41 | if rect is not None: 42 | ret_image = screen_shot.crop( 43 | (int(float(rect.left)), int(float(rect.top)), int(float(rect.right)), int(float(rect.bottom)))) 44 | 45 | if to_clipboard: 46 | output = BytesIO() 47 | ret_image.convert('RGB').save(output, 'BMP') 48 | data = output.getvalue()[14:] 49 | output.close() 50 | clip.OpenClipboard() 51 | clip.EmptyClipboard() 52 | clip.SetClipboardData(win32con.CF_DIB, data) 53 | clip.CloseClipboard() 54 | 55 | return ret_image 56 | 57 | @classmethod 58 | def snapshot_cv(cls, rect): 59 | """ 60 | 根据跟定的ScreenRect区域截图 61 | Args: 62 | rect: 遵从一般系统坐标系的矩形区域(左上角为0,0点), autogui和opencv都适用 63 | 64 | Returns: 65 | 返回opencv格式的指定区域截图 66 | """ 67 | pil_image = cls.snapshot(rect) 68 | return ActionImage.pil_to_cv(pil_image) 69 | 70 | @classmethod 71 | def snapshot_pil(cls, rect=None, to_clipboard=False): 72 | return cls.snapshot(rect, to_clipboard) 73 | 74 | @staticmethod 75 | def pick_color(x, y): 76 | """ 77 | 获取屏幕上指定位置像素的颜色 78 | Args: 79 | x: 指定位置x坐标 80 | y: 指定位置y坐标 81 | 82 | Returns: 83 | List[int]: 长度为4的整形数组，分别用0~255之间的数字，表示R,G,B,A四个通道 84 | """ 85 | return autogui.pixel(x, y) 86 | 87 | @staticmethod 88 | def pixel_matches_color(x, y, color_tuple, tolerance=0): 89 | """ 90 | 检查给定的颜色和屏幕上特定坐标点的颜色是否匹配 91 | Args: 92 | x: 指定位置x坐标 93 | y: 指定位置y坐标 94 | color_tuple (Tuple[int]): (R,G,B) 组成的颜色值 95 | tolerance: 容忍度 96 | 97 | Returns: 98 | bool: 是否匹配 99 | """ 100 | return autogui.pixelMatchesColor(x, y, color_tuple, tolerance) 101 | -------------------------------------------------------------------------------- /simplerpa/core/monitor/ImageMonitor.py: -------------------------------------------------------------------------------- 1 | import cv2 2 | 3 | from simplerpa.core.action.ActionImage import ActionImage 4 | from simplerpa.core.action.ActionScreen import ActionScreen 5 | from simplerpa.core.action.ActionSystem import ActionSystem 6 | from .Monitor import Monitor 7 | from ..const import MONITOR_RESULT 8 | from ..data.Action import Action 9 | from ..data.ScreenRect import ScreenRect 10 | 11 | 12 | class ChangeResult(object): 13 | rate: float = 0 14 | position: float = None 15 | scroll: str = None 16 | 17 | def __init__(self, rate, position=None, scroll=None): 18 | self.rate = rate 19 | self.position = position 20 | 21 | 22 | class ImageMonitor(Monitor): 23 | monitor_diff: bool = True 24 | snapshot: ScreenRect 25 | interval: float = 1 26 | times: int = None 27 | threshold: float = 0.1 28 | 29 | def __init__(self): 30 | super().__init__() 31 | self.pre_snapshot = None 32 | 33 | def do(self, source_image=None): 34 | # if source_image is None: 35 | # snapshot_image = ActionScreen.snapshot(self.snapshot.evaluate()) 36 | # source_image = ActionImage.pil_to_cv(snapshot_image) 37 | i = 0 38 | change = None 39 | rect = self.snapshot.evaluate() 40 | while self.times is None or i < self.times: 41 | change = self._check_change(rect) 42 | if change is not None: 43 | break 44 | if self.debug: 45 | print("No change, wait...") 46 | # ActionImage.log_image('monitor_pre_snapshot', self.pre_snapshot, self.debug) 47 | ActionSystem.wait(self.interval) 48 | i += 1 49 | if self.debug: 50 | print("Monitoring change, rate:{}, position:{}".format(change.rate, change.position)) 51 | Action.save_call_env({MONITOR_RESULT: change}) 52 | 53 | return change 54 | 55 | def _check_change(self, rect): 56 | snapshot_image = ActionImage.pil_to_cv(ActionScreen.snapshot(rect)) 57 | if self.pre_snapshot is None: 58 | self.pre_snapshot = snapshot_image 59 | return None 60 | diff = ActionImage.diff(self.pre_snapshot, snapshot_image) 61 | 62 | rate = cv2.countNonZero(diff) / diff.size 63 | if rate > self.threshold: 64 | ActionImage.log_image('monitor_pre', self.pre_snapshot, self.debug) 65 | # 等半秒，以防消息连续发送 66 | ActionSystem.wait(0.5) 67 | scroll = "up" 68 | height = self.pre_snapshot.shape[0] 69 | feature_img = self.pre_snapshot[height - 65:height, :] 70 | res_list = ActionImage.find_all_template(snapshot_image, feature_img, 0.8) 71 | 72 | if len(res_list) == 0: 73 | scroll = 'down' 74 | feature_img = self.pre_snapshot[0:65, :] 75 | res_list = ActionImage.find_all_template(snapshot_image, feature_img, 0.8) 76 | 77 | if len(res_list) == 0: 78 | position = None 79 | else: 80 | if scroll == "up": 81 | position = res_list[0].rect.bottom 82 | else: 83 | position = res_list[0].rect.top 84 | 85 | ActionImage.log_image('monitor_now', snapshot_image, self.debug) 86 | self.pre_snapshot = None 87 | return ChangeResult(rate, position, scroll) 88 | else: 89 | self.pre_snapshot = snapshot_image 90 | return None 91 | -------------------------------------------------------------------------------- /docs/simplerpa_class_diagram.md: -------------------------------------------------------------------------------- 1 | simplrpa 数据类型是通过class中的class attribute来定义的，而不是instance attribute，所以下图中看到的属性，都定义在class中，而不是__init__方法里： 2 | 3 | ```puml 4 | @startuml 5 | skinparam monochrome true 6 | skinparam classAttributeIconSize 0 7 | scale 2 8 | 9 | namespace simplerpa { 10 | 11 | namespace core { 12 | 13 | namespace data { 14 | class Project{ 15 | name: 名称 16 | ver: 版本 17 | screen_width: 屏幕宽度-横向分辨率 18 | screen_height: 屏幕高度-纵向分辨率 19 | states: 状态集合 20 | } 21 | 22 | class State{ 23 | name: 名称 24 | id: 微一标识 25 | check: Find-确认页面状态 26 | find: Find-查找页面内容 27 | action: 进入页面后要执行的动作 28 | transition: 迁移 29 | foreach: 遍历 30 | } 31 | 32 | class Find { 33 | image: 图像检测 34 | ocr: 文本检测 35 | color: 色彩检测 36 | window: 窗口句柄检测 37 | scroll: 检测的同时滚动窗口 38 | fail_action: 检测失败以后采取的动作 39 | find_all: bool = 查找所有结果 40 | } 41 | 42 | class Detection { 43 | template: 目标模板 44 | } 45 | 46 | class ImageDetection { 47 | snapshot: 截图区域 48 | confidence: 置信度 49 | keep_clip: 指定额外的截图 50 | } 51 | 52 | class OcrDetection { 53 | snapshot: 截图区域 54 | confidence: 置信度 55 | text: 目标文本 56 | } 57 | 58 | class WindowDetection { 59 | hwnd: 窗口句柄 60 | } 61 | 62 | class ColorDetection { 63 | pos: 像素点位置 64 | color: 像素点颜色 65 | tolerance: 误差容忍度 66 | } 67 | 68 | class Transition { 69 | action: 迁移的触发动作 70 | wait_before: 执行action之前的等待时间 71 | wait: 执行action之后的等待时间 72 | to: 迁移目标状态 73 | sub_states: 子状态集合 74 | max_time: 最多迁移次数(用于终止循环) 75 | } 76 | 77 | class Action { 78 | func_dict: 定义了所有可用的自动化控制函数 79 | } 80 | 81 | class Evaluation { 82 | # 有返回值的Action 83 | } 84 | class Execution { 85 | # 无返回值的Action 86 | } 87 | 88 | class ForEach { 89 | in_items: 需要遍历的集合表达式 90 | item: 遍历集合时,单个元素的变量名 91 | action: 遍历每个元素时执行的动作 92 | sub_states: 遍历每个元素经历的子状态 93 | } 94 | 95 | Detection <|-- ImageDetection 96 | Detection <|-- OcrDetection 97 | Detection <|-- WindowDetection 98 | Detection <|-- ColorDetection 99 | 100 | Action <|-- Execution 101 | Action <|-- Evaluation 102 | 103 | Project "1" o-- "*" State 104 | State "1" o-- "*" Find: check 105 | State "1" o-- "*" Find: find 106 | State "1" o-- "1" Transition 107 | State "1" o-- "1" ForEach 108 | State "1" o-- "*" Execution : action 109 | 110 | Find "1" o-- "*" Detection : image 111 | Find "1" o-- "*" Detection : ocr 112 | Find "1" o-- "*" Detection : window 113 | Find "1" o-- "*" Detection : color 114 | 115 | Find "1" o-- "*" Execution : fail_action 116 | 117 | Transition "1" o-- "*" Execution : action 118 | Transition "1" o-- "*" State : sub_states 119 | 120 | ForEach "1" o-- "*" Evaluation : in_items 121 | ForEach "1" o-- "*" State : sub_states 122 | ForEach "1" o-- "*" Execution : action 123 | } 124 | } 125 | } 126 | @enduml 127 | ``` 128 | -------------------------------------------------------------------------------- /simplerpa/core/action/ActionWindow.py: -------------------------------------------------------------------------------- 1 | import win32api 2 | import win32con 3 | import win32gui 4 | 5 | from simplerpa.core.data import ScreenRect 6 | 7 | 8 | class ActionWindow: 9 | screen_width: int = None 10 | screen_height: int = None 11 | 12 | @classmethod 13 | def get_current_window(cls): 14 | """ 15 | 获取当前窗口句柄 16 | Returns: 17 | int: 当前窗口句柄（hwnd） 18 | """ 19 | return win32gui.GetForegroundWindow() 20 | 21 | @classmethod 22 | def set_current_window(cls, hwnd): 23 | """ 24 | 将指定的窗口设置为当前窗口 25 | Args: 26 | hwnd (int): 指定的窗口句柄 27 | 28 | Returns: 29 | None 30 | """ 31 | if win32gui.IsIconic(hwnd): 32 | # 如果窗口被最小化了，先恢复 33 | win32gui.ShowWindow(hwnd, win32con.SW_RESTORE) 34 | win32gui.SetForegroundWindow(hwnd) 35 | 36 | @classmethod 37 | def get_window_title(cls, hwnd): 38 | """ 39 | 获取指定窗口标题 40 | Args: 41 | hwnd (int): 指定窗口句柄 42 | 43 | Returns: 44 | str: 窗口标题 45 | """ 46 | return win32gui.GetWindowText(hwnd) 47 | 48 | @classmethod 49 | def get_window_class(cls, hwnd): 50 | """ 51 | 获取指定窗口的class name 52 | Args: 53 | hwnd (int): 指定窗口句柄 54 | 55 | Returns: 56 | str: 窗口class name(windows api中的概念，可以看做window系统内部对窗口的标识名） 57 | """ 58 | return win32gui.GetClassName(hwnd) 59 | 60 | @classmethod 61 | def get_current_window_title(cls): 62 | """ 63 | 获取当前窗口标题 64 | 65 | Returns: 66 | str: 窗口标题 67 | """ 68 | return cls.get_window_title(cls.get_current_window()) 69 | 70 | @classmethod 71 | def find_window(cls, title=None, win_class=None): 72 | """ 73 | 根据窗口的class_name或者标题，查找窗口 74 | Args: 75 | title (str): 窗口标题 76 | win_class (str): 窗口class_name 77 | 78 | Returns: 79 | int,None: 找到的窗口句柄（hwnd），如果没找到则返回None 80 | """ 81 | try: 82 | hwnd = win32gui.FindWindow(win_class, title) 83 | if hwnd == 0: 84 | return None 85 | else: 86 | return hwnd 87 | except Exception as ex: 88 | print('error calling win32gui.FindWindow ' + str(ex)) 89 | return None 90 | 91 | @classmethod 92 | def get_window_rect(cls, hwnd): 93 | """ 94 | 获取指定窗口的位置和大小 95 | Args: 96 | hwnd (int): 指定窗口句柄 97 | 98 | Returns: 99 | ScreenRect: 窗口的位置和大小 100 | """ 101 | left, top, right, bottom = win32gui.GetWindowRect(hwnd) 102 | return ScreenRect.ScreenRect(left, right, top, bottom) 103 | 104 | @classmethod 105 | def set_window_pos(cls, hwnd, x, y, width, height): 106 | """ 107 | 设置窗口的位置和大小 108 | Args: 109 | hwnd (int): 指定窗口句柄 110 | x (int): 窗口左上角x坐标 111 | (int): 窗口左上角y坐标 112 | width (int): 窗口宽度 113 | height (int): 窗宽高度 114 | 115 | Returns: 116 | 117 | """ 118 | win32gui.SetWindowPos(hwnd, win32con.HWND_TOPMOST, x, y, width, height, win32con.SWP_SHOWWINDOW) 119 | 120 | @classmethod 121 | def get_screen_resolution(cls): 122 | if cls.screen_width is None or cls.screen_height is None: 123 | cls.refresh_screen_resolution() 124 | return cls.screen_width, cls.screen_height 125 | 126 | @classmethod 127 | def refresh_screen_resolution(cls): 128 | cls.screen_width = win32api.GetSystemMetrics(win32con.SM_CXSCREEN) 129 | cls.screen_height = win32api.GetSystemMetrics(win32con.SM_CYSCREEN) 130 | -------------------------------------------------------------------------------- /simplerpa/conf/project.yaml: -------------------------------------------------------------------------------- 1 | name: "整理钉钉周报内容到记事本" 2 | #ver: 0.1 3 | #screen_width: 1920 4 | #screen_height: 1080 5 | #range: !rect l:0, r:1000, t:0, b:800 6 | #time_scale: 1 7 | states: 8 | - name: "桌面" 9 | id: 1 10 | check: 11 | - image: 12 | snapshot: !rect l:160, r:200, t:160, b:200 13 | template: states/1/dingding_logo.png 14 | confidence: 0.8 15 | fail_action: raise_error('找不到钉钉图标') 16 | - image: 17 | snapshot: !rect l:160, r:200, t:160, b:200 18 | template: states/1/notepad_logo.png 19 | fail_action: raise_error('找不到记事本图标') 20 | transition: 21 | # 打开记事本 22 | action: DbClick($check_result[1].center_x, $check_result[1].center_y) 23 | wait: 5 24 | to: next 25 | 26 | - name: "记事本主界面" 27 | id: 2 28 | check: 29 | image: 30 | snapshot: !rect l:10, r:20, t:25, b:35 31 | template: states/2/notepad_title.png 32 | fail_action: raise_error('没能成功启动记事本') 33 | transition: 34 | # 打开钉钉 35 | action: DbClick($state[1].check_result[0].center_x, $state[1].check_result[0].center_y) 36 | wait: 5 37 | 38 | - name: "钉钉主界面" 39 | id: 3 40 | check: 41 | image: 42 | snapshot: !rect l:10, r:20, t:25, b:35 43 | template: states/2/tzding_logo.png 44 | fail_action: raise_error('没能成功启动钉钉,或者当前企业不是天助定') 45 | find: 46 | image: 47 | snapshot: !rect l:30, r:120, t:35, b:800 48 | template: states/2/work_report.png 49 | scroll: 50 | one_page: 200 51 | page_count: 5 52 | fail_action: raise_error('没找到工作汇报消息') 53 | transition: 54 | action: Click($find_result.center_x, $find_result.rect.bottom) 55 | wait: 2 56 | 57 | - name: "工作台日志列表" 58 | id: 4 59 | check: 60 | ocr: 61 | # "看日志" 文本 62 | snapshot: !rect l:30, r:50, t:35, b:55 63 | text: "看日志" 64 | fail_action: raise_error('没能成功进入日志列表页面') 65 | find: 66 | # image: 67 | # # "已读" "未读" 标记 作为行标识 68 | # snapshot: !rect l:80, r:120, t:55, b:800 69 | # template: 70 | # - states/3/read.png 71 | # - states/3/unread.png 72 | # strategy: any 73 | image: 74 | # 查找"的周报"标记, 作为行标识 75 | snapshot: !rect l:70, r:90, t:55, b:800 76 | template: states/3/work_report_text_snippet.png 77 | find_all: True 78 | fail_action: raise_error('没找到任何人的周报') 79 | transition: 80 | foreach: row in $find_result 81 | action: Click($row.rect.left - 50, $row.center_y) 82 | wait: 1 83 | sub_states: 84 | - name: "某人的日报内容" 85 | id: 40 86 | transition: 87 | action: RightClick(240, 180) 88 | wait: 0.3 89 | - name: "某人的日报内容上覆盖右键菜单" 90 | id: 41 91 | find: 92 | image: 93 | snapshot: !rect l:$mouse.x, r:$mouse.x + 50, t:$mouse.y, b:$mouse.y + 100 94 | template: states/30/context_select_all.png 95 | transition: 96 | action: Click($find_result.center_x, $find_result.center_y) 97 | wait: 0.2 98 | - name: "某人的日报内容全选" 99 | id: 42 100 | action: HotKey("ctrl","c") 101 | transition: 102 | action: HotKey("alt","tab") 103 | wait: 0.2 104 | - name: "记事本在最前" 105 | id: 43 106 | action: 107 | - HotKey("ctrl","v") 108 | - HotKey("return") 109 | transition: 110 | # 最后切换回钉钉在最前, 这里如果不放心, 可以加一个状态检查 111 | action: HotKey("alt","tab") 112 | wait: 0.2 113 | 114 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # SimpleRPA 2 | 基于图像识别的开源RPA工具，理论上可以支持所有windows软件和网页的自动化 3 | 4 | ## 简介 5 | SimpleRPA是一款python语言编写的开源RPA工具（桌面自动控制工具），用户可以通过配置yaml格式的文件，来实现桌面软件的自动化控制，简化繁杂重复的工作，比如运营人员给用户发消息，打标签，给店铺插旗；项目管理人员采集数据；测试人员实现简单的自动化测试等等。 6 | 7 | ## 为什么是SimpleRPA 8 | * 这是一个基于MIT协议的开源项目，对商业应用友好 9 | * 市面上常见的RPA工具，虽然功能强大完善，但基本上都基于过程控制的理念，实际上成了图形化编程工具，面对稍微复杂的场景，就需要编制大量的判断跳转和子流程嵌套；而SimpleRPA针对实际RPA场景做出了合理的抽象，虽然使用YAML格式配置，实际上是一种桌面自动控制的DSL，可以更便捷地表达自动化场景。 10 | * 支持配置文件内嵌Python代码，可以实现更灵活的逻辑 11 | * 基于图像采集、智能匹配和OCR识别，可以支持任何类型的桌面应用，而无需手工分析页面结构。 12 | 13 | ## 状态机概念 14 | 我们做屏幕自动化任务的时候，通常都会经历这样几个步骤： 15 | 1. 检查当前桌面上是否显示了需要的页面（比如查看特定位置的图像，或者比对OCR识别出的文字） 16 | 2. 如果确实是，就收集一些文字或图像的信息（这一步未必会有，要看具体任务类型，有些自动化只要把页面流程走通就可以） 17 | 3. 查找页面上特定的控件（比如某个按钮），对它进行操作（如点击） 18 | 4. 跳转到下一个页面，回到步骤1，反复循环，直到最终页面出现 19 | 20 | SimpleRPA把这个过程，抽象为一个状态机模型：每个页面是一个状态（state），通过“action”触发，可以跳转到下一个状态； 21 | 在每一个State内部，可以做check（检查是否需要的页面），可以find（查找特定控件，或者收集信息）； 22 | 针对find的结果，还可以形成子状态，来实现复杂的操作。 23 | 24 | ## 示例 25 | SimpleRPA的自动化脚本，由一个yaml配置文件，和子文件夹构成，文件夹中通常存放要查找的图像模板。 26 | 27 | ### 示例1——自动刷新页面 28 | 一个简单的配置文件示例如下： 29 | ```yaml 30 | # 有一个特定的浏览器页面，我们需要定时刷新，以便更新它的状态 31 | name: "浏览器自动刷新" 32 | ver: 0.1 33 | # 默认不会调整屏幕分辨率，所有内容里指定的坐标，都是相对于当前屏幕左上角； 34 | # 但如果这里指定了屏幕宽度或高度，就会在开始运行内容之前，调整分辨率 35 | # screen_width: 3440 36 | # screen_height: 1440 37 | states: 38 | - name: "当前窗口" 39 | # 为了简化，这里假设当前桌面刚刚从浏览器窗口切换到脚本运行窗口，所以一启动就先用alt+tab键切换回去 40 | id: 1 41 | transition: 42 | # 通过点击热键这个action, 迁移到下一个状态 43 | action: hotkey('alt', 'tab') 44 | wait: 1 45 | - name: "浏览器窗口" 46 | id: 2 47 | check: 48 | image: 49 | snapshot: !rect l:0, r:60, t:113, b:182 50 | template: auto_test/detect_logo.png 51 | # debug: True 52 | fail_action: raise_error('当前页面不是期待的页面') 53 | transition: 54 | # 通过点击F5实现浏览器刷新，迁移前先等待60s； 55 | # 没有其他页面需要显示了，所以还是迁移到当前状态，无限循环 56 | action: hotkey('f5') 57 | wait: 60 58 | to: 2 59 | ``` 60 | 上面这个示例可以用流程图表示如下: 61 | ```mermaid 62 | graph TD; 63 | 1[当前窗口] -- Alt+Tab --> 2[浏览器窗口] 64 | 2 -- F5 --> 2 65 | ``` 66 | 67 | 这里states是一个列表，每个列表项是一个状态，每个状态有一个id属性作为唯一标识。状态之间的迁移，通过transition属性的to来指定。 68 | to指定的内容可以是某一个state的id，也可以是next（缺省值），next意味着迁移到下一个状态（按列表定义顺序，而不是id编号顺序）。 69 | 70 | transition的action是表示触发迁移的动作，支持键盘鼠标、屏幕、剪贴板、窗口引用（目前只支持windows）等一系列操作。 71 | transition的wait表示动作执行以后，等待的时间。 72 | 73 | 这里的check属性里面定义了image，用来检测屏幕上特定区域是否显示了指定的图案，如果图案存在，说明正确进入了当前状态； 74 | 如果不存在，会触发fail_action的执行。 75 | 76 | ### 示例2—— 77 | 自动归档trello任务。一个典型的trello归档页面如下： 78 | ![trello看板归档](simplerpa/conf/auto_trello/trello.png) 79 | 80 | 下面的脚本，可以帮用户自动归档所有已完成的任务。 81 | 82 | ```yaml 83 | name: "自动归档Trello" 84 | ver: 0.5 85 | #screen_width: 3440 86 | #screen_height: 1440 87 | range: !rect l:0, r:1920, t:0, b:1080 88 | time_scale: 1 89 | states: 90 | - name: "点击获取窗口焦点" 91 | id: 1 92 | transition: 93 | # 点击 94 | action: click(300, 20) 95 | wait: 1.5 96 | to: next 97 | - name: "已完成列表" 98 | id: 2 99 | transition: 100 | # 右击第一个卡片 101 | action: rightclick(1540, 290) 102 | wait: 1 103 | to: next 104 | - name: "右键菜单" 105 | id: 3 106 | find: 107 | image: 108 | snapshot: !rect l:1415, r:1805, t:239, b:609 109 | template: auto_trello/detect_target.png 110 | confidence: 0.8 111 | fail_action: raise_error('找不到归档按钮') 112 | transition: 113 | # 左击归档按钮 114 | action: click(1415 + state.find_result.center_x, 239 + state.find_result.center_y) 115 | wait: 1 116 | to: 2 117 | max_time: 2 118 | ``` 119 | 120 | 121 | ## 配置类 122 | 实际上，每个配置项，都有对应的数据类型定义，SimpleRPA读取配置文件的时候，会通过objtyping把yaml数据转换为对应的类实例。 123 | 124 | 数据类型定义，请参照 [SimpleRPA 类图](docs/simplerpa_class_diagram.md) 125 | 126 | ![plantuml代理生成的SimpleRPA 类图](http://www.plantuml.com/plantuml/proxy?cache=no&src=https://raw.githubusercontent.com/songofhawk/simplerpa/main/docs/simplerpa_class_diagram.md) 127 | 128 | 129 | 本文档开头实例中的配置文件，转换之后的实例关系图如下：[SimpleRPA 示例对象图](docs/simplerpa_sample_object_diagram.md) 130 | 131 | ![plantuml代理生成的SimpleRPA 对象图](http://www.plantuml.com/plantuml/proxy?cache=no&src=https://raw.githubusercontent.com/songofhawk/simplerpa/main/docs/simplerpa_sample_object_diagram.md) 132 | 133 | 134 | ## 待实现 135 | * 更方便的数据读取和采集模型（目前只能基于键盘鼠标操作实现） 136 | * 图形化设计器（会先放出一个辅助截图工具） 137 | * 可扩展的操作（这样就可以自己实现 138 | * 发布到PyPI库，支持pip install 安装 139 | 140 | -------------------------------------------------------------------------------- /test/test_selenium/pexels_download.py: -------------------------------------------------------------------------------- 1 | import os 2 | import re 3 | import time 4 | from option import Option 5 | 6 | import pandas as pd 7 | import wget as wget 8 | from selenium import webdriver 9 | from selenium.webdriver.common.by import By 10 | from selenium.webdriver.support.ui import WebDriverWait 11 | import download 12 | 13 | __author__ = 'Song Hui' # 作者名 14 | 15 | 16 | def get_options_from_command_line(): 17 | import argparse 18 | # Initialize parser 19 | parser = argparse.ArgumentParser() 20 | 21 | # Adding optional argument 22 | parser.add_argument("-o", "--output_dir", help="to set directory of all output files") 23 | parser.add_argument("-r", "--result_file", help="to set result CSV file name") 24 | parser.add_argument("-l", "--limit", help="to set max file count") 25 | 26 | # Read arguments from command line 27 | args = parser.parse_args() 28 | 29 | if args: 30 | print("parsing arguments: {}".format(args)) 31 | return Option(args.output_dir, args.result_file, int(args.limit) if args.limit is not None else None) 32 | 33 | 34 | if __name__ == '__main__': 35 | # 获取命令行参数 36 | option = get_options_from_command_line() 37 | 38 | if not os.path.exists(option.output_dir): 39 | os.mkdir(option.output_dir) 40 | # 设置从url中获取名字的正则表达式 41 | exp_img = re.compile(r'videos/\d+/(.*?)\?') 42 | exp_video = re.compile(r'external/(.*?)\?') 43 | 44 | web_options = webdriver.ChromeOptions() 45 | web_options.add_argument("--enable-javascript") 46 | # web_options.add_argument('--always-authorize-plugins=true') 47 | with webdriver.Chrome(options=web_options) as driver: 48 | # with webdriver.Chrome(chrome_options=options) as driver: 49 | wait = WebDriverWait(driver, 10) 50 | 51 | # 访问首页 52 | driver.get("https://www.pexels.com/videos/") 53 | # 获取所有视频描述节点 54 | results = [] 55 | while len(results) < option.limit: 56 | results = driver.find_elements(By.CSS_SELECTOR, 57 | "div.photos>div.photos__column>div>article") 58 | driver.execute_script("window.scrollTo(0,document.body.scrollHeight)") 59 | time.sleep(5) 60 | 61 | times = 0 62 | # 准备数据表 63 | df = pd.DataFrame(columns=['title', 'img_name', 'video_name']) 64 | for ele in results: 65 | title = ele.get_attribute('data-meta-title') 66 | print(title) 67 | 68 | # 获取图片节点 69 | img_ele = ele.find_element(By.CSS_SELECTOR, 70 | "a.js-photo-link>img.photo-item__img") 71 | img_url = img_ele.get_attribute('src') 72 | # print(img_url) 73 | img_name = re.search(exp_img, img_url).group(1) 74 | print(img_name) 75 | 76 | # 这里是最终是视频节点 77 | source_ele = ele.find_element(By.CSS_SELECTOR, 78 | "a.js-photo-link>video.photo-item__video>source") 79 | 80 | video_url = source_ele.get_attribute('src') 81 | # print(video_url) 82 | 83 | video_name = re.search(exp_video, video_url).group(1) 84 | print(video_name) 85 | 86 | if os.path.exists(option.output_dir + video_name): 87 | # 如果目标路径中，对应的视频文件已经存在，那么跳过该记录 88 | continue 89 | 90 | # 下载对应的图片和视频文件 91 | # 这里之所以用不同方式下载，是因为wget是最简洁稳定的方式，还能显示进度，但它不支持伪装user-agent 92 | # 而图片的目标网站拒绝自动机器人下载 93 | try: 94 | download.urlretrieve(img_url, option.output_dir + img_name) 95 | except Exception as e: 96 | print('下载图片"{}"异常：{}'.format(img_url, e)) 97 | continue 98 | 99 | try: 100 | wget.download(video_url, option.output_dir + video_name) 101 | except Exception as e: 102 | print('下载视频"{}"异常：{}'.format(video_url, e)) 103 | continue 104 | 105 | # 新增1条数据记录 106 | df = df.append({'title': title, 'img_name': img_name, 'video_name': video_name}, ignore_index=True) 107 | # 检查数据上限 108 | times += 1 109 | if times >= option.limit: 110 | break 111 | # # driver.get(ele.get_attribute('src')) 112 | time.sleep(0.5) 113 | # 保存数据文件 114 | with open(option.output_result, 'a', encoding="utf-8", newline='') as f: 115 | # 如果文件存在，则添加数据 116 | df.to_csv(f, header=f.tell() == 0) 117 | -------------------------------------------------------------------------------- /simplerpa/core/detection/WindowDetection.py: -------------------------------------------------------------------------------- 1 | from simplerpa.core.action.ActionSystem import ActionSystem 2 | from simplerpa.core.action.ActionWindow import ActionWindow 3 | from simplerpa.core.data.ScreenRect import ScreenRect 4 | from simplerpa.core.detection.Detection import Detection 5 | 6 | 7 | class WindowResult: 8 | """ 9 | WindowDetection的返回值对象 10 | 11 | Attributes: 12 | hwnd (int): 窗口句柄 13 | """ 14 | hwnd: int 15 | 16 | def __init__(self, hwnd): 17 | self.hwnd = hwnd 18 | 19 | 20 | class WindowDetection(Detection): 21 | """ 22 | 窗口检测，检测在当前环境中，是否有指定的窗口。 23 | 如果current_only设置为True（缺省值），那么只检测当前窗口是否符合title或者win_class配置； 24 | 如果current_only设置为False，那么就在整个桌面中寻找指定title或者win_class的窗口。 25 | 如果结果找到了，就返回一个WindowResult对象，里面含有一个hwnd属性，保存了窗口句柄； 26 | 如果结果没到到，则返回None 27 | 28 | Example: 29 | ```yaml 30 | # State的一个属性 31 | find: 32 | window: 33 | title: 钉钉 34 | debug: True 35 | fail_action: raise_error('没有找到钉钉窗口') 36 | ``` 37 | 或者 38 | ```yaml 39 | # State的一个属性 40 | check: 41 | - window: 42 | win_class: StandardFrame_DingTalk 43 | debug: True 44 | fail_action: locate_state('没有找到钉钉窗口') 45 | ``` 46 | Attributes: 47 | current_only (bool): 是否只检测当前窗口，缺省为True；如果设置为False，则会在桌面所有打开的窗口中查找 48 | win_class (str): 窗口的win_class，可用spy++等工具查看 49 | title (str): 窗口标题，通常是显示在任务栏上的文字 50 | 51 | ```yaml 52 | result: 如果找到了，就返回一个WindowResult对象，如果没有找到，就返回None 53 | ``` 54 | """ 55 | current_only: True 56 | win_class: str = None 57 | title: str = None 58 | activate: bool = False 59 | wait: float = 1 60 | set_pos: ScreenRect = None 61 | 62 | def do_detection(self, find_all=False): 63 | if self.current_only: 64 | check_pass = True 65 | hwnd = ActionWindow.get_current_window() 66 | msg = '' 67 | if self.title: 68 | title = ActionWindow.get_window_title(hwnd) 69 | if title != self.title: 70 | check_pass = check_pass or False 71 | else: 72 | check_pass = check_pass or True 73 | if self.debug: 74 | msg += '实际title:"{}", 预期title:"{}"'.format(title, self.title) 75 | 76 | if self.win_class: 77 | win_class = ActionWindow.get_window_class(hwnd) 78 | if win_class != self.win_class: 79 | check_pass = check_pass or False 80 | else: 81 | check_pass = check_pass or True 82 | if self.debug: 83 | msg += '' if msg == '' else ', ' 84 | msg += '实际class:"{}", 预期class:"{}"'.format(win_class, self.win_class) 85 | 86 | if check_pass: 87 | if self.debug: 88 | print('检查当前窗口成功，{}'.format(msg)) 89 | self._set_position(hwnd) 90 | return WindowResult(hwnd) 91 | else: 92 | if self.debug: 93 | print('检查当前窗口失败，{}'.format(msg)) 94 | return None 95 | else: 96 | hwnd = ActionWindow.find_window(self.title, self.win_class) 97 | if hwnd is None: 98 | if self.debug: 99 | print('检测窗口失败，预期title:"{}", 预期class:"{}'.format(self.title, self.win_class)) 100 | return None 101 | else: 102 | if self.debug: 103 | print('检测窗口成功，预期title:"{}", 预期class:"{}'.format(self.title, self.win_class)) 104 | if self.activate: 105 | ActionWindow.set_current_window(hwnd) 106 | ActionSystem.wait(self.wait) 107 | self._set_position(hwnd) 108 | return WindowResult(hwnd) 109 | 110 | def _set_position(self, hwnd): 111 | pos = self.set_pos 112 | # if isinstance(pos, ScreenRect): 113 | # 这里很奇怪，用isinstance判断总是返回False，但如果debug，pos的类型明明就是ScreenRect 114 | if pos is None: 115 | if self.debug: 116 | print('self.set_pos is None, ignore this operation!') 117 | return 118 | 119 | ActionWindow.set_window_pos(hwnd, 120 | pos.left, 121 | pos.top, 122 | pos.right - pos.left, 123 | pos.bottom - pos.top) 124 | ActionSystem.wait(self.wait) 125 | -------------------------------------------------------------------------------- /test/test_selenium/facebook_download.py: -------------------------------------------------------------------------------- 1 | import os 2 | import re 3 | import time 4 | from option import Option 5 | 6 | import pandas as pd 7 | import wget as wget 8 | from selenium import webdriver 9 | from selenium.webdriver.common.by import By 10 | from selenium.webdriver.support.ui import WebDriverWait 11 | import download 12 | 13 | __author__ = 'Song Hui' # 作者名 14 | 15 | 16 | def get_options_from_command_line(): 17 | import argparse 18 | # Initialize parser 19 | parser = argparse.ArgumentParser() 20 | 21 | # Adding optional argument 22 | parser.add_argument("-o", "--output_dir", help="to set directory of all output files") 23 | parser.add_argument("-r", "--result_file", help="to set result CSV file name") 24 | parser.add_argument("-l", "--limit", help="to set max file count") 25 | 26 | # Read arguments from command line 27 | args = parser.parse_args() 28 | 29 | if args: 30 | print("parsing arguments: {}".format(args)) 31 | return Option(args.output_dir, args.result_file, int(args.limit) if args.limit is not None else None) 32 | 33 | 34 | if __name__ == '__main__': 35 | # 获取命令行参数 36 | option = get_options_from_command_line() 37 | 38 | if not os.path.exists(option.output_dir): 39 | os.mkdir(option.output_dir) 40 | # 设置从url中获取名字的正则表达式 41 | exp_img = re.compile(r'v/.+?/(.*?)\?') 42 | exp_video = re.compile(r'v/.+?/(.*?)\?') 43 | 44 | web_options = webdriver.ChromeOptions() 45 | web_options.add_argument("--enable-javascript") 46 | # web_options.add_argument('--always-authorize-plugins=true') 47 | with webdriver.Chrome(options=web_options) as driver: 48 | # with webdriver.Chrome(chrome_options=options) as driver: 49 | wait = WebDriverWait(driver, 10) 50 | 51 | # 访问首页 52 | driver.get( 53 | "https://www.facebook.com/ads/library/?active_status=active&ad_type=all&country=ALL&q=clothing&sort_data[direction]=desc&sort_data[mode]=relevancy_monthly_grouped&start_date[min]=2021-11-25&start_date[max]=2021-11-26&search_type=keyword_unordered&media_type=video") 54 | # 获取所有视频描述节点 55 | results = [] 56 | while len(results) < option.limit: 57 | results = driver.find_elements(By.CSS_SELECTOR, 58 | "div._99s5") 59 | driver.execute_script("window.scrollTo(0,document.body.scrollHeight)") 60 | time.sleep(5) 61 | 62 | times = 0 63 | # 准备数据表 64 | columns = ['title', 'img_name', 'video_name', 'desc'] 65 | df = pd.DataFrame(columns=columns) 66 | for ele in results: 67 | ele_item = ele.find_element(By.CSS_SELECTOR, "div.iajz466s div._7jyg") 68 | 69 | # 获取标题节点 70 | title_item = ele_item.find_element(By.CSS_SELECTOR, 71 | "div._8nsi a.aa8h9o0m>span.a53abz89") 72 | title = title_item.text 73 | print(title) 74 | 75 | # 获取描述节点 76 | desc_item = ele_item.find_element(By.CSS_SELECTOR, 77 | "div._7jyr>span div._4ik4>div") 78 | desc = desc_item.text 79 | 80 | # 获取视频图片节点 81 | video_item = ele_item.find_element(By.CSS_SELECTOR, 82 | "div._8o0a>video") 83 | img_url = video_item.get_attribute('poster') 84 | # print(img_url) 85 | img_name = re.search(exp_img, img_url).group(1) 86 | print(img_name) 87 | 88 | video_url = video_item.get_attribute('src') 89 | # print(video_url) 90 | 91 | video_name = re.search(exp_video, video_url).group(1) + 'mp4' 92 | # 网站给出的视频文件没有扩展名，这里随便加一个，应该就可以播放了 93 | print(video_name) 94 | 95 | if os.path.exists(option.output_dir + video_name): 96 | # 如果目标路径中，对应的视频文件已经存在，那么跳过该记录 97 | continue 98 | 99 | # 下载对应的图片和视频文件 100 | try: 101 | wget.download(img_url, option.output_dir + img_name) 102 | except Exception as e: 103 | print('下载图片"{}"异常：{}'.format(img_url, e)) 104 | continue 105 | 106 | try: 107 | wget.download(video_url, option.output_dir + video_name) 108 | except Exception as e: 109 | print('下载视频"{}"异常：{}'.format(video_url, e)) 110 | continue 111 | 112 | # 新增1条数据记录 113 | df = df.append({'title': title, 'img_name': img_name, 'video_name': video_name, 'desc': desc}, 114 | ignore_index=True) 115 | # 检查数据上限 116 | times += 1 117 | if times >= option.limit: 118 | break 119 | # # driver.get(ele.get_attribute('src')) 120 | time.sleep(0.5) 121 | # 保存数据文件 122 | with open(option.output_result, 'a', encoding="utf-8", newline='') as f: 123 | # 如果文件存在，则添加数据 124 | df.to_csv(f, header=f.tell() == 0) 125 | -------------------------------------------------------------------------------- /simplerpa/core/extractor/Extractor.py: -------------------------------------------------------------------------------- 1 | from simplerpa.core.action.ActionData import ActionData 2 | from simplerpa.core.action.ActionImage import ActionImage 3 | from simplerpa.core.action.ActionScreen import ActionScreen 4 | from simplerpa.core.data.Action import Evaluation 5 | from simplerpa.core.data.ScreenRect import ScreenRect 6 | from simplerpa.core.data.StateBlockBase import StateBlockBase 7 | from simplerpa.core.data.Transition import Transition 8 | from simplerpa.core.detection.ImageDetection import ImageDetection 9 | 10 | 11 | class PartSplitter(StateBlockBase): 12 | start: ImageDetection = None 13 | end: ImageDetection = None 14 | 15 | def split_parts(self, snapshot, image): 16 | image_current = image 17 | if self.start is None and self.end is None: 18 | print("start and end template are all none in splitter '{}'".format(self.name)) 19 | return 20 | 21 | start_found_list = None 22 | if self.start is not None: 23 | self.start.snapshot = snapshot 24 | start_found_list = self.start.do_detection(image_current) 25 | if start_found_list is None: 26 | print("start template not found in splitter '{}'".format(self.name)) 27 | if start_found_list is None: 28 | return None 29 | start_found_list.sort(key=lambda x: x.rect_on_image.top, reverse=False) 30 | 31 | end_found_list = None 32 | if self.end is not None: 33 | self.end.snapshot = snapshot 34 | end_found_list = self.end.do_detection(image_current) 35 | if end_found_list is None: 36 | print("end template not found in splitter '{}'".format(self.name)) 37 | if end_found_list is None: 38 | return None 39 | end_found_list.sort(key=lambda x: x.rect_on_image.top, reverse=False) 40 | 41 | pre_top = None 42 | pre_bottom = None 43 | parts = [] 44 | if self.start is not None and self.end is None: 45 | for start_found in start_found_list: 46 | start_top = start_found.rect_on_image.top 47 | if pre_top is not None: 48 | part = image[pre_top:start_top, :] 49 | parts.append(part) 50 | pre_top = start_top 51 | elif self.start is None and self.end is not None: 52 | for end_found in end_found_list: 53 | end_bottom = end_found.rect_on_image.bottom 54 | if pre_bottom is not None: 55 | part = image[pre_bottom:end_bottom, :] 56 | parts.append(part) 57 | pre_bottom = end_bottom 58 | else: 59 | start_len = len(start_found_list) 60 | end_len = len(end_found_list) 61 | start_i = 0 62 | end_i = 0 63 | while True: 64 | start_top = start_found_list[start_i].rect_on_image.top 65 | end_bottom = end_found_list[end_i].rect_on_image.bottom 66 | if start_top >= end_bottom: 67 | end_i += 1 68 | continue 69 | part = image[start_top:end_bottom, :] 70 | parts.append(part) 71 | start_i += 1 72 | end_i += 1 73 | if start_i >= start_len or end_i >= end_len: 74 | break 75 | return parts 76 | 77 | 78 | class Extractor(StateBlockBase): 79 | snapshot: ScreenRect = None 80 | part_splitter: PartSplitter = None 81 | file: str = "result.csv" 82 | in_data: Evaluation = None 83 | fail: Transition = None 84 | 85 | def __init__(self): 86 | self.image = None 87 | self.in_data_dict = None 88 | 89 | def prepare(self): 90 | data_dict = self.in_data.call_once() 91 | self.in_data_dict = data_dict 92 | df = ActionData.create_dataframe(data_dict.keys() if data_dict is not None else None) 93 | return df 94 | 95 | def do_once(self, image): 96 | raise RuntimeError("do_once method must bu implemented by sub class of Extractor!") 97 | 98 | def do(self): 99 | if self.snapshot is None: 100 | raise RuntimeError('There should be a "snapshot" attribute in extractor named "{}"!'.format(self.name)) 101 | df = self.prepare() 102 | 103 | screen_image = ActionScreen.snapshot(self.snapshot.evaluate()) 104 | image_source = ActionImage.pil_to_cv(screen_image) 105 | 106 | if self.part_splitter is not None: 107 | image_parts = self.part_splitter.split_parts(self.snapshot, image_source) 108 | if image_parts is None: 109 | return None 110 | for index, part in enumerate(image_parts): 111 | ActionImage.log_image('part-{}'.format(index), part) 112 | data_dict = self.do_once(part) 113 | data_dict.update(self.in_data_dict) 114 | df = df.append(data_dict, ignore_index=True) 115 | else: 116 | data_dict = self.do_once(image_source) 117 | data_dict.update(self.in_data_dict) 118 | df = df.append(data_dict, ignore_index=True) 119 | 120 | with open(self.file, 'a', encoding="utf-8", newline='') as f: 121 | df.to_csv(f, header=f.tell() == 0) 122 | 123 | return df 124 | -------------------------------------------------------------------------------- /simplerpa/core/data/Find.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from typing import List 4 | 5 | import simplerpa.core.data.Transition 6 | from simplerpa.core.action import ActionMouse 7 | from simplerpa.core.data.Action import Execution, Action 8 | from simplerpa.core.detection.ColorDetection import ColorDetection 9 | from simplerpa.core.detection.ImageDetection import ImageDetection 10 | from simplerpa.core.detection.OcrDetection import OcrDetection 11 | from simplerpa.core.detection.WindowDetection import WindowDetection 12 | from simplerpa.core.share import list_util 13 | 14 | from . import Misc 15 | from .StateBlockBase import StateBlockBase 16 | from ..action.ActionSystem import ActionSystem 17 | from ..const import FIND_RESULT 18 | from ..detection.Detection import Detection 19 | 20 | 21 | class Scroll(StateBlockBase): 22 | """ 23 | 在查找（Find）过程中的滚动配置 24 | Attributes: 25 | one_page (int): 每页滚动的距离，单位是虚拟像素（根据屏幕分辨率可能有缩放） 26 | page_count (int): 滚动页数 27 | find_mode (str): 是否要在滚动的过程中，找出所有结果，缺省为"Any"； 28 | 如果为"All"，表示要完成所有滚动，并且在每一页执行detection，保存检测结果； 29 | 如果为"Any"，则只要有一页检测通过，就不再滚动了 30 | """ 31 | one_page: int # 32 | page_count: int # 33 | find_mode: str = "Any" 34 | 35 | 36 | class Repeat(StateBlockBase): 37 | """ 38 | 重复性地执行detections 39 | Attributes: 40 | interval (float): 重复间隔时间，单位为秒 41 | times (int): 重复执行次数，如果等于None则一直重复下去，直到detections找到结果为止 42 | """ 43 | interval: float = 1 44 | times: int = None 45 | 46 | 47 | class Find(StateBlockBase): 48 | """ 49 | 用于查找的基础配置，可以有不同的查找模式，在State节点中，它如果是check属性，则不保存查找结果，如果是find属性，则把查找结果，临时存入find_result 50 | 51 | Attributes: 52 | image (ImageDetection) : 图像检测，在当前页面中找指定图像片段，不一定要完全一致，可以指定相似度 53 | ocr (OcrDetection) : 文本检测，在当前页面的指定位置做OCR识别，然后查看是否有指定的文本 54 | color (ColorDetection) : 颜色检测，在当前页面的指定像素位置，查看是否符合定义的颜色 55 | window (WindowDetection) : 窗口检测，在当前页面查找指定title或者name的窗口 56 | 57 | scroll (Scroll) : 查找的时候，如果没找到，就滚动当前窗口，继续查找 58 | fail_action (Execution) : 如果什么没有找到，需要执行的操作 59 | result_name (str): 给检测结果一个变量名 60 | """ 61 | detections: List[Detection] = None 62 | image: ImageDetection 63 | ocr: OcrDetection 64 | color: ColorDetection 65 | window: WindowDetection 66 | 67 | mode: str = "any" 68 | order: str = "asc" # to support 'desc' and 'rand' 69 | result_name: str = None 70 | foreach: Misc.ForEach = None 71 | fail: simplerpa.core.data.Transition.Transition = None 72 | scroll: Scroll 73 | 74 | repeat: Repeat = None 75 | 76 | def __init__(self): 77 | self._prepared = False 78 | self.detections = [] 79 | self.fail = simplerpa.core.data.Transition.Transition(action_str='raise_error("find failed!")') 80 | 81 | def _prepare(self): 82 | if self._prepared: 83 | return 84 | if self.image is not None: 85 | self.detections.append(self.image) 86 | if self.ocr is not None: 87 | self.detections.append(self.ocr) 88 | if self.color is not None: 89 | self.detections.append(self.color) 90 | if self.window is not None: 91 | self.detections.append(self.window) 92 | self._prepared = True 93 | 94 | def do(self): 95 | self._prepare() 96 | results = [] 97 | 98 | times = 0 99 | max_times = 1 if self.repeat is None else self.repeat.times 100 | while times < max_times: 101 | for detect_one in self.detections: 102 | res_list = self._detect_once(detect_one) 103 | list_util.append_to(results, res_list) 104 | if len(res_list) > 0 and self.mode == "any": 105 | break 106 | if len(results) > 0: 107 | final_result = results if len(results) > 1 else results[0] 108 | Action.save_call_env({FIND_RESULT: final_result}) 109 | if self.result_name is not None: 110 | Action.save_call_env({self.result_name: final_result}) 111 | break 112 | else: 113 | Action.save_call_env({FIND_RESULT: None}) 114 | if self.repeat is not None: 115 | ActionSystem.wait(self.repeat.interval) 116 | 117 | times += 1 118 | return results 119 | 120 | def _detect_once(self, detection): 121 | if detection is None: 122 | return None 123 | results = [] 124 | if self.scroll is None: 125 | detect_res = detection.do() 126 | list_util.append_to(results, detect_res) 127 | else: 128 | scroll = self.scroll 129 | # 有滚动的话，就按滚动页数执行循环 130 | count = scroll.page_count 131 | page = 0 132 | while True: 133 | # 如果滚动的时候，找到即返回，那么就检查detect_res是否为None 134 | # 如果滚动到指定页数，返回所有找到的结果，那么就不用检查detect_res了 135 | detect_res = detection.do() 136 | list_util.append_to(results, detect_res) 137 | if detect_res is not None and scroll.find_mode == "Any": 138 | break 139 | page += 1 140 | if page < count: 141 | # print('before scroll {}'.format(self.scroll.one_page)) 142 | ActionMouse.scroll(self.scroll.one_page) 143 | ActionSystem.wait(1.5) 144 | # print('-- after scroll') 145 | 146 | if self.foreach is not None: 147 | self.foreach.do() 148 | 149 | return results 150 | -------------------------------------------------------------------------------- /simplerpa/objtyping/objtyping.py: -------------------------------------------------------------------------------- 1 | import inspect 2 | from typing import get_type_hints, TypeVar, List 3 | 4 | 5 | class DataObject(object): 6 | pass 7 | 8 | 9 | def get_obj_definition(obj): 10 | types_in_class = get_type_hints(type(obj)) 11 | # 用实例的类型，来覆盖定义的类型，也许是不对的，定义就是定义 12 | # instance_in_obj = obj.__dict__ 13 | # for k, v in instance_in_obj.items(): 14 | # types_in_class[k] = type(v) 15 | 16 | # 这里有个合并两个字典的方法，要求python >= 3.5 17 | # types = {**types_in_class, **types_in_obj} 18 | return types_in_class 19 | 20 | 21 | def has_init_argument(clazz): 22 | signature = inspect.signature(clazz.__init__) 23 | for name, parameter in signature.parameters.items(): 24 | # print(clazz.__name__, name, parameter.default, parameter.annotation, parameter.kind) 25 | if name not in ['self', 'args', 'kwargs'] and parameter.default is inspect.Parameter.empty: 26 | return True 27 | return False 28 | 29 | 30 | def _parse_tuple_str(tuple_str): 31 | striped_str = tuple_str.strip() 32 | if striped_str.startswith('(') and striped_str.endswith(')'): 33 | return eval(striped_str, {"__builtins__": {}}, {}) 34 | else: 35 | raise RuntimeError('需要的项目') 36 | 37 | 38 | T = TypeVar('T') 39 | 40 | 41 | def from_dict_list(dict_list_obj, clazz: T, reserve_extra_attr=True, init_empty_attr=True, reserved_classes=None) -> T: 42 | if clazz is None and not reserve_extra_attr: 43 | return None 44 | 45 | if reserved_classes is not None and clazz in reserved_classes: 46 | return dict_list_obj 47 | 48 | elif isinstance(dict_list_obj, list): 49 | new_list = [] 50 | if clazz is None: 51 | item_type = None 52 | elif hasattr(clazz, '__origin__'): 53 | # __origin__ 是泛型对应的原始类型，比如list或是dict 54 | item_type = clazz.__args__[0] 55 | # __args__ 是泛型参数数组，对于list来说，它只有一个元素，表示了list中保存的是什么类型的数据，如果是dict，它应该有两个参数，分别表示key和value的数据类型 56 | # 这里获取了list中应当保存的数据类型 57 | else: 58 | # 之前考虑预期类型应该和实际类型匹配，也就是'class.__origin__ is list'， 59 | # 但为了更灵活一些，有些节点是既可以是类实例，也可以是该实例组成的数组的，比如check节点 60 | # 所以改成了即使预期定义不是list，这里也按list解析 61 | item_type = clazz 62 | 63 | for item in dict_list_obj: 64 | typed_obj = from_dict_list(item, item_type, reserve_extra_attr, init_empty_attr, reserved_classes) 65 | if typed_obj is not None: 66 | new_list.append(typed_obj) 67 | return new_list 68 | 69 | elif isinstance(dict_list_obj, dict): 70 | if has_init_argument(clazz): 71 | raise TypeError('类 {} 的构造函数需要参数，无法通过dict实例化！\r\n {}'.format(clazz.__name__, dict_list_obj)) 72 | if clazz is None: 73 | obj = DataObject() 74 | types = None 75 | else: 76 | clazz = find_sub_class(dict_list_obj, clazz) 77 | # 如果定义的类包含子类，那么根据dict中的key来推断使用哪个子类 78 | obj = clazz() 79 | types = get_obj_definition(obj) 80 | 81 | for k, v in dict_list_obj.items(): 82 | if types is not None and k in types: 83 | attr_type = types[k] 84 | else: 85 | attr_type = None 86 | typed_obj = from_dict_list(v, attr_type, reserve_extra_attr, init_empty_attr, reserved_classes) 87 | if typed_obj is not None: 88 | setattr(obj, k, typed_obj) 89 | 90 | '''初始化那些在types中存在，dict_list数据中没有属性''' 91 | if init_empty_attr and types is not None: 92 | for k in types.keys(): 93 | if k not in dict_list_obj and not hasattr(obj, k): 94 | # 注意一下，如果有个属性在构造函数中初始化了，它也会在types中存在 95 | setattr(obj, k, None) 96 | 97 | return obj 98 | elif is_basic_type(dict_list_obj): 99 | if clazz is None: 100 | return dict_list_obj 101 | elif type(dict_list_obj) == clazz: 102 | return dict_list_obj 103 | elif hasattr(clazz, '__origin__') and clazz.__origin__ == tuple and isinstance(dict_list_obj, str): 104 | return _parse_tuple_str(dict_list_obj) 105 | else: 106 | # 如果dict_list_obj是个基本类型(比如字符串),但对应的是定义clazz一个类, 那么假设该类的构造函数, 正好接收这个类的参数 107 | return clazz(dict_list_obj) 108 | else: 109 | # raise TypeError('需要转换的对象，是一个出乎意料的类型：{}，\n\r{}'.format(type(yaml_obj), yaml_obj)) 110 | # 对于实现了from_yaml的类，可以直接得到对象实例 111 | return dict_list_obj 112 | 113 | 114 | def is_basic_type(obj): 115 | if isinstance(obj, str) or \ 116 | isinstance(obj, int) or isinstance(obj, float) or isinstance(obj, bool) or isinstance(obj, complex): 117 | return True 118 | else: 119 | return False 120 | 121 | 122 | def to_dict_list(obj): 123 | if isinstance(obj, list): 124 | list1 = [] 125 | for item in obj: 126 | list1.append(to_dict_list(item)) 127 | return list1 128 | elif isinstance(obj, dict): 129 | dict1 = {} 130 | for k, v in obj.items(): 131 | dict1[k] = to_dict_list(v) 132 | elif is_basic_type(obj): 133 | return obj 134 | else: 135 | dict1 = {} 136 | for k, v in obj.__dict__.items(): 137 | dict1[k] = to_dict_list(v) 138 | return dict1 139 | 140 | 141 | def find_sub_class(dict_obj, clazz): 142 | sub_list = clazz.__subclasses__() 143 | if sub_list is None or len(sub_list) == 0: 144 | return clazz 145 | got_the_clazz = True 146 | sub_clazz = None 147 | for sub_clazz in sub_list: 148 | types = get_type_hints(sub_clazz) 149 | got_the_clazz = True 150 | for k in dict_obj.keys(): 151 | if k not in types: 152 | got_the_clazz = False 153 | break 154 | if got_the_clazz: 155 | break 156 | if got_the_clazz: 157 | return sub_clazz 158 | else: 159 | return clazz 160 | -------------------------------------------------------------------------------- /simplerpa/conf/test_rect.yaml: -------------------------------------------------------------------------------- 1 | name: "测试rect" 2 | ver: 0.5 3 | #screen_width: 3440 4 | #screen_height: 1440 5 | range: ScreenRect(3,5,8,9) 6 | #range: !rect l:0, r:1920, t:0, b:1080 7 | time_scale: 1 8 | states: 9 | # - id: 1 10 | # name: "测试find_rect" 11 | # check: 12 | # image: 13 | # snapshot: !rect l:3, r:522, t:169, b:829 14 | # rect: !vector x:519, y:9 15 | # color: (245,245,245) 16 | # debug: True 17 | # detect_all: True 18 | # - id: 2 19 | # name: "测试查找不准确的图片" 20 | # find: 21 | # image: 22 | # snapshot: !rect l:1470, r:1970, t:120, b:1320 23 | # template: auto_wechat/line_in_snapshot.png 24 | # to_binary: 25 | ## background: (255,255,255) 26 | # foreground: (239,239,239) 27 | # tolerance: 0.02 28 | ## grayscale: True 29 | ## confidence: 0.9 30 | ## priority: color 31 | ## color: (232,102,35) 32 | # find_all: True 33 | # debug: True 34 | # auto_scale: (0.6,1.5) 35 | ## detect_all: True 36 | # - id: 3 37 | # name: "测试窗口提取信息" 38 | # form: 39 | # snapshot: !rect l:1470, r:1970, t:120, b:1320 40 | # fields: 41 | # - name: time 42 | # feature: 43 | # template: auto_wechat/icon_time.png 44 | # auto_scale: (0.8,1.5) 45 | # to_binary: 46 | # foreground: (155,155,155) 47 | # tolerance: 0.1 48 | # position: feature_rect.snap_right(330, 6) 49 | # foreground: (155,155,155) 50 | # tolerance: 0.3 51 | # - name: start 52 | # feature: 53 | # template: auto_wechat/icon_start_big.png 54 | # auto_scale: (0.8,1.5) 55 | # to_binary: 56 | # foreground: (64,61,80) 57 | # tolerance: 0.1 58 | # position: feature_rect.snap_right(400) 59 | # foreground: (25,25,25) 60 | # tolerance: 0.1 61 | # - name: end 62 | # feature: 63 | # template: auto_wechat/icon_end_big.png 64 | # auto_scale: (0.8,1.5) 65 | # to_binary: 66 | # foreground: (237,104,44) 67 | # tolerance: 0.2 68 | # position: feature_rect.snap_right(400) 69 | # foreground: (25,25,25) 70 | # tolerance: 0.1 71 | # - name: price 72 | # feature: 73 | # template: auto_wechat/icon_rmb_small.png 74 | # to_binary: 75 | # foreground: (237,104,44) 76 | # tolerance: 0.3 77 | # auto_scale: (0.8,2) 78 | # confidence: 0.65 79 | # debug: True 80 | # debug: True 81 | # position: feature_rect.snap_right(100) 82 | # foreground: (237,104,44) 83 | # tolerance: 0.3 84 | ## debug: True 85 | # 86 | # part_splitter: 87 | # name: "用分割符切割窗口" 88 | # start: 89 | # template: auto_wechat/form_start.png 90 | # to_binary: 91 | # foreground: (153,153,153) 92 | ## background: (255,255,255) 93 | # tolerance: 0.1 94 | # detect_all: True 95 | ## debug: True 96 | # auto_scale: (1,1.5) 97 | # end: 98 | # template: auto_wechat/form_end.png 99 | # to_binary: 100 | # foreground: (239,99,31) 101 | # tolerance: 0.3 102 | # detect_all: True 103 | # auto_scale: (0.8,2) 104 | # confidence: 0.65 105 | # debug: True 106 | 107 | # - id: 4 108 | # name: "测试在二值图像中查找icon_time" 109 | # check: 110 | # image: 111 | # snapshot: !rect l:1443, r:1943, t:669, b:945 112 | # template: auto_wechat/icon_time_bin.png 113 | ## confidence: 0.3 114 | # debug: True 115 | # - id: 5 116 | # name: "找到网约沟通窗口" 117 | # find: 118 | # window: 119 | # win_class: "ChatWnd" 120 | # title: "天助定【网约沟通】" 121 | # fail_action: raise_error('没有找到【网约沟通】！') 122 | # activate: True 123 | # wait: 1 124 | # transition: 125 | # # 点击 126 | # action: 127 | # - chat = get_window_rect(find_result.hwnd) 128 | # - print(chat) 129 | # wait: 1 130 | # - id: 6 131 | # name: "测试监控窗口变化" 132 | # monitor: 133 | # snapshot: ScreenRect(chat.left, chat.right, chat.top+65, chat.top+415) 134 | # interval: 0.3 135 | # time: None 136 | # scroll: up 137 | ## confidence: 0.3 138 | # debug: True 139 | # - id: 10 140 | # name: "测试查找多个图像" 141 | # check: 142 | # detections: 143 | # - snapshot: !rect l:0, r:1600, t:200, b:1400 144 | # template: auto_playvalkyr/veryweak.png 145 | ## debug: True 146 | # - snapshot: !rect l:0, r:1600, t:200, b:1400 147 | # template: auto_playvalkyr/weak.png 148 | # debug: True 149 | # fail: 150 | # action: 151 | # - print('not found') 152 | # to: 12 153 | # transition: 154 | # to: next 155 | # - id: 11 156 | # name: "测试查找多个图像-成功" 157 | # action: 158 | # - print(find_result) 159 | # - print("The End") 160 | # - id: 12 161 | # name: "测试查找多个图像-失败" 162 | # action: 163 | # - print(find_result) 164 | # - print("Failed") 165 | 166 | - id: 1 167 | name: "桌面" 168 | find: 169 | # 查找浏览器PlayValkyr窗口 170 | window: 171 | title: "Play Valkyrio - Google Chrome" 172 | # win_class: 173 | fail_action: raise_error('没有找到Play Valkyrio窗口！') 174 | activate: True 175 | wait: 1 176 | set_pos: !rect l:0, r:1440, t:0, b:1200 177 | transition: 178 | action: 179 | - win_rect = get_window_rect(find_result.hwnd) 180 | - id: 2 181 | name: "测试play界面中的repeat find" 182 | find: 183 | # 查找240分 184 | ocr: 185 | snapshot: win_rect.topleft.offset_rect(263, 270, 28, 13) 186 | to_binary: 187 | forground: (255,255,255) 188 | tolerance: 0.1 189 | text: "111" 190 | # fail_action: raise_error('没找到指定的分数') 191 | debug: True 192 | repeat: 193 | times: 1000 194 | interval: 3 195 | transition: 196 | action: 197 | # - print(found) 198 | to: end 199 | -------------------------------------------------------------------------------- /simplerpa/conf/auto_dingding.yaml: -------------------------------------------------------------------------------- 1 | name: "整理钉钉周报内容到记事本" 2 | ver: 1 3 | #screen_width: 3440 4 | #screen_height: 1440 5 | range: !rect l:0, r:1000, t:0, b:800 6 | time_scale: 1 7 | states: 8 | - name: "当前窗口" 9 | id: 0 10 | transition: 11 | # 显示桌面的热键 12 | action: hotkey('win', 'd') 13 | wait: 1 14 | - name: "桌面" 15 | id: 1 16 | find: 17 | image: 18 | snapshot: !rect l:210, r:316, t:468, b:593 19 | template: auto_dingding/notepad_logo.png 20 | # debug: True 21 | fail_action: raise_error('找不到记事本图标') 22 | transition: 23 | # 双击记事本图标 24 | action: dbclick(find_result.rect_on_screen.center_x, find_result.rect_on_screen.center_y) 25 | wait: 1 26 | 27 | - name: "记事本主界面" 28 | id: 2 29 | # check: 30 | # ocr: 31 | # snapshot: !rect l:1055, r:1097, t:3, b:30 32 | # template: auto_dingding/notepad_title.png 33 | # # ocr本来不需要源图，这里放一个模板，是为了给人一个参照 34 | # text: 记事本 35 | # fail_action: raise_error('没能成功启动记事本') 36 | find: 37 | image: 38 | snapshot: !rect l:213, r:320, t:310, b:440 39 | template: auto_dingding/dingding_logo.png 40 | fail_action: raise_error('找不到钉钉图标') 41 | transition: 42 | # 双击钉钉图标 43 | action: dbclick(find_result.rect_on_screen.center_x, find_result.rect_on_screen.center_y) 44 | wait: 5 45 | 46 | - name: "钉钉主界面" 47 | id: 3 48 | check: 49 | - window: 50 | win_class: StandardFrame_DingTalk 51 | debug: True 52 | fail_action: locate_state('没有找到钉钉窗口') 53 | find: 54 | window: 55 | title: 钉钉 56 | debug: True 57 | fail_action: raise_error('没有找到钉钉窗口') 58 | transition: 59 | action: 60 | - wait(2) 61 | - set_window_pos(find_result.hwnd, 0, 0, 1024, 768) 62 | wait: 3 63 | 64 | - name: "钉钉主界面已放置左上角" 65 | id: 4 66 | check: 67 | - image: 68 | snapshot: !rect l:9, r:58, t:42, b:86 69 | template: auto_dingding/tzding_logo_on_dingding.png 70 | # debug: True 71 | fail_action: raise_error('没能成功启动钉钉,或者当前企业不是天助定') 72 | - color: 73 | pos: (12,92) 74 | color: (209, 211, 213) 75 | debug: True 76 | fail_action: locate_state('当前不在钉钉首页') 77 | find: 78 | image: 79 | snapshot: !rect l:70, r:250, t:76, b:760 80 | template: auto_dingding/work_report.png 81 | # debug: True 82 | scroll: 83 | one_page: -200 # 负数是假设目前看到的是列表顶端，内容向上滚 84 | page_count: 20 85 | fail_action: raise_error('没找到工作汇报消息') 86 | transition: 87 | action: click(find_result.rect_on_screen.center_x, find_result.rect_on_screen.bottom) 88 | wait: 1 89 | 90 | - name: "工作台日志列表" 91 | id: 5 92 | check: 93 | ocr: 94 | # "看日志" 文本 95 | snapshot: !rect l:75, r:137, t:129, b:157 96 | text: "看日志" 97 | # debug: True 98 | fail_action: raise_error('没能成功进入日志列表页面') 99 | find: 100 | # image: 101 | # # "已读" "未读" 标记 作为行标识 102 | # snapshot: !rect l:80, r:120, t:55, b:800 103 | # template: 104 | # - states/3/read.png 105 | # - states/3/unread.png 106 | # strategy: any 107 | image: 108 | # 查找"周报"标记, 作为行标识 109 | snapshot: !rect l:65, r:298, t:221, b:761 110 | template: auto_dingding/work_report_text_snippet.png 111 | confidence: 0.8 112 | keep_clip: result.rect_on_image.snap_left(45) 113 | # debug: True 114 | find_all: True 115 | scroll: 116 | one_page: -800 # 负数是假设目前看到的是列表顶端，内容向上滚 117 | page_count: 1 118 | find_mode: all 119 | fail_action: raise_error('没找到任何人的周报') 120 | foreach: 121 | in_items: find_result 122 | item: week_report 123 | action: 124 | - x = ocr(week_report.clip) 125 | - copy(x) 126 | - hotkey("alt","tab") 127 | - wait(0.5) 128 | - hotkey("ctrl","v") 129 | - wait(0.8) 130 | - hotkey("alt","tab") 131 | - wait(0.5) 132 | # - print(week_report.rect_on_screen) 133 | - snapshot_rect = ScreenRect(65, 298, 221, 761) 134 | - snapshot_image = snapshot(snapshot_rect) 135 | # - print(snapshot_image.shape) 136 | # - print(week_report.clip.shape) 137 | # - log_image("snapshot", snapshot_image) 138 | # - log_image("clip", week_report.clip) 139 | - clip_rect = find_template(snapshot_image, week_report.clip).rect 140 | - clip_on_screen = clip_rect.offset_from(snapshot_rect) 141 | # - print(x) 142 | # - print(clip_on_screen.center_x, clip_on_screen.center_y) 143 | - click(clip_on_screen.center_x, clip_on_screen.center_y) 144 | - wait(1) 145 | sub_states: 146 | - name: "某人的日报内容" 147 | id: 40 148 | transition: 149 | wait_before: 2 150 | action: rightclick(500, 400) 151 | wait: 0.3 152 | - name: "某人的日报内容上覆盖右键菜单" 153 | id: 41 154 | find: 155 | image: 156 | snapshot: !rect l:mouse.x, r:mouse.x + 200, t:mouse.y, b:mouse.y + 170 157 | template: auto_dingding/context_select_all.png 158 | # debug: True 159 | fail_action: raise_error('没找到右键菜单中的全选项') 160 | transition: 161 | action: click(find_result.rect_on_screen.center_x, find_result.rect_on_screen.center_y) 162 | wait: 0.2 163 | - name: "某人的日报内容全选" 164 | id: 42 165 | action: 166 | - hotkey("ctrl","c") 167 | transition: 168 | wait_before: 1.5 169 | action: hotkey("alt","tab") 170 | wait: 1.5 171 | - name: "记事本在最前" 172 | id: 43 173 | action: 174 | - print("粘贴 {} 的日志信息".format(x)) 175 | - type('--------------') 176 | - hotkey("ctrl","v") 177 | - hotkey("enter") 178 | - hotkey("enter") 179 | - hotkey("enter") 180 | transition: 181 | # 最后切换回钉钉在最前, 这里如果不放心, 可以加一个状态检查 182 | wait_before: 1.5 183 | action: hotkey("alt","tab") 184 | wait: 1.5 185 | 186 | -------------------------------------------------------------------------------- /simplerpa/core/data/Action.py: -------------------------------------------------------------------------------- 1 | from __future__ import annotations 2 | 3 | from simplerpa.core.action.ActionSystem import ActionSystem 4 | from .StateBlockBase import StateBlockBase 5 | from simplerpa.core.action import ActionMouse 6 | from simplerpa.core.action.ActionClipboard import ActionClipboard 7 | from simplerpa.core.action.ActionError import ActionError 8 | from simplerpa.core.action.ActionData import ActionData 9 | from simplerpa.core.action.ActionImage import ActionImage 10 | from simplerpa.core.action.ActionKeyboard import ActionKeyboard 11 | from simplerpa.core.action.ActionScreen import ActionScreen 12 | from simplerpa.core.action.ActionWindow import ActionWindow 13 | from simplerpa.core.const import MOUSE 14 | from simplerpa.core.data.ScreenRect import ScreenRect 15 | 16 | 17 | class Action(StateBlockBase): 18 | """ 19 | 执行操作基础类，有Evaluation和Execution两个子类，分别对应有返回结果和没有返回结果的操作。在本基础类中，还定义了所有具体的操作方法名，这些方法可以在Action类型的配置节点中直接调用。具体的参数会在对应的类中介绍 20 | 21 | Attributes: 22 | move (ActionMouse.move): 移动鼠标，例如 move(354,267) 23 | raise_error (ActionError.trigger): 引发异常，例如raise_error('钉钉不是当前页面') 24 | click (ActionMouse.click): 点击鼠标，例如 click(354,267)，click执行的时候，鼠标指针也会被移动到指定位置 25 | dbclick (ActionMouse.dbclick): 双击鼠标，例如 dbclick(354,267) 26 | rightclick (ActionMouse.rightclick): 右键点击鼠标，例如 rightclick(354,267) 27 | hotkey (ActionKeyboard.hotkey): 点击键盘热键，例如 hotkey("alt","tab") 28 | type (ActionKeyboard.type): 输入字符串（模拟键盘输入，参数是整个字符串），例如 type("张三") 29 | press (ActionKeyboard.press): 点击键盘（指定一个键），例如 press("a"), press("ctrl", 2) 30 | ocr (ActionImage.ocr): 识别给定图片中的文字，例如 x = ocr(week_report.clip)，这里week_report.clip是上一步截图得到的图像数据，x是识别出来的文字 31 | print (print): 打印信息到控制台，就是python内置的print方法 32 | find_template (ActionImage.find_one_template): 在指定图像中，查找另外一幅图像，例如 find_template(snapshot_image, week_report.clip).rect，这里要从图像snapshot_image中找到图像week_report.clip， 返回结果里有有个rect属性，标识了找到的片段在屏幕上的位置 33 | snapshot (ActionScreen.snapshot_cv): 屏幕截图，例如 snapshot(snapshot_rect)，传入一个表示屏幕矩形位置的ScreenRect对象，返回对应位置的图像 34 | log_image (ActionImage.log_image): 保存图片到文件，例如 log_image("snapshot", snapshot_image)，把图像保存到指定文件 35 | ScreenRect (ScreenRect): 新建一个ScreenRect对象，例如 snapshot_rect = ScreenRect(65, 298, 221, 761)，这个对象接受4个参数，分别是left, right, top, bottom 36 | wait (time.sleep): 等待指定时间（秒），例如 wait(0.5)，执行到这里会等待（休眠）指定的秒数 37 | copy (ActionClipboard.copy): 复制，例如 copy(x)，把指定的字符串复制到剪贴板 38 | paste (ActionClipboard.paste): 粘贴，例如 x = paste(), 把剪贴板里的内容变成字符串返回（注意并不是在界面当前位置粘贴，如果需要这个效果，可以调用 hotkey("ctrl","v") ） 39 | locate_state (ActionError.locate_state): 定位当前处于哪个State，例如 locate_state('没有找到钉钉窗口')，此时程序会中止正常的状态迁移，而是遍历当前状态层级，查找有可能处于哪个状态，并跳转到那个状态继续执行；参数字符串会输出到控制台， 40 | set_window_pos (ActionWindow.set_window_pos): 设置窗口的位置和大小，例如 set_window_pos(find_result.hwnd, 0, 0, 1024, 768)，这里的hwnd通常是WindowDetection返回的窗口句柄，通过这个函数，指定窗口的top,left,width,height坐标 41 | 42 | """ 43 | func_dict = { 44 | 'move': ActionMouse.move, 45 | 'raise_error': ActionError.trigger, 46 | 'click': ActionMouse.click, 47 | 'dbclick': ActionMouse.dbclick, 48 | 'rightclick': ActionMouse.rightclick, 49 | 'hotkey': ActionKeyboard.hotkey, 50 | 'type': ActionKeyboard.type, 51 | 'press': ActionKeyboard.press, 52 | 'ocr': ActionImage.ocr, 53 | 'print': print, 54 | 'find_template': ActionImage.find_all_template, 55 | 'snapshot': ActionScreen.snapshot_cv, 56 | 'snapshot_pil': ActionScreen.snapshot_pil, 57 | 'log_image': ActionImage.log_image, 58 | 'ScreenRect': ScreenRect, 59 | 'wait': ActionSystem.wait, 60 | 'copy': ActionClipboard.copy, 61 | 'paste': ActionClipboard.paste, 62 | 'locate_state': ActionError.locate_state, 63 | 'level_return': ActionError.level_return, 64 | 'set_window_pos': ActionWindow.set_window_pos, 65 | 'get_window_rect': ActionWindow.get_window_rect, 66 | 'set_current_window': ActionWindow.set_current_window, 67 | 'create_dataframe': ActionData.create_dataframe, 68 | } 69 | 70 | _call_env = {**func_dict, **{}} 71 | 72 | test = '5' 73 | 74 | def __init__(self, action_str): 75 | self._action_str = action_str 76 | if action_str.startswith('locate_state'): 77 | self.is_flow_control = True 78 | self.is_locate_state = True 79 | self.is_level_return = False 80 | elif action_str.startswith('level_return'): 81 | self.is_flow_control = True 82 | self.is_locate_state = False 83 | self.is_level_return = True 84 | else: 85 | self.is_flow_control = False 86 | self.is_locate_state = False 87 | self.is_level_return = False 88 | 89 | 90 | def call_once(self, call_env=None): 91 | if call_env is not None: 92 | if isinstance(call_env, dict): 93 | self.save_call_env(call_env) 94 | else: 95 | raise RuntimeError( 96 | 'The argument in "call" method of "Action" object, should be a dict, but {} is passed'.format( 97 | type(call_env))) 98 | return self.evaluate_exp() 99 | 100 | def evaluate_exp(self): 101 | raise RuntimeError('_evaluate_exp is an abstract method, can not be called in an Action object!') 102 | 103 | def _prepare_exp(self): 104 | self._call_env[MOUSE] = ActionMouse.position() 105 | 106 | @classmethod 107 | def save_call_env(cls, call_env): 108 | cls._call_env.update(call_env) 109 | 110 | @classmethod 111 | def get_call_env(cls, name): 112 | return cls._call_env[name] 113 | 114 | @classmethod 115 | def call(cls, actions: Action, call_env=None): 116 | if actions is None: 117 | return None 118 | if isinstance(actions, list): 119 | results = [] 120 | for action in actions: 121 | result = action.call_once(call_env) 122 | results.append(result) 123 | return results 124 | elif isinstance(actions, Action): 125 | return actions.call_once(call_env) 126 | 127 | 128 | class Evaluation(Action): 129 | """ 130 | Action操作类的子类，执行以后会有返回值 131 | """ 132 | 133 | def __init__(self, action_str): 134 | super().__init__(action_str) 135 | self.exp = compile(action_str, '', 'eval') 136 | 137 | def evaluate_exp(self): 138 | super()._prepare_exp() 139 | return eval(self.exp, {"__builtins__": {}}, self._call_env) 140 | 141 | 142 | class Execution(Action): 143 | """ 144 | Action操作类的子类，执行以后没有返回值 145 | """ 146 | 147 | def __init__(self, action_str): 148 | super().__init__(action_str) 149 | self.exp = compile(action_str, '', 'exec') 150 | 151 | def evaluate_exp(self): 152 | super()._prepare_exp() 153 | return exec(self.exp, {"__builtins__": {}}, self._call_env) 154 | -------------------------------------------------------------------------------- /test/find_template/find_all_template.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | import cv2 4 | 5 | from top_k import top_k 6 | 7 | 8 | def _to_gray(image): 9 | channel = 1 if len(image.shape) == 2 else image.shape[2] 10 | if channel == 1: 11 | # if the image is gray, then keep it 12 | image_gray = image 13 | elif channel == 3: 14 | # if it's colorful, then convert it to gray 15 | image_gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) 16 | elif channel == 4: 17 | # if it's colorful with transparent channel, then convert it to gray 18 | image_gray = cv2.cvtColor(image, cv2.COLOR_BGRA2GRAY) 19 | else: 20 | raise RuntimeError('im_search have {} channel, which is unexpected!'.format(channel)) 21 | return image_gray 22 | 23 | 24 | def find_all_template_v1(im_source, im_template, threshold=0.5, maxcnt=0, edge=False, debug=False): 25 | """ 26 | 用 cv2.templateFind 方法, 在im_source中查找im_search的匹配位置，源图和 27 | 28 | Use pixel match to find pictures. 29 | 30 | Args: 31 | im_source(string): 图像、素材 32 | im_template(string): 需要查找的图片 33 | threshold: 阈值，当匹配度小于该阈值的时候，就忽略掉 34 | maxcnt: 最大查找数量, 缺省为0, 即不限 35 | edge: 是否做边缘提取后再匹配 36 | 37 | Returns: 38 | A tuple of found [(point, score), ...] 39 | 40 | Raises: 41 | IOError: when file read error 42 | """ 43 | 44 | if debug: 45 | start_time = time.time() 46 | gray_template = _to_gray(im_template) 47 | gray_source = _to_gray(im_source) 48 | if debug: 49 | print("to_gray time: {}".format(time.time() - start_time)) 50 | 51 | # 边界提取(来实现背景去除的功能) 52 | if edge: 53 | if debug: 54 | start_time = time.time() 55 | gray_template = cv2.Canny(gray_template, 100, 200) 56 | gray_source = cv2.Canny(gray_source, 100, 200) 57 | if debug: 58 | print("Canny time: {}".format(time.time() - start_time)) 59 | 60 | if debug: 61 | start_time = time.time() 62 | res = cv2.matchTemplate(gray_source, gray_template, cv2.TM_CCOEFF_NORMED) 63 | if debug: 64 | print("matchTemplate time: {}".format(time.time() - start_time)) 65 | 66 | if debug: 67 | start_time = time.time() 68 | w, h = im_template.shape[1], im_template.shape[0] 69 | sw, sh = im_source.shape[1], im_source.shape[0] 70 | 71 | result = [] 72 | while True: 73 | min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(res) 74 | top_left = max_loc 75 | if max_val < threshold: 76 | break 77 | 78 | left = top_left[0] 79 | top = top_left[1] 80 | middle_point = (left + w / 2, top + h / 2) 81 | result.append(dict( 82 | result=middle_point, 83 | rectangle=(top_left, (left, top + h), (left + w, top), 84 | (left + w, top + h)), 85 | confidence=max_val 86 | )) 87 | if maxcnt and len(result) >= maxcnt: 88 | break 89 | # 用最小值填充当前结果的周边区域，避免下次找到重叠的结果 90 | x1 = left - w + 1 if left - w + 1 > 0 else 0 91 | x2 = left + w - 1 if left + w - 1 < sw else sw 92 | y1 = top - h + 1 if top - h + 1 > 0 else 0 93 | y2 = top + h - 1 if top + h - 1 < sh else sh 94 | res[y1:y2, x1:x2] = -1000 95 | 96 | if debug: 97 | print("find max time: {}".format(time.time() - start_time)) 98 | 99 | return result 100 | 101 | 102 | def find_all_template_v2(im_source, im_search, threshold=0.5, maxcnt=100, rgb=False, bgremove=False): 103 | ''' 104 | Locate image position with cv2.templateFind 105 | 106 | Use pixel match to find pictures. 107 | 108 | Args: 109 | im_source(string): 图像、素材 110 | im_search(string): 需要查找的图片 111 | threshold: 阈值，当相识度小于该阈值的时候，就忽略掉 112 | 113 | Returns: 114 | A tuple of found [(point, score), ...] 115 | 116 | Raises: 117 | IOError: when file read error 118 | ''' 119 | # method = cv2.TM_CCORR_NORMED 120 | # method = cv2.TM_SQDIFF_NORMED 121 | method = cv2.TM_CCOEFF_NORMED 122 | 123 | if rgb: 124 | s_bgr = cv2.split(im_search) # Blue Green Red 125 | i_bgr = cv2.split(im_source) 126 | weight = (0.3, 0.3, 0.4) 127 | resbgr = [0, 0, 0] 128 | for i in range(3): # bgr 129 | resbgr[i] = cv2.matchTemplate(i_bgr[i], s_bgr[i], method) 130 | res = resbgr[0] * weight[0] + resbgr[1] * weight[1] + resbgr[2] * weight[2] 131 | else: 132 | start_time = time.time() 133 | channel = 1 if len(im_search.shape) == 2 else im_search.shape[2] 134 | if channel == 1: 135 | # if the image is gray, then keep it 136 | s_gray = im_search 137 | elif channel == 3: 138 | # if it's colorful, then convert it to gray 139 | s_gray = cv2.cvtColor(im_search, cv2.COLOR_BGR2GRAY) 140 | elif channel == 4: 141 | # if it's colorful with transparent channel, then convert it to gray 142 | s_gray = cv2.cvtColor(im_search, cv2.COLOR_BGRA2GRAY) 143 | else: 144 | raise RuntimeError('im_search have {} channel, which is unexpected!'.format(channel)) 145 | 146 | channel = 1 if len(im_source.shape) == 2 else im_source.shape[2] 147 | if channel == 1: 148 | # if the image is gray, then keep it 149 | i_gray = im_source 150 | elif channel == 3: 151 | # if it's colorful, then convert it to gray 152 | i_gray = cv2.cvtColor(im_source, cv2.COLOR_BGR2GRAY) 153 | elif channel == 4: 154 | # if it's colorful with transparent channel, then convert it to gray 155 | i_gray = cv2.cvtColor(im_source, cv2.COLOR_BGRA2GRAY) 156 | else: 157 | raise RuntimeError('im_source have {} channel, which is unexpected!'.format(channel)) 158 | print("cvtColor time: {}".format(time.time() - start_time)) 159 | 160 | # 边界提取(来实现背景去除的功能) 161 | if bgremove: 162 | s_gray = cv2.Canny(s_gray, 100, 200) 163 | i_gray = cv2.Canny(i_gray, 100, 200) 164 | 165 | start_time = time.time() 166 | res = cv2.matchTemplate(i_gray, s_gray, method) 167 | print("matchTemplate time: {}".format(time.time() - start_time)) 168 | 169 | w, h = im_search.shape[1], im_search.shape[0] 170 | 171 | start_time = time.time() 172 | 173 | rows, cols = top_k(res, maxcnt) 174 | result = [] 175 | # flatten = rows.flatten() 176 | for index in range(rows.size): 177 | row = rows[index] 178 | col = cols[index] 179 | val = res[row][col] 180 | if val < threshold: 181 | break 182 | middle_point = (col + h / 2, row + w / 2) 183 | result.append(dict( 184 | result=middle_point, 185 | rectangle=((col, row), (col, row + h), (col + w, row), 186 | (col + w, row + h)), 187 | confidence=val 188 | )) 189 | if maxcnt and len(result) >= maxcnt: 190 | break 191 | # floodfill the already found area 192 | print("find max time: {}".format(time.time() - start_time)) 193 | 194 | return result 195 | -------------------------------------------------------------------------------- /simplerpa/core/Executor.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | from simplerpa.core.Variable import Variable 4 | from simplerpa.core.action.ActionSystem import ActionSystem 5 | from simplerpa.core.data.Action import Action 6 | from simplerpa.core.action.ActionScreen import ActionScreen 7 | from simplerpa.core.const import STATE 8 | from simplerpa.core.data.Misc import ForEach 9 | from simplerpa.core.data.Project import Project 10 | from simplerpa.core.extractor.Extractor import Extractor 11 | 12 | 13 | class Env(object): 14 | @classmethod 15 | def adjust_resolution(cls, screen_width, screen_height): 16 | ActionScreen.change_resolution((screen_width, screen_height)) 17 | 18 | 19 | class Executor: 20 | project: Project = None 21 | variables: Variable = Variable() 22 | 23 | def __init__(self, project: Project): 24 | self.project = project 25 | project.executor = self 26 | self._current_state = None 27 | self._current_index = None 28 | self._current_states = None 29 | 30 | # 用列表实现栈 31 | self._currents = [] 32 | 33 | def run(self): 34 | if self.project is None: 35 | return 36 | 37 | self._load_env() 38 | states = self.project.states 39 | if states is None or len(states) == 0: 40 | return 41 | self._current_states = states 42 | init_state = states[0] 43 | self._current_index = 0 44 | self._current_state = init_state 45 | while self._current_state is not None: 46 | self._one_state(self._current_state) 47 | 48 | def _load_env(self): 49 | if self.project.screen_width is not None and \ 50 | self.project.screen_height is not None: 51 | Env.adjust_resolution(self.project.screen_width, self.project.screen_height) 52 | 53 | # def _run_from_state(self, init_state): 54 | # self._one_state(init_state) 55 | 56 | def _one_state(self, the_state): 57 | print('enter state "{}"-{}'.format(the_state.name, the_state.id)) 58 | Action.save_call_env({STATE: the_state}) 59 | 60 | # check中如果触发了影响流程的fail_action，那么就退出当前状态，直接按照该action指定的状态迁移 61 | if not self._do_find(the_state.check): 62 | fail_trans = the_state.check.fail 63 | self._do_transition(fail_trans) 64 | return 65 | 66 | # monitor 67 | if not self._do_monitor(the_state.monitor): 68 | raise RuntimeError('monitor failed in state "{}"'.format(the_state.name)) 69 | 70 | Action.call(the_state.action) 71 | 72 | # 暂时find先不执行影响流程的fail_action（is_flow_control==True) 73 | found = self._do_find(the_state.find) 74 | if found: 75 | transition = the_state.transition 76 | form = the_state.form 77 | if isinstance(form, Extractor): 78 | if form.do() is None: 79 | fail_trans = form.fail 80 | self._do_transition(fail_trans) 81 | return 82 | foreach = the_state.foreach 83 | if isinstance(foreach, ForEach): 84 | foreach.do() 85 | else: 86 | transition = the_state.find.fail 87 | 88 | self._do_transition(transition) 89 | 90 | def _do_transition(self, transition): 91 | '''从这里开始，处理transition模块''' 92 | if transition is None: 93 | # 改成transition为空的话，什么也不做 94 | return 95 | if transition.reach_max_time(): 96 | print('reach max time({}) of transition, terminate!'.format(transition.max_time)) 97 | self._current_state = None 98 | self._current_index = None 99 | return 100 | 101 | if transition.wait_before is not None: 102 | time.sleep(transition.wait_before) 103 | 104 | # 首先调用触发transition的动作 105 | Action.call(transition.action) 106 | 107 | if transition.wait is not None: 108 | ActionSystem.wait(transition.wait) 109 | 110 | # 如果有子状态集，进入首选子状态 111 | # sub_states = transition.sub_states 112 | # if sub_states is not None and len(sub_states) > 0: 113 | # self.drill_into_substates(sub_states) 114 | 115 | # 迁移到下一个状态 116 | transition.count() 117 | trans_to = transition.to 118 | next_state, next_index = self._get_next_state(trans_to) 119 | if next_state is None: 120 | self._current_state = None 121 | self._current_index = None 122 | return 123 | 124 | self._current_index = next_index 125 | self._current_state = next_state 126 | 127 | def drill_into_substates(self, sub_states): 128 | self.push_currents() 129 | 130 | self._current_index = 0 131 | sub_init_state = sub_states[0] 132 | self._current_states = sub_states 133 | self._current_state = sub_init_state 134 | while self._current_state is not None: 135 | self._one_state(self._current_state) 136 | 137 | self.pop_currents() 138 | 139 | def _get_next_state(self, trans_to): 140 | if trans_to.is_next: 141 | if self._current_index < len(self._current_states) - 1: 142 | next_index = self._current_index + 1 143 | next_state = self._current_states[next_index] 144 | else: 145 | next_state = None 146 | next_index = None 147 | elif trans_to.id is not None: 148 | next_state, next_index = self._find_state_by_id(self._current_states, trans_to.id) 149 | else: 150 | next_state = None 151 | next_index = None 152 | return next_state, next_index 153 | 154 | @staticmethod 155 | def _find_state_by_id(current_states, state_id): 156 | for i, state in enumerate(current_states): 157 | if state.id == state_id: 158 | return state, i 159 | return None, None 160 | 161 | def push_currents(self): 162 | self._currents.append((self._current_state, self._current_index, self._current_states)) 163 | 164 | def pop_currents(self): 165 | (self._current_state, self._current_index, self._current_states) = self._currents.pop() 166 | 167 | @staticmethod 168 | def _do_find(find): 169 | """ 170 | check的目标是检查页面是否中的处于当前状态，所以只要有一条不符，就失败。此时如果fail_action是locate_state，就会从当前状态组中逐一检查处于哪个状态，并且直接跳转到对应的状态执行 171 | :param find: 172 | :return: 173 | """ 174 | if find is None: 175 | return True 176 | 177 | results = find.do() 178 | if len(results) > 0: 179 | return True 180 | 181 | return False 182 | 183 | @staticmethod 184 | def _do_monitor(monitor): 185 | if monitor is None: 186 | return True 187 | 188 | change = monitor.do() 189 | if change is None: 190 | return False 191 | else: 192 | return True 193 | 194 | def _locate_state(self): 195 | # 执行定位状态 196 | print("即将定位当前界面处于那个状态...") 197 | for one_state in self._current_states[::-1]: 198 | # 逆序遍历状态列表,也可以用reversed方法 199 | print("-- 测试是否属于状态'{}'".format(one_state.name)) 200 | locate_check = one_state.check 201 | if locate_check is None: 202 | continue 203 | check_pass, _ = self._get_find_result(locate_check, False) 204 | if check_pass: 205 | print("---- 找到了,就是这个状态!") 206 | self._current_state = one_state 207 | self._current_index = one_state.id 208 | return False 209 | self._current_state = None 210 | self._current_index = None 211 | return False 212 | -------------------------------------------------------------------------------- /simplerpa/conf/auto_wechat.yaml: -------------------------------------------------------------------------------- 1 | name: "微信获取数据" 2 | ver: 0.1 3 | #screen_width: 3440 4 | #screen_height: 1440 5 | range: !rect l:0, r:1920, t:0, b:1080 6 | time_scale: 1 7 | states: 8 | - name: "微信聊天窗口" 9 | id: 1 10 | find: 11 | window: 12 | win_class: "ChatWnd" 13 | title: "天助定【网约沟通】" 14 | fail_action: raise_error('没有找到微信！') 15 | activate: True 16 | wait: 1 17 | transition: 18 | # 点击 19 | action: 20 | - wechat = get_window_rect(find_result.hwnd) 21 | - print(wechat) 22 | wait: 1 23 | - name: "内容上沿" 24 | id: 3 25 | find: 26 | image: 27 | template: auto_wechat/wangyue_title.png 28 | snapshot: wechat 29 | fail_action: raise_error('没找到网约沟通标题') 30 | transition: 31 | action: 32 | - content_top = find_result.rect_on_screen.bottom 33 | - name: "内容下沿" 34 | id: 4 35 | find: 36 | image: 37 | template: auto_wechat/chat_toolbar.png 38 | snapshot: wechat 39 | fail_action: raise_error('没找到聊天工具栏') 40 | transition: 41 | action: 42 | - content_bottom = find_result.rect_on_screen.top 43 | - content_left = find_result.rect_on_screen.left 44 | - content = ScreenRect(content_left, content_left+430, content_top, content_bottom) 45 | - name: "内容监控" 46 | id: 5 47 | monitor: 48 | snapshot: content 49 | interval: 2 50 | # times: 100 51 | scroll: up 52 | threshold: 0.01 53 | # debug: True 54 | actin: 55 | - print('content:{}'.format(content)) 56 | - print('position:{}'.format(monitor_result.position)) 57 | find: 58 | image: 59 | template: auto_wechat/chat_head.png 60 | snapshot: ScreenRect(content.left, content.left+70, content.top+monitor_result.position if monitor_result.position is not None else content.top, content.bottom) 61 | to_binary: 62 | background: (242,242,242) 63 | tolerance: 0.1 64 | confidence: 0.5 65 | # debug: True 66 | detect_all: True 67 | # scroll: 68 | # one_page: 800 # 负数是假设目前看到的是列表顶端，内容向上滚 69 | # page_count: 20 70 | # fail_action: raise_error('没找到人物头像') 71 | fail: 72 | action: 73 | - print('没找到头像, 返回监控') 74 | to: 5 75 | foreach: 76 | in_items: find_result 77 | item: head 78 | action: 79 | - head_rect = head.rect_on_screen 80 | sub_states: 81 | - id: 60 82 | name: 1条聊天记录 83 | check: 84 | image: 85 | snapshot: head_rect.snap_right(30, 5) 86 | template: auto_wechat/angle.png 87 | for_not_exist: True 88 | # debug: True 89 | fail: 90 | to: end 91 | transition: 92 | to: next 93 | - id: 62 94 | name: 缩略图 95 | check: 96 | image: 97 | snapshot: head_rect.topright.offset_rect(0,0, 90, 180) 98 | template: auto_wechat/snapshot_thumbnail.png 99 | confidence: 0.6 100 | # debug: True 101 | fail: 102 | to: end 103 | transition: 104 | action: 105 | - data_to_form = {} 106 | - data_to_form['name'] = ocr(snapshot(head_rect.topright.offset_rect(0,-5, 120, 17))) 107 | - print('data_to_form:{}'.format(data_to_form)) 108 | - click(head_rect.bottomright.offset(45,80)) 109 | - wait(1) 110 | - id: 63 111 | name: 检测图片窗口 112 | # check: 113 | # image: 114 | # snapshot: ScreenRect.center_expand(500,1200) 115 | # template: auto_wechat/completed.png 116 | # confidence: 0.5 117 | # auto_scale: (0.5,2) 118 | ## debug: True 119 | # result_name: complete_text 120 | find: 121 | window: 122 | win_class: "ImagePreviewWnd" 123 | title: "图片查看" 124 | wait: 1 125 | fail: 126 | action: print('没找到图片查看窗口') 127 | to: end 128 | transition: 129 | # 点击 130 | action: 131 | - print('找到图片查看窗口，hwnd：{}'.format(find_result.hwnd)) 132 | - form_rect = get_window_rect(find_result.hwnd) 133 | - print(wechat) 134 | - move(form_rect.center) 135 | - print('data_to_form:{}'.format(data_to_form)) 136 | - wait(0.5) 137 | wait: 1 138 | - id: 64 139 | name: "提取form信息" 140 | action: 141 | - print('data_to_form:{}'.format(data_to_form)) 142 | form: 143 | file: "result.csv" 144 | snapshot: form_rect 145 | in_data: data_to_form 146 | part_splitter: 147 | name: "用分割符切割窗口" 148 | start: 149 | template: auto_wechat/form_start.png 150 | to_binary: 151 | foreground: (153,153,153) 152 | # background: (255,255,255) 153 | tolerance: 0.15 154 | detect_all: True 155 | # debug: True 156 | auto_scale: (1,1.5) 157 | end: 158 | template: auto_wechat/form_end.png 159 | to_binary: 160 | foreground: (239,99,31) 161 | tolerance: 0.3 162 | detect_all: True 163 | auto_scale: (0.8,2) 164 | confidence: 0.6 165 | # debug: True 166 | fail: 167 | action: 168 | - print('提取失败') 169 | - hotkey('alt','f4') 170 | - wait(0.5) 171 | to: end 172 | 173 | fields: 174 | - name: time 175 | feature: 176 | template: auto_wechat/icon_time.png 177 | auto_scale: (0.8,1.5) 178 | to_binary: 179 | foreground: (155,155,155) 180 | tolerance: 0.1 181 | position: feature_rect.snap_right(330, 6) 182 | foreground: (155,155,155) 183 | tolerance: 0.3 184 | - name: start 185 | feature: 186 | template: auto_wechat/icon_start_big.png 187 | auto_scale: (0.8,1.5) 188 | # debug: True 189 | to_binary: 190 | foreground: (64,61,80) 191 | tolerance: 0.1 192 | confidence: 0.7 193 | position: feature_rect.snap_right(400) 194 | foreground: (25,25,25) 195 | tolerance: 0.1 196 | # debug: True 197 | - name: end 198 | feature: 199 | template: auto_wechat/icon_end_big.png 200 | auto_scale: (0.8,1.5) 201 | to_binary: 202 | foreground: (237,104,44) 203 | tolerance: 0.2 204 | confidence: 0.65 205 | position: feature_rect.snap_right(400) 206 | foreground: (25,25,25) 207 | tolerance: 0.1 208 | - name: price 209 | feature: 210 | template: auto_wechat/icon_rmb_small.png 211 | to_binary: 212 | foreground: (237,104,44) 213 | tolerance: 0.3 214 | auto_scale: (1,2) 215 | confidence: 0.6 216 | # debug: True 217 | # debug: True 218 | position: feature_rect.snap_right(100) 219 | foreground: (237,104,44) 220 | tolerance: 0.3 221 | # debug: True 222 | transition: 223 | action: 224 | - hotkey('alt','f4') 225 | - wait(0.5) 226 | - print("----完成一组form识别---") 227 | transition: 228 | to: 5 229 | 230 | -------------------------------------------------------------------------------- /simplerpa/core/detection/ImageDetection.py: -------------------------------------------------------------------------------- 1 | from typing import Tuple 2 | 3 | import PIL 4 | import cv2 5 | import numpy as np 6 | from simplerpa.core.data.StateBlockBase import StateBlockBase 7 | from simplerpa.core.action.ActionImage import ActionImage 8 | from simplerpa.core.action.ActionScreen import ActionScreen 9 | from simplerpa.core.data import Action 10 | from simplerpa.core.data.ScreenRect import ScreenRect, Vector 11 | from simplerpa.core.detection.Detection import Detection, DetectResult 12 | 13 | 14 | class ImageDetectResult(DetectResult): 15 | """ 16 | Attributes: 17 | rect_on_image: 结果图像在指定区域内的相对位置 18 | rect_on_screen: 结果图像在屏幕上的位置 19 | image: 结果图像 20 | clip: 额外截取的图像片段 21 | clip_on_image: clip图像在指定区域内的相对位置 22 | clip_on_screen: clip图像在屏幕上的位置 23 | scale: 在什么缩放比例下，得到的当前检测结果 24 | """ 25 | rect_on_image: ScreenRect = None 26 | rect_on_screen: ScreenRect = None 27 | image = None 28 | clip = None 29 | clip_on_image: ScreenRect = None 30 | clip_on_screen: ScreenRect = None 31 | scale: float = None 32 | priority: float = None 33 | 34 | 35 | class ToBinary(StateBlockBase): 36 | background: Tuple[int, int, int] = None 37 | foreground: Tuple[int, int, int] = None 38 | tolerance: float = 0.01 39 | 40 | def __init__(self): 41 | pass 42 | 43 | def convert(self, image): 44 | # bk_bgr = np.array([self.background[2], self.background[1], self.background[0]]) 45 | return ActionImage.to_binary(image, self.foreground, self.background, self.tolerance) 46 | 47 | 48 | class ImageDetection(Detection): 49 | """ 50 | 图像检测，在当前页面的指定范围内，查找匹配的图像。如果找到了，会返回一个result结构， 51 | 52 | Example: 53 | ```yaml 54 | # State的一个属性 55 | image: 56 | snapshot: !rect l:65, r:298, t:221, b:761 57 | template: auto_dingding/work_report_text_snippet.png 58 | confidence: 0.8 59 | keep_clip: result.rect_on_image.snap_left(45) 60 | ``` 61 | 62 | Attributes: 63 | snapshot (ScreenRect): 屏幕截图位置，限定查找范围，可以指定得大一些，程序会在指定范围内查找图像 64 | template (str): 要查找的图像模板文件（支持相对路径） 65 | rect (Vector): 和color是一组，表示要检测一个纯色块，Vector里的x表示宽，y表示高 66 | color (Tuple[int,int,int]): 和rect是一组，表示要检测一个纯色块，color按照(r,g,b)的顺序指定颜色 67 | confidence (float): 置信度，查找图像的时候，并不需要严格一致，程序会模糊匹配，并返回一个置信度（0 ~ 1之间的一个数值），置信度大于给定的数值，才会认为找到了 68 | keep_clip (Action.Evaluation): 根据返回结果，额外保持一个截图片段（比如已经查到图像左侧100个像素的区域） 69 | 70 | """ 71 | template: str 72 | rect: Vector 73 | color: Tuple[int, int, int] = (0, 0, 0) 74 | snapshot: ScreenRect 75 | confidence: float = 0.8 76 | keep_clip: Action.Evaluation 77 | auto_scale: Tuple[float, float] = None 78 | scale: Action.Evaluation = 1 79 | priority: str = None 80 | grayscale: bool = False 81 | to_binary: ToBinary = None 82 | 83 | def do_detection(self, source_image=None): 84 | if source_image is None: 85 | snapshot_image = ActionScreen.snapshot(self.snapshot.evaluate()) 86 | source_image = ActionImage.pil_to_cv(snapshot_image) 87 | 88 | if self.scale is not None: 89 | scale = self.scale.evaluate_exp() if isinstance(self.scale, Action.Evaluation) else self.scale 90 | else: 91 | scale = 1 92 | 93 | if self.template is not None: 94 | res = self.image_in(self._get_template_full_path(), source_image, self.confidence, self.auto_scale, 95 | scale) 96 | elif self.rect is not None: 97 | res = self.rect_in(self.rect, self.color, source_image, self.confidence) 98 | else: 99 | raise RuntimeError( 100 | "ImageDetection should has either template or rect property, but all are None: {}".format(self)) 101 | 102 | if res is None: 103 | return None 104 | elif isinstance(res, list): 105 | results = [] 106 | for one in res: 107 | result = self._gen_result(one, source_image) 108 | results.append(result) 109 | return results 110 | else: 111 | return self._gen_result(res, source_image) # rect是相对偏移量 112 | 113 | def _gen_result(self, res, screen_image): 114 | result = ImageDetectResult() 115 | result.rect_on_image = res.rect 116 | result.rect_on_screen = res.rect.offset_from(self.snapshot) if self.snapshot is not None else None 117 | result.image = screen_image 118 | result.scale = res.scale 119 | self.get_clip(result) 120 | return result 121 | 122 | def image_in(self, template_file_path, big_image, min_confidence, auto_scale, scale): 123 | """ 124 | 检查两幅图是否相似 125 | :param min_confidence: 最低可信度, 不足这个可信度的结果将被忽略 126 | :param template_file_path: 要查找的图文件路径位置 127 | :param big_image: 大图 128 | :param auto_scale: 自动缩放模板图，来寻找匹配 129 | :param scale: 指定要缩放模板图的比例 130 | :return:相似度，完全相同是1，完全不同是0 131 | 目标图像需要是pillow格式的，将在函数中被转换为opencv格式的，最后用aircv的find_template方法比较是否相似 132 | """ 133 | if isinstance(big_image, PIL.Image.Image): 134 | image_current = ActionImage.pil_to_cv(big_image) 135 | else: 136 | image_current = big_image 137 | 138 | print('image detection: \r\n\tsnapshot:{}, template:{}'.format(self.snapshot, self.template)) 139 | 140 | if self.grayscale: 141 | image_current = ActionImage.to_grayscale(image_current, high_contrast=True, keep3channel=True) 142 | if self.to_binary is not None: 143 | image_current = self.to_binary.convert(image_current) 144 | ActionImage.log_image('current', image_current, debug=self.debug) 145 | 146 | image_template = ActionImage.load_from_file(template_file_path) 147 | if image_template.shape[0] > image_current.shape[0] or image_template.shape[1] > image_current.shape[1]: 148 | # 如果源图比模板图还小，那肯定找不到了，直接返回空 149 | return None 150 | 151 | if self.grayscale: 152 | image_template = ActionImage.to_grayscale(image_template, high_contrast=True, keep3channel=True) 153 | if self.to_binary is not None: 154 | image_template = self.to_binary.convert(image_template) 155 | ActionImage.log_image('template', image_template, debug=self.debug) 156 | 157 | result_list = ActionImage.find_all_template(image_current, image_template, min_confidence, auto_scale, scale) 158 | if self.priority is not None: 159 | for result in result_list: 160 | if self.priority == "color": 161 | rect = result.rect 162 | sim = ActionImage.get_color_sim(image_current, self.color, rect.center) 163 | result.priority = sim 164 | result_list.sort(key=lambda x: x.priority, reverse=True) 165 | if self.debug: 166 | if result_list is None: 167 | print('image detection result_list: found None') 168 | else: 169 | size = len(result_list) 170 | print('image detection result_list: found {}'.format(size)) 171 | img_result = image_current.copy() 172 | for index, result in enumerate(result_list): 173 | rect = result.rect 174 | print('result-{}: confidence-{}, scale-{}, priority-{}， {}'.format(index, 175 | result.confidence if result is not None else None, 176 | result.scale, 177 | result.priority if hasattr( 178 | result, 179 | 'priority') else None, 180 | rect if result is not None else None)) 181 | cv2.rectangle(img_result, (rect.left, rect.top), (rect.right, rect.bottom), (0, 0, 220), 2) 182 | ActionImage.log_image('result', img_result, debug=self.debug) 183 | 184 | if result_list is None or len(result_list) == 0: 185 | return None 186 | elif self.detect_all: 187 | return result_list 188 | else: 189 | return result_list[0] 190 | 191 | def rect_in(self, rect, color, big_image, min_confidence): 192 | print('image detection: \r\n\tsnapshot:{}, rect:{}， color:{}'.format(self.snapshot, self.rect, self.color)) 193 | result_list = ActionImage.find_rect(big_image, rect, color, find_all=self.detect_all, debug=False) 194 | 195 | if self.debug: 196 | size = len(result_list) 197 | print('image detection result_list: found {}'.format(size)) 198 | for index, result in enumerate(result_list): 199 | res_rect = result.rect 200 | print('result-{}: top-{}, left-{}'.format(index, 201 | res_rect.top if result is not None else None, 202 | res_rect.left if result is not None else None)) 203 | cv2.rectangle(big_image, (res_rect.left, res_rect.top), (res_rect.right, res_rect.bottom), 204 | (0, 0, 220), 2) 205 | ActionImage.log_image('result', big_image, debug=self.debug) 206 | return result_list 207 | 208 | def get_clip(self, res): 209 | if self.keep_clip is None: 210 | return None 211 | call_env: dict = {'result': res} 212 | rect = self.keep_clip.call_once(call_env) 213 | image = res.image 214 | clip = image[rect.top:rect.bottom, rect.left:rect.right] 215 | 216 | res.clip = clip 217 | res.clip_on_image = rect 218 | res.clip_on_screen = res.clip_on_image.offset_from(self.snapshot) 219 | 220 | return clip 221 | 222 | def _get_template_full_path(self): 223 | if self._template_full_path is not None: 224 | pass 225 | elif self.project.path_root is not None: 226 | self._template_full_path = self.project.path_root + '/' + self.template 227 | else: 228 | self._template_full_path = self.template 229 | 230 | return self._template_full_path 231 | -------------------------------------------------------------------------------- /simplerpa/core/data/ScreenRect.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | from ruamel.yaml import yaml_object 4 | 5 | from simplerpa.core.action import ActionWindow 6 | from simplerpa.core.data import Action 7 | from simplerpa.core.share.yaml import yaml 8 | 9 | 10 | @yaml_object(yaml) 11 | class Vector(object): 12 | yaml_tag = u'!vector' 13 | 14 | def __init__(self, x, y): 15 | self.x = x 16 | self.y = y 17 | 18 | def offset_rect(self, x, y, width, height): 19 | """ 20 | 按照x,y偏移左上角位置， 21 | 按照width, height改变矩形框大小 22 | """ 23 | left = self.x + x 24 | top = self.y + y 25 | right = left + width 26 | bottom = top + height 27 | return ScreenRect(left, right, top, bottom) 28 | 29 | def offset(self, x, y): 30 | """ 31 | 大小不变，位置偏移一段距离 32 | """ 33 | return Vector(self.x + x, self.y + y) 34 | 35 | @classmethod 36 | def to_yaml(cls, representer, node): 37 | return representer.represent_scalar(cls.yaml_tag, 38 | 'x:{}, y:{}'.format(node.x, node.y)) 39 | 40 | @classmethod 41 | def from_yaml(cls, constructor, node): 42 | result = re.search(r"x:(\S*)\s*,\s*y:(\S*)\s*", node.value) 43 | if result is None: 44 | raise RuntimeError('ScreenRect string is not formatted well: ""!'.format(node.value)) 45 | else: 46 | v = result.groups() 47 | v = list(map(int, v)) 48 | # splits = node.value.split(', ') 49 | # test = list(map(lambda x: x + '_sss', splits)) 50 | # v = list(map(lambda x: int(x[1]) if x[1].isdigit() else x[1], map(methodcaller("split", ":"), splits))) 51 | # print(v) 52 | return cls(x=v[0], y=v[1]) 53 | 54 | 55 | @yaml_object(yaml) 56 | class ScreenRect(object): 57 | yaml_tag = u'!rect' 58 | snapshot = None 59 | 60 | def __init__(self, left=None, right=None, top=None, bottom=None): 61 | # super(ScreenRect, self).__init__([left, right, top, bottom]) 62 | # self._inner_list = [left, right, top, bottom] 63 | self.has_exp = False 64 | self.exp = None 65 | if isinstance(left, str): 66 | self.has_exp = True 67 | self.left = None 68 | if right is None and top is None and bottom is None: 69 | # 如果只有第一个参数，其他参数都没有传，说明整个字符串，是一个ScreenRect表达式 70 | self.exp = Action.Evaluation(left) 71 | self._exp_str = left 72 | else: 73 | self.left_exp: Action.Evaluation = Action.Evaluation(left) 74 | else: 75 | self.left = left 76 | self.left_exp = None 77 | 78 | if isinstance(right, str): 79 | self.has_exp = True 80 | self.right_exp: Action.Evaluation = Action.Evaluation(right) 81 | self.right = None 82 | else: 83 | self.right = right 84 | self.right_exp = None 85 | 86 | if isinstance(top, str): 87 | self.has_exp = True 88 | self.top_exp: Action.Evaluation = Action.Evaluation(top) 89 | self.top = None 90 | else: 91 | self.top = top 92 | self.top_exp = None 93 | 94 | if isinstance(bottom, str): 95 | self.has_exp = True 96 | self.bottom_exp: Action.Evaluation = Action.Evaluation(bottom) 97 | self.bottom = None 98 | else: 99 | self.bottom = bottom 100 | self.bottom_exp = None 101 | 102 | self.center_x = 0 103 | self.center_y = 0 104 | if not self.has_exp: 105 | self._compute() 106 | 107 | def _compute(self): 108 | self.center_x = (self.left + self.right) // 2 109 | self.center_y = (self.top + self.bottom) // 2 110 | self.center = Vector(self.center_x, self.center_y) 111 | self.width = self.right - self.left 112 | self.height = self.bottom - self.top 113 | self.area = self.width * self.height 114 | 115 | def evaluate(self): 116 | if not self.has_exp: 117 | return self 118 | 119 | if self.exp is not None: 120 | temp_rect = self.exp.evaluate_exp() 121 | self.left = temp_rect.left 122 | self.right = temp_rect.right 123 | self.top = temp_rect.top 124 | self.bottom = temp_rect.bottom 125 | self._compute() 126 | return self 127 | 128 | if self.left_exp is not None: 129 | self.left = self.left_exp.evaluate_exp() 130 | if self.right_exp is not None: 131 | self.right = self.right_exp.evaluate_exp() 132 | if self.top_exp is not None: 133 | self.top = self.top_exp.evaluate_exp() 134 | if self.bottom_exp is not None: 135 | self.bottom = self.bottom_exp.evaluate_exp() 136 | 137 | self._compute() 138 | return self 139 | 140 | def swap_top_bottom(self): 141 | temp_top = self.top 142 | screen_width, screen_height = ActionWindow.ActionWindow.get_screen_resolution() 143 | top = screen_height - temp_top if temp_top < screen_height else 0 144 | bottom = screen_height - self.bottom if self.bottom < screen_height else 0 145 | return ScreenRect(self.left, self.right, top, bottom) 146 | 147 | @classmethod 148 | def center_expand(cls, width, height): 149 | screen_width, screen_height = ActionWindow.ActionWindow.get_screen_resolution() 150 | left = (screen_width - width) / 2 151 | right = screen_width - left 152 | top = (screen_height - height) / 2 153 | bottom = screen_height - top 154 | return ScreenRect(left, right, top, bottom) 155 | 156 | def __str__(self): 157 | return 'l:{}, r:{}, t:{}, b:{}'.format(self.left, self.right, self.top, self.bottom) 158 | 159 | @classmethod 160 | def to_yaml(cls, representer, node): 161 | return representer.represent_scalar(cls.yaml_tag, 162 | 'l:{}, r:{}, t:{}, b:{}'.format(node.left, node.right, node.top, 163 | node.bottom)) 164 | # return {'l': self.left, 'r': self.right, 't': self.top, 'b': self.bottom} 165 | 166 | @classmethod 167 | def from_yaml(cls, constructor, node): 168 | result = re.search(r"l:(\S*)\s*,\s*r:(\S*)\s*,\s*t:(\S*)\s*,\s*b:(\S*)", node.value) 169 | if result is None: 170 | raise RuntimeError('ScreenRect string is not formatted well: ""!'.format(node.value)) 171 | else: 172 | v = result.groups() 173 | v = list(map(int, v)) 174 | # splits = node.value.split(', ') 175 | # test = list(map(lambda x: x + '_sss', splits)) 176 | # v = list(map(lambda x: int(x[1]) if x[1].isdigit() else x[1], map(methodcaller("split", ":"), splits))) 177 | # print(v) 178 | return cls(left=v[0], right=v[1], top=v[2], bottom=v[3]) 179 | 180 | def offset_from(self, other_rect): 181 | if (isinstance(self.left, float) or isinstance(self.left, int)) \ 182 | and (isinstance(other_rect.left, float) or isinstance(other_rect.left, int)): 183 | left = self.left + other_rect.left 184 | else: 185 | raise RuntimeError("left must be an integer or float number for '+' operator") 186 | if (isinstance(self.top, float) or isinstance(self.top, int)) \ 187 | and (isinstance(other_rect.top, float) or isinstance(other_rect.top, int)): 188 | top = self.top + other_rect.top 189 | else: 190 | raise RuntimeError("top must be an integer or float number for '+' operator") 191 | 192 | if (isinstance(self.right, float) or isinstance(self.right, int)) \ 193 | and (isinstance(other_rect.left, float) or isinstance(other_rect.left, int)): 194 | right = self.right + other_rect.left 195 | else: 196 | raise RuntimeError("right must be an integer or float number for '+' operator") 197 | if (isinstance(self.bottom, float) or isinstance(self.bottom, int)) \ 198 | and (isinstance(other_rect.top, float) or isinstance(other_rect.top, int)): 199 | bottom = self.bottom + other_rect.top 200 | else: 201 | raise RuntimeError("bottom must be an integer or float number for '+' operator") 202 | 203 | return ScreenRect(left, right, top, bottom) 204 | 205 | def snap_left(self, width, offset_y=0): 206 | return ScreenRect(self.left - width, self.left, self.top - offset_y, self.bottom + offset_y) 207 | 208 | def snap_right(self, width, offset_y=0): 209 | return ScreenRect(self.right, self.right + width, self.top - offset_y, self.bottom + offset_y) 210 | 211 | def snap_top(self, height, offset_x=0): 212 | return ScreenRect(self.left - offset_x, self.right + offset_x, self.top - height, self.top) 213 | 214 | def snap_bottom(self, height, offset_x=0): 215 | return ScreenRect(self.left - offset_x, self.right + offset_x, self.bottom, self.bottom + height) 216 | 217 | def expand_left(self, width, height=None): 218 | if height is not None: 219 | offset_y = (height - (self.bottom - self.top)) / 2 220 | else: 221 | offset_y = 0 222 | 223 | return ScreenRect(self.left - width, self.right, self.top - offset_y, self.bottom + offset_y) 224 | 225 | def expand_right(self, width, height=None): 226 | if height is not None: 227 | offset_y = (height - (self.bottom - self.top)) / 2 228 | else: 229 | offset_y = 0 230 | 231 | return ScreenRect(self.left, self.right + width, self.top - offset_y, self.bottom + offset_y) 232 | 233 | def expand_top(self, height, width=None): 234 | if width is not None: 235 | offset_x = (width - (self.right - self.left)) / 2 236 | else: 237 | offset_x = 0 238 | 239 | return ScreenRect(self.left - offset_x, self.right + offset_x, self.top - height, self.bottom) 240 | 241 | def expand_bottom(self, height, width=None): 242 | if width is not None: 243 | offset_x = (width - (self.right - self.left)) / 2 244 | else: 245 | offset_x = 0 246 | 247 | return ScreenRect(self.left - offset_x, self.right + offset_x, self.top, self.bottom + height) 248 | 249 | @property 250 | def topleft(self): 251 | return Vector(self.left, self.top) 252 | 253 | @property 254 | def topright(self): 255 | return Vector(self.right, self.top) 256 | 257 | @property 258 | def bottomleft(self): 259 | return Vector(self.left, self.bottom) 260 | 261 | @property 262 | def bottomright(self): 263 | return Vector(self.right, self.bottom) 264 | --------------------------------------------------------------------------------