├── .gitignore ├── LICENSE ├── README.md ├── datasets ├── README.md └── new │ ├── Adobe │ ├── 1.jpg │ ├── 1.json │ ├── 6.jpg │ └── 6.json │ └── Bitget1 │ ├── 1.jpg │ ├── 1.json │ ├── 3.jpg │ └── 3.json ├── environment.yml ├── experiments ├── rq1_screen_inconsistency │ ├── .env.example │ ├── main.py │ ├── mutate │ │ ├── __init__.py │ │ ├── deletion.py │ │ ├── insertion.py │ │ ├── substitution.py │ │ └── utils.py │ └── utils.py ├── rq2_flow_inconsistency │ ├── .env.example │ ├── README.md │ ├── action_completion.user.prompt │ ├── actions │ │ ├── __init__.py │ │ ├── automator.py │ │ ├── recorder.py │ │ └── translator.py │ ├── main.py │ └── utils.py ├── rq3_component_wise_evaluation │ └── README.md └── rq4_case_study │ ├── .env.example │ ├── action_completion.user.prompt │ ├── actions │ ├── __init__.py │ └── translator.py │ ├── main.py │ └── utils.py ├── guipilot ├── __init__.py ├── agent │ ├── __init__.py │ ├── action_completion.system.prompt │ └── agent.py ├── checker │ ├── __init__.py │ ├── checker.py │ ├── guipilot.py │ └── gvt.py ├── entities │ ├── __init__.py │ ├── constants.py │ ├── process.py │ ├── screen.py │ └── widget.py ├── matcher │ ├── __init__.py │ ├── guipilotv2.py │ ├── gvt.py │ ├── matcher.py │ └── utils.py └── models │ ├── __init__.py │ ├── detector │ ├── detector.py │ └── yolo │ │ ├── README.md │ │ ├── __init__.py │ │ ├── dataset.yaml │ │ ├── main.py │ │ └── shapes.py │ └── ocr │ └── ocr.py └── setup.py /.gitignore: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/GUIPilot/HEAD/.gitignore -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/GUIPilot/HEAD/LICENSE -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/GUIPilot/HEAD/README.md -------------------------------------------------------------------------------- /datasets/README.md: -------------------------------------------------------------------------------- 1 | # Datasets 2 | 3 | -------------------------------------------------------------------------------- /datasets/new/Adobe/1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/GUIPilot/HEAD/datasets/new/Adobe/1.jpg -------------------------------------------------------------------------------- /datasets/new/Adobe/1.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/GUIPilot/HEAD/datasets/new/Adobe/1.json -------------------------------------------------------------------------------- /datasets/new/Adobe/6.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/GUIPilot/HEAD/datasets/new/Adobe/6.jpg -------------------------------------------------------------------------------- /datasets/new/Adobe/6.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/GUIPilot/HEAD/datasets/new/Adobe/6.json -------------------------------------------------------------------------------- /datasets/new/Bitget1/1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/GUIPilot/HEAD/datasets/new/Bitget1/1.jpg -------------------------------------------------------------------------------- /datasets/new/Bitget1/1.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/GUIPilot/HEAD/datasets/new/Bitget1/1.json -------------------------------------------------------------------------------- /datasets/new/Bitget1/3.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/GUIPilot/HEAD/datasets/new/Bitget1/3.jpg -------------------------------------------------------------------------------- /datasets/new/Bitget1/3.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/GUIPilot/HEAD/datasets/new/Bitget1/3.json -------------------------------------------------------------------------------- /environment.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/GUIPilot/HEAD/environment.yml -------------------------------------------------------------------------------- /experiments/rq1_screen_inconsistency/.env.example: -------------------------------------------------------------------------------- 1 | DATASET_PATH= -------------------------------------------------------------------------------- /experiments/rq1_screen_inconsistency/main.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/GUIPilot/HEAD/experiments/rq1_screen_inconsistency/main.py -------------------------------------------------------------------------------- /experiments/rq1_screen_inconsistency/mutate/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/GUIPilot/HEAD/experiments/rq1_screen_inconsistency/mutate/__init__.py -------------------------------------------------------------------------------- /experiments/rq1_screen_inconsistency/mutate/deletion.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/GUIPilot/HEAD/experiments/rq1_screen_inconsistency/mutate/deletion.py -------------------------------------------------------------------------------- /experiments/rq1_screen_inconsistency/mutate/insertion.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/GUIPilot/HEAD/experiments/rq1_screen_inconsistency/mutate/insertion.py -------------------------------------------------------------------------------- /experiments/rq1_screen_inconsistency/mutate/substitution.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/GUIPilot/HEAD/experiments/rq1_screen_inconsistency/mutate/substitution.py -------------------------------------------------------------------------------- /experiments/rq1_screen_inconsistency/mutate/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/GUIPilot/HEAD/experiments/rq1_screen_inconsistency/mutate/utils.py -------------------------------------------------------------------------------- /experiments/rq1_screen_inconsistency/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/GUIPilot/HEAD/experiments/rq1_screen_inconsistency/utils.py -------------------------------------------------------------------------------- /experiments/rq2_flow_inconsistency/.env.example: -------------------------------------------------------------------------------- 1 | OPENAI_KEY= 2 | DATASET_PATH= -------------------------------------------------------------------------------- /experiments/rq2_flow_inconsistency/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/GUIPilot/HEAD/experiments/rq2_flow_inconsistency/README.md -------------------------------------------------------------------------------- /experiments/rq2_flow_inconsistency/action_completion.user.prompt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/GUIPilot/HEAD/experiments/rq2_flow_inconsistency/action_completion.user.prompt -------------------------------------------------------------------------------- /experiments/rq2_flow_inconsistency/actions/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/GUIPilot/HEAD/experiments/rq2_flow_inconsistency/actions/__init__.py -------------------------------------------------------------------------------- /experiments/rq2_flow_inconsistency/actions/automator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/GUIPilot/HEAD/experiments/rq2_flow_inconsistency/actions/automator.py -------------------------------------------------------------------------------- /experiments/rq2_flow_inconsistency/actions/recorder.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/GUIPilot/HEAD/experiments/rq2_flow_inconsistency/actions/recorder.py -------------------------------------------------------------------------------- /experiments/rq2_flow_inconsistency/actions/translator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/GUIPilot/HEAD/experiments/rq2_flow_inconsistency/actions/translator.py -------------------------------------------------------------------------------- /experiments/rq2_flow_inconsistency/main.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/GUIPilot/HEAD/experiments/rq2_flow_inconsistency/main.py -------------------------------------------------------------------------------- /experiments/rq2_flow_inconsistency/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/GUIPilot/HEAD/experiments/rq2_flow_inconsistency/utils.py -------------------------------------------------------------------------------- /experiments/rq3_component_wise_evaluation/README.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /experiments/rq4_case_study/.env.example: -------------------------------------------------------------------------------- 1 | OPENAI_KEY= 2 | DATASET_PATH= -------------------------------------------------------------------------------- /experiments/rq4_case_study/action_completion.user.prompt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/GUIPilot/HEAD/experiments/rq4_case_study/action_completion.user.prompt -------------------------------------------------------------------------------- /experiments/rq4_case_study/actions/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/GUIPilot/HEAD/experiments/rq4_case_study/actions/__init__.py -------------------------------------------------------------------------------- /experiments/rq4_case_study/actions/translator.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/GUIPilot/HEAD/experiments/rq4_case_study/actions/translator.py -------------------------------------------------------------------------------- /experiments/rq4_case_study/main.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/GUIPilot/HEAD/experiments/rq4_case_study/main.py -------------------------------------------------------------------------------- /experiments/rq4_case_study/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/GUIPilot/HEAD/experiments/rq4_case_study/utils.py -------------------------------------------------------------------------------- /guipilot/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /guipilot/agent/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/GUIPilot/HEAD/guipilot/agent/__init__.py -------------------------------------------------------------------------------- /guipilot/agent/action_completion.system.prompt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/GUIPilot/HEAD/guipilot/agent/action_completion.system.prompt -------------------------------------------------------------------------------- /guipilot/agent/agent.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/GUIPilot/HEAD/guipilot/agent/agent.py -------------------------------------------------------------------------------- /guipilot/checker/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/GUIPilot/HEAD/guipilot/checker/__init__.py -------------------------------------------------------------------------------- /guipilot/checker/checker.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/GUIPilot/HEAD/guipilot/checker/checker.py -------------------------------------------------------------------------------- /guipilot/checker/guipilot.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/GUIPilot/HEAD/guipilot/checker/guipilot.py -------------------------------------------------------------------------------- /guipilot/checker/gvt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/GUIPilot/HEAD/guipilot/checker/gvt.py -------------------------------------------------------------------------------- /guipilot/entities/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/GUIPilot/HEAD/guipilot/entities/__init__.py -------------------------------------------------------------------------------- /guipilot/entities/constants.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/GUIPilot/HEAD/guipilot/entities/constants.py -------------------------------------------------------------------------------- /guipilot/entities/process.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/GUIPilot/HEAD/guipilot/entities/process.py -------------------------------------------------------------------------------- /guipilot/entities/screen.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/GUIPilot/HEAD/guipilot/entities/screen.py -------------------------------------------------------------------------------- /guipilot/entities/widget.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/GUIPilot/HEAD/guipilot/entities/widget.py -------------------------------------------------------------------------------- /guipilot/matcher/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/GUIPilot/HEAD/guipilot/matcher/__init__.py -------------------------------------------------------------------------------- /guipilot/matcher/guipilotv2.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/GUIPilot/HEAD/guipilot/matcher/guipilotv2.py -------------------------------------------------------------------------------- /guipilot/matcher/gvt.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/GUIPilot/HEAD/guipilot/matcher/gvt.py -------------------------------------------------------------------------------- /guipilot/matcher/matcher.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/GUIPilot/HEAD/guipilot/matcher/matcher.py -------------------------------------------------------------------------------- /guipilot/matcher/utils.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/GUIPilot/HEAD/guipilot/matcher/utils.py -------------------------------------------------------------------------------- /guipilot/models/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/GUIPilot/HEAD/guipilot/models/__init__.py -------------------------------------------------------------------------------- /guipilot/models/detector/detector.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/GUIPilot/HEAD/guipilot/models/detector/detector.py -------------------------------------------------------------------------------- /guipilot/models/detector/yolo/README.md: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/GUIPilot/HEAD/guipilot/models/detector/yolo/README.md -------------------------------------------------------------------------------- /guipilot/models/detector/yolo/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /guipilot/models/detector/yolo/dataset.yaml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/GUIPilot/HEAD/guipilot/models/detector/yolo/dataset.yaml -------------------------------------------------------------------------------- /guipilot/models/detector/yolo/main.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/GUIPilot/HEAD/guipilot/models/detector/yolo/main.py -------------------------------------------------------------------------------- /guipilot/models/detector/yolo/shapes.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/GUIPilot/HEAD/guipilot/models/detector/yolo/shapes.py -------------------------------------------------------------------------------- /guipilot/models/ocr/ocr.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/GUIPilot/HEAD/guipilot/models/ocr/ocr.py -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/code-philia/GUIPilot/HEAD/setup.py --------------------------------------------------------------------------------